From 99ad846fff9c3206cac87abb6e2aad2086c62935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Poniedzia=C5=82ek?= Date: Wed, 14 Feb 2024 09:31:53 +0100 Subject: [PATCH 1/3] Upgrade scala to 2.13 --- .../registry/apirequest/Input.scala | 4 +- .../common/utils/JsonPath.scala | 69 +++++++++++-------- .../registry/apirequest/InputSpec.scala | 3 +- .../utils/JsonPathSpec.scala | 9 ++- project/BuildSettings.scala | 2 +- project/Dependencies.scala | 3 - 6 files changed, 48 insertions(+), 42 deletions(-) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Input.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Input.scala index 851c5a34c..9ed656b5c 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Input.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Input.scala @@ -16,7 +16,7 @@ import cats.data.ValidatedNel import cats.implicits._ import io.circe.{Json => JSON, DecodingFailure, Decoder} -import io.gatling.jsonpath.{JsonPath => GatlingJsonPath} +import com.jayway.jsonpath.{JsonPath => JaywayJsonPath} import com.snowplowanalytics.iglu.core.{SchemaCriterion, SelfDescribingData} import com.snowplowanalytics.snowplow.badrows.igluSchemaCriterionDecoder @@ -34,7 +34,7 @@ sealed trait Input extends Product with Serializable { // We could short-circuit enrichment process on invalid JSONPath, // but it won't give user meaningful error message - def validatedJsonPath: Either[String, GatlingJsonPath] = + def validatedJsonPath: Either[String, JaywayJsonPath] = this match { case json: Input.Json => compileQuery(json.jsonPath) case _ => "No JSON Path given".asLeft diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonPath.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonPath.scala index 00e2dd004..978dbd15c 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonPath.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonPath.scala @@ -11,51 +11,66 @@ package com.snowplowanalytics.snowplow.enrich.common.utils import cats.syntax.either._ +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature} +import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider +import com.jayway.jsonpath.{Configuration, JsonPath => JaywayJsonPath, Option => JOption} import io.circe._ -import io.gatling.jsonpath.{JsonPath => GatlingJsonPath} +import io.circe.jackson.{circeToJackson, jacksonToCirce} -/** Wrapper for `io.gatling.jsonpath` for circe and scalaz */ +import scala.jdk.CollectionConverters.asScalaIteratorConverter + +/** Wrapper for `com.jayway.jsonpath` for circe */ object JsonPath { - /** - * Wrapper method for not throwing an exception on JNothing, representing it as invalid JSON - * @param json JSON value, possibly JNothing - * @return successful POJO on any JSON except JNothing - */ - def convertToJson(json: Json): Object = - io.circe.jackson.mapper.convertValue(json, classOf[Object]) + private val JacksonNodeJsonObjectMapper = { + val objectMapper = new ObjectMapper() + objectMapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) + objectMapper + } + private val JsonPathConf = + Configuration + .builder() + .options(JOption.SUPPRESS_EXCEPTIONS) + .options(JOption.ALWAYS_RETURN_LIST) + .jsonProvider(new JacksonJsonNodeJsonProvider(JacksonNodeJsonObjectMapper)) + .build() /** * Pimp-up JsonPath class to work with JValue * Unlike `query(jsonPath, json)` it gives empty list on any error (like JNothing) * @param jsonPath precompiled with [[compileQuery]] JsonPath object */ - implicit class CirceExtractor(jsonPath: GatlingJsonPath) { - def circeQuery(json: Json): List[Json] = { - val pojo = convertToJson(json) - jsonPath.query(pojo).map(anyToJson).toList - } + implicit class CirceExtractor(jsonPath: JaywayJsonPath) { + def circeQuery(json: Json): List[Json] = + arrayNodeToCirce(jsonPath.read[ArrayNode](circeToJackson(json), JsonPathConf)) } /** * Query some JSON by `jsonPath`. It always return List, even for single match. * Unlike `json.circeQuery(stringPath)` it gives error if JNothing was given */ - def query(jsonPath: String, json: Json): Either[String, List[Json]] = { - val pojo = convertToJson(json) - GatlingJsonPath.query(jsonPath, pojo) match { - case Right(iterator) => iterator.map(anyToJson).toList.asRight - case Left(error) => error.reason.asLeft + def query(jsonPath: String, json: Json): Either[String, List[Json]] = + Either.catchNonFatal { + JaywayJsonPath + .using(JsonPathConf) + .parse(circeToJackson(json)) + .read[ArrayNode](jsonPath) + } match { + case Right(jacksonArrayNode) => + arrayNodeToCirce(jacksonArrayNode).asRight + case Left(error) => + error.getMessage.asLeft } - } /** * Precompile JsonPath query + * * @param query JsonPath query as a string * @return valid JsonPath object either error message */ - def compileQuery(query: String): Either[String, GatlingJsonPath] = - GatlingJsonPath.compile(query).leftMap(_.reason) + def compileQuery(query: String): Either[String, JaywayJsonPath] = + Either.catchNonFatal(JaywayJsonPath.compile(query)).leftMap(_.getMessage) /** * Wrap list of values into JSON array if several values present @@ -70,12 +85,6 @@ object JsonPath { case many => Json.fromValues(many) } - /** - * Convert POJO to JValue with `jackson` mapper - * @param any raw JVM type representing JSON - * @return Json - */ - private[utils] def anyToJson(any: Any): Json = - if (any == null) Json.Null - else CirceUtils.mapper.convertValue(any, classOf[Json]) + private def arrayNodeToCirce(jacksonArrayNode: ArrayNode): List[Json] = + jacksonArrayNode.elements().asScala.toList.map(jacksonToCirce) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/InputSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/InputSpec.scala index 3f88c50d8..419999865 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/InputSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/InputSpec.scala @@ -223,7 +223,8 @@ class InputSpec extends Specification with ValidatedMatchers with CatsEffect { unstructEvent = None ) templateContext must beInvalid.like { - case errors => errors.toList must have length 3 + case errors => + errors.toList must have length 2 // TODO it's not 3 anymore because `"*.invalidJsonPath"` path doesn't fail during jsonpath compilation } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonPathSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonPathSpec.scala index eab03196d..638ac60cb 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonPathSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonPathSpec.scala @@ -21,7 +21,7 @@ class JsonPathSpec extends Specification { test query of non-exist value $e2 test query of empty array $e3 test primitive JSON type (JString) $e6 - invalid JSONPath (JQ syntax) must fail $e4 + test invalid JSONPath (JQ syntax) $e4 invalid JSONPath must fail $e5 test query of long $e7 test query of integer $e8 @@ -81,13 +81,12 @@ class JsonPathSpec extends Specification { JsonPath.query("$.store.unicorns", someJson) must beRight(Nil) def e4 = - JsonPath.query(".notJsonPath", someJson) must beLeft.like { - case f => f must beEqualTo("'$' expected but '.' found") - } + //TODO it's not failure anymore because `.notJsonPath` is not treated as invalid jsonpath by jayway + JsonPath.query(".notJsonPath", someJson) must beRight(Nil) def e5 = JsonPath.query("$.store.book[a]", someJson) must beLeft.like { - case f => f must beEqualTo("':' expected but 'a' found") + case f => f must beEqualTo("Could not parse token starting at position 12. Expected ?, ', 0-9, * ") } def e6 = diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala index 7001f53cf..1d7aabf3c 100644 --- a/project/BuildSettings.scala +++ b/project/BuildSettings.scala @@ -28,7 +28,7 @@ object BuildSettings { lazy val projectSettings = Seq( organization := "com.snowplowanalytics", - scalaVersion := "2.12.15", + scalaVersion := "2.13.12", licenses += ("Apache-2.0", url("http://www.apache.org/licenses/LICENSE-2.0.html")) ) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6397f3f80..f707d325f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -58,7 +58,6 @@ object Dependencies { val circeJackson = "0.14.0" val scalaForex = "3.0.0" val scalaWeather = "2.0.0" - val gatlingJsonpath = "0.6.14" val scalaUri = "1.5.1" val badRows = "2.3.0" val igluClient = "3.1.0" @@ -132,7 +131,6 @@ object Dependencies { val circeOptics = "io.circe" %% "circe-optics" % V.circeOptics val circeJackson = "io.circe" %% "circe-jackson210" % V.circeJackson val scalaUri = "io.lemonlabs" %% "scala-uri" % V.scalaUri - val gatlingJsonpath = "io.gatling" %% "jsonpath" % V.gatlingJsonpath val scalaForex = "com.snowplowanalytics" %% "scala-forex" % V.scalaForex val refererParser = "com.snowplowanalytics" %% "scala-referer-parser" % V.refererParser val maxmindIplookups = "com.snowplowanalytics" %% "scala-maxmind-iplookups" % V.maxmindIplookups @@ -232,7 +230,6 @@ object Dependencies { scalaUri, scalaForex, scalaWeather, - gatlingJsonpath, badRows, igluClient, snowplowRawEvent, From b47cb477304ae3f56e11c3f9ea20ae47ac533644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Poniedzia=C5=82ek?= Date: Wed, 14 Feb 2024 12:17:19 +0100 Subject: [PATCH 2/3] Fix everything --- .github/workflows/test.yml | 12 +-- .../snowplow/enrich/aws/S3Client.scala | 2 +- .../common/fs2/test/CollectorPayloadGen.scala | 74 +++++++++++++------ .../snowplow/enrich/common/fs2/Assets.scala | 4 +- .../snowplow/enrich/common/fs2/Enrich.scala | 2 +- .../snowplow/enrich/common/fs2/Run.scala | 2 - .../enrich/common/fs2/io/FileSystem.scala | 2 +- .../common/fs2/blackbox/BlackBoxTesting.scala | 2 +- .../common/adapters/registry/Adapter.scala | 6 +- .../registry/GoogleAnalyticsAdapter.scala | 10 ++- .../adapters/registry/OlarkAdapter.scala | 2 +- .../registry/StatusGatorAdapter.scala | 2 +- .../adapters/registry/UnbounceAdapter.scala | 2 +- .../registry/snowplow/RedirectAdapter.scala | 1 + .../enrichments/EnrichmentManager.scala | 2 +- .../registry/AnonIpEnrichment.scala | 1 + .../registry/YauaaEnrichment.scala | 6 +- .../apirequest/ApiRequestEnrichment.scala | 2 +- .../pii/PiiPseudonymizerEnrichment.scala | 10 +-- .../registry/sqlquery/DbExecutor.scala | 2 +- .../common/loaders/CollectorPayload.scala | 2 +- .../common/loaders/Loader.scala | 2 +- .../common/loaders/ThriftLoader.scala | 2 +- .../common/utils/ConversionUtils.scala | 4 +- .../common/utils/JsonPath.scala | 2 +- .../common/utils/MapTransformer.scala | 5 +- .../enrich/CirceJsonDeserializer.scala | 2 +- .../enrich/kafka/test/EnrichKafkaSpec.scala | 48 ++++++------ .../Main.scala | 1 + .../enrich/kinesis/DynamoDbConfig.scala | 5 +- .../snowplow/enrich/kinesis/Sink.scala | 2 +- .../snowplow/enrich/nsq/Main.scala | 4 +- .../snowplow/enrich/nsq/Sink.scala | 2 +- .../snowplow/enrich/pubsub/Main.scala | 4 +- project/BuildSettings.scala | 27 +++---- project/plugins.sbt | 12 ++- 36 files changed, 147 insertions(+), 123 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 43e503e63..83ce22824 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,7 +43,7 @@ jobs: env: PGPASSWORD: supersecret1 - name: Run tests - run: SBT_OPTS="-Xms1G -Xmx8G -Xss4M -XX:MaxMetaspaceSize=1024M" sbt coverage +test + run: SBT_OPTS="-Xms1G -Xmx8G -Xss4M -XX:MaxMetaspaceSize=1024M" sbt +test env: OER_KEY: ${{ secrets.OER_KEY }} - name: Check Scala formatting @@ -62,12 +62,4 @@ jobs: sbt "project kafka" IntegrationTest/test docker-compose -f integration-tests/enrich-kafka/docker-compose.yml down - name: Run integration tests for enrich-nsq - run: sbt "project nsqDistroless" IntegrationTest/test - - name: Generate coverage report - run: sbt coverageReport - - name: Aggregate coverage data - run: sbt coverageAggregate - - name: Submit coveralls data - run: sbt coveralls - env: - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} \ No newline at end of file + run: sbt "project nsqDistroless" IntegrationTest/test \ No newline at end of file diff --git a/modules/cloudutils/aws/src/main/scala/com/snowplowanalytics/snowplow/enrich/aws/S3Client.scala b/modules/cloudutils/aws/src/main/scala/com/snowplowanalytics/snowplow/enrich/aws/S3Client.scala index d5c7da502..608d47092 100644 --- a/modules/cloudutils/aws/src/main/scala/com/snowplowanalytics/snowplow/enrich/aws/S3Client.scala +++ b/modules/cloudutils/aws/src/main/scala/com/snowplowanalytics/snowplow/enrich/aws/S3Client.scala @@ -31,7 +31,7 @@ object S3Client { def mk[F[_]: Async]: Resource[F, Client[F]] = for { - s3Client <- Resource.fromAutoCloseable(Sync[F].delay(S3AsyncClient.builder().region(getRegion).build())) + s3Client <- Resource.fromAutoCloseable(Sync[F].delay(S3AsyncClient.builder().region(getRegion()).build())) store <- Resource.eval(S3Store.builder[F](s3Client).build.toEither.leftMap(_.head).pure[F].rethrow) } yield new Client[F] { def canDownload(uri: URI): Boolean = diff --git a/modules/common-fs2/src/it/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/CollectorPayloadGen.scala b/modules/common-fs2/src/it/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/CollectorPayloadGen.scala index f8c9157d5..f32aaea2c 100644 --- a/modules/common-fs2/src/it/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/CollectorPayloadGen.scala +++ b/modules/common-fs2/src/it/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/CollectorPayloadGen.scala @@ -27,7 +27,7 @@ import org.apache.thrift.TSerializer import java.util.Base64 -import com.snowplowanalytics.iglu.core.{ SelfDescribingData, SchemaKey, SchemaVer } +import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData} import com.snowplowanalytics.iglu.core.circe.CirceIgluCodecs._ import com.snowplowanalytics.snowplow.enrich.common.loaders.CollectorPayload @@ -40,7 +40,9 @@ object CollectorPayloadGen { generateRaw(nbGoodEvents, nbBadRows).map(_.toThrift).map(new TSerializer().serialize) def generateRaw[F[_]: Sync](nbGoodEvents: Long, nbBadRows: Long): Stream[F, CollectorPayload] = - Stream.repeatEval(runGen(collectorPayloadGen(true))).take(nbGoodEvents) ++ Stream.repeatEval(runGen(collectorPayloadGen(false))).take(nbBadRows) + Stream.repeatEval(runGen(collectorPayloadGen(true))).take(nbGoodEvents) ++ Stream + .repeatEval(runGen(collectorPayloadGen(false))) + .take(nbBadRows) private def collectorPayloadGen(valid: Boolean): Gen[CollectorPayload] = for { @@ -74,46 +76,74 @@ object CollectorPayloadGen { aid <- Gen.const("enrich-kinesis-integration-tests").withKey("aid") e <- Gen.const("ue").withKey("e") tv <- Gen.oneOf("scala-tracker_1.0.0", "js_2.0.0", "go_1.2.3").withKey("tv") - uePx <- - if(valid) - ueGen.map(_.toString).map(str => base64Encoder.encodeToString(str.getBytes)).withKey("ue_px") - else - Gen.const("foo").withKey("ue_px") + uePx <- if (valid) + ueGen.map(_.toString).map(str => base64Encoder.encodeToString(str.getBytes)).withKey("ue_px") + else + Gen.const("foo").withKey("ue_px") } yield SelfDescribingData( - SchemaKey("com.snowplowanalytics.snowplow", "payload_data", "jsonschema", SchemaVer.Full(1,0,4)), - List(asObject(List(p, aid, e, uePx, tv))).asJson + SchemaKey("com.snowplowanalytics.snowplow", "payload_data", "jsonschema", SchemaVer.Full(1, 0, 4)), + List(asObject(List(p, aid, e, uePx, tv))).asJson ).asJson.toString private def ueGen = for { sdj <- Gen.oneOf(changeFormGen, clientSessionGen) } yield SelfDescribingData( - SchemaKey("com.snowplowanalytics.snowplow", "unstruct_event", "jsonschema", SchemaVer.Full(1,0,0)), + SchemaKey("com.snowplowanalytics.snowplow", "unstruct_event", "jsonschema", SchemaVer.Full(1, 0, 0)), sdj.asJson ).asJson - private def changeFormGen = for { - formId <- strGen(32, Gen.alphaNumChar).withKey("formId") + formId <- strGen(32, Gen.alphaNumChar).withKey("formId") elementId <- strGen(32, Gen.alphaNumChar).withKey("elementId") - nodeName <- Gen.oneOf(List("INPUT", "TEXTAREA", "SELECT")).withKey("nodeName") - `type` <- Gen.option(Gen.oneOf(List("button", "checkbox", "color", "date", "datetime", "datetime-local", "email", "file", "hidden", "image", "month", "number", "password", "radio", "range", "reset", "search", "submit", "tel", "text", "time", "url", "week"))).withKeyOpt("type") - value <- Gen.option(strGen(16, Gen.alphaNumChar)).withKeyNull("value") + nodeName <- Gen.oneOf(List("INPUT", "TEXTAREA", "SELECT")).withKey("nodeName") + `type` <- Gen + .option( + Gen.oneOf( + List( + "button", + "checkbox", + "color", + "date", + "datetime", + "datetime-local", + "email", + "file", + "hidden", + "image", + "month", + "number", + "password", + "radio", + "range", + "reset", + "search", + "submit", + "tel", + "text", + "time", + "url", + "week" + ) + ) + ) + .withKeyOpt("type") + value <- Gen.option(strGen(16, Gen.alphaNumChar)).withKeyNull("value") } yield SelfDescribingData( - SchemaKey("com.snowplowanalytics.snowplow", "change_form", "jsonschema", SchemaVer.Full(1,0,0)), + SchemaKey("com.snowplowanalytics.snowplow", "change_form", "jsonschema", SchemaVer.Full(1, 0, 0)), asObject(List(formId, elementId, nodeName, `type`, value)) ) private def clientSessionGen = for { - userId <- Gen.uuid.withKey("userId") - sessionId <- Gen.uuid.withKey("sessionId") - sessionIndex <- Gen.choose(0, 2147483647).withKey("sessionIndex") + userId <- Gen.uuid.withKey("userId") + sessionId <- Gen.uuid.withKey("sessionId") + sessionIndex <- Gen.choose(0, 2147483647).withKey("sessionIndex") previousSessionId <- Gen.option(Gen.uuid).withKeyNull("previousSessionId") - storageMechanism <- Gen.oneOf(List("SQLITE", "COOKIE_1", "COOKIE_3", "LOCAL_STORAGE", "FLASH_LSO")).withKey("storageMechanism") + storageMechanism <- Gen.oneOf(List("SQLITE", "COOKIE_1", "COOKIE_3", "LOCAL_STORAGE", "FLASH_LSO")).withKey("storageMechanism") } yield SelfDescribingData( - SchemaKey("com.snowplowanalytics.snowplow", "client_session", "jsonschema", SchemaVer.Full(1,0,1)), + SchemaKey("com.snowplowanalytics.snowplow", "client_session", "jsonschema", SchemaVer.Full(1, 0, 1)), asObject(List(userId, sessionId, sessionIndex, previousSessionId, storageMechanism)) ) @@ -159,7 +189,7 @@ object CollectorPayloadGen { implicit class GenOps[A](gen: Gen[A]) { def withKey[B](name: String)(implicit enc: Encoder[A]): Gen[Option[(String, Json)]] = - gen.map { a => Some((name -> a.asJson)) } + gen.map(a => Some((name -> a.asJson))) } implicit class GenOptOps[A](gen: Gen[Option[A]]) { diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Assets.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Assets.scala index 90dba7903..25411007d 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Assets.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Assets.scala @@ -14,7 +14,6 @@ import java.net.URI import scala.concurrent.ExecutionContext import scala.concurrent.duration._ -import scala.util.control.NonFatal import cats.Applicative import cats.implicits._ @@ -259,8 +258,7 @@ object Assets { def worthRetrying[F[_]: Applicative](e: Throwable): F[Boolean] = e match { case _: Clients.RetryableFailure => Applicative[F].pure(true) - case _: IllegalArgumentException => Applicative[F].pure(false) - case NonFatal(_) => Applicative[F].pure(false) + case _ => Applicative[F].pure(false) } def onError[F[_]: Sync](error: Throwable, details: RetryDetails): F[Unit] = diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Enrich.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Enrich.scala index 89753726b..c6719960f 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Enrich.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Enrich.scala @@ -152,7 +152,7 @@ object Enrich { payload.fold(_.asJson.noSpaces, _.map(_.toBadRowPayload.asJson.noSpaces).getOrElse("None")) /** Log an error, turn the problematic `CollectorPayload` into `BadRow` and notify Sentry if configured */ - def sendToSentry[F[_]: Sync: Clock]( + def sendToSentry[F[_]: Sync]( original: Array[Byte], sentry: Option[SentryClient], processor: Processor, diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Run.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Run.scala index 746c582af..cdc76b447 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Run.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Run.scala @@ -18,8 +18,6 @@ import cats.implicits._ import fs2.Stream -import scala.concurrent.ExecutionContext - import cats.effect.kernel.{Async, Resource, Sync} import cats.effect.ExitCode diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/FileSystem.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/FileSystem.scala index 0d72eb666..23e4159fa 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/FileSystem.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/FileSystem.scala @@ -12,7 +12,7 @@ package com.snowplowanalytics.snowplow.enrich.common.fs2.io import java.nio.file.{Files, Path} -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.io.{Source => SSource} import cats.data.EitherT diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/BlackBoxTesting.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/BlackBoxTesting.scala index b20a225ca..dedcd02f8 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/BlackBoxTesting.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/BlackBoxTesting.scala @@ -10,7 +10,7 @@ */ package com.snowplowanalytics.snowplow.enrich.common.fs2.blackbox -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import org.specs2.mutable.Specification diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/Adapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/Adapter.scala index c9f551ebb..566aecd30 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/Adapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/Adapter.scala @@ -165,7 +165,7 @@ trait Adapter { formatter: FormatterFunc, platform: String ): RawEventParameters = { - val params = formatter(parameters - ("nuid", "aid", "cv", "p")) + val params = formatter(parameters -- List("nuid", "aid", "cv", "p")) val json = toUnstructEvent(SelfDescribingData(schema, params)).noSpaces buildUnstructEventParams(tracker, platform, parameters, json) } @@ -182,7 +182,7 @@ trait Adapter { "p" -> parameters.getOrElse("p", Option(platform)), // Required field "ue_pr" -> Option(json) ) ++ - parameters.filterKeys(AcceptedQueryParameters) + parameters.view.filterKeys(AcceptedQueryParameters).toMap /** * Creates a Snowplow unstructured event by nesting the provided JValue in a self-describing @@ -375,7 +375,7 @@ trait Adapter { */ private[registry] def camelCase(snakeOrDash: String) = snakeCaseOrDashTokenCapturingRegex.replaceAllIn( - Character.toLowerCase(snakeOrDash.charAt(0)) + snakeOrDash.substring(1), + Character.toString(Character.toLowerCase(snakeOrDash.charAt(0))) + snakeOrDash.substring(1), m => m.group(1).capitalize ) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/GoogleAnalyticsAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/GoogleAnalyticsAdapter.scala index ecd0fa344..999fd31ad 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/GoogleAnalyticsAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/GoogleAnalyticsAdapter.scala @@ -12,7 +12,7 @@ package com.snowplowanalytics.snowplow.enrich.common.adapters.registry import scala.annotation.tailrec -import cats.{Applicative, Functor, Monad} +import cats.{Applicative, Monad} import cats.data.{NonEmptyList, ValidatedNel} import cats.implicits._ @@ -531,7 +531,7 @@ case class GoogleAnalyticsAdapter(schemas: GoogleAnalyticsSchemas) extends Adapt ) schemaVal = lookupSchema( hitType.some, - unstructEventData.mapValues(_.schemaKey) + unstructEventData.view.mapValues(_.schemaKey).toMap ).toValidatedNel simpleContexts = buildContexts(params, contextData, fieldToSchemaMap) compositeContexts = buildCompositeContexts( @@ -675,7 +675,9 @@ case class GoogleAnalyticsAdapter(schemas: GoogleAnalyticsSchemas) extends Adapt // composite params have digits in their key composite <- originalParams .collect { case (k, Some(v)) => (k, v) } + .view .filterKeys(k => k.exists(_.isDigit)) + .toMap .asRight brokenDown <- composite.toList.sorted.map { case (k, v) => breakDownCompField(k, v, indicator) @@ -684,7 +686,9 @@ case class GoogleAnalyticsAdapter(schemas: GoogleAnalyticsSchemas) extends Adapt // we additionally make sure we have a rectangular dataset grouped = (partitioned._2 ++ removeConsecutiveDuplicates(partitioned._1)).flatten .groupBy(_._1) + .view .mapValues(_.map(_._2)) + .toMap translated <- { val m = grouped .foldLeft( @@ -821,7 +825,7 @@ case class GoogleAnalyticsAdapter(schemas: GoogleAnalyticsSchemas) extends Adapt case head => head :: transpose(l.collect { case _ :: tail => tail }) } - private def traverseMap[G[_]: Functor: Applicative, K, V](m: Map[K, G[V]]): G[Map[K, V]] = + private def traverseMap[G[_]: Applicative, K, V](m: Map[K, G[V]]): G[Map[K, V]] = m.toList .traverse { case (name, vnel) => diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/OlarkAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/OlarkAdapter.scala index 0a5c03e0f..338f463a5 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/OlarkAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/OlarkAdapter.scala @@ -13,7 +13,7 @@ package com.snowplowanalytics.snowplow.enrich.common.adapters.registry import java.net.URI import java.nio.charset.StandardCharsets.UTF_8 -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.util.{Try, Success => TS, Failure => TF} import cats.Monad diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/StatusGatorAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/StatusGatorAdapter.scala index 84f308957..a62151df2 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/StatusGatorAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/StatusGatorAdapter.scala @@ -13,7 +13,7 @@ package com.snowplowanalytics.snowplow.enrich.common.adapters.registry import java.net.URI import java.nio.charset.StandardCharsets.UTF_8 -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.util.{Try, Success => TS, Failure => TF} import cats.Monad diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UnbounceAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UnbounceAdapter.scala index b2a3b86eb..30b5736fc 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UnbounceAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UnbounceAdapter.scala @@ -13,7 +13,7 @@ package com.snowplowanalytics.snowplow.enrich.common.adapters.registry import java.net.URI import java.nio.charset.StandardCharsets.UTF_8 -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.util.{Try, Success => TS, Failure => TF} import cats.Monad diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/RedirectAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/RedirectAdapter.scala index 588f27022..c6e95e490 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/RedirectAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/RedirectAdapter.scala @@ -98,6 +98,7 @@ object RedirectAdapter extends Adapter { case (None, Some(Some(co))) if co == "" => newCo.asRight case (None, Some(Some(co))) => addToExistingCo(json, co).map(str => Map("co" -> str)) case (Some(Some(cx)), _) => addToExistingCx(json, cx).map(str => Map("cx" -> str)) + case other => throw new IllegalStateException(s"Illegal state: $other") } } else // Add URI redirect as an unstructured event diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala index 1d238fdc2..c47143103 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala @@ -287,7 +287,7 @@ object EnrichmentManager { def setCollectorTstamp(event: EnrichedEvent, timestamp: Option[DateTime]): Either[FailureDetails.EnrichmentFailure, Unit] = EE.formatCollectorTstamp(timestamp).map { t => event.collector_tstamp = t - ().asRight + () } def setUseragent( diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala index be2342ae9..7a5fa9bf3 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala @@ -150,6 +150,7 @@ final case class AnonIpEnrichment(ipv4Octets: AnonIPv4Octets.AnonIPv4Octets, ipv .map { case _: Inet4Address => anonymizeIpV4(ip) case ipv6: Inet6Address => anonymizeIpV6(ipv6.getHostAddress) + case _ => throw new IllegalStateException(s"Illegal state") } .getOrElse(tryAnonymizingInvalidIp(ip)) }.orNull diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala index 55127907c..13d28d42b 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala @@ -10,7 +10,7 @@ */ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import cats.data.ValidatedNel import cats.syntax.either._ @@ -64,7 +64,7 @@ object YauaaEnrichment extends ParseableEnrichment { s match { case _ if s.isEmpty => s case _ if s.length == 1 => s.toLowerCase - case _ => s.charAt(0).toLower + s.substring(1) + case _ => Character.toString(s.charAt(0).toLower) + s.substring(1) } } @@ -112,7 +112,9 @@ final case class YauaaEnrichment(cacheSize: Option[Int]) extends Enrichment { parsedUA.getAvailableFieldNamesSorted.asScala .map(field => decapitalize(field) -> parsedUA.getValue(field)) .toMap + .view .filterKeys(validFields) + .toMap } /** Yauaa 7.x added many new fields which are not in the 1-0-4 schema */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala index 90126baeb..895138742 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala @@ -93,7 +93,7 @@ object ApiRequestEnrichment extends ParseableEnrichment { UUID.nameUUIDFromBytes(contentKey.getBytes).toString } - def create[F[_]: Async: Clock]( + def create[F[_]: Async]( schemaKey: SchemaKey, inputs: List[Input], api: HttpApi, diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/PiiPseudonymizerEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/PiiPseudonymizerEnrichment.scala index 955bf6ac7..c4f61995c 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/PiiPseudonymizerEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/PiiPseudonymizerEnrichment.scala @@ -10,8 +10,8 @@ */ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.pii -import scala.collection.JavaConverters._ -import scala.collection.mutable.MutableList +import scala.jdk.CollectionConverters._ +import scala.collection.mutable.ListBuffer import cats.data.ValidatedNel import cats.implicits._ @@ -216,7 +216,7 @@ final case class PiiJson( val jObjectMap = obj.toMap val contextMapped = jObjectMap.map(mapContextTopFields(_, strategy)) ( - Json.obj(contextMapped.mapValues(_._1).toList: _*), + Json.obj(contextMapped.view.mapValues(_._1).toList: _*), contextMapped.values.flatMap(_._2) ) } @@ -282,7 +282,7 @@ final case class PiiJson( ): (Json, List[JsonModifiedField]) = { val objectNode = io.circe.jackson.mapper.valueToTree[ObjectNode](json) val documentContext = JJsonPath.using(JsonPathConf).parse(objectNode) - val modifiedFields = MutableList[JsonModifiedField]() + val modifiedFields = ListBuffer[JsonModifiedField]() Option(documentContext.read[AnyRef](jsonPath)) match { // check that json object not null case None => (jacksonToCirce(documentContext.json[JsonNode]()), modifiedFields.toList) case _ => @@ -297,7 +297,7 @@ final case class PiiJson( private final case class ScrambleMapFunction( strategy: PiiStrategy, - modifiedFields: MutableList[JsonModifiedField], + modifiedFields: ListBuffer[JsonModifiedField], fieldName: String, jsonPath: String, schema: String diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/DbExecutor.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/DbExecutor.scala index 567c586f0..d9856e18b 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/DbExecutor.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/DbExecutor.scala @@ -193,6 +193,6 @@ object DbExecutor { if (intMap.keys.size == placeholderCount) true else false } - def getConnection[F[_]: Monad: DbExecutor](dataSource: DataSource): Resource[F, Connection] = + def getConnection[F[_]: DbExecutor](dataSource: DataSource): Resource[F, Connection] = DbExecutor[F].getConnection(dataSource) } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/CollectorPayload.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/CollectorPayload.scala index 3ba901d9b..bd417e176 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/CollectorPayload.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/CollectorPayload.scala @@ -12,7 +12,7 @@ package com.snowplowanalytics.snowplow.enrich.common.loaders import java.util.UUID -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import cats.syntax.either._ import cats.syntax.option._ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/Loader.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/Loader.scala index 8df5099d0..bc4391934 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/Loader.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/Loader.scala @@ -13,7 +13,7 @@ package com.snowplowanalytics.snowplow.enrich.common.loaders import java.net.URI import java.nio.charset.Charset -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import cats.data.ValidatedNel import cats.syntax.either._ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/ThriftLoader.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/ThriftLoader.scala index 55e9099fd..e9de549bc 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/ThriftLoader.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/ThriftLoader.scala @@ -14,7 +14,7 @@ import java.nio.charset.Charset import java.time.Instant import java.util.UUID -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.util.control.NonFatal import cats.data.{NonEmptyList, ValidatedNel} diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ConversionUtils.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ConversionUtils.scala index 39866edcb..72ea8d1c6 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ConversionUtils.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ConversionUtils.scala @@ -19,7 +19,7 @@ import java.nio.charset.StandardCharsets.UTF_8 import java.util.UUID import java.io.{PrintWriter, StringWriter} -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.util.Try import scala.util.control.NonFatal @@ -182,7 +182,7 @@ object ConversionUtils { val validateInteger: (String, String) => Either[FailureDetails.EnrichmentFailure, String] = (field, str) => { Either - .catchNonFatal { str.toInt; str } + .catchNonFatal { str.toInt: Unit; str } .leftMap { _ => val f = FailureDetails.EnrichmentFailureMessage.InputData( field, diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonPath.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonPath.scala index 978dbd15c..923ab6150 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonPath.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonPath.scala @@ -18,7 +18,7 @@ import com.jayway.jsonpath.{Configuration, JsonPath => JaywayJsonPath, Option => import io.circe._ import io.circe.jackson.{circeToJackson, jacksonToCirce} -import scala.jdk.CollectionConverters.asScalaIteratorConverter +import scala.jdk.CollectionConverters._ /** Wrapper for `com.jayway.jsonpath` for circe */ object JsonPath { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/MapTransformer.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/MapTransformer.scala index 6bc76be1c..00402b307 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/MapTransformer.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/MapTransformer.scala @@ -100,7 +100,7 @@ object MapTransformer { * @param obj Any Object * @return the new Transformable class, with manifest attached */ - implicit def makeTransformable[T <: AnyRef](obj: T)(implicit m: Manifest[T]) = + implicit def makeTransformable[T <: AnyRef](obj: T)(implicit m: Manifest[T]): TransformableClass[T] = new TransformableClass[T](obj) /** A pimped object, now transformable by using the transform method. */ @@ -166,6 +166,7 @@ object MapTransformer { setters(f3).invoke(obj, result._3) setters(f4).invoke(obj, result._4) 4.asRight // +4 to the count of fields successfully set + case other => throw new IllegalStateException(s"Illegal state: $other") } } case None => 0.asRight // Key not found: zero fields updated @@ -204,5 +205,7 @@ object MapTransformer { c.getDeclaredMethods .filter(_.getName.startsWith("set")) .groupBy(setterToFieldName(_)) + .view .mapValues(_.head) + .toMap } diff --git a/modules/common/src/main/scala/io/circe/jackson/enrich/CirceJsonDeserializer.scala b/modules/common/src/main/scala/io/circe/jackson/enrich/CirceJsonDeserializer.scala index d6824a3f2..d8854bcb4 100644 --- a/modules/common/src/main/scala/io/circe/jackson/enrich/CirceJsonDeserializer.scala +++ b/modules/common/src/main/scala/io/circe/jackson/enrich/CirceJsonDeserializer.scala @@ -19,7 +19,7 @@ import io.circe.jackson.{DeserializerContext, JacksonCompat, ReadingList, Readin import io.circe.{Json, JsonBigDecimal, JsonLong} import scala.annotation.{switch, tailrec} -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ private[jackson] final class CirceJsonDeserializer(klass: Class[_]) extends JsonDeserializer[Object] with JacksonCompat { override def isCachable: Boolean = true diff --git a/modules/kafka/src/it/scala/com/snowplowanalytics/snowplow/enrich/kafka/test/EnrichKafkaSpec.scala b/modules/kafka/src/it/scala/com/snowplowanalytics/snowplow/enrich/kafka/test/EnrichKafkaSpec.scala index ea30c37c6..11afcac2f 100644 --- a/modules/kafka/src/it/scala/com/snowplowanalytics/snowplow/enrich/kafka/test/EnrichKafkaSpec.scala +++ b/modules/kafka/src/it/scala/com/snowplowanalytics/snowplow/enrich/kafka/test/EnrichKafkaSpec.scala @@ -44,8 +44,8 @@ class EnrichKafkaSpec extends Specification with CatsEffect { val enrichedStream = "it-enrich-kinesis-enriched" val badRowsStream = "it-enrich-kinesis-bad" - val nbGood = 100l - val nbBad = 10l + val nbGood = 100L + val nbBad = 10L type AggregateGood = List[Event] type AggregateBad = List[String] @@ -55,12 +55,12 @@ class EnrichKafkaSpec extends Specification with CatsEffect { val bootstrapServers = s"localhost:$kafkaPort" val consumerConf: Map[String, String] = Map( - "group.id" -> "it-enrich", - "auto.offset.reset" -> "earliest", - "key.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer", - "value.deserializer" -> "org.apache.kafka.common.serialization.ByteArrayDeserializer", - "security.protocol" -> "PLAINTEXT", - "sasl.mechanism" -> "GSSAPI" + "group.id" -> "it-enrich", + "auto.offset.reset" -> "earliest", + "key.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer", + "value.deserializer" -> "org.apache.kafka.common.serialization.ByteArrayDeserializer", + "security.protocol" -> "PLAINTEXT", + "sasl.mechanism" -> "GSSAPI" ) val producerConf: Map[String, String] = Map( @@ -71,11 +71,13 @@ class EnrichKafkaSpec extends Specification with CatsEffect { def run(): IO[Aggregates] = { - val resources = Sink.init[IO](OutKafka(collectorPayloadsStream, bootstrapServers, "", Set.empty, producerConf), classOf[SourceAuthHandler].getName) + val resources = + Sink.init[IO](OutKafka(collectorPayloadsStream, bootstrapServers, "", Set.empty, producerConf), classOf[SourceAuthHandler].getName) resources.use { sink => val generate = - CollectorPayloadGen.generate[IO](nbGood, nbBad) + CollectorPayloadGen + .generate[IO](nbGood, nbBad) .evalMap(events => sink(List(events))) .onComplete(fs2.Stream.eval(Logger[IO].info(s"Random data has been generated and sent to $collectorPayloadsStream"))) @@ -83,10 +85,16 @@ class EnrichKafkaSpec extends Specification with CatsEffect { consumeGood(refGood).merge(consumeBad(refBad)) def consumeGood(ref: Ref[IO, AggregateGood]): Stream[IO, Unit] = - Source.init[IO](InKafka(enrichedStream, bootstrapServers, consumerConf), classOf[GoodSinkAuthHandler].getName).map(_.record.value).evalMap(aggregateGood(_, ref)) + Source + .init[IO](InKafka(enrichedStream, bootstrapServers, consumerConf), classOf[GoodSinkAuthHandler].getName) + .map(_.record.value) + .evalMap(aggregateGood(_, ref)) def consumeBad(ref: Ref[IO, AggregateBad]): Stream[IO, Unit] = - Source.init[IO](InKafka(badRowsStream, bootstrapServers, consumerConf), classOf[BadSinkAuthHandler].getName).map(_.record.value).evalMap(aggregateBad(_, ref)) + Source + .init[IO](InKafka(badRowsStream, bootstrapServers, consumerConf), classOf[BadSinkAuthHandler].getName) + .map(_.record.value) + .evalMap(aggregateBad(_, ref)) def aggregateGood(r: Array[Byte], ref: Ref[IO, AggregateGood]): IO[Unit] = for { @@ -94,12 +102,11 @@ class EnrichKafkaSpec extends Specification with CatsEffect { _ <- ref.update(updateAggregateGood(_, e)) } yield () - def aggregateBad(r: Array[Byte], ref: Ref[IO, AggregateBad]): IO[Unit] = { + def aggregateBad(r: Array[Byte], ref: Ref[IO, AggregateBad]): IO[Unit] = for { br <- IO(new String(r)) _ <- ref.update(updateAggregateBad(_, br)) } yield () - } def updateAggregateGood(aggregate: AggregateGood, e: Event): AggregateGood = e :: aggregate @@ -110,13 +117,12 @@ class EnrichKafkaSpec extends Specification with CatsEffect { for { refGood <- Ref.of[IO, AggregateGood](Nil) refBad <- Ref.of[IO, AggregateBad](Nil) - _ <- - generate - .merge(consume(refGood, refBad)) - .interruptAfter(30.seconds) - .attempt - .compile - .drain + _ <- generate + .merge(consume(refGood, refBad)) + .interruptAfter(30.seconds) + .attempt + .compile + .drain aggregateGood <- refGood.get aggregateBad <- refBad.get } yield Aggregates(aggregateGood, aggregateBad) diff --git a/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/Main.scala b/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/Main.scala index 0a9ef88e0..0d97dcb4f 100644 --- a/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/Main.scala +++ b/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/Main.scala @@ -74,6 +74,7 @@ object Main extends IOApp { else records .groupBy(_.record.partition) + .view .mapValues(_.maxBy(_.record.offset)) .values .toList diff --git a/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/DynamoDbConfig.scala b/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/DynamoDbConfig.scala index 35d9f6ca2..bd751b985 100644 --- a/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/DynamoDbConfig.scala +++ b/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/DynamoDbConfig.scala @@ -10,9 +10,8 @@ */ package com.snowplowanalytics.snowplow.enrich.kinesis -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.concurrent.duration.DurationLong -import scala.util.control.NonFatal import cats.effect.kernel.{Async, Resource, Sync} @@ -147,7 +146,7 @@ object DynamoDbConfig { private def worthRetrying[F[_]: Applicative](e: Throwable): F[Boolean] = e match { case ace: AmazonClientException if ace.isRetryable => Applicative[F].pure(true) - case NonFatal(_) => Applicative[F].pure(false) + case _ => Applicative[F].pure(false) } private def onError[F[_]: Sync](error: Throwable, details: RetryDetails): F[Unit] = diff --git a/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Sink.scala b/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Sink.scala index 6196250a1..14ed3034c 100644 --- a/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Sink.scala +++ b/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Sink.scala @@ -13,7 +13,7 @@ package com.snowplowanalytics.snowplow.enrich.kinesis import java.nio.ByteBuffer import java.util.UUID -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import cats.implicits._ import cats.{Monoid, Parallel} diff --git a/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Main.scala b/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Main.scala index 93fd912f9..85e9f68b9 100644 --- a/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Main.scala +++ b/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Main.scala @@ -16,7 +16,7 @@ import cats.Parallel import cats.implicits._ import cats.effect.{ExitCode, IO, IOApp} -import cats.effect.kernel.{Resource, Sync} +import cats.effect.kernel.Resource import cats.effect.metrics.CpuStarvationWarningMetrics import org.typelevel.log4cats.Logger @@ -66,7 +66,7 @@ object Main extends IOApp { None ) - private def checkpoint[F[_]: Parallel: Sync](records: List[Record[F]]): F[Unit] = + private def checkpoint[F[_]: Parallel](records: List[Record[F]]): F[Unit] = records.parTraverse_(_.ack) private def createBlobStorageClient(conf: BlobStorageClients): List[Resource[IO, Client[IO]]] = { diff --git a/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Sink.scala b/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Sink.scala index 7016a2b76..ba8c66a1e 100644 --- a/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Sink.scala +++ b/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Sink.scala @@ -10,7 +10,7 @@ */ package com.snowplowanalytics.snowplow.enrich.nsq -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import cats.effect.kernel.{Async, Resource, Sync} diff --git a/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Main.scala b/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Main.scala index daee27114..033023d05 100644 --- a/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Main.scala +++ b/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Main.scala @@ -15,7 +15,7 @@ import scala.concurrent.duration._ import cats.Parallel import cats.implicits._ -import cats.effect.kernel.{Resource, Sync} +import cats.effect.kernel.Resource import cats.effect.{ExitCode, IO, IOApp} import cats.effect.metrics.CpuStarvationWarningMetrics @@ -68,6 +68,6 @@ object Main extends IOApp { None ) - private def checkpoint[F[_]: Parallel: Sync](records: List[ConsumerRecord[F, Array[Byte]]]): F[Unit] = + private def checkpoint[F[_]: Parallel](records: List[ConsumerRecord[F, Array[Byte]]]): F[Unit] = records.parTraverse_(_.ack) } diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala index 1d7aabf3c..2559020e1 100644 --- a/project/BuildSettings.scala +++ b/project/BuildSettings.scala @@ -20,7 +20,6 @@ import com.typesafe.sbt.packager.docker.DockerPlugin.autoImport._ import com.typesafe.sbt.packager.linux.LinuxPlugin.autoImport._ import com.typesafe.sbt.packager.universal.UniversalPlugin.autoImport._ import org.scalafmt.sbt.ScalafmtPlugin.autoImport._ -import scoverage.ScoverageKeys._ object BuildSettings { @@ -159,14 +158,6 @@ object BuildSettings { // TESTS - lazy val scoverageSettings = Seq( - coverageMinimumStmtTotal := 50, - coverageFailOnMinimum := false, - (Test / test) := { - (coverageReport dependsOn (Test / test)).value - } - ) - lazy val noParallelTestExecution = Seq(Test / parallelExecution := false) /** Add example config for integration tests. */ @@ -187,35 +178,35 @@ object BuildSettings { // Build and publish publishSettings ++ // Tests - scoverageSettings ++ noParallelTestExecution + noParallelTestExecution } lazy val commonFs2BuildSettings = { // Project commonFs2ProjectSettings ++ buildSettings ++ // Tests - scoverageSettings ++ noParallelTestExecution ++ addExampleConfToTestCp + noParallelTestExecution ++ addExampleConfToTestCp } lazy val awsUtilsBuildSettings = { // Project awsUtilsProjectSettings ++ buildSettings ++ // Tests - scoverageSettings ++ noParallelTestExecution ++ addExampleConfToTestCp + noParallelTestExecution ++ addExampleConfToTestCp } lazy val gcpUtilsBuildSettings = { // Project gcpUtilsProjectSettings ++ buildSettings ++ // Tests - scoverageSettings ++ noParallelTestExecution ++ addExampleConfToTestCp + noParallelTestExecution ++ addExampleConfToTestCp } lazy val azureUtilsBuildSettings = { // Project azureUtilsProjectSettings ++ buildSettings ++ // Tests - scoverageSettings ++ noParallelTestExecution ++ addExampleConfToTestCp + noParallelTestExecution ++ addExampleConfToTestCp } lazy val pubsubBuildSettings = { @@ -225,7 +216,7 @@ object BuildSettings { dockerSettingsFocal ++ Seq(Docker / packageName := "snowplow-enrich-pubsub") ++ // Tests - scoverageSettings ++ noParallelTestExecution ++ addExampleConfToTestCp + noParallelTestExecution ++ addExampleConfToTestCp } lazy val pubsubDistrolessBuildSettings = pubsubBuildSettings.diff(dockerSettingsFocal) ++ dockerSettingsDistroless @@ -237,7 +228,7 @@ object BuildSettings { dockerSettingsFocal ++ Seq(Docker / packageName := "snowplow-enrich-kinesis") ++ // Tests - scoverageSettings ++ noParallelTestExecution ++ Seq(Test / fork := true) ++ addExampleConfToTestCp + noParallelTestExecution ++ Seq(Test / fork := true) ++ addExampleConfToTestCp } lazy val kinesisDistrolessBuildSettings = kinesisBuildSettings.diff(dockerSettingsFocal) ++ dockerSettingsDistroless @@ -249,7 +240,7 @@ object BuildSettings { dockerSettingsFocal ++ Seq(Docker / packageName := "snowplow-enrich-kafka") ++ // Tests - scoverageSettings ++ noParallelTestExecution ++ addExampleConfToTestCp + noParallelTestExecution ++ addExampleConfToTestCp } lazy val kafkaDistrolessBuildSettings = kafkaBuildSettings.diff(dockerSettingsFocal) ++ dockerSettingsDistroless @@ -261,7 +252,7 @@ object BuildSettings { dockerSettingsFocal ++ Seq(Docker / packageName := "snowplow-enrich-nsq") ++ // Tests - scoverageSettings ++ noParallelTestExecution ++ addExampleConfToTestCp + noParallelTestExecution ++ addExampleConfToTestCp } lazy val nsqDistrolessBuildSettings = nsqBuildSettings.diff(dockerSettingsFocal) ++ dockerSettingsDistroless diff --git a/project/plugins.sbt b/project/plugins.sbt index 5ed1caaa7..ccee45e47 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,7 +1,5 @@ -addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.9.0") -addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.2") -addSbtPlugin("io.github.davidgregory084" % "sbt-tpolecat" % "0.1.17") -addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.3.1") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.2") -addSbtPlugin("com.geirsson" % "sbt-ci-release" % "1.5.7") -addSbtPlugin("com.snowplowanalytics" % "sbt-snowplow-release" % "0.3.1") +addSbtPlugin("io.github.davidgregory084" % "sbt-tpolecat" % "0.4.2") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6") +addSbtPlugin("com.snowplowanalytics" % "sbt-snowplow-release" % "0.3.1") +addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.11.0") +addSbtPlugin("com.dwijnand" % "sbt-dynver" % "4.1.1") \ No newline at end of file From 3f1f3aa518dc5f17e003250d689b1e54ed49233b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Poniedzia=C5=82ek?= Date: Thu, 15 Feb 2024 08:51:41 +0100 Subject: [PATCH 3/3] WIP --- .../registry/GoogleAnalyticsAdapterSpec.scala | 2 +- .../registry/MailgunAdapterSpec.scala | 8 ++-- .../adapters/registry/OlarkAdapterSpec.scala | 11 ++--- .../registry/SendgridAdapterSpec.scala | 42 +++++++++---------- .../enrichments/CachingEvaluatorSpec.scala | 6 +-- .../enrichments/SchemaEnrichmentSpec.scala | 2 +- .../registry/AnonIpEnrichmentSpec.scala | 9 ++++ .../EventFingerprintEnrichmentSpec.scala | 2 +- .../JavascriptScriptEnrichmentSpec.scala | 5 ++- .../apirequest/ApiRequestEnrichmentSpec.scala | 2 +- .../pii/PiiPseudonymizerEnrichmentSpec.scala | 1 - .../registry/sqlquery/InputSpec.scala | 7 +--- .../SqlQueryEnrichmentIntegrationTest.scala | 4 +- .../enrichments/web/ExtractPageUriSpec.scala | 2 +- .../utils/IgluUtilsSpec.scala | 18 ++++---- .../utils/JsonPathSpec.scala | 8 ++-- .../utils/JsonUtilsSpec.scala | 3 +- .../utils/conversionUtilsSpecs.scala | 17 ++++---- project/BuildSettings.scala | 1 + 19 files changed, 80 insertions(+), 70 deletions(-) diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/GoogleAnalyticsAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/GoogleAnalyticsAdapterSpec.scala index d08072084..575cd8d3a 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/GoogleAnalyticsAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/GoogleAnalyticsAdapterSpec.scala @@ -117,7 +117,7 @@ class GoogleAnalyticsAdapterSpec extends Specification with DataTables with Vali .InputData("t", "unknown".some, "no matching hit type"), FailureDetails.AdapterFailure.SchemaMapping( "unknown".some, - adapterWithDefaultSchemas.unstructEventData.mapValues(_.schemaKey), + adapterWithDefaultSchemas.unstructEventData.view.mapValues(_.schemaKey).toMap, "no schema associated with the provided type parameter" ) ) diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailgunAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailgunAdapterSpec.scala index 937551956..52b4ccf56 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailgunAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailgunAdapterSpec.scala @@ -138,7 +138,7 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat val expected = NonEmptyList.one( RawEvent( Shared.api, - Map("tv" -> "com.mailgun-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> parse(expectedJson).map(_.noSpaces).right.get).toOpt, + Map("tv" -> "com.mailgun-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> parse(expectedJson).map(_.noSpaces).toOption.get).toOpt, ContentType.some, Shared.cljSource, Shared.context @@ -205,7 +205,7 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat val expected = NonEmptyList.one( RawEvent( Shared.api, - Map("tv" -> "com.mailgun-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> parse(expectedJson).map(_.noSpaces).right.get).toOpt, + Map("tv" -> "com.mailgun-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> parse(expectedJson).map(_.noSpaces).toOption.get).toOpt, ContentType.some, Shared.cljSource, Shared.context @@ -237,7 +237,7 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat val expected = NonEmptyList.one( RawEvent( Shared.api, - Map("tv" -> "com.mailgun-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> parse(expectedJson).map(_.noSpaces).right.get).toOpt, + Map("tv" -> "com.mailgun-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> parse(expectedJson).map(_.noSpaces).toOption.get).toOpt, ContentType.some, Shared.cljSource, Shared.context @@ -354,7 +354,7 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat val expected = NonEmptyList.one( RawEvent( Shared.api, - Map("tv" -> "com.mailgun-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> parse(expectedJson).map(_.noSpaces).right.get).toOpt, + Map("tv" -> "com.mailgun-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> parse(expectedJson).map(_.noSpaces).toOption.get).toOpt, Some("application/json"), Shared.cljSource, Shared.context diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/OlarkAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/OlarkAdapterSpec.scala index ebd9cb0f2..14b3c631a 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/OlarkAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/OlarkAdapterSpec.scala @@ -315,11 +315,12 @@ class OlarkAdapterSpec extends Specification with DataTables with ValidatedMatch .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) .map(_ must beInvalid.like { case nel => - nel.size must_== 1 - nel.head must haveClass[FailureDetails.AdapterFailure.NotJson] - val f = nel.head.asInstanceOf[FailureDetails.AdapterFailure.NotJson] - f.field must_== "data" - f.error must_== """invalid json: expected json value got 'kind":...' (line 1, column 1)""" + (nel.size must_== 1) and + (nel.head must haveClass[FailureDetails.AdapterFailure.NotJson]) and { + val f = nel.head.asInstanceOf[FailureDetails.AdapterFailure.NotJson] + f.field must_== "data" + f.error must_== """invalid json: expected json value got 'kind":...' (line 1, column 1)""" + } }) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/SendgridAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/SendgridAdapterSpec.scala index 712540b31..a9fa3802e 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/SendgridAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/SendgridAdapterSpec.scala @@ -49,7 +49,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email":"example@test.com", "timestamp":1446549615, - "smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>", "event":"processed", "category":"cat facts", "sg_event_id":"sZROwMGMagFgnOEmSdvhig==", @@ -62,7 +62,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email":"example@test.com", "timestamp":1446549615, - "smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>", "event":"deferred", "category":"cat facts", "sg_event_id":"jWmZXTZbtHTV2-S47asrww==", @@ -77,7 +77,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email":"example@test.com", "timestamp":1446549615, - "smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>", "event":"delivered", "category":"cat facts", "sg_event_id":"cikAODhD-ffTphZ7xixsRw==", @@ -91,7 +91,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email":"example@test.com", "timestamp":1446549615, - "smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>", "event":"open", "category":"cat facts", "sg_event_id":"VGRrZCh-qMkOaAmuxcFujA==", @@ -106,7 +106,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email":"example@test.com", "timestamp":1446549615, - "smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>", "event":"click", "category":"cat facts", "sg_event_id":"QjGWYpcksoD31aVQAONfAg==", @@ -122,7 +122,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email":"example@test.com", "timestamp":1446549615, - "smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>", "event":"bounce", "category":"cat facts", "sg_event_id":"PQmsSRnaTMVde4mu4TUgTQ==", @@ -137,7 +137,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email":"example@test.com", "timestamp":1446549615, - "smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>", "event":"dropped", "category":"cat facts", "sg_event_id":"BP0-vnv2BjDPzwaldo-XVg==", @@ -152,7 +152,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email":"example@test.com", "timestamp":1446549615, - "smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>", "event":"spamreport", "category":"cat facts", "sg_event_id":"ApWZolLiPe04wm5jAhFifA==", @@ -165,7 +165,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email":"example@test.com", "timestamp":1446549615, - "smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>", "event":"unsubscribe", "category":"cat facts", "sg_event_id":"HoBsy5C1Tcoc1dJNsy5SfA==", @@ -178,7 +178,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email":"example@test.com", "timestamp":1446549615, - "smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>", "event":"group_unsubscribe", "category":"cat facts", "sg_event_id":"hew55AFBIgLbd33pcviQTQ==", @@ -195,7 +195,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email":"example@test.com", "timestamp":1446549615, - "smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>", "event":"group_resubscribe", "category":"cat facts", "sg_event_id":"TDlqEy7cUfKLVMY3EAVCag==", @@ -322,7 +322,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email": "example@test.com", "timestamp": 1446549615, - "smtp-id": "\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id": "<14c5d75ce93.dfd.64b469@ismtpd-555>", "event": "moon landing", "category": "cat facts", "sg_event_id": "sZROwMGMagFgnOEmSdvhig==", @@ -386,7 +386,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email": "example@test.com", "timestamp": 1446549615, - "smtp-id": "\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id": "<14c5d75ce93.dfd.64b469@ismtpd-555>", "event": "processed", "category": "cat facts", "sg_event_id": "sZROwMGMagFgnOEmSdvhig==", @@ -395,7 +395,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email": "example@test.com", "timestamp": 1446549615, - "smtp-id": "\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id": "<14c5d75ce93.dfd.64b469@ismtpd-555>", "category": "cat facts", "sg_event_id": "sZROwMGMagFgnOEmSdvhig==", "sg_message_id": "14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0" @@ -434,7 +434,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email": "example@test.com", "timestamp": 1446549615, - "smtp-id": "\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id": "<14c5d75ce93.dfd.64b469@ismtpd-555>", "event": "processed", "category": "cat facts", "sg_event_id": "sZROwMGMagFgnOEmSdvhig==", @@ -457,7 +457,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats ) val expectedJson = - """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.sendgrid/processed/jsonschema/3-0-0","data":{"timestamp":"2015-11-03T11:20:15.000Z","email":"example@test.com","marketing_campaign_name":"campaign name","sg_event_id":"sZROwMGMagFgnOEmSdvhig==","smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e","marketing_campaign_version":"B","marketing_campaign_id":12345,"marketing_campaign_split_id":13471,"category":"cat facts","sg_message_id":"14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}}}""" + """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.sendgrid/processed/jsonschema/3-0-0","data":{"timestamp":"2015-11-03T11:20:15.000Z","email":"example@test.com","marketing_campaign_name":"campaign name","sg_event_id":"sZROwMGMagFgnOEmSdvhig==","smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>","marketing_campaign_version":"B","marketing_campaign_id":12345,"marketing_campaign_split_id":13471,"category":"cat facts","sg_message_id":"14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}}}""" adapterWithDefaultSchemas .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) @@ -488,7 +488,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email":"example@test.com", "timestamp":1446549615, - "smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>", "event":"processed", "category":"cat facts", "sg_event_id":"sZROwMGMagFgnOEmSdvhig==", @@ -501,7 +501,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email":"example@test.com", "timestamp":1446549615, - "smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>", "event":"processed", "category":"cat facts", "sg_event_id":"sZROwMGMagFgnOEmSdvhig==", @@ -539,7 +539,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email":"example@test.com", "timestamp":1446549615, - "smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>", "event":"processed", "category":"cat facts", "sg_event_id":"sZROwMGMagFgnOEmSdvhig==", @@ -552,7 +552,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats { "email":"example@test.com", "timestamp":1446549615, - "smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", + "smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>", "event":"deferred", "category":"cat facts", "sg_event_id":"jWmZXTZbtHTV2-S47asrww==", @@ -584,7 +584,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats ) val expectedJsonProcessed = - """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.custom/processed/jsonschema/1-2-3","data":{"timestamp":"2015-11-03T11:20:15.000Z","email":"example@test.com","marketing_campaign_name":"campaign name","sg_event_id":"sZROwMGMagFgnOEmSdvhig==","smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e","marketing_campaign_version":"B","marketing_campaign_id":12345,"marketing_campaign_split_id":13471,"category":"cat facts","sg_message_id":"14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}}}""" + """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.custom/processed/jsonschema/1-2-3","data":{"timestamp":"2015-11-03T11:20:15.000Z","email":"example@test.com","marketing_campaign_name":"campaign name","sg_event_id":"sZROwMGMagFgnOEmSdvhig==","smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>","marketing_campaign_version":"B","marketing_campaign_id":12345,"marketing_campaign_split_id":13471,"category":"cat facts","sg_message_id":"14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}}}""" val expectedJsonDeferred = """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.sendgrid/deferred/jsonschema/3-0-0","data":{"timestamp":"2015-11-03T11:20:15.000Z","email":"example@test.com","marketing_campaign_name":"campaign name","sg_event_id":"jWmZXTZbtHTV2-S47asrww==","smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>","marketing_campaign_version":"B","response":"400 try again later","marketing_campaign_id":12345,"marketing_campaign_split_id":13471,"category":"cat facts","attempt":"5","sg_message_id":"14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}}}""" diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/CachingEvaluatorSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/CachingEvaluatorSpec.scala index 34df2a432..faa485138 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/CachingEvaluatorSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/CachingEvaluatorSpec.scala @@ -70,7 +70,7 @@ class CachingEvaluatorSpec extends Specification with CatsEffect { v1 must beRight(json""" { "field": "value1" } """) v2 must beRight(json""" { "field": "value2" } """) } - } + }: Unit "1 call - success, 2 call - error => fallback to previous success, still TTL for errors in force " in { for { @@ -85,7 +85,7 @@ class CachingEvaluatorSpec extends Specification with CatsEffect { v2 must beRight(json""" { "field": "value1" } """) v3 must beRight(json""" { "field": "value2" } """) } - } + }: Unit "1 call - error, 2 call - error => use new error" in { for { @@ -111,7 +111,7 @@ class CachingEvaluatorSpec extends Specification with CatsEffect { } } } - } + }: Unit private def getValue(context: TestContext, ifEvaluated: GetResult[IO, Json]): IO[Either[Throwable, Json]] = { implicit val clock: TestClock = context.clock diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/SchemaEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/SchemaEnrichmentSpec.scala index 0d9a47908..85ced7c54 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/SchemaEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/SchemaEnrichmentSpec.scala @@ -62,7 +62,7 @@ class SchemaEnrichmentSpec extends Specification with DataTables { SchemaVer.Full(1, 0, 0) ) | "unstruct event" !! unstructEvent(signupFormSubmitted) ! Some( - SpecHelpers.jsonStringToSDJ(signupFormSubmitted).right.get + SpecHelpers.jsonStringToSDJ(signupFormSubmitted).toOption.get ) ! SchemaKey( "com.snowplowanalytics.snowplow-website", "signup_form_submitted", diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/AnonIpEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/AnonIpEnrichmentSpec.scala index e60d05007..ace27d255 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/AnonIpEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/AnonIpEnrichmentSpec.scala @@ -14,9 +14,12 @@ import org.specs2.{ScalaCheck, Specification} import org.specs2.matcher.DataTables import org.scalacheck._ import org.scalacheck.Prop.forAll + import java.net.{Inet4Address, Inet6Address} import com.google.common.net.{InetAddresses => GuavaInetAddress} +import scala.annotation.nowarn + /** * Tests the anonymzeIp function */ @@ -85,6 +88,8 @@ class AnonIpEnrichmentSpec extends Specification with DataTables with ScalaCheck ip <- Gen.listOfN(segNum, segmentGen) } yield ip.updated(segNum - 1, s":${ip(segNum - 1)}")).map(_.mkString(":")) + //"disable `a type was inferred to be `AnyVal`; this may indicate a programming error` for `countProp and validIpProp`") + @nowarn def e2 = forAll(ipv4Gen, octetsNumGen) { case (ip, octetsNum) => @@ -96,6 +101,8 @@ class AnonIpEnrichmentSpec extends Specification with DataTables with ScalaCheck countProp and validIpProp } + //"disable `a type was inferred to be `AnyVal`; this may indicate a programming error` for `countProp and validIpProp`") + @nowarn def e3 = forAll(ipv6Gen, segmentsNumGen) { case (ip, segemntsNum) => @@ -108,6 +115,8 @@ class AnonIpEnrichmentSpec extends Specification with DataTables with ScalaCheck countProp and validIpProp } + //"disable `a type was inferred to be `AnyVal`; this may indicate a programming error` for `countProp and validIpProp`") + @nowarn def e4 = forAll(shortenedIPv6Gen, segmentsNumGen) { case (ip, segemntsNum) => diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EventFingerprintEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EventFingerprintEnrichmentSpec.scala index b3bdef2f8..105b3fd5d 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EventFingerprintEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EventFingerprintEnrichmentSpec.scala @@ -28,7 +28,7 @@ class EventFingerprintEnrichmentSpec extends Specification { val standardConfig = EventFingerprintEnrichment( - EventFingerprintEnrichment.getAlgorithm("MD5").right.get, + EventFingerprintEnrichment.getAlgorithm("MD5").toOption.get, List("stm", "eid") ) diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/JavascriptScriptEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/JavascriptScriptEnrichmentSpec.scala index b7a424028..66efd4230 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/JavascriptScriptEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/JavascriptScriptEnrichmentSpec.scala @@ -21,6 +21,7 @@ import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData import com.snowplowanalytics.snowplow.badrows.FailureDetails import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent +import io.circe.Json class JavascriptScriptEnrichmentSpec extends Specification { def is = s2""" @@ -144,7 +145,7 @@ class JavascriptScriptEnrichmentSpec extends Specification { var a = 42 // no-op }""" - JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched()) must beRight(Nil) + JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched()) must beRight(List[SelfDescribingData[Json]]()) } def e10 = { @@ -153,7 +154,7 @@ class JavascriptScriptEnrichmentSpec extends Specification { return null }""" - JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched()) must beRight(Nil) + JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched()) must beRight(List[SelfDescribingData[Json]]()) } def e11 = { diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentSpec.scala index 1f686ca04..76a04f839 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentSpec.scala @@ -457,7 +457,7 @@ class ApiRequestEnrichmentSpec extends Specification with ValidatedMatchers with failingLookup(ignoreOnError = false).map(_ must beInvalid) def e7 = - failingLookup(ignoreOnError = true).map(_ must beValid(List.empty)) + failingLookup(ignoreOnError = true).map(_ must beValid(List.empty[SelfDescribingData[Json]])) private def failingLookup(ignoreOnError: Boolean): IO[ValidatedNel[FailureDetails.EnrichmentFailure, List[SelfDescribingData[Json]]]] = { val inputs = List() diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/pii/PiiPseudonymizerEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/pii/PiiPseudonymizerEnrichmentSpec.scala index 53a0894be..9519d01e5 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/pii/PiiPseudonymizerEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/pii/PiiPseudonymizerEnrichmentSpec.scala @@ -46,7 +46,6 @@ import com.snowplowanalytics.snowplow.enrich.common.loaders._ import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent import com.snowplowanalytics.snowplow.enrich.common.AcceptInvalid -import com.snowplowanalytics.snowplow.enrich.common.SpecHelpers import com.snowplowanalytics.snowplow.enrich.common.SpecHelpers._ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidatedMatchers with CatsEffect { diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/InputSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/InputSpec.scala index e1131e2b3..7556a6037 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/InputSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/InputSpec.scala @@ -11,16 +11,13 @@ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.sqlquery import scala.collection.immutable.IntMap - import io.circe.DecodingFailure import io.circe.literal._ import io.circe.parser._ - import org.specs2.Specification import org.specs2.matcher.ValidatedMatchers - import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer, SelfDescribingData} - +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.sqlquery.Input.ExtractedValueMap import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent class InputSpec extends Specification with ValidatedMatchers { @@ -175,7 +172,7 @@ class InputSpec extends Specification with ValidatedMatchers { customContexts = List(cookieContext, overriderContext), unstructEvent = Some(unstructEvent) ) - placeholderMap must beRight(None) + placeholderMap must beRight[Option[ExtractedValueMap]](None) } def e8 = { diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentIntegrationTest.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentIntegrationTest.scala index f499c7a52..629db2d1b 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentIntegrationTest.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentIntegrationTest.scala @@ -409,7 +409,7 @@ class SqlQueryEnrichmentIntegrationTest extends Specification with ValidatedMatc for { enrichment <- SqlQueryEnrichment.parse(configuration, SCHEMA_KEY).map(_.enrichment[IO](shift)).toOption.get contexts <- enrichment.lookup(event, Nil, Nil, None) - } yield contexts must beValid(Nil) + } yield contexts must beValid(List[SelfDescribingData[Json]]()) } } @@ -425,7 +425,7 @@ class SqlQueryEnrichmentIntegrationTest extends Specification with ValidatedMatc } def e5 = - invalidCreds(ignoreOnError = true).map(_ must beRight(List.empty)) + invalidCreds(ignoreOnError = true).map(_ must beRight(List.empty[SelfDescribingData[Json]])) private def invalidCreds(ignoreOnError: Boolean) = { val configuration = diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ExtractPageUriSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ExtractPageUriSpec.scala index 862805309..3686c934b 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ExtractPageUriSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ExtractPageUriSpec.scala @@ -25,7 +25,7 @@ class ExtractPageUriSpec extends Specification with DataTables { // No URI def e1 = - PageEnrichments.extractPageUri(None, None) must beRight(None) + PageEnrichments.extractPageUri(None, None) must beRight[Option[URI]](None) // Valid URI combinations val originalUri = "http://www.mysite.com/shop/session/_internal/checkout" diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala index f60d5778a..494c18cc5 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala @@ -135,7 +135,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect "return None if unstruct_event field is empty" >> { IgluUtils .extractAndValidateUnstructEvent(new EnrichedEvent, SpecHelpers.client, SpecHelpers.registryLookup) - .map(_ must beValid(None)) + .map(_ must beValid[Option[IgluUtils.SdjExtractResult]](None)) } "return a SchemaViolation.NotJson if unstruct_event does not contain a properly formatted JSON string" >> { @@ -282,7 +282,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect "return Nil if contexts field is empty" >> { IgluUtils .extractAndValidateInputContexts(new EnrichedEvent, SpecHelpers.client, SpecHelpers.registryLookup) - .map(_ must beValid(Nil)) + .map(_ must beValid(List[IgluUtils.SdjExtractResult]())) } "return a SchemaViolation.NotJson if .contexts does not contain a properly formatted JSON string" >> { @@ -442,7 +442,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect "validateEnrichmentsContexts" should { "return a BadRow.EnrichmentFailures with one expected failure for one invalid context" >> { val contexts = List( - SpecHelpers.jsonStringToSDJ(invalidEmailSent).right.get + SpecHelpers.jsonStringToSDJ(invalidEmailSent).toOption.get ) IgluUtils @@ -467,8 +467,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect "return a BadRow.EnrichmentFailures 2 expected failures for 2 invalid contexts" >> { val contexts = List( - SpecHelpers.jsonStringToSDJ(invalidEmailSent).right.get, - SpecHelpers.jsonStringToSDJ(noSchema).right.get + SpecHelpers.jsonStringToSDJ(invalidEmailSent).toOption.get, + SpecHelpers.jsonStringToSDJ(noSchema).toOption.get ) IgluUtils @@ -501,8 +501,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect "return a BadRow.EnrichmentFailures with an expected failure for 1 valid context and one invalid" >> { val contexts = List( - SpecHelpers.jsonStringToSDJ(invalidEmailSent).right.get, - SpecHelpers.jsonStringToSDJ(emailSent1).right.get + SpecHelpers.jsonStringToSDJ(invalidEmailSent).toOption.get, + SpecHelpers.jsonStringToSDJ(emailSent1).toOption.get ) IgluUtils @@ -527,8 +527,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect "not return any error for 2 valid contexts" >> { val contexts = List( - SpecHelpers.jsonStringToSDJ(emailSent1).right.get, - SpecHelpers.jsonStringToSDJ(emailSent2).right.get + SpecHelpers.jsonStringToSDJ(emailSent1).toOption.get, + SpecHelpers.jsonStringToSDJ(emailSent2).toOption.get ) IgluUtils diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonPathSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonPathSpec.scala index 638ac60cb..0bba1b3e5 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonPathSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonPathSpec.scala @@ -75,14 +75,14 @@ class JsonPathSpec extends Specification { beRight(List(Json.fromDoubleOrNull(12.99))) def e2 = - JsonPath.query("$.store.book[5].price", someJson) must beRight(Nil) + JsonPath.query("$.store.book[5].price", someJson) must beRight(List[Json]()) def e3 = - JsonPath.query("$.store.unicorns", someJson) must beRight(Nil) + JsonPath.query("$.store.unicorns", someJson) must beRight(List[Json]()) def e4 = //TODO it's not failure anymore because `.notJsonPath` is not treated as invalid jsonpath by jayway - JsonPath.query(".notJsonPath", someJson) must beRight(Nil) + JsonPath.query(".notJsonPath", someJson) must beRight(List[Json]()) def e5 = JsonPath.query("$.store.book[a]", someJson) must beLeft.like { @@ -90,7 +90,7 @@ class JsonPathSpec extends Specification { } def e6 = - JsonPath.query("$.store.book[2]", Json.fromString("somestring")) must beRight(List()) + JsonPath.query("$.store.book[2]", Json.fromString("somestring")) must beRight(List[Json]()) def e7 = { val q1 = JsonPath.query("$.empId", json"""{ "empId": 2147483649 }""") must beRight(List(Json.fromLong(2147483649L))) diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonUtilsSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonUtilsSpec.scala index 2b9a4310a..ed703bc25 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonUtilsSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonUtilsSpec.scala @@ -80,8 +80,7 @@ class JsonUtilsSpec extends Specification { val malformedDate = "2020-09-02" val correctDate = "2020-09-02T22:00:00.000Z" - val exp1 = JsonUtils.toJson(key, Option(malformedDate), Nil, Nil, Some(NonEmptyList.one(key) -> formatter)) must - be !== (key -> Json.fromString(malformedDate)) + val exp1 = JsonUtils.toJson(key, Option(malformedDate), Nil, Nil, Some(NonEmptyList.one(key) -> formatter)) mustNotEqual(key -> Json.fromString(malformedDate)) val exp2 = JsonUtils.toJson(key, Option(correctDate), Nil, Nil, Some(NonEmptyList.one(key) -> formatter)) must beEqualTo(key -> Json.fromString(correctDate)) diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/conversionUtilsSpecs.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/conversionUtilsSpecs.scala index a6dcd1c39..86b6d110c 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/conversionUtilsSpecs.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/conversionUtilsSpecs.scala @@ -13,6 +13,9 @@ package com.snowplowanalytics.snowplow.enrich.common.utils import java.net.{Inet6Address, InetAddress, URI} import java.nio.ByteBuffer import java.nio.charset.StandardCharsets +import java.lang.Integer +import java.lang.{Byte => JByte} + import cats.syntax.either._ import cats.syntax.option._ @@ -394,11 +397,11 @@ class StringToJIntegerSpec extends Specification with DataTables { def e2 = "SPEC NAME" || "INPUT STR" | "EXPECTED" | - "Integer #1" !! "0" ! 0 | - "Integer #2" !! "23" ! 23 | - "Negative integer #1" !! "-2012103" ! -2012103 | - "Negative integer #2" !! "-1" ! -1 | - "Null" !! null ! null |> { (_, str, expected) => + "Integer #1" !! "0" ! Integer.valueOf(0) | + "Integer #2" !! "23" ! Integer.valueOf(23) | + "Negative integer #1" !! "-2012103" ! Integer.valueOf(-2012103) | + "Negative integer #2" !! "-1" ! Integer.valueOf(-1) | + "Null" !! null ! null |> { (_, str: String, expected: Integer) => ConversionUtils.stringToJInteger(str) must beRight(expected) } } @@ -435,8 +438,8 @@ class StringToBooleanLikeJByteSpec extends Specification with DataTables { def e2 = "SPEC NAME" || "INPUT STR" | "EXPECTED" | - "True aka 1" !! "1" ! 1.toByte | - "False aka 0" !! "0" ! 0.toByte |> { (_, str, expected) => + "True aka 1" !! "1" ! (1.toByte: JByte) | + "False aka 0" !! "0" ! (0.toByte: JByte) |> { (_, str, expected) => ConversionUtils.stringToBooleanLikeJByte(FieldName, str) must beRight(expected) } } diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala index 2559020e1..e26014bae 100644 --- a/project/BuildSettings.scala +++ b/project/BuildSettings.scala @@ -28,6 +28,7 @@ object BuildSettings { lazy val projectSettings = Seq( organization := "com.snowplowanalytics", scalaVersion := "2.13.12", + scalacOptions := List("-Wconf:cat=lint:s"), licenses += ("Apache-2.0", url("http://www.apache.org/licenses/LICENSE-2.0.html")) )