Feature/1693 api v3 versioned model - Dataset v3 implementation

data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/NamedLatestVersion.scala

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.enceladus.model.versionedModel
+
+case class NamedLatestVersion(name: String, version: Int)

data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala

@@ -15,4 +15,8 @@
 
 package za.co.absa.enceladus.model.versionedModel
 
-case class VersionedSummary(_id: String, latestVersion: Int)
+case class VersionedSummary(_id: String, latestVersion: Int) {
+  def toNamedLatestVersion: NamedLatestVersion = NamedLatestVersion(_id, latestVersion)
+}
+
+

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala

@@ -17,7 +17,7 @@ package za.co.absa.enceladus.rest_api
 
 import com.google.common.base.Predicate
 import com.google.common.base.Predicates.or
-import org.springframework.context.annotation.{Bean, Configuration}
+import org.springframework.context.annotation.{Bean, Configuration, Primary, Profile}
 import springfox.documentation.builders.PathSelectors.regex
 import springfox.documentation.builders.{ApiInfoBuilder, RequestHandlerSelectors}
 import springfox.documentation.spi.DocumentationType
@@ -28,27 +28,52 @@ import za.co.absa.enceladus.utils.general.ProjectMetadata
 @Configuration
 @EnableSwagger2
 class SpringFoxConfig extends ProjectMetadata {
+
+  import org.springframework.beans.factory.annotation.Value
+
+  @Value("${spring.profiles.active:}")
+  private val activeProfiles: String = null
+
   @Bean
   def api(): Docket = {
+    val isDev = activeProfiles.split(",").map(_.toLowerCase).contains("dev")
+
     new Docket(DocumentationType.SWAGGER_2)
-      .apiInfo(apiInfo)
+      .apiInfo(apiInfo(isDev))
       .select
       .apis(RequestHandlerSelectors.any)
-      .paths(filteredPaths)
+      .paths(filteredPaths(isDev))
       .build
   }
 
-  private def filteredPaths: Predicate[String] =
-    or[String](regex("/api/dataset.*"), regex("/api/schema.*"),
+  private def filteredPaths(isDev: Boolean): Predicate[String] = {
+    val v2Paths = Seq(
+      regex("/api/dataset.*"), regex("/api/schema.*"),
       regex("/api/mappingTable.*"), regex("/api/properties.*"),
-      regex("/api/monitoring.*"),regex("/api/runs.*"),
+      regex("/api/monitoring.*"), regex("/api/runs.*"),
       regex("/api/user.*"), regex("/api/spark.*"),
       regex("/api/configuration.*")
     )
 
-  private def apiInfo =
+    val v3paths = Seq(
+      regex("/api-v3/datasets.*"), regex("/api-v3/schemas.*"),
+      regex("/api-v3/mapping-tables.*"), regex("/api-v3/property-definitions.*")
+    )
+
+    val paths: Seq[Predicate[String]] = if (isDev) {
+      v2Paths ++ v3paths
+    } else {
+      v3paths
+    }
+
+    or[String](
+      paths: _*
+    )
+  }
+
+  private def apiInfo(isDev: Boolean) =
     new ApiInfoBuilder()
-      .title("Menas API")
+      .title(s"Menas API${ if (isDev) " - DEV " else ""}")
       .description("Menas API reference for developers")
       .license("Apache 2.0 License")
       .version(projectVersion) // api or project?

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala

@@ -64,7 +64,7 @@ class DatasetController @Autowired()(datasetService: DatasetService)
       latestVersion <- datasetService.getLatestVersionValue(datasetName)
       res <- latestVersion match {
         case Some(version) => datasetService.addConformanceRule(user.getUsername, datasetName, version, rule).map {
-          case Some(ds) => ds
+          case Some((ds, validation)) => ds // v2 disregarding validation
           case _        => throw notFound()
         }
         case _ => throw notFound()
@@ -113,7 +113,7 @@ class DatasetController @Autowired()(datasetService: DatasetService)
   def replaceProperties(@AuthenticationPrincipal principal: UserDetails,
                         @PathVariable datasetName: String,
                         @RequestBody newProperties: Optional[Map[String, String]]): CompletableFuture[ResponseEntity[Option[Dataset]]] = {
-    datasetService.replaceProperties(principal.getUsername, datasetName, newProperties.toScalaOption).map {
+    datasetService.updateProperties(principal.getUsername, datasetName, newProperties.toScalaOption).map {
       case None => throw notFound()
       case Some(dataset) =>
         val location: URI = new URI(s"/api/dataset/${dataset.name}/${dataset.version}")

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/MappingTableController.scala

@@ -40,7 +40,7 @@ class MappingTableController @Autowired() (mappingTableService: MappingTableServ
     @RequestBody upd: MenasObject[Array[DefaultValue]]): CompletableFuture[MappingTable] = {
     mappingTableService.updateDefaults(user.getUsername, upd.id.name,
       upd.id.version, upd.value.toList).map {
-        case Some(entity) => entity
+        case Some(entity) => entity._1 // v2 disregarding validation
         case None         => throw notFound()
       }
   }
@@ -51,7 +51,7 @@ class MappingTableController @Autowired() (mappingTableService: MappingTableServ
     @RequestBody newDefault: MenasObject[DefaultValue]): CompletableFuture[MappingTable] = {
     mappingTableService.addDefault(user.getUsername, newDefault.id.name,
       newDefault.id.version, newDefault.value).map {
-        case Some(entity) => entity
+        case Some(entity) => entity._1 // v2 disregarding validation
         case None         => throw notFound()
       }
   }

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala

@@ -44,6 +44,11 @@ class RestExceptionHandler {
 
   private val logger = LoggerFactory.getLogger(this.getClass)
 
+  @ExceptionHandler(value = Array(classOf[IllegalArgumentException]))
+  def handleIllegalArgumentException(exception: IllegalArgumentException): ResponseEntity[Any] = {
+    ResponseEntity.badRequest().body(exception.getMessage)
+  }
+
   @ExceptionHandler(value = Array(classOf[AsyncRequestTimeoutException]))
   def handleAsyncRequestTimeoutException(exception: AsyncRequestTimeoutException): ResponseEntity[Any] = {
     val message = Option(exception.getMessage).getOrElse("Request timeout expired.")

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/SchemaController.scala

@@ -148,7 +148,7 @@ class SchemaController @Autowired()(
         // the parsing of sparkStruct can fail, therefore we try to save it first before saving the attachment
         update <- schemaService.schemaUpload(username, menasAttachment.refName, menasAttachment.refVersion - 1, sparkStruct)
         _ <- attachmentService.uploadAttachment(menasAttachment)
-      } yield update
+      } yield update.map(_._1) // v2 disregarding the validation
     } catch {
       case e: SchemaParsingException => throw e.copy(schemaType = schemaType) //adding schema type
     }

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala

@@ -17,7 +17,6 @@ package za.co.absa.enceladus.rest_api.controllers
 
 import java.util.Optional
 import java.util.concurrent.CompletableFuture
-
 import com.mongodb.client.result.UpdateResult
 import org.springframework.http.HttpStatus
 import org.springframework.security.core.annotation.AuthenticationPrincipal
@@ -29,6 +28,7 @@ import za.co.absa.enceladus.rest_api.exceptions.NotFoundException
 import za.co.absa.enceladus.rest_api.services.VersionedModelService
 import za.co.absa.enceladus.model.menas.audit._
 
+
 abstract class VersionedModelController[C <: VersionedModel with Product with Auditable[C]](versionedModelService: VersionedModelService[C])
   extends BaseController {
 
@@ -39,7 +39,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au
   @GetMapping(Array("/list", "/list/{searchQuery}"))
   @ResponseStatus(HttpStatus.OK)
   def getList(@PathVariable searchQuery: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = {
-    versionedModelService.getLatestVersionsSummary(searchQuery.toScalaOption)
+    versionedModelService.getLatestVersionsSummarySearch(searchQuery.toScalaOption)
   }
 
   @GetMapping(Array("/searchSuggestions"))
@@ -114,7 +114,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au
   @ResponseStatus(HttpStatus.CREATED)
   def importSingleEntity(@AuthenticationPrincipal principal: UserDetails,
                          @RequestBody importObject: ExportableObject[C]): CompletableFuture[C] = {
-    versionedModelService.importSingleItem(importObject.item, principal.getUsername, importObject.metadata).map {
+    versionedModelService.importSingleItemV2(importObject.item, principal.getUsername, importObject.metadata).map {
       case Some(entity) => entity
       case None         => throw notFound()
     }
@@ -130,7 +130,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au
         versionedModelService.create(item, principal.getUsername)
       }
     }.map {
-      case Some(entity) => entity
+      case Some((entity, validation)) => entity // v2 does not support validation-warnings on create
       case None         => throw notFound()
     }
   }
@@ -140,7 +140,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au
   def edit(@AuthenticationPrincipal user: UserDetails,
       @RequestBody item: C): CompletableFuture[C] = {
     versionedModelService.update(user.getUsername, item).map {
-      case Some(entity) => entity
+      case Some((entity, validation)) => entity // v2 disregarding validation on edit
       case None         => throw notFound()
     }
   }

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.enceladus.rest_api.controllers.v3
+
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.http.{HttpStatus, ResponseEntity}
+import org.springframework.security.core.annotation.AuthenticationPrincipal
+import org.springframework.security.core.userdetails.UserDetails
+import org.springframework.web.bind.annotation._
+import za.co.absa.enceladus.model.Validation
+import za.co.absa.enceladus.model.conformanceRule.ConformanceRule
+import za.co.absa.enceladus.rest_api.services.v3.DatasetServiceV3
+import za.co.absa.enceladus.rest_api.utils.implicits._
+
+import java.util.concurrent.CompletableFuture
+import javax.servlet.http.HttpServletRequest
+import scala.concurrent.ExecutionContext.Implicits.global
+
+@RestController
+@RequestMapping(path = Array("/api-v3/datasets"))
+class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3)
+  extends VersionedModelControllerV3(datasetService) {
+
+  @GetMapping(Array("/{name}/{version}/properties"))
+  @ResponseStatus(HttpStatus.OK)
+  def getAllPropertiesForVersion(@PathVariable name: String,
+                                 @PathVariable version: String): CompletableFuture[Map[String, String]] = {
+    forVersionExpression(name, version)(datasetService.getVersion).map {
+      case Some(entity) => entity.propertiesAsMap
+      case None => throw notFound()
+    }
+  }
+
+  @PutMapping(Array("/{name}/{version}/properties"))
+  @ResponseStatus(HttpStatus.OK)
+  def updateProperties(@AuthenticationPrincipal principal: UserDetails,
+                       @PathVariable name: String,
+                       @PathVariable version: String,
+                       @RequestBody newProperties: java.util.Map[String, String],
+                       request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = {
+    forVersionExpression(name, version) { case (dsName, dsVersion) =>
+      datasetService.updateProperties(principal.getUsername, dsName, dsVersion, newProperties.toScalaMap).map {
+
+        case Some((entity, validation)) =>
+          // stripping last 3 segments (/dsName/dsVersion/properties), instead of /api-v3/dastasets/dsName/dsVersion/properties we want /api-v3/dastasets/dsName/dsVersion/properties
+          createdWithNameVersionLocationBuilder(entity.name, entity.version, request, stripLastSegments = 3, suffix = "/properties")
+            .body(validation) // todo include in tests
+        case None => throw notFound()
+      }
+    }
+  }
+
+  // todo putIntoInfoFile switch needed?
+
+  @GetMapping(Array("/{name}/{version}/rules"))
+  @ResponseStatus(HttpStatus.OK)
+  def getConformanceRules(@PathVariable name: String,
+                          @PathVariable version: String): CompletableFuture[Seq[ConformanceRule]] = {
+    forVersionExpression(name, version)(datasetService.getVersion).map {
+      case Some(entity) => entity.conformance
+      case None => throw notFound()
+    }
+  }
+
+  @PostMapping(Array("/{name}/{version}/rules"))
+  @ResponseStatus(HttpStatus.CREATED)
+  def addConformanceRule(@AuthenticationPrincipal user: UserDetails,
+                         @PathVariable name: String,
+                         @PathVariable version: String,
+                         @RequestBody rule: ConformanceRule,
+                         request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = {
+    forVersionExpression(name, version)(datasetService.getVersion).flatMap {
+      case Some(entity) => datasetService.addConformanceRule(user.getUsername, name, entity.version, rule).map {
+        case Some((updatedDs, validation)) =>
+          val addedRuleOrder = updatedDs.conformance.last.order
+          createdWithNameVersionLocationBuilder(name, updatedDs.version, request, stripLastSegments = 3, // strip: /{name}/{version}/rules
+            suffix = s"/rules/$addedRuleOrder").body(validation)
+        case _ => throw notFound()
+      }
+      case None => throw notFound()
+    }
+  }
+
+  @GetMapping(Array("/{name}/{version}/rules/{order}"))
+  @ResponseStatus(HttpStatus.OK)
+  def getConformanceRuleByOrder(@PathVariable name: String,
+                                @PathVariable version: String,
+                                @PathVariable order: Int): CompletableFuture[ConformanceRule] = {
+    for {
+      optDs <- forVersionExpression(name, version)(datasetService.getVersion)
+      ds = optDs.getOrElse(throw notFound())
+      rule = ds.conformance.find(_.order == order).getOrElse(throw notFound())
+    } yield rule
+  }
+
+}
+
+