Feature/1693 api v3 versioned model - Dataset v3 implementation

data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala

@@ -16,3 +16,5 @@
 package za.co.absa.enceladus.model.versionedModel
 
 case class VersionedSummary(_id: String, latestVersion: Int)
+
+case class VersionsList(_id: String, versions: Seq[Int])

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala

@@ -39,11 +39,17 @@ class SpringFoxConfig extends ProjectMetadata {
   }
 
   private def filteredPaths: Predicate[String] =
-    or[String](regex("/api/dataset.*"), regex("/api/schema.*"),
+    or[String](
+      // api v2
+      regex("/api/dataset.*"), regex("/api/schema.*"),
       regex("/api/mappingTable.*"), regex("/api/properties.*"),
       regex("/api/monitoring.*"),regex("/api/runs.*"),
       regex("/api/user.*"), regex("/api/spark.*"),
-      regex("/api/configuration.*")
+      regex("/api/configuration.*"),
+
+      // api v3
+        regex("/api-v3/datasets.*")
+
     )
 
   private def apiInfo =

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala

@@ -113,7 +113,7 @@ class DatasetController @Autowired()(datasetService: DatasetService)
   def replaceProperties(@AuthenticationPrincipal principal: UserDetails,
                         @PathVariable datasetName: String,
                         @RequestBody newProperties: Optional[Map[String, String]]): CompletableFuture[ResponseEntity[Option[Dataset]]] = {
-    datasetService.replaceProperties(principal.getUsername, datasetName, newProperties.toScalaOption).map {
+    datasetService.updatePropertiesV2(principal.getUsername, datasetName, newProperties.toScalaOption).map {
       case None => throw notFound()
       case Some(dataset) =>
         val location: URI = new URI(s"/api/dataset/${dataset.name}/${dataset.version}")

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala

@@ -44,6 +44,11 @@ class RestExceptionHandler {
 
   private val logger = LoggerFactory.getLogger(this.getClass)
 
+  @ExceptionHandler(value = Array(classOf[IllegalArgumentException]))
+  def handleIllegalArgumentException(exception: IllegalArgumentException): ResponseEntity[Any] = {
+    ResponseEntity.badRequest().body(exception.getMessage)
+  }
+
   @ExceptionHandler(value = Array(classOf[AsyncRequestTimeoutException]))
   def handleAsyncRequestTimeoutException(exception: AsyncRequestTimeoutException): ResponseEntity[Any] = {
     val message = Option(exception.getMessage).getOrElse("Request timeout expired.")

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala

@@ -17,7 +17,6 @@ package za.co.absa.enceladus.rest_api.controllers
 
 import java.util.Optional
 import java.util.concurrent.CompletableFuture
-
 import com.mongodb.client.result.UpdateResult
 import org.springframework.http.HttpStatus
 import org.springframework.security.core.annotation.AuthenticationPrincipal
@@ -29,6 +28,8 @@ import za.co.absa.enceladus.rest_api.exceptions.NotFoundException
 import za.co.absa.enceladus.rest_api.services.VersionedModelService
 import za.co.absa.enceladus.model.menas.audit._
 
+import scala.concurrent.Future
+
 abstract class VersionedModelController[C <: VersionedModel with Product with Auditable[C]](versionedModelService: VersionedModelService[C])
   extends BaseController {
 

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala

@@ -0,0 +1,110 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.enceladus.rest_api.controllers.v3
+
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.http.{HttpStatus, ResponseEntity}
+import org.springframework.security.core.annotation.AuthenticationPrincipal
+import org.springframework.security.core.userdetails.UserDetails
+import org.springframework.web.bind.annotation._
+import za.co.absa.enceladus.model.Dataset
+import za.co.absa.enceladus.model.conformanceRule.ConformanceRule
+import za.co.absa.enceladus.rest_api.services.v3.DatasetServiceV3
+import za.co.absa.enceladus.rest_api.utils.implicits._
+
+import java.util.concurrent.CompletableFuture
+import javax.servlet.http.HttpServletRequest
+import scala.concurrent.ExecutionContext.Implicits.global
+
+@RestController
+@RequestMapping(path = Array("/api-v3/datasets"))
+class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3)
+  extends VersionedModelControllerV3(datasetService) {
+
+  @GetMapping(Array("/{name}/{version}/properties"))
+  @ResponseStatus(HttpStatus.OK)
+  def getAllPropertiesForVersion(@PathVariable name: String,
+                                 @PathVariable version: String): CompletableFuture[Map[String, String]] = {
+    forVersionExpression(name, version)(datasetService.getVersion).map {
+      case Some(entity) => entity.propertiesAsMap
+      case None => throw notFound()
+    }
+  }
+
+  @PutMapping(Array("/{name}/{version}/properties"))
+  @ResponseStatus(HttpStatus.OK)
+  def updateProperties(@AuthenticationPrincipal principal: UserDetails,
+                       @PathVariable name: String,
+                       @PathVariable version: String,
+                       @RequestBody newProperties: java.util.Map[String, String],
+                       request: HttpServletRequest): CompletableFuture[ResponseEntity[Nothing]] = {
+    forVersionExpression(name, version) { case (dsName, dsVersion) =>
+      datasetService.updateProperties(principal.getUsername, dsName, dsVersion, newProperties.toScalaMap).map {
+
+        case Some(entity) =>
+          // stripping last 3 segments (/dsName/dsVersion/properties), instead of /api-v3/dastasets/dsName/dsVersion/properties we want /api-v3/dastasets/dsName/dsVersion/properties
+          createdWithNameVersionLocation(entity.name, entity.version, request, stripLastSegments = 3, suffix = "/properties")
+        case None => throw notFound()
+      }
+    }
+  }
+
+  // todo putIntoInfoFile switch needed?
+
+  @GetMapping(Array("/{name}/{version}/rules"))
+  @ResponseStatus(HttpStatus.OK)
+  def getConformanceRules(@PathVariable name: String,
+                          @PathVariable version: String): CompletableFuture[Seq[ConformanceRule]] = {
+    forVersionExpression(name, version)(datasetService.getVersion).map {
+      case Some(entity) => entity.conformance
+      case None => throw notFound()
+    }
+  }
+
+  @PostMapping(Array("/{name}/{version}/rules"))
+  @ResponseStatus(HttpStatus.CREATED)
+  def addConformanceRule(@AuthenticationPrincipal user: UserDetails,
+                         @PathVariable name: String,
+                         @PathVariable version: String,
+                         @RequestBody rule: ConformanceRule,
+                         request: HttpServletRequest): CompletableFuture[ResponseEntity[Nothing]] = {
+    forVersionExpression(name, version)(datasetService.getVersion).flatMap {
+      case Some(entity) => datasetService.addConformanceRule(user.getUsername, name, entity.version, rule).map {
+        case Some(updatedDs) =>
+          val addedRuleOrder = updatedDs.conformance.last.order
+          createdWithNameVersionLocation(name, updatedDs.version, request, stripLastSegments = 3, // strip: /{name}/{version}/rules
+            suffix = s"/rules/$addedRuleOrder")
+        case _ => throw notFound()
+      }
+      case None => throw notFound()
+    }
+  }
+
+  @GetMapping(Array("/{name}/{version}/rules/{order}"))
+  @ResponseStatus(HttpStatus.OK)
+  def getConformanceRuleByOrder(@PathVariable name: String,
+                                @PathVariable version: String,
+                                @PathVariable order: Int): CompletableFuture[ConformanceRule] = {
+    for {
+      optDs <- forVersionExpression(name, version)(datasetService.getVersion)
+      ds = optDs.getOrElse(throw notFound())
+      rule = ds.conformance.find(_.order == order).getOrElse(throw notFound())
+    } yield rule
+  }
+
+}
+
+

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala

@@ -0,0 +1,217 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.enceladus.rest_api.controllers.v3
+
+import com.mongodb.client.result.UpdateResult
+import org.springframework.http.{HttpStatus, ResponseEntity}
+import org.springframework.security.core.annotation.AuthenticationPrincipal
+import org.springframework.security.core.userdetails.UserDetails
+import org.springframework.web.bind.annotation._
+import org.springframework.web.servlet.support.ServletUriComponentsBuilder
+import za.co.absa.enceladus.model.menas.audit._
+import za.co.absa.enceladus.model.versionedModel._
+import za.co.absa.enceladus.model.{ExportableObject, UsedIn, Validation}
+import za.co.absa.enceladus.rest_api.controllers.BaseController
+import za.co.absa.enceladus.rest_api.controllers.v3.VersionedModelControllerV3.LatestVersionKey
+import za.co.absa.enceladus.rest_api.services.VersionedModelService
+
+import java.net.URI
+import java.util
+import java.util.Optional
+import java.util.concurrent.CompletableFuture
+import javax.servlet.http.HttpServletRequest
+import scala.concurrent.Future
+import scala.util.{Failure, Success, Try}
+
+abstract class VersionedModelControllerV3[C <: VersionedModel with Product
+  with Auditable[C]](versionedModelService: VersionedModelService[C]) extends BaseController {
+
+  import za.co.absa.enceladus.rest_api.utils.implicits._
+
+  import scala.concurrent.ExecutionContext.Implicits.global
+
+  // todo maybe offset/limit?
+  @GetMapping(Array(""))
+  @ResponseStatus(HttpStatus.OK)
+  def getList(@RequestParam searchQuery: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = {
+    versionedModelService.getLatestVersionsSummary(searchQuery.toScalaOption)
+  }
+
+  @GetMapping(Array("/{name}"))
+  @ResponseStatus(HttpStatus.OK)
+  def getVersionsList(@PathVariable name: String): CompletableFuture[VersionsList] = {
+    versionedModelService.getAllVersionsValues(name) map {
+      case Some(entity) => entity
+      case None => throw notFound()
+    }
+  }
+
+  @GetMapping(Array("/{name}/{version}"))
+  @ResponseStatus(HttpStatus.OK)
+  def getVersionDetail(@PathVariable name: String,
+                       @PathVariable version: String): CompletableFuture[C] = {
+    forVersionExpression(name, version)(versionedModelService.getVersion).map {
+      case Some(entity) => entity
+      case None => throw notFound()
+    }
+  }
+
+
+  @GetMapping(Array("/{name}/audit-trail"))
+  @ResponseStatus(HttpStatus.OK)
+  def getAuditTrail(@PathVariable name: String): CompletableFuture[AuditTrail] = {
+    versionedModelService.getAuditTrail(name)
+  }
+
+  @GetMapping(Array("/{name}/{version}/used-in"))
+  @ResponseStatus(HttpStatus.OK)
+  def usedIn(@PathVariable name: String,
+             @PathVariable version: String): CompletableFuture[UsedIn] = {
+    forVersionExpression(name, version) { case (name, versionInt) => versionedModelService.getUsedIn(name, Some(versionInt)) }
+  }
+
+  @GetMapping(Array("/{name}/{version}/export"))
+  @ResponseStatus(HttpStatus.OK)
+  def exportSingleEntity(@PathVariable name: String, @PathVariable version: String): CompletableFuture[String] = {
+    forVersionExpression(name, version)(versionedModelService.exportSingleItem)
+  }
+
+  @GetMapping(Array("/{name}/export"))
+  @ResponseStatus(HttpStatus.OK)
+  def exportLatestEntity(@PathVariable name: String): CompletableFuture[String] = {
+    versionedModelService.exportLatestItem(name) // todo: remove in favor of the above? (that supports /{name}/latest/export)
+  }
+
+  @PostMapping(Array("/{name}/import"))
+  @ResponseStatus(HttpStatus.CREATED)
+  def importSingleEntity(@AuthenticationPrincipal principal: UserDetails,
+                         @PathVariable name: String,
+                         @RequestBody importObject: ExportableObject[C],
+                         request: HttpServletRequest): CompletableFuture[ResponseEntity[Nothing]] = {
+    if (name != importObject.item.name) {
+      Future.failed(new IllegalArgumentException(s"URL and payload entity name mismatch: '$name' != '${importObject.item.name}'"))
+    } else {
+      versionedModelService.importSingleItem(importObject.item, principal.getUsername, importObject.metadata).map {
+        case Some(entity) =>
+          // stripping two last segments, instead of /api-v3/dastasets/dsName/import + /dsName/dsVersion we want /api-v3/dastasets + /dsName/dsVersion
+          createdWithNameVersionLocation(entity.name, entity.version, request, stripLastSegments = 2)
+        case None => throw notFound()
+      }
+    }
+  }
+
+  @GetMapping(Array("/{name}/{version}/validation"))
+  @ResponseStatus(HttpStatus.OK)
+  def validation(@PathVariable name: String,
+                 @PathVariable version: String): CompletableFuture[Validation] = {
+    forVersionExpression(name, version)(versionedModelService.validate)
+  }
+
+  @PostMapping(Array(""))
+  @ResponseStatus(HttpStatus.CREATED)
+  def create(@AuthenticationPrincipal principal: UserDetails,
+             @RequestBody item: C,
+             request: HttpServletRequest): CompletableFuture[ResponseEntity[Nothing]] = {
+    versionedModelService.isDisabled(item.name).flatMap { isDisabled =>
+      if (isDisabled) {
+        versionedModelService.recreate(principal.getUsername, item)
+      } else {
+        versionedModelService.create(item, principal.getUsername)
+      }
+    }.map {
+      case Some(entity) => createdWithNameVersionLocation(entity.name, entity.version, request)
+      case None => throw notFound()
+    }
+  }
+
+  @PutMapping(Array("/{name}/{version}"))
+  @ResponseStatus(HttpStatus.NO_CONTENT)
+  def edit(@AuthenticationPrincipal user: UserDetails,
+           @PathVariable name: String,
+           @PathVariable version: Int,
+           @RequestBody item: C,
+           request: HttpServletRequest): CompletableFuture[ResponseEntity[Nothing]] = {
+
+    if (name != item.name) {
+      Future.failed(new IllegalArgumentException(s"URL and payload entity name mismatch: '$name' != '${item.name}'"))
+    } else if (version != item.version) {
+      Future.failed(new IllegalArgumentException(s"URL and payload version mismatch: ${version} != ${item.version}"))
+    } else {
+      versionedModelService.update(user.getUsername, item).map {
+        case Some(entity) => createdWithNameVersionLocation(entity.name, entity.version, request, stripLastSegments = 2)
+        case None => throw notFound()
+      }
+    }
+  }
+
+  @DeleteMapping(Array("/{name}", "/{name}/{version}"))
+  @ResponseStatus(HttpStatus.OK)
+  def disable(@PathVariable name: String,
+              @PathVariable version: Optional[String]): CompletableFuture[UpdateResult] = {
+    val v = if (version.isPresent) {
+      // For some reason Spring reads the Optional[Int] param as a Optional[String] and then throws ClassCastException
+      Some(version.get.toInt)
+    } else {
+      None
+    }
+    versionedModelService.disableVersion(name, v)
+  }
+
+  /**
+   * For entity's name and version expression (either a number or 'latest'), the forVersionFn is called.
+   *
+   * @param name
+   * @param versionStr
+   * @param forVersionFn
+   * @return
+   */
+  protected def forVersionExpression[T](name: String, versionStr: String)
+                                       (forVersionFn: (String, Int) => Future[T]): Future[T] = {
+    versionStr.toLowerCase match {
+      case LatestVersionKey =>
+        versionedModelService.getLatestVersionValue(name).flatMap {
+          case None => Future.failed(notFound())
+          case Some(actualLatestVersion) => forVersionFn(name, actualLatestVersion)
+        }
+      case nonLatestVersionString => Try(nonLatestVersionString.toInt) match {
+        case Success(actualVersion) => forVersionFn(name, actualVersion)
+        case Failure(exception) =>
+          Future.failed(new IllegalArgumentException(s"Cannot convert '$versionStr' to a valid version expression. " +
+            s"Either use 'latest' or an actual version number. Underlying problem: ${exception.getMessage}"))
+      }
+    }
+  }
+
+  protected def createdWithNameVersionLocation(name: String, version: Int, request: HttpServletRequest,
+                                               stripLastSegments: Int = 0, suffix: String = ""): ResponseEntity[Nothing] = {
+    val strippingPrefix = Range(0, stripLastSegments).map(_ => "/..").mkString
+
+    val location: URI = ServletUriComponentsBuilder.fromRequest(request)
+      .path(s"$strippingPrefix/{name}/{version}$suffix")
+      .buildAndExpand(name, version.toString)
+      .normalize() // will normalize `/one/two/../three` into `/one/tree`
+      .toUri() // will create location e.g. http:/domain.ext/api-v3/dataset/MyExampleDataset/1
+
+    // hint on "/.." + normalize https://github.com/spring-projects/spring-framework/issues/14905#issuecomment-453400918
+
+    ResponseEntity.created(location).build()
+  }
+
+}
+
+object VersionedModelControllerV3 {
+  val LatestVersionKey = "latest"
+}