Avoid unpack conflict (#217)

Avoid unpacking conflict by using dedicated folder.
Drop avroUseNamespace task

README.md

@@ -45,7 +45,6 @@ to generate the avro classes.
 | `avroFieldVisibility`          | `public`                                                                 | Field visibility for the properties. Possible values: `private`, `public`.              |
 | `avroOptionalGetters`          | `false` (requires avro `1.10+`)                                          | Generate getters that return `Optional` for nullable fields.                            |
 | `avroStringType`               | `CharSequence`                                                           | Type for representing strings. Possible values: `CharSequence`, `String`, `Utf8`.       |
-| `avroUseNamespace`             | `false`                                                                  | Validate that directory layout reflects namespaces, i.e. `com/myorg/MyRecord.avsc`.     |
 | `avroVersion`                  | `1.12.0`                                                                 | Avro version to use in the project.                                                     |
 
 ### Scoped settings (Compile/Test)

api/src/main/java/com/github/sbt/avro/AvroCompiler.java

@@ -6,13 +6,12 @@ public interface AvroCompiler {
 
     void setStringType(String stringType);
     void setFieldVisibility(String fieldVisibility);
-    void setUseNamespace(boolean useNamespace);
     void setEnableDecimalLogicalType(boolean enableDecimalLogicalType);
     void setCreateSetters(boolean createSetters);
     void setOptionalGetters(boolean optionalGetters);
 
     void recompile(Class<?>[] records, File target) throws Exception;
     void compileIdls(File[] idls, File target) throws Exception;
-    void compileAvscs(AvroFileRef[] avscs, File target) throws Exception;
+    void compileAvscs(File[] avscs, File target) throws Exception;
     void compileAvprs(File[] avprs, File target) throws Exception;
 }

bridge/src/main/java/com/github/sbt/avro/AvroCompilerBridge.java

@@ -21,7 +21,6 @@ public class AvroCompilerBridge implements AvroCompiler {
 
     protected StringType stringType;
     protected FieldVisibility fieldVisibility;
-    protected boolean useNamespace;
     protected boolean enableDecimalLogicalType;
     protected boolean createSetters;
     protected boolean optionalGetters;
@@ -41,11 +40,6 @@ public void setFieldVisibility(String fieldVisibility) {
         this.fieldVisibility = FieldVisibility.valueOf(fieldVisibility);
     }
 
-    @Override
-    public void setUseNamespace(boolean useNamespace) {
-        this.useNamespace = useNamespace;
-    }
-
     @Override
     public void setEnableDecimalLogicalType(boolean enableDecimalLogicalType) {
         this.enableDecimalLogicalType = enableDecimalLogicalType;
@@ -101,21 +95,16 @@ public void compileIdls(File[] idls, File target) throws Exception {
     }
 
     @Override
-    public void compileAvscs(AvroFileRef[] avscs, File target) throws Exception {
-        List<AvroFileRef> files = new ArrayList<>(avscs.length);
-        for (AvroFileRef ref : avscs) {
-            System.out.println("Compiling Avro schema: " + ref.getFile());
-            files.add(ref);
-        }
-        Map<AvroFileRef, Schema> schemas = parser.parseFiles(files);
-        if (useNamespace) {
-            for (Map.Entry<AvroFileRef, Schema> s: schemas.entrySet()) {
-                validateParsedSchema(s.getKey(), s.getValue());
-            }
+    public void compileAvscs(File[] avscs, File target) throws Exception {
+        List<File> files = new ArrayList<>(avscs.length);
+        for (File schema : avscs) {
+            System.out.println("Compiling Avro schema: " + schema);
+            files.add(schema);
         }
+        Map<File, Schema> schemas = parser.parseFiles(files);
 
-        for (Map.Entry<AvroFileRef, Schema> entry: schemas.entrySet()) {
-            File file = entry.getKey().getFile();
+        for (Map.Entry<File, Schema> entry: schemas.entrySet()) {
+            File file = entry.getKey();
             Schema schema = entry.getValue();
             SpecificCompiler compiler = new SpecificCompiler(schema);
             configureCompiler(compiler);
@@ -133,24 +122,4 @@ public void compileAvprs(File[] avprs, File target) throws Exception {
             compiler.compileToDestination(null, target);
         }
     }
-
-    private void validateParsedSchema(AvroFileRef src, Schema schema) {
-        if (useNamespace) {
-            if (schema.getType() != Schema.Type.RECORD && schema.getType() != Schema.Type.ENUM) {
-                throw new SchemaGenerationException(String.format(
-                        "Error compiling schema file %s. "
-                                + "Only one root RECORD or ENUM type is allowed per file.",
-                        src
-                ));
-            } else if (!src.pathToClassName().equals(schema.getFullName())) {
-                throw new SchemaGenerationException(String.format(
-                        "Error compiling schema file %s. "
-                                + "File class name %s does not match record class name %s",
-                        src,
-                        src.pathToClassName(),
-                        schema.getFullName()
-                ));
-            }
-        }
-    }
 }

bridge/src/main/java/com/github/sbt/avro/AvscFilesParser.java

@@ -4,6 +4,7 @@
 import org.apache.avro.Schema;
 
 import java.io.IOException;
+import java.io.File;
 import java.util.*;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -30,20 +31,20 @@ public void addTypes(Iterable<Schema> types) {
     }
   }
 
-  public Map<AvroFileRef, Schema> parseFiles(Collection<AvroFileRef> files) {
-    Set<AvroFileRef> unparsedFiles = new HashSet<>(files);
-    Map<AvroFileRef, Schema> parsedFiles = new HashMap<>();
-    Map<AvroFileRef, Exception> parseExceptions = new HashMap<>();
+  public Map<File, Schema> parseFiles(Collection<File> files) {
+    Set<File> unparsedFiles = new HashSet<>(files);
+    Map<File, Schema> parsedFiles = new HashMap<>();
+    Map<File, Exception> parseExceptions = new HashMap<>();
 
     Schema.Parser parser = unstashParser();
     boolean progressed = true;
     while (progressed && !unparsedFiles.isEmpty()) {
       progressed = false;
       parseExceptions.clear();
 
-      for (AvroFileRef file : unparsedFiles) {
+      for (File file : unparsedFiles) {
         try {
-          Schema schema = parser.parse(file.getFile());
+          Schema schema = parser.parse(file);
           parsedFiles.put(file, schema);
           progressed = true;
           stashParser(parser);
@@ -64,7 +65,7 @@ public Map<AvroFileRef, Schema> parseFiles(Collection<AvroFileRef> files) {
                 String message = Optional.ofNullable(parseExceptions.get(f))
                         .map(Exception::getMessage)
                         .orElse("Unknown error");
-                return f.getFile().getName() + ": " + message;
+                return f.getName() + ": " + message;
               })
               .collect(Collectors.joining(",\n"));
 

bridge/src/test/scala/com/github/sbt/avro/AvscFilesParserSpec.scala

@@ -16,19 +16,19 @@ class AvscFilesParserSpec extends Specification {
   "It should be possible to compile types depending on others if source files are provided in right order" >> {
     val parser = new AvscFilesParser()
     val fullyQualifiedNames = Seq(
-      new AvroFileRef(sourceDir, "a.avsc"),
-      new AvroFileRef(sourceDir, "b.avsc"),
-      new AvroFileRef(sourceDir, "c.avsc"),
-      new AvroFileRef(sourceDir, "d.avsc"),
-      new AvroFileRef(sourceDir, "e.avsc")
+      new File(sourceDir, "a.avsc"),
+      new File(sourceDir, "b.avsc"),
+      new File(sourceDir, "c.avsc"),
+      new File(sourceDir, "d.avsc"),
+      new File(sourceDir, "e.avsc")
     )
 
     val simpleNames = Seq(
-      new AvroFileRef(sourceDir, "_a.avsc"),
-      new AvroFileRef(sourceDir, "_b.avsc"),
-      new AvroFileRef(sourceDir, "_c.avsc"),
-      new AvroFileRef(sourceDir, "_d.avsc"),
-      new AvroFileRef(sourceDir, "_e.avsc")
+      new File(sourceDir, "_a.avsc"),
+      new File(sourceDir, "_b.avsc"),
+      new File(sourceDir, "_c.avsc"),
+      new File(sourceDir, "_d.avsc"),
+      new File(sourceDir, "_e.avsc")
     )
 
     val sourceFiles = fullyQualifiedNames ++ simpleNames
@@ -66,7 +66,7 @@ class AvscFilesParserSpec extends Specification {
           |  ]
           |}""".stripMargin
     )
-    val parent = new AvroFileRef(sourceDir, "test_records.avsc")
+    val parent = new File(sourceDir, "test_records.avsc")
     parser.addTypes(Seq(dependant).asJava)
     val schemas = parser.parseFiles(Seq(parent).asJava)
     val names = schemas.asScala.values.map(_.getFullName)

plugin/src/main/scala/com/github/sbt/avro/SbtAvro.scala

@@ -41,7 +41,6 @@ object SbtAvro extends AutoPlugin {
     val avroSource = settingKey[File]("Default Avro source directory for *.avsc, *.avdl and *.avpr files.")
     val avroStringType = settingKey[String]("Type for representing strings. Possible values: CharSequence, String, Utf8.")
     val avroUnmanagedSourceDirectories = settingKey[Seq[File]]("Unmanaged Avro source directories, which contain manually created sources.")
-    val avroUseNamespace = settingKey[Boolean]("Validate that directory layout reflects namespaces, i.e. com/myorg/MyRecord.avsc.")
     val avroVersion = settingKey[String]("Avro version to use in the project.")
 
     val avroGenerate = taskKey[Seq[File]]("Generate Java sources for Avro schemas.")
@@ -60,7 +59,6 @@ object SbtAvro extends AutoPlugin {
       avroFieldVisibility := "public",
       avroOptionalGetters := false,
       avroStringType := "CharSequence",
-      avroUseNamespace := false,
 
       // addArtifact doesn't take publishArtifact setting in account
       artifacts ++= Classpaths.artifactDefs(avroArtifactTasks).value,
@@ -142,11 +140,13 @@ object SbtAvro extends AutoPlugin {
         inStyle = FilesInfo.lastModified,
         outStyle = FilesInfo.exists
       ) { deps =>
-        IO.createDirectory(extractTarget)
+        // dedicated directory per artifact to avoid name conflicts
+        val depTarget = extractTarget / jar.base
+        IO.createDirectory(depTarget)
         deps.flatMap { dep =>
           val filter = includeFilter -- excludeFilter
           val (avroSpecs, filtered) = IO
-            .unzip(dep, extractTarget, AvroFilter)
+            .unzip(dep, depTarget, AvroFilter)
             .partition(filter.accept)
           IO.delete(filtered)
           if (avroSpecs.nonEmpty) {
@@ -178,14 +178,19 @@ object SbtAvro extends AutoPlugin {
       .toSeq
       .map { case (_, _, _, f) => f }
 
-    unpack(
+    val unpacked = unpack(
       cacheBaseDirectory = cacheBaseDirectory,
       deps = avroArtifacts,
       extractTarget = (key / target).value,
       includeFilter = (key / includeFilter).value,
       excludeFilter = (key / excludeFilter).value,
       streams = (key / streams).value
     )
+
+    val previouslyUnpacked = key.previous.toSeq.flatten
+    IO.delete(previouslyUnpacked.diff(unpacked))
+
+    unpacked
   }
 
   private def sourceGeneratorTask(key: TaskKey[Seq[File]]) = Def.task {
@@ -239,18 +244,13 @@ object SbtAvro extends AutoPlugin {
 
                 compiler.setStringType(avroStringType.value)
                 compiler.setFieldVisibility(avroFieldVisibility.value.toUpperCase)
-                compiler.setUseNamespace(avroUseNamespace.value)
                 compiler.setEnableDecimalLogicalType(avroEnableDecimalLogicalType.value)
                 compiler.setCreateSetters(avroCreateSetters.value)
                 compiler.setOptionalGetters(avroOptionalGetters.value)
 
                 val recs = records.map(avroClassLoader.loadClass)
                 val avdls = srcDirs.flatMap(d => (d ** AvroAvdlFilter).get())
-                val avscs = srcDirs.flatMap(d =>
-                  (d ** AvroAvscFilter)
-                    .get()
-                    .map(avsc => new AvroFileRef(d, avsc.relativeTo(d).get.toString))
-                )
+                val avscs = srcDirs.flatMap(d => (d ** AvroAvscFilter).get())
                 val avprs = srcDirs.flatMap(d => (d ** AvroAvrpFilter).get())
 
                 out.log.info(

plugin/src/sbt-test/sbt-avro/publishing/build.sbt

@@ -69,11 +69,11 @@ lazy val root: Project = project
     Compile / avroUnpackDependencies / excludeFilter := (Compile / avroUnpackDependencies / excludeFilter).value || "exclude.avsc",
 
     Compile / checkUnpacked := {
-      exists(crossTarget.value / "src_managed" / "avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "external" / "avdl.avdl")
-      exists(crossTarget.value / "src_managed" / "avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "external" / "avpr.avpr")
-      exists(crossTarget.value / "src_managed" / "avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "external" / "avsc.avsc")
-      absent(crossTarget.value / "src_managed" / "avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "external" / "exclude.avsc")
-      exists(crossTarget.value / "src_managed" / "avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "transitive" / "avsc.avsc")
+      exists(crossTarget.value / "src_managed" / "avro" / "main" / "external-avro" / "avdl.avdl")
+      exists(crossTarget.value / "src_managed" / "avro" / "main" / "external-avro" / "avpr.avpr")
+      exists(crossTarget.value / "src_managed" / "avro" / "main" / "external-avro" / "avsc.avsc")
+      absent(crossTarget.value / "src_managed" / "avro" / "main" / "external-avro" / "exclude.avsc")
+      exists(crossTarget.value / "src_managed" / "avro" / "main" / "transitive-avro" / "avsc.avsc")
     },
     Compile / checkGenerated := {
       exists(crossTarget.value / "src_managed" / "compiled_avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "external" / "Avdl.java")
@@ -82,7 +82,7 @@ lazy val root: Project = project
       exists(crossTarget.value / "src_managed" / "compiled_avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "transitive" / "Avsc.java")
     },
     Test / checkUnpacked := {
-      exists(crossTarget.value / "src_managed" / "avro" / "test" / "test.avsc")
+      exists(crossTarget.value / "src_managed" / "avro" / "test" / "transitive-tests" / "test.avsc")
     },
     Test / checkGenerated := {
       exists(crossTarget.value / "src_managed" / "compiled_avro" / "test" / "com" / "github" / "sbt" / "avro" / "test" / "transitive" / "Test.java")