Merge pull request #1 from kg-construct/test-fnml

Sanity checking KGC Challenge test cases

pom.xml

@@ -4,7 +4,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>BURP</groupId>
 	<artifactId>BURP</artifactId>
-	<version>0.0.1-SNAPSHOT</version>
+	<version>0.1.1-SNAPSHOT</version>
 	<name>Basic and Unassuming RML Processor</name>
 
 	<properties>

src/main/java/burp/ls/LogicalSourceFactory.java

@@ -231,7 +231,7 @@ private static String getFile(Resource ls) {
 			return Util.downloadFile(url);
 		}
 
-		throw new RuntimeException("Source from other way not yet implemented");
+		throw new RuntimeException("Source from this logical source type not yet implemented");
 	}
 
 	private static Resource getCompression(Resource ls) {

src/main/java/burp/util/Util.java

@@ -14,6 +14,7 @@
 
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
 import org.apache.commons.io.IOUtils;
 import org.apache.jena.iri.IRIFactory;
 import org.apache.jena.iri.Violation;
@@ -133,7 +134,7 @@ public static String getDecompressedFile(String file, Resource compression) {
 			OutputStream out = new FileOutputStream(temp);
 			FileInputStream fin = new FileInputStream(file);
 			InputStream in = null;
-			
+
 			if(RML.zip.equals(compression)) {
 				ZipInputStream a = new ZipInputStream(fin);
 				a.getNextEntry();
@@ -143,17 +144,17 @@ public static String getDecompressedFile(String file, Resource compression) {
 			} else if(RML.targz.equals(compression)) {
 				// Suppress warning because we do close it
 				@SuppressWarnings("resource")
-				TarArchiveInputStream a = new TarArchiveInputStream(fin);
+				TarArchiveInputStream a = new TarArchiveInputStream(new GzipCompressorInputStream(fin));
 				a.getNextEntry();
 				in = a;
 			} else if(RML.tarxz.equals(compression)) {
 				// Suppress warning because we do close it
 				@SuppressWarnings("resource")
-				TarArchiveInputStream a = new TarArchiveInputStream(fin);
+				TarArchiveInputStream a = new TarArchiveInputStream(new XZCompressorInputStream(fin));
 				a.getNextEntry();
 				in = a;
 			}
-			
+
 			IOUtils.copy(in, out);
 			in.close();
 			out.close();

src/test/java/burp/TestRMLFNML.java

@@ -13,7 +13,7 @@
 import org.junit.jupiter.api.Test;
 
 public class TestRMLFNML {
-	private static String base = "./src/test/resources/rml-fnml/";
+	private static String base = "./src/test/resources/rml-fnml-burp/";
 
 	// UUID test should be removed from the test cases
 	// @Test public void RMLFNOTC0000CSV() throws IOException { testForOK("RMLFNOTC0000-CSV"); }

src/test/java/burp/TestRMLFNMLChallenge.java

@@ -0,0 +1,82 @@
+
+package burp;
+
+import org.apache.jena.rdf.model.Model;
+import org.apache.jena.riot.RDFDataMgr;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestRMLFNMLChallenge {
+	private static String base = "./src/test/resources/rml-fnml/";
+
+	// UUID test should be removed from the test cases
+	@Test public void RMLFNOTC0000CSV() throws IOException { testForOK("RMLFNOTC0000-CSV"); }
+	@Test public void RMLFNOTC0000CSVb() throws IOException { testForOK("RMLFNOTC0000b-CSV"); }
+	@Test public void RMLFNOTC0001CSV() throws IOException { testForOK("RMLFNOTC0001-CSV"); }
+	@Test public void RMLFNOTC0002CSV() throws IOException { testForNotOK("RMLFNOTC0002-CSV"); }
+	@Test public void RMLFNOTC0003CSV() throws IOException { testForOK("RMLFNOTC0003-CSV"); }
+	@Test public void RMLFNOTC0004CSV() throws IOException { testForOK("RMLFNOTC0004-CSV"); }
+	@Test public void RMLFNOTC0004CSVb() throws IOException { testForOK("RMLFNOTC0004b-CSV"); }
+	@Test public void RMLFNOTC0005CSV() throws IOException { testForOK("RMLFNOTC0005-CSV"); }
+	@Test public void RMLFNOTC0005CSVb() throws IOException { testForOK("RMLFNOTC0005b-CSV"); }
+	@Test public void RMLFNOTC0006CSV() throws IOException { testForOK("RMLFNOTC0006-CSV"); }
+	@Test public void RMLFNOTC0006CSVb() throws IOException { testForOK("RMLFNOTC0006b-CSV"); }
+	@Test public void RMLFNOTC0008CSV() throws IOException { testForOK("RMLFNOTC0008-CSV"); }
+	@Test public void RMLFNOTC0009CSV() throws IOException { testForOK("RMLFNOTC0009-CSV"); }
+	@Test public void RMLFNOTC0010CSV() throws IOException { testForOK("RMLFNOTC0010-CSV"); }
+
+
+
+	public void testForOK(String f) throws IOException {
+		System.out.println(String.format("Now processing %s", f));
+		String m = new File(base + f, "data/shared/mapping.ttl").getAbsolutePath().toString();
+		String r = Files.createTempFile(null, ".nq").toString();
+		System.out.println(String.format("Writing output to %s", r));
+
+		System.out.println("This test should generate a graph.");
+		String o = new File(base + f, "data/shared/expected/output.nq").getAbsolutePath().toString();
+
+		int exit = Main.doMain(new String[] { "-m", m, "-o", r, "-b", "http://example.com/base/" });
+
+		Model expected = RDFDataMgr.loadModel(o);
+		Model actual = RDFDataMgr.loadModel(r);
+
+		if (!expected.isIsomorphicWith(actual)) {
+			expected.write(System.out, "Turtle");
+			System.out.println("---");
+			actual.write(System.out, "Turtle");
+		}
+
+		assertEquals(0, exit);
+
+		System.out.println(expected.isIsomorphicWith(actual) ? "OK" : "NOK");
+
+		assertTrue(expected.isIsomorphicWith(actual));
+	}
+
+	public void testForNotOK(String f) throws IOException {
+		System.out.println(String.format("Now processing %s", f));
+		String m = new File(base + f, "data/shared/mapping.ttl").getAbsolutePath().toString();
+		String r = Files.createTempFile(null, ".nq").toString();
+		System.out.println(String.format("Writing output to %s", r));
+
+		System.out.println("This test should NOT generate a graph.");
+		int exit = Main.doMain(new String[] { "-m", m, "-o", r });
+		System.out.println(Files.size(Paths.get(r)) == 0 ? "OK" : "NOK");
+		Model actual = RDFDataMgr.loadModel(r);
+		actual.write(System.out, "NQ");
+
+		assertTrue(exit > 0);
+		assertTrue(Files.size(Paths.get(r)) == 0);
+
+		System.out.println();
+	}
+
+}

src/test/java/burp/TestRMLIOSource.java

@@ -38,7 +38,7 @@ public class TestRMLIOSource {
 	@Test public void RMLSTC0006a() throws IOException { testForOK("RMLSTC0006a"); }
 	@Test public void RMLSTC0006b() throws IOException { testForOK("RMLSTC0006b"); }
 
-	// This test requires a SPARQL endpoint
+	// This test requires a SPARQL endpoint [disabled]
 	// @Test public void RMLSTC0006c() throws IOException { testForOK("RMLSTC0006c"); }
 
 	@Test public void RMLSTC0006d() throws IOException { testForOK("RMLSTC0006d"); }

src/test/resources/rml-fnml/RMLFNOTC0000-CSV/data/shared/expected/output.nq

@@ -0,0 +1 @@
+<http://example.com/Venus> <http://xmlns.com/foaf/0.1/name> "e4dcc7ee-8e2a-4012-92cc-9a74dd545e89" .

src/test/resources/rml-fnml/RMLFNOTC0000-CSV/data/shared/mapping.rml.rml.ttl

@@ -0,0 +1,12 @@
+@prefix foaf: <http://xmlns.com/foaf/0.1/> .
+@prefix morph-kgc: <https://github.com/morph-kgc/morph-kgc/function/built-in.ttl#> .
+@prefix rml: <http://w3id.org/rml/> .
+
+<http://example.com/base/TriplesMap1> rml:logicalSource [ rml:referenceFormulation rml:CSV ;
+            rml:source "student.csv" ] ;
+    rml:predicateObjectMap [ rml:objectMap [ rml:functionExecution <http://example.com/base/#Execution> ] ;
+            rml:predicate foaf:name ] ;
+    rml:subjectMap [ rml:template "http://example.com/{Name}" ] .
+
+<http://example.com/base/#Execution> rml:function morph-kgc:uuid .
+

src/test/resources/rml-fnml/RMLFNOTC0000-CSV/data/shared/mapping.ttl

@@ -0,0 +1,31 @@
+@prefix foaf: <http://xmlns.com/foaf/0.1/> .
+@prefix ex: <http://example.com/> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+@prefix rml: <http://w3id.org/rml/> .
+@prefix fno: <https://w3id.org/function/ontology#> .
+@prefix morph-kgc: <https://github.com/morph-kgc/morph-kgc/function/built-in.ttl#> .
+@prefix grel: <http://users.ugent.be/~bjdmeest/function/grel.ttl#> .
+@prefix idlab-fn: <http://example.com/idlab/function/> .
+
+@base <http://example.com/base/> .
+
+<TriplesMap1>
+    rml:logicalSource [
+        rml:source [ a rml:RelativePathSource;
+          rml:root rml:MappingDirectory;
+          rml:path "student.csv"
+        ];
+        rml:referenceFormulation rml:CSV
+    ];
+    rml:subjectMap [
+        rml:template "http://example.com/{Name}"
+    ];
+    rml:predicateObjectMap [
+        rml:predicate foaf:name;
+        rml:objectMap [
+            rml:functionExecution <#Execution> ;
+        ]
+    ] .
+
+<#Execution>
+    rml:function morph-kgc:uuid .

src/test/resources/rml-fnml/RMLFNOTC0000-CSV/data/shared/student.csv

@@ -0,0 +1,2 @@
+Id,Name,Comment,Class
+1,Venus,A&B,A

src/test/resources/rml-fnml/RMLFNOTC0000-CSV/metadata.json

@@ -0,0 +1,30 @@
+{
+    "@id": "http://w3id.org/kg-construct/rml-fnml/test-cases/RMLFNOTC0000-CSV",
+    "name": "rml-fnml test-case RMLFNOTC0000-CSV",
+    "description": "Execute test-case RMLFNOTC0000-CSV of rml-fnml",
+    "steps": [
+        {
+            "@id": "http://w3id.org/kg-construct/rml-fnml/test-cases/RMLFNOTC0000-CSV#step1",
+            "name": "Execute RML mapping",
+            "resource": "BURP",
+            "command": "execute_mapping",
+            "parameters": {
+                "mapping_file": "mapping.rml.rml.ttl",
+                "output_file": "output.nq",
+                "serialization": "nquads"
+            },
+            "expect_failure": false
+        },
+        {
+            "@id": "http://w3id.org/kg-construct/rml-fnml/test-cases/RMLFNOTC0000-CSV#step2",
+            "name": "Validate rml-fnml/RMLFNOTC0000-CSV/output.nq",
+            "resource": "Validate",
+            "command": "compare_graphs",
+            "parameters": {
+                "graph_file": "output.nq",
+                "expected_graph_file": "expected/output.nq"
+            },
+            "expect_failure": false
+        }
+    ]
+}

src/test/resources/rml-fnml/RMLFNOTC0000-CSV/results/run_1/case-info.txt

@@ -0,0 +1,40 @@
+===> CASE <===
+Name: rml-fnml test-case RMLFNOTC0000-CSV
+Timestamp: 2024-04-15T10:06:12.861597+00:00
+Directory: /home/dylan/Projects/challenge-2024/track1-v6-burp/rml-fnml/RMLFNOTC0000-CSV
+Run: 1
+Number of steps: 2
+
+===> HARDWARE <===
+System
+	Hostname: dell-laptop
+	OS name: Fedora Linux
+	OS version: 39 (Workstation Edition)
+	Kernel: Linux-6.8.5-201.fc39.x86_64-x86_64-with-glibc2.38
+	Architecture: x86_64
+CPU
+	Name: Intel(R) Core(TM) i7-8650U CPU @ 1.90GHz
+	Cores: 8
+	Minimum frequency: 400 Hz
+	Maximum frequency: 4200 Hz
+Memory
+	RAM memory: 16615 MB
+	SWAP memory: 8589 MB
+Storage
+	Disk "/": 510.48 GB
+	Disk "/home": 510.48 GB
+	Disk "/boot": 1.02 GB
+	Disk "/boot/efi": 0.54 GB
+Network
+	Interface "lo"
+	Interface "enp0s31f6"
+	Interface "wlp2s0"
+	Interface "docker0"
+	Interface "br-8b7c5071df4c"
+
+===> DOCKER <===
+Version: 26.0.1
+Root directory: /var/lib/docker
+Drivers:
+	Storage: btrfs
+	CgroupFS: systemd v2

src/test/resources/rml-fnml/RMLFNOTC0000-CSV/results/run_1/log.txt

@@ -0,0 +1,13 @@
+[2024-04-15 12:06:14,699] ERROR    SLF4J(W): No SLF4J providers were found.
+[2024-04-15 12:06:14,699] ERROR    SLF4J(W): Defaulting to no-operation (NOP) logger implementation
+[2024-04-15 12:06:14,699] ERROR    SLF4J(W): See https://www.slf4j.org/codes.html#noProviders for further details.
+[2024-04-15 12:06:14,699] ERROR    Node=_:B8551d3750e727742b41567f52593bcdf
+[2024-04-15 12:06:14,699] ERROR    Path=<http://w3id.org/rml/source>
+[2024-04-15 12:06:14,699] ERROR    Value: "student.csv"
+[2024-04-15 12:06:14,699] ERROR    Message:
+[2024-04-15 12:06:14,699] ERROR    rml:source must be provided to locate the data source.
+[2024-04-15 12:06:14,699] ERROR    rml:source must be provided exactly once as an URI.
+[2024-04-15 12:06:14,699] ERROR    
+[2024-04-15 12:06:14,700] ERROR    Mapping did not satisfy shapes.
+[2024-04-15 12:06:14,700] ERROR    System exiting with code: 1
+[2024-04-15 12:06:14,700] ERROR    Command failed while waiting for exit with status code: 1

src/test/resources/rml-fnml/RMLFNOTC0000-CSV/results/run_1/metrics.csv

@@ -0,0 +1,20 @@
+name,run,index,step,timestamp,version,cpu_user,cpu_system,cpu_user_system,cpu_idle,cpu_iowait,memory_ram,memory_swap,memory_ram_swap,disk_read_count,disk_write_count,disk_read_bytes,disk_write_bytes,disk_read_time,disk_write_time,disk_busy_time,network_received_count,network_sent_count,network_received_bytes,network_sent_bytes,network_received_error,network_sent_error,network_received_drop,network_sent_drop
+rml-fnml test-case RMLFNOTC0000-CSV,1,1,1,0.0,3,0.0,0.0,0.0,0.0,0.0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,2,1,0.1023,3,0.05,0.1,0.15000000000000002,0.65,0.01,5384126464,2797568,5386924032,2,835,32768,39780352,16,63,86,0,0,0,0,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,3,1,0.2044,3,0.15,0.24,0.39,1.18,0.04,5394284544,2797568,5397082112,2,2088,32768,71720960,16,206,255,0,0,0,0,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,4,1,0.306,3,0.23,0.3,0.53,1.81,0.07,5412413440,2797568,5415211008,2,2201,32768,72622080,16,267,323,0,0,0,0,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,5,1,0.4081,3,0.3,0.35,0.6499999999999999,2.46,0.08,5398827008,2797568,5401624576,12,2229,450560,72859648,20,309,363,2,5,166,498,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,6,1,0.5095,3,0.39,0.54,0.93,2.95,0.11,5439885312,2797568,5442682880,717,2279,53207040,73269248,583,341,493,2,7,166,798,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,7,1,0.6109,3,0.51,0.56,1.07,3.59,0.13,5448888320,2797568,5451685888,923,2279,67944448,73269248,628,341,624,2,7,166,798,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,8,1,0.7128,3,0.69,0.58,1.27,4.18,0.15,5460668416,2797568,5463465984,1117,2299,79601664,73433088,657,352,734,2,7,166,798,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,9,1,0.8141,3,0.84,0.6,1.44,4.79,0.16,5474664448,2797568,5477462016,1265,2299,83812352,73433088,693,352,818,2,9,166,906,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,10,1,0.9161,3,0.95,0.66,1.6099999999999999,5.42,0.18,5490372608,2797568,5493170176,1549,2299,96690176,73433088,825,352,923,2,10,166,996,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,11,1,1.0174,3,1.13,0.7,1.8299999999999998,5.98,0.21,5500100608,2797568,5502898176,1822,2299,109142016,73433088,974,352,1052,2,10,166,996,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,12,1,1.1191,3,1.32,0.73,2.05,6.54,0.23,5513560064,2797568,5516357632,2013,2299,117497856,73433088,1005,352,1167,2,11,166,1082,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,13,1,1.2205,3,1.48,0.76,2.24,7.14,0.24,5517799424,2797568,5520596992,2091,2299,123633664,73433088,1043,352,1212,4,11,324,1082,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,14,1,1.3218,3,1.73,0.79,2.52,7.65,0.25,5526822912,2797568,5529620480,2119,2299,124067840,73433088,1059,352,1234,7,12,636,1196,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,15,1,1.4233,3,1.89,0.81,2.7,8.29,0.25,5535576064,2797568,5538373632,2123,2299,124387328,73433088,1064,352,1242,7,12,636,1196,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,16,1,1.5246,3,1.89,0.81,2.7,9.08,0.25,5534498816,2797568,5537296384,2123,2299,124387328,73433088,1064,352,1242,8,15,822,1560,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,17,1,1.6259,3,1.89,0.82,2.71,9.88,0.25,5534498816,2797568,5537296384,2123,2301,124387328,73441280,1064,366,1258,13,18,1817,2013,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,18,1,1.7274,3,1.91,0.85,2.76,10.65,0.25,5445509120,2797568,5448306688,2123,2303,124387328,73449472,1064,368,1262,13,18,1817,2013,0,0,0,0
+rml-fnml test-case RMLFNOTC0000-CSV,1,19,1,1.8292,3,1.97,0.89,2.86,11.29,0.3,5400670208,2797568,5403467776,2123,2471,124387328,74792960,1064,473,1378,10,11,1551,1339,0,0,0,0

src/test/resources/rml-fnml/RMLFNOTC0000b-CSV/data/shared/expected/output.nq

@@ -0,0 +1 @@
+<http://example.com/Venus> <http://xmlns.com/foaf/0.1/name> "Hello World!" .

src/test/resources/rml-fnml/RMLFNOTC0000b-CSV/data/shared/mapping.ttl

@@ -0,0 +1,31 @@
+@prefix foaf: <http://xmlns.com/foaf/0.1/> .
+@prefix ex: <http://example.com/> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+@prefix rml: <http://w3id.org/rml/> .
+@prefix fno: <https://w3id.org/function/ontology#> .
+@prefix fns: <http://example.com/functions/> .
+@prefix grel: <http://users.ugent.be/~bjdmeest/function/grel.ttl#> .
+@prefix idlab-fn: <http://example.com/idlab/function/> .
+
+@base <http://example.com/base/> .
+
+<TriplesMap1>
+    rml:logicalSource [
+        rml:source [ a rml:RelativePathSource;
+          rml:root rml:MappingDirectory;
+          rml:path "student.csv"
+        ];
+        rml:referenceFormulation rml:CSV
+    ];
+    rml:subjectMap [
+        rml:template "http://example.com/{Name}"
+    ];
+    rml:predicateObjectMap [
+        rml:predicate foaf:name;
+        rml:objectMap [
+            rml:functionExecution <#Execution> ;
+        ]
+    ] .
+
+<#Execution>
+    rml:function fns:helloworld .

src/test/resources/rml-fnml/RMLFNOTC0000b-CSV/data/shared/student.csv

@@ -0,0 +1,2 @@
+Id,Name,Comment,Class
+1,Venus,A&B,A

src/test/resources/rml-fnml/RMLFNOTC0000b-CSV/metadata.json

@@ -0,0 +1,30 @@
+{
+    "@id": "http://w3id.org/kg-construct/rml-fnml/test-cases/RMLFNOTC0000b-CSV",
+    "name": "rml-fnml test-case RMLFNOTC0000b-CSV",
+    "description": "Execute test-case RMLFNOTC0000b-CSV of rml-fnml",
+    "steps": [
+        {
+            "@id": "http://w3id.org/kg-construct/rml-fnml/test-cases/RMLFNOTC0000b-CSV#step1",
+            "name": "Execute RML mapping",
+            "resource": "BURP",
+            "command": "execute_mapping",
+            "parameters": {
+                "mapping_file": "mapping.ttl",
+                "output_file": "output.nq",
+                "serialization": "nquads"
+            },
+            "expect_failure": false
+        },
+        {
+            "@id": "http://w3id.org/kg-construct/rml-fnml/test-cases/RMLFNOTC0000b-CSV#step2",
+            "name": "Validate rml-fnml/RMLFNOTC0000b-CSV/output.nq",
+            "resource": "Validate",
+            "command": "compare_graphs",
+            "parameters": {
+                "graph_file": "output.nq",
+                "expected_graph_file": "expected/output.nq"
+            },
+            "expect_failure": false
+        }
+    ]
+}