From 05199a336044beae39a35183c629abffdc5a3293 Mon Sep 17 00:00:00 2001 From: Benedek Horvath Date: Fri, 27 Mar 2020 08:33:35 +0100 Subject: [PATCH 1/2] #32 QueryPerformanceTest and RelativeQueryPerformanceTest are added to the tests in order to benchmark VQL queries in a MagicDraw project. --- .../.classpath | 1 + .../performance/QueryPerformanceTest.java | 31 +++ .../QueryPerformanceTestAdapter.java | 30 +++ .../viatra/QueryPerformanceTest.xtend | 255 ++++++++++++++++++ .../viatra/RelativeQueryPerformanceTest.xtend | 94 +++++++ .../plugin/example/test/suite/AllTests.java | 4 +- 6 files changed, 414 insertions(+), 1 deletion(-) create mode 100644 com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/QueryPerformanceTest.java create mode 100644 com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/QueryPerformanceTestAdapter.java create mode 100644 com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/QueryPerformanceTest.xtend create mode 100644 com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/RelativeQueryPerformanceTest.xtend diff --git a/com.incquerylabs.magicdraw.plugin.example/.classpath b/com.incquerylabs.magicdraw.plugin.example/.classpath index 8878ab2..06e2690 100644 --- a/com.incquerylabs.magicdraw.plugin.example/.classpath +++ b/com.incquerylabs.magicdraw.plugin.example/.classpath @@ -4,6 +4,7 @@ + diff --git a/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/QueryPerformanceTest.java b/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/QueryPerformanceTest.java new file mode 100644 index 0000000..52df786 --- /dev/null +++ b/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/QueryPerformanceTest.java @@ -0,0 +1,31 @@ +package com.incquerylabs.magicdraw.plugin.example.test.performance; + +import java.io.File; + +import org.eclipse.viatra.query.runtime.api.GenericQueryGroup; +import org.eclipse.viatra.query.runtime.api.IQueryGroup; +import org.junit.Test; + +import com.nomagic.magicdraw.core.Project; +import com.nomagic.magicdraw.tests.MagicDrawTestCase; +import com.nomagic.magicdraw.tests.common.TestEnvironment; + +import util.CommonLibrary; + +public class QueryPerformanceTest extends MagicDrawTestCase { + + private static final IQueryGroup SYSML = GenericQueryGroup.of(CommonLibrary.instance()); + + // path: src/test/data + protected static String projectToUse = "Python_Codegen_Example.mdzip"; + + private Project project; + + @Test + public void runMeasurements() { + project = loadProject(new File(TestEnvironment.getResourceDir(), projectToUse).getAbsolutePath()); + QueryPerformanceTestAdapter adapter = new QueryPerformanceTestAdapter(project, SYSML); + adapter.queryPerformance(); + } + +} \ No newline at end of file diff --git a/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/QueryPerformanceTestAdapter.java b/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/QueryPerformanceTestAdapter.java new file mode 100644 index 0000000..20e494c --- /dev/null +++ b/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/QueryPerformanceTestAdapter.java @@ -0,0 +1,30 @@ +package com.incquerylabs.magicdraw.plugin.example.test.performance; + +import org.eclipse.viatra.query.runtime.api.AdvancedViatraQueryEngine; +import org.eclipse.viatra.query.runtime.api.IQueryGroup; + +import com.incquerylabs.magicdraw.plugin.example.test.performance.viatra.RelativeQueryPerformanceTest; +import com.incquerylabs.v4md.ViatraQueryAdapter; +import com.nomagic.magicdraw.core.Project; + +public class QueryPerformanceTestAdapter extends RelativeQueryPerformanceTest { + + private Project project; + private IQueryGroup queryGroup; + + public QueryPerformanceTestAdapter(Project project, IQueryGroup queryGroup) { + this.project = project; + this.queryGroup = queryGroup; + } + + @Override + public IQueryGroup getQueryGroup() { + return queryGroup; + } + + @Override + public AdvancedViatraQueryEngine getEngine() { + return ViatraQueryAdapter.getOrCreateAdapter(project).getEngine(); + } + +} diff --git a/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/QueryPerformanceTest.xtend b/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/QueryPerformanceTest.xtend new file mode 100644 index 0000000..d70c8a0 --- /dev/null +++ b/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/QueryPerformanceTest.xtend @@ -0,0 +1,255 @@ +/******************************************************************************* + * Copyright (c) 2010-2016, Abel Hegedus, IncQuery Labs Ltd. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-v20.html. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package com.incquerylabs.magicdraw.plugin.example.test.performance.viatra + +import com.google.common.base.Function +import com.google.common.base.Stopwatch +import com.google.common.collect.Maps +import java.util.Map +import java.util.concurrent.TimeUnit +import org.apache.log4j.Level +import org.apache.log4j.Logger +import org.eclipse.viatra.query.runtime.api.AdvancedViatraQueryEngine +import org.eclipse.viatra.query.runtime.api.IPatternMatch +import org.eclipse.viatra.query.runtime.api.IQueryGroup +import org.eclipse.viatra.query.runtime.api.IQuerySpecification +import org.eclipse.viatra.query.runtime.api.ViatraQueryMatcher +import org.eclipse.viatra.query.runtime.api.scope.QueryScope +import org.eclipse.viatra.query.runtime.util.ViatraQueryLoggingUtil +import org.eclipse.xtend.lib.annotations.Data +import org.junit.Test + +/** + * This abstract test class can be used to measure the steady-state memory requirements of the base index and + * Rete networks of individual queries on a given {@link QueryScope} and with a given query group. + * + *

+ * This test case prepares a ViatraQueryEngine on the given scope and with the provided query group. + * After the initial preparation is done, the engine is wiped (deletes the Rete network but keeps the base index). + * Next, the following is performed for each query in the group: + *

+ *

    + *
  1. Wipe the engine
    2. + *
  2. Create the matcher and count matches
    3. + *
  3. Wipe the engine
    4. + *
+ * + * After each step, the used, total and free heap space is logged in MBytes after 5 GC calls and 1 second of waiting. + * Note that even this does not always provide an absolute steady state or a precise result, but can be useful for + * finding problematic queries. + */ +abstract class QueryPerformanceTest { + + protected static extension Logger logger = ViatraQueryLoggingUtil.getLogger(QueryPerformanceTest) + + /** + * @since 1.3 + */ + @Data + protected static class QueryPerformanceData implements Comparable { + int sequence + int countMatches + long usedHeapBefore + long usedHeapAfter + public long usedHeap + long elapsed + + new(int sequence, int countMatches, long usedHeapBefore, long usedHeapAfter, long usedHeap, long elapsed) { + this.sequence = sequence + this.countMatches = countMatches + this.usedHeapBefore = usedHeapBefore + this.usedHeapAfter = usedHeapAfter + this.usedHeap = usedHeap + this.elapsed = elapsed + } + + override compareTo(QueryPerformanceData o) { + if (o === null) { + return -1 + } else { + return Integer.compare(this.sequence, o.sequence) + } + } + + } + + /** + * @since 1.3 + */ + protected AdvancedViatraQueryEngine queryEngine + protected Map results = Maps.newTreeMap + + /** + * This method shall return the query engine to be used to evaluate the queries. + * @throws ViatraQueryRuntimeException + */ + def AdvancedViatraQueryEngine getEngine() + + /** + * This method shall return the query group that contains the set of queries to evaluate. + * @throws ViatraQueryRuntimeException + */ + def IQueryGroup getQueryGroup() + + protected def prepare() { + info("Preparing query performance test") + + logMemoryProperties("Scope prepared") + + queryEngine = engine + queryGroup.prepare(queryEngine) + logMemoryProperties("Base index created") + queryEngine.wipe() + logMemoryProperties("VIATRA Query engine wiped") + info("Prepared query performance test") + } + + /** + * This test case executes the performance evaluation on the given scope and with the provided query group. + */ + @Test + def queryPerformance() { + logger.level = Level.DEBUG + prepare() + + info("Starting query performance test") + + measureEntireGroup() + + info("Finished query performance test") + + printResults() + } + + /** + * @since 1.5 + */ + protected def measureEntireGroup() { + val specifications = queryGroup.getSpecifications + val numOfSpecifications = specifications.length + var current = 0 + for (IQuerySpecification _specification : specifications) { + current++ + debug("Measuring query " + _specification.getFullyQualifiedName + "(" + current + "/" + numOfSpecifications + ")") + val usedHeapBefore = _specification.wipe + performMeasurements(_specification as IQuerySpecification>, current, usedHeapBefore) + } + } + + /** + * @since 1.3 + */ + def wipe(IQuerySpecification _specification) { + wipeAndMeasure + } + + /** + * @since 1.5 + */ + def wipeAndMeasure() { + queryEngine.wipe + val usedHeapBefore = logMemoryProperties("Wiped engine before building") + return usedHeapBefore + } + + /** + * @since 1.3 + */ + def > performMeasurements(IQuerySpecification specification, int current, long usedHeapBefore) { + return performMeasurements(specification.getFullyQualifiedName, current, usedHeapBefore) [getMatcher(specification)] + } + + /** + * The measured action may OPTIONALLY return a matcher, in which case the matches will be counted. + * @since 1.5 + */ + def performMeasurements(String queryName, int sequence, long usedHeapBefore, + Function> measuredAction + ) { + debug("Building Rete") + val watch = Stopwatch.createStarted + val matcher = measuredAction.apply(queryEngine) + watch.stop() + val countMatches = if (matcher === null) -1 else matcher.countMatches + val elapsed = watch.elapsed(TimeUnit.MILLISECONDS) + + return concludeMeasurement(queryName, sequence, countMatches, elapsed, usedHeapBefore) + } + + /** + * Includes logging and wipe. + * @since 1.5 + */ + def concludeMeasurement(String queryName, int sequence, int countMatches, long elapsed, long usedHeapBefore) { + val usedHeapAfter = logMemoryProperties("Matcher created") + val usedHeap = usedHeapAfter - usedHeapBefore + val result = new QueryPerformanceData(sequence, countMatches, usedHeapBefore, usedHeapAfter, usedHeap, elapsed) + results.put(queryName, result) + info( + "Query " + queryName + "( " + countMatches + " matches, used " + usedHeap + + " kByte heap, took " + elapsed + " ms)") + + queryEngine.wipe + logMemoryProperties("Wiped engine after building") + debug("\n-------------------------------------------\n") + + return result + } + + protected def printResults() { + + val resultSB = new StringBuilder("\n\nPerformance test results:\n") + resultSB.append( + "pattern, sequence, matches count, heap before (kb), heap after (kb), used heap (kb), elapsed (ms)\n"); + results.entrySet.forEach [ entry | + resultSB.append(entry.key) + resultSB.append(", "); + resultSB.append(entry.value.sequence) + resultSB.append(", "); + resultSB.append(entry.value.countMatches) + resultSB.append(", "); + resultSB.append(entry.value.usedHeapBefore) + resultSB.append(", "); + resultSB.append(entry.value.usedHeapAfter) + resultSB.append(", "); + resultSB.append(entry.value.usedHeap) + resultSB.append(", "); + resultSB.append(entry.value.elapsed) + resultSB.append("\n") + ] + info(resultSB) + + } + + /** + * Calls garbage collector 5 times, sleeps 1 second and logs the used, total and free heap sizes in MByte. + * + * @param logger + * @return The amount of used heap memory in kBytes + */ + protected def static logMemoryProperties(String status) { + (0 .. 4).forEach[Runtime.getRuntime().gc()] + + try { + Thread.sleep(1000) + } catch (InterruptedException e) { + trace("Sleep after GC interrupted") + } + + val totalHeapKB = Runtime.getRuntime().totalMemory() / 1024; + val freeHeapKB = Runtime.getRuntime().freeMemory() / 1024; + val usedHeapKB = totalHeapKB - freeHeapKB; + debug( + status + ": Used Heap size: " + usedHeapKB / 1024 + " MByte (Total: " + totalHeapKB / 1024 + + " MByte, Free: " + freeHeapKB / 1024 + " MByte)") + + usedHeapKB + } +} + \ No newline at end of file diff --git a/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/RelativeQueryPerformanceTest.xtend b/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/RelativeQueryPerformanceTest.xtend new file mode 100644 index 0000000..e92c262 --- /dev/null +++ b/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/RelativeQueryPerformanceTest.xtend @@ -0,0 +1,94 @@ +/******************************************************************************* + * Copyright (c) 2010-2016, Gabor Bergmann, IncQuery Labs Ltd. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-v20.html. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package com.incquerylabs.magicdraw.plugin.example.test.performance.viatra + +import com.google.common.base.Stopwatch +import com.google.common.collect.Maps +import java.util.Map +import java.util.concurrent.TimeUnit +import org.eclipse.viatra.query.runtime.api.GenericQueryGroup +import org.eclipse.viatra.query.runtime.api.IPatternMatch +import org.eclipse.viatra.query.runtime.api.IQuerySpecification +import org.eclipse.viatra.query.runtime.api.ViatraQueryMatcher + +/** + * This abstract test class can be used to measure the steady-state memory requirements + * of the Rete networks of individual queries on a given QueryScope, + * relative to the network built to evaluate its dependencies. + * In other words, the "local cost" of a query is measured; this is the memory footprint of the query + * imposed on top of the memory footprint of all other queries invoked by it. + * + *

+ * This test case prepares an ViatraEngine on the given scope and with the provided query group. + * After the initial preparation is done, the engine is wiped (deletes the Rete network but keeps the base index). + * Next, the following is performed for each query in the group: + *

+ *

    + *
  1. Wipe the engine
    2. + *
  2. Prepare all queries referenced by the query under consideration, then measure memory
    3. + *
  3. Create the matcher for the query and count matches, then measure memory again to see the difference
    4. + *
  4. Wipe the engine
    5. + *
+ * + * After each step, the used, total and free heap space is logged in MBytes after 5 GC calls and 1 second of waiting. + * Note that even this does not always provide an absolute steady state or a precise result, but can be useful for + * finding problematic queries. + * + * @since 1.3 + */ +abstract class RelativeQueryPerformanceTest extends QueryPerformanceTest { + + Map absoluteHeapResults = Maps.newTreeMap() + Map relativeHeapResults = Maps.newTreeMap() + + override > performMeasurements(IQuerySpecification specification, int current, long usedHeapBefore) { + + val prerequisites = specification.directPrerequisites + var prerequisitesHeap = 0L + { + debug("Building Prerequisites") + val watch = Stopwatch.createStarted + queryEngine.prepareGroup(prerequisites, null) + watch.stop() + val usedHeapAfter = logMemoryProperties("Prerequisites built") + + prerequisitesHeap = usedHeapAfter - usedHeapBefore + info( + "Prerequisites of query " + specification.fullyQualifiedName + "(used " + prerequisitesHeap + + " kByte heap, took " + watch.elapsed(TimeUnit.MILLISECONDS) + " ms)") + // / incQueryEngine.wipe + // logMemoryProperties("Wiped engine after building prerequisites") + } + + val result = super.performMeasurements(specification, current, usedHeapBefore) + + absoluteHeapResults.put(specification.getFullyQualifiedName, result.usedHeap) + relativeHeapResults.put(specification.getFullyQualifiedName, result.usedHeap - prerequisitesHeap) + + return result + } + + protected override printResults() { + super.printResults + val resultSB = new StringBuilder("\nAbsoluteHeap[kB]\trelativeHeap[kB]\tquery\n") + absoluteHeapResults.entrySet.forEach [ entry | + val query = entry.key + resultSB.append(String.format("%12d\t%12d\t%s\n", entry.value, relativeHeapResults.get(query), query)) + ] + info(resultSB) + + } + + protected static def getDirectPrerequisites(IQuerySpecification query) { + val referredQueries = query.internalQueryRepresentation.disjunctBodies.directReferredQueries + val referredSpecifications = referredQueries.map[publishedAs.filter(IQuerySpecification)].flatten + GenericQueryGroup.of(referredSpecifications) + } + +} diff --git a/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/suite/AllTests.java b/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/suite/AllTests.java index eba93b0..4dde8a1 100644 --- a/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/suite/AllTests.java +++ b/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/suite/AllTests.java @@ -5,10 +5,12 @@ import org.junit.runners.Suite.SuiteClasses; import com.incquerylabs.magicdraw.plugin.example.test.DummyTest; +import com.incquerylabs.magicdraw.plugin.example.test.performance.QueryPerformanceTest; @RunWith(Suite.class) @SuiteClasses({ - DummyTest.class + DummyTest.class, + QueryPerformanceTest.class }) public class AllTests {} From d58e2280b65796a9775a5258bfa87752c374f348 Mon Sep 17 00:00:00 2001 From: Benedek Horvath Date: Fri, 3 Apr 2020 17:15:30 +0200 Subject: [PATCH 2/2] #32 reimplement QueryPerformanceTest and RelativeQueryPerformanceTest in pure Java, without Guava dependency --- .../.classpath | 1 - .../viatra/QueryPerformanceTest.java | 279 ++++++++++++++++++ .../viatra/QueryPerformanceTest.xtend | 255 ---------------- .../viatra/RelativeQueryPerformanceTest.java | 105 +++++++ .../viatra/RelativeQueryPerformanceTest.xtend | 94 ------ 5 files changed, 384 insertions(+), 350 deletions(-) create mode 100644 com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/QueryPerformanceTest.java delete mode 100644 com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/QueryPerformanceTest.xtend create mode 100644 com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/RelativeQueryPerformanceTest.java delete mode 100644 com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/RelativeQueryPerformanceTest.xtend diff --git a/com.incquerylabs.magicdraw.plugin.example/.classpath b/com.incquerylabs.magicdraw.plugin.example/.classpath index 06e2690..8878ab2 100644 --- a/com.incquerylabs.magicdraw.plugin.example/.classpath +++ b/com.incquerylabs.magicdraw.plugin.example/.classpath @@ -4,7 +4,6 @@ - diff --git a/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/QueryPerformanceTest.java b/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/QueryPerformanceTest.java new file mode 100644 index 0000000..e101103 --- /dev/null +++ b/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/QueryPerformanceTest.java @@ -0,0 +1,279 @@ +/******************************************************************************* + * Copyright (c) 2010-2016, Abel Hegedus, IncQuery Labs Ltd. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-v20.html. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package com.incquerylabs.magicdraw.plugin.example.test.performance.viatra; + +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.function.Function; + +import org.apache.commons.lang.time.StopWatch; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.eclipse.viatra.query.runtime.api.AdvancedViatraQueryEngine; +import org.eclipse.viatra.query.runtime.api.IPatternMatch; +import org.eclipse.viatra.query.runtime.api.IQueryGroup; +import org.eclipse.viatra.query.runtime.api.IQuerySpecification; +import org.eclipse.viatra.query.runtime.api.ViatraQueryMatcher; +import org.eclipse.viatra.query.runtime.api.scope.QueryScope; +import org.eclipse.viatra.query.runtime.matchers.ViatraQueryRuntimeException; +import org.eclipse.viatra.query.runtime.util.ViatraQueryLoggingUtil; +import org.junit.Test; + +/** + * This abstract test class can be used to measure the steady-state memory + * requirements of the base index and Rete networks of individual queries on a + * given {@link QueryScope} and with a given query group. + * + *

+ * This test case prepares a ViatraQueryEngine on the given scope and with the + * provided query group. After the initial preparation is done, the engine is + * wiped (deletes the Rete network but keeps the base index). Next, the + * following is performed for each query in the group: + *

+ *

    + *
  1. Wipe the engine
    2. + *
  2. Create the matcher and count matches
    3. + *
  3. Wipe the engine
    4. + *
+ * + * After each step, the used, total and free heap space is logged in MBytes + * after 5 GC calls and 1 second of waiting. Note that even this does not always + * provide an absolute steady state or a precise result, but can be useful for + * finding problematic queries. + */ +public abstract class QueryPerformanceTest { + protected static Logger logger = ViatraQueryLoggingUtil.getLogger(QueryPerformanceTest.class); + + /** + * @since 1.3 + */ + protected static class QueryPerformanceData implements Comparable { + private int sequence; + private int countMatches; + private long usedHeapBefore; + private long usedHeapAfter; + public long usedHeap; + private long elapsed; + + public QueryPerformanceData(int sequence, int countMatches, long usedHeapBefore, long usedHeapAfter, + long usedHeap, long elapsed) { + this.sequence = sequence; + this.countMatches = countMatches; + this.usedHeapBefore = usedHeapBefore; + this.usedHeapAfter = usedHeapAfter; + this.usedHeap = usedHeap; + this.elapsed = elapsed; + } + + @Override + public int compareTo(QueryPerformanceData o) { + if (o == null) { + return -1; + } else { + return Integer.compare(this.sequence, o.sequence); + } + } + } + + /** + * @since 1.3 + */ + protected AdvancedViatraQueryEngine queryEngine; + protected Map results = new TreeMap<>(); + + /** + * This method shall return the query engine to be used to evaluate the queries. + * + * @throws ViatraQueryRuntimeException + */ + public abstract AdvancedViatraQueryEngine getEngine(); + + /** + * This method shall return the query group that contains the set of queries to + * evaluate. + * + * @throws ViatraQueryRuntimeException + */ + public abstract IQueryGroup getQueryGroup(); + + protected void prepare() { + logger.info("Preparing query performance test"); + + logMemoryProperties("Scope prepared"); + + queryEngine = getEngine(); + getQueryGroup().prepare(queryEngine); + logMemoryProperties("Base index created"); + queryEngine.wipe(); + logMemoryProperties("VIATRA Query engine wiped"); + logger.info("Prepared query performance test"); + } + + /** + * This test case executes the performance evaluation on the given scope and + * with the provided query group. + */ + @Test + public void queryPerformance() { + logger.setLevel(Level.DEBUG); + prepare(); + + logger.info("Starting query performance test"); + + measureEntireGroup(); + + logger.info("Finished query performance test"); + + printResults(); + } + + /** + * @since 1.5 + */ + @SuppressWarnings("unchecked") + protected void measureEntireGroup() { + Set> specifications = getQueryGroup().getSpecifications(); + int numOfSpecifications = specifications.size(); + int current = 0; + for (IQuerySpecification _specification : specifications) { + current++; + logger.debug("Measuring query " + _specification.getFullyQualifiedName() + "(" + current + "/" + + numOfSpecifications + ")"); + long usedHeapBefore = wipe(_specification); + IQuerySpecification> specification = (IQuerySpecification>) _specification; + performMeasurements(specification, current, usedHeapBefore); + } + } + + /** + * @since 1.3 + */ + public long wipe(IQuerySpecification _specification) { + return wipeAndMeasure(); + } + + /** + * @since 1.5 + */ + public long wipeAndMeasure() { + queryEngine.wipe(); + long usedHeapBefore = logMemoryProperties("Wiped engine before building"); + return usedHeapBefore; + } + + /** + * @since 1.3 + */ + public > QueryPerformanceData performMeasurements( + IQuerySpecification specification, int current, long usedHeapBefore) { + Function> measuredAction = new Function>() { + @Override + public ViatraQueryMatcher apply(AdvancedViatraQueryEngine it) { + return it.getMatcher(specification); + } + }; + + return performMeasurements(specification.getFullyQualifiedName(), current, usedHeapBefore, measuredAction); + } + + /** + * The measured action may OPTIONALLY return a matcher, in which case the + * matches will be counted. + * + * @since 1.5 + */ + public QueryPerformanceData performMeasurements(String queryName, int sequence, long usedHeapBefore, + Function> measuredAction) { + logger.debug("Building Rete"); + StopWatch watch = new StopWatch(); + watch.start(); + ViatraQueryMatcher matcher = measuredAction.apply(queryEngine); + watch.stop(); + int countMatches = -1; + if (matcher != null) { + countMatches = matcher.countMatches(); + } + long elapsed = watch.getTime(); + + return concludeMeasurement(queryName, sequence, countMatches, elapsed, usedHeapBefore); + } + + /** + * Includes logging and wipe. + * + * @since 1.5 + */ + public QueryPerformanceData concludeMeasurement(String queryName, int sequence, int countMatches, long elapsed, + long usedHeapBefore) { + long usedHeapAfter = logMemoryProperties("Matcher created"); + long usedHeap = usedHeapAfter - usedHeapBefore; + QueryPerformanceData result = new QueryPerformanceData(sequence, countMatches, usedHeapBefore, usedHeapAfter, + usedHeap, elapsed); + results.put(queryName, result); + logger.info("Query " + queryName + "( " + countMatches + " matches, used " + usedHeap + " kByte heap, took " + + elapsed + " ms)"); + + queryEngine.wipe(); + logMemoryProperties("Wiped engine after building"); + logger.debug("\n-------------------------------------------\n"); + + return result; + } + + protected void printResults() { + StringBuilder resultSB = new StringBuilder("\n\nPerformance test results:\n"); + resultSB.append( + "pattern, sequence, matches count, heap before (kb), heap after (kb), used heap (kb), elapsed (ms)\n"); + results.entrySet().forEach(entry -> { + resultSB.append(entry.getKey()); + resultSB.append(", "); + resultSB.append(entry.getValue().sequence); + resultSB.append(", "); + resultSB.append(entry.getValue().countMatches); + resultSB.append(", "); + resultSB.append(entry.getValue().usedHeapBefore); + resultSB.append(", "); + resultSB.append(entry.getValue().usedHeapAfter); + resultSB.append(", "); + resultSB.append(entry.getValue().usedHeap); + resultSB.append(", "); + resultSB.append(entry.getValue().elapsed); + resultSB.append("\n"); + }); + logger.info(resultSB); + } + + /** + * Calls garbage collector 5 times, sleeps 1 second and logs the used, total and + * free heap sizes in MByte. + * + * @param logger + * @return The amount of used heap memory in kBytes + */ + protected static long logMemoryProperties(String status) { + for (int i = 0; i <= 4; i++) { + Runtime.getRuntime().gc(); + } + + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + logger.trace("Sleep after GC interrupted"); + } + + long totalHeapKB = Runtime.getRuntime().totalMemory() / 1024; + long freeHeapKB = Runtime.getRuntime().freeMemory() / 1024; + long usedHeapKB = totalHeapKB - freeHeapKB; + logger.debug(status + ": Used Heap size: " + usedHeapKB / 1024 + " MByte (Total: " + totalHeapKB / 1024 + + " MByte, Free: " + freeHeapKB / 1024 + " MByte)"); + + return usedHeapKB; + } +} diff --git a/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/QueryPerformanceTest.xtend b/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/QueryPerformanceTest.xtend deleted file mode 100644 index d70c8a0..0000000 --- a/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/QueryPerformanceTest.xtend +++ /dev/null @@ -1,255 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2010-2016, Abel Hegedus, IncQuery Labs Ltd. - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v. 2.0 which is available at - * http://www.eclipse.org/legal/epl-v20.html. - * - * SPDX-License-Identifier: EPL-2.0 - *******************************************************************************/ -package com.incquerylabs.magicdraw.plugin.example.test.performance.viatra - -import com.google.common.base.Function -import com.google.common.base.Stopwatch -import com.google.common.collect.Maps -import java.util.Map -import java.util.concurrent.TimeUnit -import org.apache.log4j.Level -import org.apache.log4j.Logger -import org.eclipse.viatra.query.runtime.api.AdvancedViatraQueryEngine -import org.eclipse.viatra.query.runtime.api.IPatternMatch -import org.eclipse.viatra.query.runtime.api.IQueryGroup -import org.eclipse.viatra.query.runtime.api.IQuerySpecification -import org.eclipse.viatra.query.runtime.api.ViatraQueryMatcher -import org.eclipse.viatra.query.runtime.api.scope.QueryScope -import org.eclipse.viatra.query.runtime.util.ViatraQueryLoggingUtil -import org.eclipse.xtend.lib.annotations.Data -import org.junit.Test - -/** - * This abstract test class can be used to measure the steady-state memory requirements of the base index and - * Rete networks of individual queries on a given {@link QueryScope} and with a given query group. - * - *

- * This test case prepares a ViatraQueryEngine on the given scope and with the provided query group. - * After the initial preparation is done, the engine is wiped (deletes the Rete network but keeps the base index). - * Next, the following is performed for each query in the group: - *

- *

    - *
  1. Wipe the engine
    2. - *
  2. Create the matcher and count matches
    3. - *
  3. Wipe the engine
    4. - *
- * - * After each step, the used, total and free heap space is logged in MBytes after 5 GC calls and 1 second of waiting. - * Note that even this does not always provide an absolute steady state or a precise result, but can be useful for - * finding problematic queries. - */ -abstract class QueryPerformanceTest { - - protected static extension Logger logger = ViatraQueryLoggingUtil.getLogger(QueryPerformanceTest) - - /** - * @since 1.3 - */ - @Data - protected static class QueryPerformanceData implements Comparable { - int sequence - int countMatches - long usedHeapBefore - long usedHeapAfter - public long usedHeap - long elapsed - - new(int sequence, int countMatches, long usedHeapBefore, long usedHeapAfter, long usedHeap, long elapsed) { - this.sequence = sequence - this.countMatches = countMatches - this.usedHeapBefore = usedHeapBefore - this.usedHeapAfter = usedHeapAfter - this.usedHeap = usedHeap - this.elapsed = elapsed - } - - override compareTo(QueryPerformanceData o) { - if (o === null) { - return -1 - } else { - return Integer.compare(this.sequence, o.sequence) - } - } - - } - - /** - * @since 1.3 - */ - protected AdvancedViatraQueryEngine queryEngine - protected Map results = Maps.newTreeMap - - /** - * This method shall return the query engine to be used to evaluate the queries. - * @throws ViatraQueryRuntimeException - */ - def AdvancedViatraQueryEngine getEngine() - - /** - * This method shall return the query group that contains the set of queries to evaluate. - * @throws ViatraQueryRuntimeException - */ - def IQueryGroup getQueryGroup() - - protected def prepare() { - info("Preparing query performance test") - - logMemoryProperties("Scope prepared") - - queryEngine = engine - queryGroup.prepare(queryEngine) - logMemoryProperties("Base index created") - queryEngine.wipe() - logMemoryProperties("VIATRA Query engine wiped") - info("Prepared query performance test") - } - - /** - * This test case executes the performance evaluation on the given scope and with the provided query group. - */ - @Test - def queryPerformance() { - logger.level = Level.DEBUG - prepare() - - info("Starting query performance test") - - measureEntireGroup() - - info("Finished query performance test") - - printResults() - } - - /** - * @since 1.5 - */ - protected def measureEntireGroup() { - val specifications = queryGroup.getSpecifications - val numOfSpecifications = specifications.length - var current = 0 - for (IQuerySpecification _specification : specifications) { - current++ - debug("Measuring query " + _specification.getFullyQualifiedName + "(" + current + "/" + numOfSpecifications + ")") - val usedHeapBefore = _specification.wipe - performMeasurements(_specification as IQuerySpecification>, current, usedHeapBefore) - } - } - - /** - * @since 1.3 - */ - def wipe(IQuerySpecification _specification) { - wipeAndMeasure - } - - /** - * @since 1.5 - */ - def wipeAndMeasure() { - queryEngine.wipe - val usedHeapBefore = logMemoryProperties("Wiped engine before building") - return usedHeapBefore - } - - /** - * @since 1.3 - */ - def > performMeasurements(IQuerySpecification specification, int current, long usedHeapBefore) { - return performMeasurements(specification.getFullyQualifiedName, current, usedHeapBefore) [getMatcher(specification)] - } - - /** - * The measured action may OPTIONALLY return a matcher, in which case the matches will be counted. - * @since 1.5 - */ - def performMeasurements(String queryName, int sequence, long usedHeapBefore, - Function> measuredAction - ) { - debug("Building Rete") - val watch = Stopwatch.createStarted - val matcher = measuredAction.apply(queryEngine) - watch.stop() - val countMatches = if (matcher === null) -1 else matcher.countMatches - val elapsed = watch.elapsed(TimeUnit.MILLISECONDS) - - return concludeMeasurement(queryName, sequence, countMatches, elapsed, usedHeapBefore) - } - - /** - * Includes logging and wipe. - * @since 1.5 - */ - def concludeMeasurement(String queryName, int sequence, int countMatches, long elapsed, long usedHeapBefore) { - val usedHeapAfter = logMemoryProperties("Matcher created") - val usedHeap = usedHeapAfter - usedHeapBefore - val result = new QueryPerformanceData(sequence, countMatches, usedHeapBefore, usedHeapAfter, usedHeap, elapsed) - results.put(queryName, result) - info( - "Query " + queryName + "( " + countMatches + " matches, used " + usedHeap + - " kByte heap, took " + elapsed + " ms)") - - queryEngine.wipe - logMemoryProperties("Wiped engine after building") - debug("\n-------------------------------------------\n") - - return result - } - - protected def printResults() { - - val resultSB = new StringBuilder("\n\nPerformance test results:\n") - resultSB.append( - "pattern, sequence, matches count, heap before (kb), heap after (kb), used heap (kb), elapsed (ms)\n"); - results.entrySet.forEach [ entry | - resultSB.append(entry.key) - resultSB.append(", "); - resultSB.append(entry.value.sequence) - resultSB.append(", "); - resultSB.append(entry.value.countMatches) - resultSB.append(", "); - resultSB.append(entry.value.usedHeapBefore) - resultSB.append(", "); - resultSB.append(entry.value.usedHeapAfter) - resultSB.append(", "); - resultSB.append(entry.value.usedHeap) - resultSB.append(", "); - resultSB.append(entry.value.elapsed) - resultSB.append("\n") - ] - info(resultSB) - - } - - /** - * Calls garbage collector 5 times, sleeps 1 second and logs the used, total and free heap sizes in MByte. - * - * @param logger - * @return The amount of used heap memory in kBytes - */ - protected def static logMemoryProperties(String status) { - (0 .. 4).forEach[Runtime.getRuntime().gc()] - - try { - Thread.sleep(1000) - } catch (InterruptedException e) { - trace("Sleep after GC interrupted") - } - - val totalHeapKB = Runtime.getRuntime().totalMemory() / 1024; - val freeHeapKB = Runtime.getRuntime().freeMemory() / 1024; - val usedHeapKB = totalHeapKB - freeHeapKB; - debug( - status + ": Used Heap size: " + usedHeapKB / 1024 + " MByte (Total: " + totalHeapKB / 1024 + - " MByte, Free: " + freeHeapKB / 1024 + " MByte)") - - usedHeapKB - } -} - \ No newline at end of file diff --git a/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/RelativeQueryPerformanceTest.java b/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/RelativeQueryPerformanceTest.java new file mode 100644 index 0000000..ed3ae26 --- /dev/null +++ b/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/RelativeQueryPerformanceTest.java @@ -0,0 +1,105 @@ +/******************************************************************************* + * Copyright (c) 2010-2016, Gabor Bergmann, IncQuery Labs Ltd. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-v20.html. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package com.incquerylabs.magicdraw.plugin.example.test.performance.viatra; + +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Stream; + +import org.apache.commons.lang.time.StopWatch; +import org.eclipse.viatra.query.runtime.api.GenericQueryGroup; +import org.eclipse.viatra.query.runtime.api.IPatternMatch; +import org.eclipse.viatra.query.runtime.api.IQueryGroup; +import org.eclipse.viatra.query.runtime.api.IQuerySpecification; +import org.eclipse.viatra.query.runtime.api.ViatraQueryMatcher; +import org.eclipse.viatra.query.runtime.matchers.psystem.queries.PQuery; + +/** + * This abstract test class can be used to measure the steady-state memory + * requirements of the Rete networks of individual queries on a given + * QueryScope, relative to the network built to evaluate its dependencies. In + * other words, the "local cost" of a query is measured; this is the memory + * footprint of the query imposed on top of the memory footprint of all other + * queries invoked by it. + * + *

+ * This test case prepares an ViatraEngine on the given scope and with the + * provided query group. After the initial preparation is done, the engine is + * wiped (deletes the Rete network but keeps the base index). Next, the + * following is performed for each query in the group: + *

+ *

    + *
  1. Wipe the engine
    2. + *
  2. Prepare all queries referenced by the query under consideration, then + * measure memory
    3. + *
  3. Create the matcher for the query and count matches, then measure memory + * again to see the difference
    4. + *
  4. Wipe the engine
    5. + *
+ * + * After each step, the used, total and free heap space is logged in MBytes + * after 5 GC calls and 1 second of waiting. Note that even this does not always + * provide an absolute steady state or a precise result, but can be useful for + * finding problematic queries. + * + * @since 1.3 + */ +public abstract class RelativeQueryPerformanceTest extends QueryPerformanceTest { + + private Map absoluteHeapResults = new TreeMap<>(); + private Map relativeHeapResults = new TreeMap<>(); + + @Override + public > QueryPerformanceData performMeasurements( + IQuerySpecification specification, int current, long usedHeapBefore) { + + IQueryGroup prerequisites = getDirectPrerequisites(specification); + long prerequisitesHeap = 0L; + + logger.debug("Building Prerequisites"); + StopWatch watch = new StopWatch(); + watch.start(); + queryEngine.prepareGroup(prerequisites, null); + watch.stop(); + long usedHeapAfter = logMemoryProperties("Prerequisites built"); + + prerequisitesHeap = usedHeapAfter - usedHeapBefore; + logger.info("Prerequisites of query " + specification.getFullyQualifiedName() + "(used " + prerequisitesHeap + + " kByte heap, took " + watch.getTime() + " ms)"); + + QueryPerformanceData result = super.performMeasurements(specification, current, usedHeapBefore); + + absoluteHeapResults.put(specification.getFullyQualifiedName(), result.usedHeap); + relativeHeapResults.put(specification.getFullyQualifiedName(), result.usedHeap - prerequisitesHeap); + + return result; + } + + @Override + protected void printResults() { + super.printResults(); + StringBuilder resultSB = new StringBuilder("\nAbsoluteHeap[kB]\trelativeHeap[kB]\tquery\n"); + absoluteHeapResults.entrySet().forEach(entry -> { + String query = entry.getKey(); + resultSB.append(String.format("%12d\t%12d\t%s\n", entry.getValue(), relativeHeapResults.get(query), query)); + }); + logger.info(resultSB); + } + + protected static IQueryGroup getDirectPrerequisites(IQuerySpecification query) { + Set referredQueries = query.getInternalQueryRepresentation().getDisjunctBodies() + .getDirectReferredQueries(); + Stream> referredSpecifications = referredQueries.stream() + .flatMap(it -> it.publishedAs().stream().filter(IQuerySpecification.class::isInstance)) + .map(IQuerySpecification.class::cast); + return GenericQueryGroup.of(referredSpecifications); + } + +} diff --git a/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/RelativeQueryPerformanceTest.xtend b/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/RelativeQueryPerformanceTest.xtend deleted file mode 100644 index e92c262..0000000 --- a/com.incquerylabs.magicdraw.plugin.example/src/test/java/com/incquerylabs/magicdraw/plugin/example/test/performance/viatra/RelativeQueryPerformanceTest.xtend +++ /dev/null @@ -1,94 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2010-2016, Gabor Bergmann, IncQuery Labs Ltd. - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v. 2.0 which is available at - * http://www.eclipse.org/legal/epl-v20.html. - * - * SPDX-License-Identifier: EPL-2.0 - *******************************************************************************/ -package com.incquerylabs.magicdraw.plugin.example.test.performance.viatra - -import com.google.common.base.Stopwatch -import com.google.common.collect.Maps -import java.util.Map -import java.util.concurrent.TimeUnit -import org.eclipse.viatra.query.runtime.api.GenericQueryGroup -import org.eclipse.viatra.query.runtime.api.IPatternMatch -import org.eclipse.viatra.query.runtime.api.IQuerySpecification -import org.eclipse.viatra.query.runtime.api.ViatraQueryMatcher - -/** - * This abstract test class can be used to measure the steady-state memory requirements - * of the Rete networks of individual queries on a given QueryScope, - * relative to the network built to evaluate its dependencies. - * In other words, the "local cost" of a query is measured; this is the memory footprint of the query - * imposed on top of the memory footprint of all other queries invoked by it. - * - *

- * This test case prepares an ViatraEngine on the given scope and with the provided query group. - * After the initial preparation is done, the engine is wiped (deletes the Rete network but keeps the base index). - * Next, the following is performed for each query in the group: - *

- *

    - *
  1. Wipe the engine
    2. - *
  2. Prepare all queries referenced by the query under consideration, then measure memory
    3. - *
  3. Create the matcher for the query and count matches, then measure memory again to see the difference
    4. - *
  4. Wipe the engine
    5. - *
- * - * After each step, the used, total and free heap space is logged in MBytes after 5 GC calls and 1 second of waiting. - * Note that even this does not always provide an absolute steady state or a precise result, but can be useful for - * finding problematic queries. - * - * @since 1.3 - */ -abstract class RelativeQueryPerformanceTest extends QueryPerformanceTest { - - Map absoluteHeapResults = Maps.newTreeMap() - Map relativeHeapResults = Maps.newTreeMap() - - override > performMeasurements(IQuerySpecification specification, int current, long usedHeapBefore) { - - val prerequisites = specification.directPrerequisites - var prerequisitesHeap = 0L - { - debug("Building Prerequisites") - val watch = Stopwatch.createStarted - queryEngine.prepareGroup(prerequisites, null) - watch.stop() - val usedHeapAfter = logMemoryProperties("Prerequisites built") - - prerequisitesHeap = usedHeapAfter - usedHeapBefore - info( - "Prerequisites of query " + specification.fullyQualifiedName + "(used " + prerequisitesHeap + - " kByte heap, took " + watch.elapsed(TimeUnit.MILLISECONDS) + " ms)") - // / incQueryEngine.wipe - // logMemoryProperties("Wiped engine after building prerequisites") - } - - val result = super.performMeasurements(specification, current, usedHeapBefore) - - absoluteHeapResults.put(specification.getFullyQualifiedName, result.usedHeap) - relativeHeapResults.put(specification.getFullyQualifiedName, result.usedHeap - prerequisitesHeap) - - return result - } - - protected override printResults() { - super.printResults - val resultSB = new StringBuilder("\nAbsoluteHeap[kB]\trelativeHeap[kB]\tquery\n") - absoluteHeapResults.entrySet.forEach [ entry | - val query = entry.key - resultSB.append(String.format("%12d\t%12d\t%s\n", entry.value, relativeHeapResults.get(query), query)) - ] - info(resultSB) - - } - - protected static def getDirectPrerequisites(IQuerySpecification query) { - val referredQueries = query.internalQueryRepresentation.disjunctBodies.directReferredQueries - val referredSpecifications = referredQueries.map[publishedAs.filter(IQuerySpecification)].flatten - GenericQueryGroup.of(referredSpecifications) - } - -}