testable phase 1: relational and testable runner (#1114)

.changeset/coffee-is-good.md

@@ -0,0 +1,5 @@
+---
+"@finos/legend-studio": minor
+---
+
+Add (WIP) global testable runner v0.

.changeset/poor-shirts-matter.md

@@ -0,0 +1,7 @@
+---
+"@finos/legend-graph": minor
+---
+
+Deprecate running service Legacy Tests. 
+Add `RelationalData` and `EqualToTDS` to support testing on relational queries.
+Support running testable tests.

.changeset/rick-shirts-matter.md

@@ -0,0 +1,4 @@
+---
+"@finos/legend-manual-tests": patch
+"@finos/legend-art": patch
+---

packages/legend-art/src/components/Icon.tsx

@@ -129,6 +129,7 @@ export {
   MdEdit as EditIcon, // to be reviewed
   MdSubject as SubjectIcon,
   MdViewHeadline as ViewHeadlineIcon,
+  MdWarning as WarningIcon,
 } from 'react-icons/md';
 export {
   VscError as ErrorIcon,

packages/legend-graph/src/MetaModelConst.ts

@@ -327,6 +327,7 @@ export enum CORE_HASH_STRUCTURE {
   EQUAL_TO_JSON_ASSERT_FAIL = 'EQUAL_TO_JSON_ASSERT_FAIL',
   EQUAL_TO = 'EQUAL_TO',
   EQUAL_TO_JSON = 'EQUAL_TO_JSON',
+  EQUAL_TO_TDS = 'EQUAL_TO_TDS',
   TEST_RESULT = 'TEST_RESULT',
   TEST_ERROR = 'TEST_ERROR',
   TEST_FAILED = 'TEST_FAILED',
@@ -337,6 +338,11 @@ export enum CORE_HASH_STRUCTURE {
   MODEL_STORE_DATA = 'MODEL_STORE_DATA',
   DATA_ELEMENT_REFERENCE = 'DATA_ELEMENT_REFERENCE',
   DATA_ELEMENT = 'DATA_ELEMENT',
+  RELATIONAL_DATA = 'RELATIONAL_DATA',
+  RELATIONAL_TDS = 'RELATIONAL_TDS',
+  RELATIONAL_DATA_TABLE = 'RELATIONAL_DATA_TABLE',
+  RELATIONAL_DATA_TABLE_ROW = 'RELATIONAL_DATA_TABLE_ROW',
+  RELATIONAL_DATA_TABLE_COLUMN = 'RELATIONAL_DATA_TABLE_COLUMN',
 }
 
 export enum MILESTONING_STEREOTYPE {

packages/legend-graph/src/graph/BasicModel.ts

@@ -65,6 +65,7 @@ import {
   getOrCreatePackage,
 } from '../helpers/DomainHelper';
 import { DataElement } from '../models/metamodels/pure/packageableElements/data/DataElement';
+import type { Testable } from '../models/metamodels/pure/test/Testable';
 
 const FORBIDDEN_EXTENSION_ELEMENT_CLASS = new Set([
   PackageableElement,
@@ -207,6 +208,14 @@ export abstract class BasicModel {
     return Array.from(this.generationSpecificationsIndex.values());
   }
 
+  get ownTestables(): Testable[] {
+    return [
+      ...this.ownServices,
+      // TODO: add mappings once supported in the backend
+      // ...this.ownMappings,
+    ];
+  }
+
   getExtensionElements<T extends PackageableElement>(
     extensionElementClass: Clazz<T>,
   ): T[] {

packages/legend-graph/src/graph/PureGraphPlugin.ts

@@ -18,6 +18,7 @@ import { AbstractPlugin, type Clazz } from '@finos/legend-shared';
 import type { PackageableElement } from '../models/metamodels/pure/packageableElements/PackageableElement';
 import type { PureModel } from './PureModel';
 import type { GraphPluginManager } from '../GraphPluginManager';
+import type { TestableExtension } from './TestableExtension';
 
 export type DeadReferencesCleaner = (graph: PureModel) => void;
 
@@ -48,4 +49,6 @@ export abstract class PureGraphPlugin extends AbstractPlugin {
    * Get the list of procedures to be done to cleanup dead references in the graph.
    */
   getExtraDeadReferencesCleaners?(): DeadReferencesCleaner[];
+
+  getExtraTestables?(): TestableExtension[];
 }

packages/legend-graph/src/graph/PureModel.ts

@@ -38,7 +38,6 @@ import type { Mapping } from '../models/metamodels/pure/packageableElements/mapp
 import type { Profile } from '../models/metamodels/pure/packageableElements/domain/Profile';
 import type { Stereotype } from '../models/metamodels/pure/packageableElements/domain/Stereotype';
 import type { Tag } from '../models/metamodels/pure/packageableElements/domain/Tag';
-import type { PackageableElement } from '../models/metamodels/pure/packageableElements/PackageableElement';
 import type { Store } from '../models/metamodels/pure/packageableElements/store/Store';
 import { DependencyManager } from '../graph/DependencyManager';
 import { ConcreteFunctionDefinition } from '../models/metamodels/pure/packageableElements/domain/ConcreteFunctionDefinition';
@@ -58,6 +57,9 @@ import {
 import type { PureGraphPlugin } from './PureGraphPlugin';
 import { createPath } from '../MetaModelUtils';
 import type { DataElement } from '../models/metamodels/pure/packageableElements/data/DataElement';
+import type { Testable } from '../models/metamodels/pure/test/Testable';
+import type { PackageableElement } from '../models/metamodels/pure/packageableElements/PackageableElement';
+import type { TestableExtension } from './TestableExtension';
 
 /**
  * CoreModel holds meta models which are constant and basic building block of the graph. Since throughout the lifetime
@@ -220,6 +222,19 @@ export class PureModel extends BasicModel {
     ];
   }
 
+  get testableExtensions(): TestableExtension[] {
+    return this.graphPlugins.flatMap(
+      (plugin) => plugin.getExtraTestables?.() ?? [],
+    );
+  }
+
+  get allOwnTestables(): Testable[] {
+    const extraTestables = this.testableExtensions.flatMap(
+      (plugin) => plugin.getExtraTestables?.(this) ?? [],
+    );
+    return [...this.ownTestables].concat(...extraTestables);
+  }
+
   getProfileStereotype = (
     path: string,
     value: string,

packages/legend-graph/src/graph/TestableExtension.ts

@@ -0,0 +1,26 @@
+/**
+ * Copyright (c) 2020-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type { Testable } from '../models/metamodels/pure/test/Testable';
+import type { PureModel } from './PureModel';
+
+export type TestableExtension = {
+  getExtraTestables?: (graph: PureModel) => Testable[];
+  getIdFromTestable?: (
+    testable: Testable,
+    graph: PureModel,
+  ) => string | undefined;
+  getTestableFromId?: (id: string, graph: PureModel) => Testable | undefined;
+};

packages/legend-graph/src/graphManager/AbstractPureGraphManager.ts

@@ -26,7 +26,6 @@ import type {
 } from './action/generation/ImportConfigurationDescription';
 import type { FileGenerationSpecification } from '../models/metamodels/pure/packageableElements/fileGeneration/FileGenerationSpecification';
 import type { GenerationOutput } from './action/generation/GenerationOutput';
-import type { ServiceTestResult } from './action/service/ServiceTestResult';
 import type { PackageableElement } from '../models/metamodels/pure/packageableElements/PackageableElement';
 import type { PureModel, CoreModel, SystemModel } from '../graph/PureModel';
 import type { Mapping } from '../models/metamodels/pure/packageableElements/mapping/Mapping';
@@ -62,6 +61,9 @@ import type { ExternalFormatDescription } from './action/externalFormat/External
 import type { ConfigurationProperty } from '../models/metamodels/pure/packageableElements/fileGeneration/ConfigurationProperty';
 import type { GraphBuilderReport } from './GraphBuilderReport';
 import type { ModelGenerationConfiguration } from '../models/ModelGenerationConfiguration';
+import type { DEPRECATED__ServiceTestResult } from './action/service/DEPRECATED__ServiceTestResult';
+import type { RunTestsTestableInput } from '../models/metamodels/pure/test/result/RunTestsTestableInput';
+import type { TestResult } from '../models/metamodels/pure/test/result/TestResult';
 
 export interface TEMPORARY__EngineSetupConfig {
   env: string;
@@ -209,6 +211,12 @@ export abstract class AbstractPureGraphManager {
     options?: { keepSourceInformation?: boolean },
   ): Promise<string>;
 
+  // ------------------------------------------- Test  -------------------------------------------
+  abstract runTests(
+    graph: PureModel,
+    testableInputs: RunTestsTestableInput[],
+  ): Promise<TestResult[]>;
+
   // ------------------------------------------- ValueSpecification  -------------------------------------------
 
   abstract buildValueSpecification(
@@ -333,15 +341,19 @@ export abstract class AbstractPureGraphManager {
     executionMode: ServiceExecutionMode,
     version: string | undefined,
   ): Promise<ServiceRegistrationResult>;
-  abstract runServiceTests(
-    service: Service,
-    graph: PureModel,
-  ): Promise<ServiceTestResult[]>;
   abstract activateService(
     serviceUrl: string,
     serviceId: string,
   ): Promise<void>;
 
+  /**
+   * @deprecated
+   */
+  abstract runLegacyServiceTests(
+    service: Service,
+    graph: PureModel,
+  ): Promise<DEPRECATED__ServiceTestResult[]>;
+
   // ------------------------------------------- Query -------------------------------------------
 
   abstract searchQueries(

packages/legend-graph/src/graphManager/action/changeDetection/DSLService_ObserverHelper.ts

@@ -52,7 +52,6 @@ import type { TestData } from '../../../models/metamodels/pure/packageableElemen
 import {
   observe_AtomicTest,
   observe_TestAssertion,
-  observe_TestSuite,
 } from './Test_ObserverHelper';
 
 export const observe_ConnectionTestData = skipObservedWithContext(
@@ -296,10 +295,7 @@ export const observe_Service = skipObservedWithContext(
     if (metamodel.test) {
       observe_ServiceTest_Legacy(metamodel.test);
     }
-    metamodel.tests.forEach((testSuite) =>
-      observe_TestSuite(testSuite, context),
-    );
-
+    metamodel.tests.forEach((m) => observe_ServiceTestSuite(m, context));
     return metamodel;
   },
 );

packages/legend-graph/src/graphManager/action/changeDetection/Data_ObserverHelper.ts

@@ -21,6 +21,12 @@ import {
   DataElementReference,
   ModelStoreData,
 } from '../../../models/metamodels/pure/data/EmbeddedData';
+import {
+  type RelationalDataTable,
+  type RelationalDataTableColumn,
+  type RelationalDataTableRow,
+  RelationalData,
+} from '../../../models/metamodels/pure/data/RelationalData';
 import type { DataElement } from '../../../models/metamodels/pure/packageableElements/data/DataElement';
 import type { EmbeddedData_PureGraphManagerPlugin_Extension } from '../../EmbeddedData_PureGraphManagerPlugin_Extension';
 import {
@@ -68,6 +74,48 @@ export const observe_ModelStoreData = skipObserved(
   },
 );
 
+export const observe_RelationalDataTableColumn = skipObserved(
+  (metamodel: RelationalDataTableColumn): RelationalDataTableColumn => {
+    makeObservable(metamodel, {
+      hashCode: computed,
+    });
+    return metamodel;
+  },
+);
+
+export const observe_RelationalDataTableRow = skipObserved(
+  (metamodel: RelationalDataTableRow): RelationalDataTableRow => {
+    makeObservable(metamodel, {
+      hashCode: computed,
+    });
+    return metamodel;
+  },
+);
+
+const observe_RelationalDataTable = skipObserved(
+  (metamodel: RelationalDataTable): RelationalDataTable => {
+    makeObservable(metamodel, {
+      columns: observable,
+      rows: observable,
+      hashCode: computed,
+    });
+    metamodel.columns.forEach(observe_RelationalDataTableColumn);
+    metamodel.rows.forEach(observe_RelationalDataTableRow);
+    return metamodel;
+  },
+);
+
+const observe_RelationalData = skipObserved(
+  (metamodel: RelationalData): RelationalData => {
+    makeObservable(metamodel, {
+      tables: observable,
+      hashCode: computed,
+    });
+    metamodel.tables.forEach(observe_RelationalDataTable);
+    return metamodel;
+  },
+);
+
 export function observe_EmbeddedData(
   metamodel: EmbeddedData,
   context: ObserverContext,
@@ -78,6 +126,8 @@ export function observe_EmbeddedData(
     return observe_ExternalFormatData(metamodel);
   } else if (metamodel instanceof ModelStoreData) {
     return observe_ModelStoreData(metamodel);
+  } else if (metamodel instanceof RelationalData) {
+    return observe_RelationalData(metamodel);
   }
   const extraEmbeddedDataObservers = context.plugins.flatMap(
     (plugin) =>