DataObject support for Scout JS

- Introduce a DataObjectTransformer which adds annotations during
  TypeScript compile time to all attributes of DataObjects. These
  annotations contain the data type of the attribute to be retrieved at
  runtime.
- Add a base class for DataObjects: BaseDoEntity
- Add ID classes in TS (number, string uuid, composite)
- Add a DoRegistry to retrieve all DataObjects known in TS
- Migrate existing DOs to the new base class including decorators.
- Add convenience methods to ajax utility to post and get DataObjects
- Add a DataObject (De)Serializer for TS. It uses the data type
  annotations added by the compile time transformer and supports:
  - Arrays
  - Maps
  - Sets
  - Records
  - Dates
  - Ids
- Enable experimental decorators for Scout JS
- Allow the unqualified ID deserializer in Java to accept over-qualified
  instances (a typeName is available event not necessary because the ID
  is concrete).
- Migrate from deprecated 'type' jQuery ajax property to the new
  'method' property.

389583

eclipse-scout-cli/scripts/DataObjectTransformer.js

@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2010, 2024 BSI Business Systems Integration AG
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+
+const ts = require('typescript');
+const ModuleDetector = require('./ModuleDetector');
+const CONSTANT_PATTERN = new RegExp('^[A-Z_0-9]+$');
+
+/**
+ * See https://github.com/itsdouges/typescript-transformer-handbook
+ */
+module.exports = class DataObjectTransformer {
+
+  constructor(program, context, namespaceResolver) {
+    this.program = program;
+    this.context = context;
+    this.moduleDetector = null; // created on first use
+    this.namespaceResolver = namespaceResolver;
+  }
+
+  transform(node) {
+    if (ts.isSourceFile(node)) {
+      const transformedFile = this._visitChildren(node); // step into top level source files
+      this.moduleDetector = null; // forget cached types for this file
+      return transformedFile;
+    }
+    if (ts.isClassDeclaration(node)) {
+      const typeNameDecorator = node.modifiers?.find(m => ts.isDecorator(m) && m.expression?.expression?.escapedText === 'typeName');
+      if (typeNameDecorator) {
+        return this._visitChildren(node); // step into DO with typeName annotation
+      }
+      return node; // no need to step into
+    }
+    if (ts.isImportDeclaration(node) || ts.isExportDeclaration(node) || ts.isVariableStatement(node) || ts.isIdentifier(node) || ts.isTypeReferenceNode(node)
+      || ts.isPropertySignature(node) || ts.isStringLiteral(node) || ts.isInterfaceDeclaration(node) || ts.isPropertyAssignment(node) || ts.isObjectLiteralExpression(node)
+      || ts.isPropertyAccessExpression(node) || ts.isTypeAliasDeclaration(node) || ts.isParameter(node) || ts.isEnumDeclaration(node)
+      || ts.isCallExpression(node) || ts.isExpressionStatement(node) || ts.isDecorator(node) || node.kind === ts.SyntaxKind.ExportKeyword) {
+      return node; // no need to step into
+    }
+
+    if (ts.isPropertyDeclaration(node) && !this._isSkipProperty(node)) {
+      const newModifiers = [
+        ...(node.modifiers || []), // existing
+        ...this._createMetaDataAnnotationsFor(node) // newly added
+      ];
+      return ts.factory.replaceDecoratorsAndModifiers(node, newModifiers);
+    }
+    return node; // no need to step into
+  }
+
+  _createMetaDataAnnotationsFor(node) {
+    const metaDataAnnotations = [];
+    metaDataAnnotations.push(this._createMetaDataAnnotation('t', this._createTypeNode(node.type)));
+    return metaDataAnnotations;
+  }
+
+  _createTypeNode(typeNode) {
+    if (typeNode.kind === ts.SyntaxKind.NumberKeyword) {
+      // primitive number
+      return ts.factory.createIdentifier('Number');
+    }
+    if (typeNode.kind === ts.SyntaxKind.StringKeyword) {
+      // primitive string
+      return ts.factory.createIdentifier('String');
+    }
+    if (typeNode.kind === ts.SyntaxKind.BooleanKeyword) {
+      // primitive boolean
+      return ts.factory.createIdentifier('Boolean');
+    }
+    // bigint is not yet supported as it is only part of ES2020 while Scout still supports ES2019
+
+    if (ts.isArrayTypeNode(typeNode)) {
+      // treat Obj[] like Array<Obj>
+      const objectType = ts.factory.createIdentifier('Array');
+      const elementType = this._createTypeNode(typeNode.elementType);
+      return this._createFieldMetaData(objectType, [elementType]);
+    }
+    if (ts.isTypeReferenceNode(typeNode)) {
+      const objectType = this._createTypeReferenceNode(typeNode);
+      if (!typeNode.typeArguments?.length) {
+        // no type arguments: directly use the type reference
+        return objectType;
+      }
+
+      // types with typeArguments like Map<string, number> or Array<MyObject>
+      const typeArgsNodes = typeNode.typeArguments.map(typeArg => this._createTypeNode(typeArg));
+      return this._createFieldMetaData(objectType, typeArgsNodes);
+    }
+    if (ts.isLiteralTypeNode(typeNode)) {
+      if (ts.isStringLiteral(typeNode.literal)) {
+        // literal types like e.g. in IDs: UuId<'scout.SimpleUuid'>
+        return ts.factory.createStringLiteral(typeNode.literal.text);
+      }
+    }
+    return ts.factory.createIdentifier('Object'); // e.g. any, void, unknown
+  }
+
+  _createFieldMetaData(objectType, typeArgsNodes) {
+    return ts.factory.createObjectLiteralExpression([
+      ts.factory.createPropertyAssignment('objectType', objectType),
+      ts.factory.createPropertyAssignment('typeArgs', ts.factory.createArrayLiteralExpression(typeArgsNodes, false))
+    ], false);
+  }
+
+  _createTypeReferenceNode(node) {
+    const name = node.typeName.escapedText;
+    if (global[name]) {
+      return ts.factory.createIdentifier(name); // Use directly the constructor for known types like Date, Number, String, Boolean, Map, Set, Array
+    }
+
+    const namespace = 'Record' === name ? null : this._detectNamespaceFor(node);
+    const qualifiedName = (!namespace || namespace === 'scout') ? name : namespace + '.' + name;
+    // use objectType as string because e.g. of TS interfaces (which do not exist at RT) and that overwrites in ObjectFactory are taken into account.
+    return ts.factory.createStringLiteral(qualifiedName);
+  }
+
+  _detectNamespaceFor(typeNode) {
+    if (!this.moduleDetector) {
+      this.moduleDetector = new ModuleDetector(typeNode);
+    }
+    const moduleName = this.moduleDetector.detectModuleOf(typeNode);
+    return this.namespaceResolver.resolveNamespace(moduleName, this.moduleDetector.sourceFile.fileName);
+  }
+
+  _createMetaDataAnnotation(key/* string */, valueNode) {
+    const reflect = ts.factory.createIdentifier('Reflect');
+    const reflectMetaData = ts.factory.createPropertyAccessExpression(reflect, ts.factory.createIdentifier('metadata'));
+    const keyNode = ts.factory.createStringLiteral('scout.m.' + key);
+    const call = ts.factory.createCallExpression(reflectMetaData, undefined, [keyNode, valueNode]);
+    return ts.factory.createDecorator(call);
+  }
+
+  _isSkipProperty(node) {
+    const propertyName = node.symbol?.escapedName;
+    if (!propertyName || propertyName.startsWith('_') || propertyName.startsWith('$') || CONSTANT_PATTERN.test(propertyName)) {
+      return true;
+    }
+    return !!node.modifiers?.some(n => n.kind === ts.SyntaxKind.StaticKeyword || n.kind === ts.SyntaxKind.ProtectedKeyword);
+  }
+
+  _visitChildren(node) {
+    return ts.visitEachChild(node, n => this.transform(n), this.context);
+  }
+};

eclipse-scout-cli/scripts/ModuleDetector.js

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2010, 2024 BSI Business Systems Integration AG
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+
+const ts = require('typescript');
+
+module.exports = class ModuleDetector {
+
+  constructor(node) {
+    this.sourceFile = this._findSourceFile(node);
+    const imports = this._findImportDeclarations(this.sourceFile);
+    this._moduleByTypeMap = this._computeImportMap(imports); // Map only contains types in 'other' modules (modules different from the one of the source file)
+  }
+
+  detectModuleOf(typeNode) {
+    const name = typeNode?.typeName?.escapedText;
+    return this._moduleByTypeMap.get(name);
+  }
+
+  _computeImportMap(imports) {
+    const moduleByTypeMap = new Map();
+    for (let imp of imports) {
+      const moduleName = imp.moduleSpecifier?.text; // e.g. '@eclipse-scout/core' or './index'
+      const isExternalModule = !moduleName?.startsWith('.'); // only store imports to other modules
+      if (isExternalModule) {
+        this._putExternalNamedBindings(imp.importClause?.namedBindings, moduleByTypeMap, moduleName);
+      }
+    }
+    return moduleByTypeMap;
+  }
+
+  _putExternalNamedBindings(namedBindings, moduleByTypeMap, moduleName) {
+    const importElements = namedBindings?.elements;
+    if (Array.isArray(importElements)) {
+      // multi import e.g.: import {a, b, c as d} from 'whatever'
+      for (let importElement of importElements) {
+        const name = importElement?.name?.escapedText;
+        if (!name) {
+          continue;
+        }
+        moduleByTypeMap.set(name, moduleName);
+      }
+    } else {
+      // single import e.g.: import * as self from './index';
+      const name = namedBindings?.name?.escapedText;
+      if (name) {
+        moduleByTypeMap.set(name, moduleName);
+      }
+    }
+  }
+
+  _findImportDeclarations(sourceFile) {
+    return sourceFile.statements.filter(s => ts.isImportDeclaration(s));
+  }
+
+  _findSourceFile(node) {
+    while (!ts.isSourceFile(node)) {
+      node = node.parent;
+    }
+    return node;
+  }
+};

eclipse-scout-cli/scripts/ModuleNamespaceResolver.js

@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2010, 2024 BSI Business Systems Integration AG
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+
+const fs = require('fs');
+const path = require('path');
+const REGISTER_NS_PATTERN = new RegExp('\\.registerNamespace\\s*\\(\'(\\w+)\'\\s*,');
+const JS_COMMENTS_PATTERN = new RegExp('\\/\\*[\\s\\S]*?\\*\\/|(?<=[^:])\\/\\/.*|^\\/\\/.*', 'g');
+
+module.exports = class ModuleNamespaceResolver {
+
+  constructor() {
+    this._rootsByFileDir = new Map();
+    this._namespaceByModuleRoot = new Map();
+    this._dependencyRootsByModuleRoot = new Set();
+    this.ownModuleNamespace = null;
+  }
+
+  resolveNamespace(moduleName, sourceFilePath) {
+    const moduleRoot = this.resolveModuleRoot(moduleName, sourceFilePath);
+    let namespace = this._namespaceByModuleRoot.get(moduleRoot);
+    if (!namespace) {
+      if (!moduleName && this.ownModuleNamespace) {
+        // use given namespace for own module if known
+        namespace = this.ownModuleNamespace;
+      } else {
+        namespace = this._resolveNamespace(moduleRoot);
+      }
+      this._namespaceByModuleRoot.set(moduleRoot, namespace);
+    }
+    return namespace;
+  }
+
+  _resolveNamespace(moduleRoot) {
+    let src = path.join(moduleRoot, 'src');
+    const mavenSrc = path.join(src, 'main/js');
+    if (fs.existsSync(mavenSrc)) {
+      src = mavenSrc;
+    }
+    return this._parseFromRegister(src);
+  }
+
+  _parseFromRegister(root) {
+    let namespace = null;
+    this._visitFiles(root, filePath => {
+      const content = fs.readFileSync(filePath, 'utf-8');
+      const result = REGISTER_NS_PATTERN.exec(content.replaceAll(JS_COMMENTS_PATTERN, ''));
+      if (result?.length === 2) {
+        namespace = result[1];
+        return false; // abort
+      }
+      return true;
+    });
+    return namespace;
+  }
+
+  _visitFiles(root, callback) {
+    const buf = this._readDir(root);
+    while (buf.length) {
+      const dirent = buf.shift(); // remove first
+      const filePath = path.join(dirent.parentPath || dirent.path, dirent.name);
+      if (dirent.isFile()) {
+        const cont = callback(filePath);
+        if (!cont) {
+          return;
+        }
+      } else {
+        buf.push(...this._readDir(filePath));
+      }
+    }
+  }
+
+  _readDir(directory) {
+    return fs.readdirSync(directory, {withFileTypes: true})
+      .filter(dirent => dirent.isDirectory() || dirent.name.endsWith('.js') || dirent.name.endsWith('.ts'))
+      .sort((a, b) => { // files first
+        if (a.isDirectory() && !b.isDirectory()) {
+          return 1;
+        }
+        if (!a.isDirectory() && b.isDirectory()) {
+          return -1;
+        }
+        return a.name.localeCompare(b.name);
+      });
+  }
+
+  resolveModuleRoot(moduleName, sourceFilePath) {
+    const directory = path.dirname(sourceFilePath);
+    const moduleRoot = this._getModuleRoot(directory);
+    if (!moduleName) {
+      return moduleRoot; // no external module name: own module
+    }
+    return this._resolveExternalModule(moduleRoot, moduleName);
+  }
+
+  _resolveExternalModule(moduleRoot, moduleName) {
+    const dependencyRoot = path.join(moduleRoot, 'node_modules', moduleName);
+    if (this._dependencyRootsByModuleRoot.has(dependencyRoot)) {
+      return dependencyRoot; // existence already verified
+    }
+
+    if (!fs.existsSync(path.join(dependencyRoot, 'package.json'))) {
+      throw new Error(`Dependency ${moduleName} not found in ${moduleRoot}.`);
+    }
+    this._dependencyRootsByModuleRoot.add(dependencyRoot);
+    return dependencyRoot;
+  }
+
+  _getModuleRoot(sourceFileDir) {
+    let root = this._rootsByFileDir.get(sourceFileDir);
+    if (!root) {
+      root = this._findModuleRoot(sourceFileDir);
+      if (!root) {
+        throw new Error(`${sourceFileDir} is not within any Node module.`);
+      }
+      this._rootsByFileDir.set(sourceFileDir, root); // remember for next files
+    }
+    return root;
+  }
+
+  _findModuleRoot(sourceFileDir) {
+    while (sourceFileDir && !fs.existsSync(path.join(sourceFileDir, 'package.json'))) {
+      sourceFileDir = path.dirname(sourceFileDir);
+    }
+    return sourceFileDir.replaceAll('\\', '/');
+  }
+};

eclipse-scout-cli/scripts/webpack-defaults.js

@@ -10,10 +10,13 @@
 const fs = require('fs');
 const path = require('path');
 const scoutBuildConstants = require('./constants');
+const DataObjectTransformer = require('./DataObjectTransformer');
+const ModuleNamespaceResolver = require('./ModuleNamespaceResolver');
 const CopyPlugin = require('copy-webpack-plugin');
 const MiniCssExtractPlugin = require('mini-css-extract-plugin');
 const AfterEmitWebpackPlugin = require('./AfterEmitWebpackPlugin');
 const {SourceMapDevToolPlugin, WatchIgnorePlugin, ProgressPlugin} = require('webpack');
+const ts = require('typescript');
 
 /**
  * @param {string} args.mode development or production
@@ -76,13 +79,22 @@ module.exports = (env, args) => {
   };
 
   const transpileOnly = typeCheck === 'fork' ? true : !typeCheck;
+  const namespaceResolver = new ModuleNamespaceResolver();
+  const getCustomTransformers = program => ({
+    before: [ctx => {
+      const doTransformer = new DataObjectTransformer(program, ctx, namespaceResolver);
+      return node => ts.visitNode(node, node => doTransformer.transform(node));
+    }]
+  });
+  getCustomTransformers.namespaceResolver = namespaceResolver;
   const tsOptions = {
     ...args.tsOptions,
     transpileOnly: transpileOnly,
     compilerOptions: {
       noEmit: false,
       ...args.tsOptions?.compilerOptions
-    }
+    },
+    getCustomTransformers
   };
 
   const config = {

eclipse-scout-core/package.json

@@ -76,6 +76,7 @@
   },
   "dependencies": {
     "jquery": "3.7.1",
+    "reflect-metadata": "0.2.2",
     "sourcemapped-stacktrace": "1.1.11"
   }
 }