diff --git a/.github/actions/slack-user-action/action.yaml b/.github/actions/slack-user-action/action.yaml index b0fc23e82e..5105fede90 100644 --- a/.github/actions/slack-user-action/action.yaml +++ b/.github/actions/slack-user-action/action.yaml @@ -41,7 +41,7 @@ runs: shell: bash run: | EMAIL=${{ inputs.emails }} - echo "DATAGROK_EMAIL=$EMAIL" >> $GITHUB_ENV + echo "DATAGROK_EMAIL=${EMAIL/mdolotova@datagrok.ai/maria.dolotova@softwarecountry.com}" >> $GITHUB_ENV # Step 2: Fetch Slack User ID by email (if email is provided) - name: Get Slack User ID by Email diff --git a/.github/workflows/docusaurus.yaml b/.github/workflows/docusaurus.yaml index 1bf184fc51..2cf873e2a6 100644 --- a/.github/workflows/docusaurus.yaml +++ b/.github/workflows/docusaurus.yaml @@ -273,7 +273,8 @@ jobs: - uses: actions/checkout@v4 with: sparse-checkout: | - docusaurus + docusaurus/docsearch.json + - uses: celsiusnarhwal/typesense-scraper@v2 continue-on-error: true with: @@ -282,3 +283,21 @@ jobs: port: '443' protocol: 'https' config: docusaurus/docsearch.json + +# TODO: Experiment with direct run to be able to cancel workflow. 05.11.2024: It fails with missing Pipfile because of working directory. +# - name: Run DocSearch Scraper +# uses: docker://typesense/docsearch-scraper:latest +# continue-on-error: true +# env: +# TYPESENSE_API_KEY: ${{ secrets.TYPESENSE_API_KEY }} +# TYPESENSE_HOST: 'typesense.datagrok.ai' +# TYPESENSE_PORT: '443' +# TYPESENSE_PROTOCOL: 'https' +# CONFIG: 'docusaurus/docsearch.json' +# WORKON_HOME: '/home/seleuser/.local/share' +# with: +# args: | +# bash -c " +# cd /home/seleuser && +# pipenv run python -m src.index +# " diff --git a/.github/workflows/grok_connect.yaml b/.github/workflows/grok_connect.yaml index 3ca4ba149c..78249ce867 100644 --- a/.github/workflows/grok_connect.yaml +++ b/.github/workflows/grok_connect.yaml @@ -164,7 +164,7 @@ jobs: - name: Upload Artifact if: github.ref != 'refs/heads/master' && startsWith(github.ref,'refs/heads/release/') != true id: docker-artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ steps.param.outputs.tar }} path: /tmp/${{ steps.param.outputs.tar }} diff --git a/.github/workflows/js-api.yml b/.github/workflows/js-api.yml index 2948c8807c..7345b33183 100644 --- a/.github/workflows/js-api.yml +++ b/.github/workflows/js-api.yml @@ -232,7 +232,7 @@ jobs: echo "Notify about version used for tests" echo "::notice title=js-api::datagrok/datagrok:$DATAGROK_VERSION SHA=$docker_sha docker version was used for tests" - id: datagrok-tools - run: npm install -g datagrok-tools@latest + run: npm install -g datagrok-tools@4.13.31 - name: Wait for Datagrok to become available timeout-minutes: 5 run: | diff --git a/.github/workflows/libraries.yaml b/.github/workflows/libraries.yaml index f0f7ce4be7..86582ea572 100644 --- a/.github/workflows/libraries.yaml +++ b/.github/workflows/libraries.yaml @@ -194,7 +194,7 @@ jobs: working-directory: libraries/${{ matrix.project }} if: ${{ matrix.build == 'build' }} - id: datagrok-tools - run: npm install -g datagrok-tools@latest + run: npm install -g datagrok-tools@4.13.31 - run: grok check id: grok_check working-directory: libraries/${{ matrix.project }} diff --git a/.github/workflows/linter.yaml b/.github/workflows/linter.yaml index 7d8b01d3f4..468a05d609 100644 --- a/.github/workflows/linter.yaml +++ b/.github/workflows/linter.yaml @@ -116,7 +116,7 @@ jobs: with: configFile: commitlint.config.js failOnWarnings: false - helpURL: https://datagrok.ai/help/develop/advanced/git-policy#commit-message-policy + helpURL: https://datagrok.ai/help/develop/dev-process/git-policy#commit-message-policy pr-name: name: PR name diff --git a/.github/workflows/packages.yaml b/.github/workflows/packages.yaml index 366ebe8c96..da60571262 100644 --- a/.github/workflows/packages.yaml +++ b/.github/workflows/packages.yaml @@ -324,7 +324,7 @@ jobs: echo "apiUrl=$apiUrl" >> $GITHUB_OUTPUT echo "alias=$alias" >> $GITHUB_OUTPUT - id: datagrok-tools - run: npm install -g datagrok-tools@latest + run: npm install -g datagrok-tools@4.13.31 - if: ${{ matrix.unpublished_deps != '' }} run: grok link working-directory: packages/${{ matrix.package }} @@ -434,7 +434,6 @@ jobs: fi done working-directory: packages/${{ matrix.package }} - - name: Set required credentials if: ${{ matrix.test == 'test' }} env: @@ -443,7 +442,7 @@ jobs: if [[ "${{ steps.datagrok-image.outputs.apiUrl }}" == *"127.0.0.1"* ]]; then if [[ "${{ matrix.package }}" == "Chemspace" ]]; then token=$(curl -s -X POST ${{ steps.datagrok-image.outputs.apiUrl }}/users/login/dev/admin | jq -r .token) - curl -s -H "Authorization: $token" -H "Content-Type": "application/json" '${{ steps.datagrok-image.outputs.apiUrl }}/credentials/for/Chemspace:Chemspace' -X POST -d "{'apiKey': '$CHEMSPACE_APIKEY'}" + curl -s -H "Authorization: $token" -H "Content-Type": "application/json" '${{ steps.datagrok-image.outputs.apiUrl }}/credentials/for/Chemspace.Chemspace' -X POST -d "{\"apiKey\": \"$CHEMSPACE_APIKEY\"}" fi fi @@ -453,36 +452,55 @@ jobs: env: DEV_TEST_DEV_KEY: ${{ secrets.DEV_TEST_DEV_KEY }} TEST_TEST_DEV_KEY: ${{ secrets.TEST_TEST_DEV_KEY }} - timeout-minutes: 10 + timeout-minutes: 20 run: | - if [[ "${{ steps.datagrok-image.outputs.alias }}" == "test" ]]; then - key="$TEST_TEST_DEV_KEY" - elif [[ "${{ steps.datagrok-image.outputs.alias }}" == "dev" ]]; then - key="$DEV_TEST_DEV_KEY" - else - key='admin' - fi - token=$(curl -s -X POST ${{ steps.datagrok-image.outputs.apiUrl }}/users/login/dev/$key | jq -r .token) - until .github/scripts/check-output.sh "curl -s -H 'Authorization: $token' '${{ steps.datagrok-image.outputs.apiUrl }}/docker/containers'" '${{ matrix.package }}' - do - sleep 1 - echo -e "\nContainer for ${{ matrix.package }} did not start yet..." - echo -e "\nRetrying '/api/docker/containers'..." + check_container=false + for dockerfile in $(echo -e ${{ matrix.docker }}); do + docker_config_file="$(dirname $dockerfile)/container.json" + if [ -f "${docker_config_file}" ]; then + on_demand=$(jq '.on_demand' ${docker_config_file}) + if [ "$on_demand" == "true" ]; then + check_container=false + else + check_container=true + break + fi + else + check_container=true + break + fi done - if [[ "${{ steps.datagrok-image.outputs.apiUrl }}" == *"127.0.0.1"* ]]; then - until .github/scripts/check-output.sh "docker ps" '${{ matrix.package }}' + + if [[ "${check_container}" == "true" ]] ; then + if [[ "${{ steps.datagrok-image.outputs.alias }}" == "test" ]]; then + key="$TEST_TEST_DEV_KEY" + elif [[ "${{ steps.datagrok-image.outputs.alias }}" == "dev" ]]; then + key="$DEV_TEST_DEV_KEY" + else + key='admin' + fi + token=$(curl -s -X POST ${{ steps.datagrok-image.outputs.apiUrl }}/users/login/dev/$key | jq -r .token) + until .github/scripts/check-output.sh "curl -s -H 'Authorization: $token' '${{ steps.datagrok-image.outputs.apiUrl }}/docker/containers'" '${{ matrix.package }}' do sleep 1 echo -e "\nContainer for ${{ matrix.package }} did not start yet..." - echo -e "\nRetrying 'docker ps'..." + echo -e "\nRetrying '/api/docker/containers'..." done + if [[ "${{ steps.datagrok-image.outputs.apiUrl }}" == *"127.0.0.1"* ]]; then + until .github/scripts/check-output.sh "docker ps" '${{ matrix.package }}' + do + sleep 1 + echo -e "\nContainer for ${{ matrix.package }} did not start yet..." + echo -e "\nRetrying 'docker ps'..." + done + fi fi - name: Test Package if: ${{ matrix.test == 'test' }} continue-on-error: ${{ contains(matrix.package, 'Tests') || matrix.package == 'Chemspace' }} timeout-minutes: 30 id: test-package - run: npm run test -- --skip-build --skip-publish --record --csv --host ${{ steps.datagrok-image.outputs.alias }} + run: npm run test -- --skip-build --skip-publish --record --csv --host ${{ steps.datagrok-image.outputs.alias }} --verbose working-directory: packages/${{ matrix.package }} - name: Upload Artifact if: always() && steps.test-package.outcome != 'skipped' diff --git a/.github/workflows/security_scan_anchore.yaml b/.github/workflows/security_scan_anchore.yaml index 0d3825d29f..409423cf06 100644 --- a/.github/workflows/security_scan_anchore.yaml +++ b/.github/workflows/security_scan_anchore.yaml @@ -62,7 +62,7 @@ jobs: with: args: sbom:./${{ matrix.target }}.sbom.syft.json -o json --file ${{ matrix.target }}.grype.json - name: Upload Artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: ${{ matrix.target }}.grype.json path: ${{ matrix.target }}.grype.json @@ -88,7 +88,7 @@ jobs: .artifact.locations[].path ]) | @csv' ${{ matrix.target }}.grype.json > ${{ matrix.target }}.report.csv - name: Upload Artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: ${{ matrix.target }}.report.csv path: ${{ matrix.target }}.report.csv diff --git a/commitlint.config.js b/commitlint.config.js index c47f1f4cee..aea2558346 100644 --- a/commitlint.config.js +++ b/commitlint.config.js @@ -76,7 +76,7 @@ const Configuration = { * Custom URL to show upon failure */ helpUrl: - 'https://datagrok.ai/help/develop/advanced/git-policy#commit-message-policy', + 'https://datagrok.ai/help/develop/dev-process/git-policy#commit-message-policy', }; module.exports = Configuration; diff --git a/connectors/grok_connect/src/main/java/grok_connect/GrokConnect.java b/connectors/grok_connect/src/main/java/grok_connect/GrokConnect.java index 49d00140d2..1c7e8b398c 100644 --- a/connectors/grok_connect/src/main/java/grok_connect/GrokConnect.java +++ b/connectors/grok_connect/src/main/java/grok_connect/GrokConnect.java @@ -6,10 +6,7 @@ import ch.qos.logback.classic.Logger; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import grok_connect.connectors_info.DataConnection; -import grok_connect.connectors_info.DataProvider; -import grok_connect.connectors_info.DataQueryRunResult; -import grok_connect.connectors_info.FuncCall; +import grok_connect.connectors_info.*; import grok_connect.providers.JdbcDataProvider; import grok_connect.table_query.TableQuery; import grok_connect.utils.*; @@ -141,17 +138,17 @@ private static void connectorsModule() { }); post("/query_table_sql", (request, response) -> { - String query = ""; + TableQuery tableQuery = null; try { DataConnection connection = gson.fromJson(request.body(), DataConnection.class); - TableQuery tableQuery = gson.fromJson(connection.get("queryTable"), TableQuery.class); + tableQuery = gson.fromJson(connection.get("queryTable"), TableQuery.class); DataProvider provider = providerManager.getByName(connection.dataSource); - query = provider.queryTableSql(connection, tableQuery); + tableQuery.query = provider.queryTableSql(connection, tableQuery); } catch (Exception ex) { PARENT_LOGGER.info(DEFAULT_LOG_EXCEPTION_MESSAGE, ex); buildExceptionResponse(response, printError(ex)); } - return query; + return tableQuery != null ? gson.toJson(new TableQueryResponse(tableQuery.query, tableQuery.params.stream().filter((p) -> p.isInput).collect(Collectors.toList()))) : ""; }); post("/foreign-keys", (request, response) -> { diff --git a/connectors/grok_connect/src/main/java/grok_connect/connectors_info/DataConnection.java b/connectors/grok_connect/src/main/java/grok_connect/connectors_info/DataConnection.java index ad9b8f7a28..68c87d5388 100644 --- a/connectors/grok_connect/src/main/java/grok_connect/connectors_info/DataConnection.java +++ b/connectors/grok_connect/src/main/java/grok_connect/connectors_info/DataConnection.java @@ -18,7 +18,11 @@ public class DataConnection public String getDb() { return (String)parameters.get(DB); } public String getPort() { Object port = parameters.get(PORT); - return port == null ? null : String.valueOf(((Double)parameters.get(PORT)).intValue()); + if (port == null) + return null; + if (port instanceof String) + return (String) port; + return String.valueOf(((Double) parameters.get(PORT)).intValue()); } public Map parameters = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); diff --git a/connectors/grok_connect/src/main/java/grok_connect/connectors_info/FuncParam.java b/connectors/grok_connect/src/main/java/grok_connect/connectors_info/FuncParam.java index f267a4e33d..252666b9a3 100644 --- a/connectors/grok_connect/src/main/java/grok_connect/connectors_info/FuncParam.java +++ b/connectors/grok_connect/src/main/java/grok_connect/connectors_info/FuncParam.java @@ -1,10 +1,14 @@ package grok_connect.connectors_info; import java.util.*; + +import com.google.gson.annotations.SerializedName; import serialization.Types; public class FuncParam { + @SerializedName("#type") + public String type = "FuncParam"; public String propertyType; public String propertySubType; public String name; diff --git a/connectors/grok_connect/src/main/java/grok_connect/connectors_info/TableQueryResponse.java b/connectors/grok_connect/src/main/java/grok_connect/connectors_info/TableQueryResponse.java new file mode 100644 index 0000000000..d100652ce8 --- /dev/null +++ b/connectors/grok_connect/src/main/java/grok_connect/connectors_info/TableQueryResponse.java @@ -0,0 +1,13 @@ +package grok_connect.connectors_info; + +import java.util.List; + +public class TableQueryResponse { + public String query; + public List params; + + public TableQueryResponse(String query, List params) { + this.query = query; + this.params = params; + } +} diff --git a/connectors/grok_connect/src/main/java/grok_connect/providers/AccessDataProvider.java b/connectors/grok_connect/src/main/java/grok_connect/providers/AccessDataProvider.java index aa6e49c14b..d3fa6a3dfc 100644 --- a/connectors/grok_connect/src/main/java/grok_connect/providers/AccessDataProvider.java +++ b/connectors/grok_connect/src/main/java/grok_connect/providers/AccessDataProvider.java @@ -80,22 +80,6 @@ public String getConnectionStringImpl(DataConnection conn) { return String.format("jdbc:ucanaccess://%s;memory=false", new File(conn.getDb()).getAbsolutePath()); } - @Override - protected String aggrToSql(GroupAggregation aggr) { - AggrFunctionInfo funcInfo = null; - for (AggrFunctionInfo info: descriptor.aggregations) { - if (info.functionName.equals(aggr.aggType)) { - funcInfo = info; - break; - } - } - if (funcInfo != null) { - String sql = funcInfo.dbFunctionName.replaceAll("#", aggr.colName); - return String.format("%s AS [%s]", sql, sql); - } else - return null; - } - @Override public DataFrame getForeignKeys(DataConnection conn, String schema) throws GrokConnectException { try (Connection connection = getConnection(conn)) { diff --git a/connectors/grok_connect/src/main/java/grok_connect/providers/JdbcDataProvider.java b/connectors/grok_connect/src/main/java/grok_connect/providers/JdbcDataProvider.java index 3a55af7b55..abd274f5c5 100644 --- a/connectors/grok_connect/src/main/java/grok_connect/providers/JdbcDataProvider.java +++ b/connectors/grok_connect/src/main/java/grok_connect/providers/JdbcDataProvider.java @@ -31,12 +31,7 @@ import grok_connect.table_query.FieldPredicate; import grok_connect.table_query.GroupAggregation; import grok_connect.table_query.TableQuery; -import grok_connect.utils.ConnectionPool; -import grok_connect.utils.GrokConnectException; -import grok_connect.utils.PatternMatcher; -import grok_connect.utils.PatternMatcherResult; -import grok_connect.utils.QueryCancelledByUser; -import grok_connect.utils.QueryMonitor; +import grok_connect.utils.*; import org.apache.commons.lang.NotImplementedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -182,11 +177,11 @@ else if (param.propertyType.equals(Types.LIST) && param.propertySubType.equals(T i = i + setArrayParamValue(statement, n + i + 1, param); } else { - if (param.value == null) { - statement.setNull(n + i + 1, java.sql.Types.VARCHAR); - } - else - statement.setObject(n + i + 1, param.value); +// if (param.value == null) { +// statement.setNull(n + i + 1, java.sql.Types.VARCHAR); +// } +// else + statement.setObject(n + i + 1, param.value); } } queryLogger.debug(EventType.STATEMENT_PARAMETERS_REPLACEMENT.getMarker(EventType.Stage.END), "Replaced designated query parameters"); @@ -584,8 +579,9 @@ protected String aggrToSql(GroupAggregation aggr) { } } if (funcInfo != null) { - String sql = funcInfo.dbFunctionName.replaceAll("#", addBrackets(aggr.colName)); - return sql + " as " + addBrackets(funcInfo.dbFunctionName.replaceAll("#", aggr.colName)); + String brackets = descriptor.nameBrackets; + String sql = GrokConnectUtil.isNotEmpty(aggr.colName) ? funcInfo.dbFunctionName.replaceAll("#", addBrackets(aggr.colName)) : funcInfo.dbFunctionName; + return sql + " as " + (GrokConnectUtil.isNotEmpty(aggr.resultColName) ? brackets.charAt(0) + aggr.resultColName + brackets.charAt(brackets.length() - 1): addBrackets(funcInfo.dbFunctionName).replaceAll("#", aggr.colName)); } else return null; @@ -595,7 +591,7 @@ public String addBrackets(String name) { String brackets = descriptor.nameBrackets; return Arrays.stream(name.split("\\.")) .map((str) -> str.startsWith(brackets.substring(0, 1)) ? str - : brackets.charAt(0) + name + brackets.substring(brackets.length() - 1)) + : brackets.charAt(0) + str + brackets.substring(brackets.length() - 1)) .collect(Collectors.joining(".")); } @@ -604,7 +600,9 @@ public String limitToSql(String query, Integer limit) { } private String patternToSql(FieldPredicate condition) { - return condition.matcher.toSql(condition.dataType, condition.field); + if (GrokConnectUtil.isNotEmpty(condition.matcher.op) && !condition.matcher.op.equals(PatternMatcher.NONE)) + return String.format("@%s(%s)", condition.getParamName(), addBrackets(condition.field)); + return String.format("%s = @%s", condition.field, condition.getParamName()); } public String queryTableSql(DataConnection conn, TableQuery query) { diff --git a/connectors/grok_connect/src/main/java/grok_connect/providers/PostgresDataProvider.java b/connectors/grok_connect/src/main/java/grok_connect/providers/PostgresDataProvider.java index de0c48edef..89a8243b1b 100644 --- a/connectors/grok_connect/src/main/java/grok_connect/providers/PostgresDataProvider.java +++ b/connectors/grok_connect/src/main/java/grok_connect/providers/PostgresDataProvider.java @@ -39,6 +39,7 @@ public PostgresDataProvider() { put("text", Types.STRING); put("boolean", Types.BOOL); put("date", Types.DATE_TIME); + put("#timestamp.*", Types.DATE_TIME); put("cidr", Types.STRING); put("ARRAY", Types.LIST); put("USER_DEFINED", Types.STRING); diff --git a/connectors/grok_connect/src/main/java/grok_connect/table_query/FieldPredicate.java b/connectors/grok_connect/src/main/java/grok_connect/table_query/FieldPredicate.java index 91d9293568..428db14d5f 100644 --- a/connectors/grok_connect/src/main/java/grok_connect/table_query/FieldPredicate.java +++ b/connectors/grok_connect/src/main/java/grok_connect/table_query/FieldPredicate.java @@ -14,6 +14,13 @@ public FieldPredicate(String field, String pattern, String dataType) { this.dataType = dataType; } + public String getParamName() { + return field + .replaceAll("\\.", "_") + .replaceAll(" ", "_") + .toLowerCase(); + } + @Override public String toString() { return field + " " + pattern; diff --git a/connectors/grok_connect/src/main/java/grok_connect/table_query/TableJoin.java b/connectors/grok_connect/src/main/java/grok_connect/table_query/TableJoin.java new file mode 100644 index 0000000000..195ddf8af7 --- /dev/null +++ b/connectors/grok_connect/src/main/java/grok_connect/table_query/TableJoin.java @@ -0,0 +1,24 @@ +package grok_connect.table_query; + +import java.util.List; + +public class TableJoin { + public String leftTableName; + public String rightTableName; + public String rightTableAlias; + public String joinType; + public List leftTableKeys; + public List rightTableKeys; + + public TableJoin () { + } + + public TableJoin(String leftTableName, String rightTableName, String rightTableAlias, String joinType, List leftTableKeys, List rightTableKeys) { + this.leftTableName = leftTableName; + this.rightTableName = rightTableName; + this.rightTableAlias = rightTableAlias; + this.joinType = joinType; + this.leftTableKeys = leftTableKeys; + this.rightTableKeys = rightTableKeys; + } +} diff --git a/connectors/grok_connect/src/main/java/grok_connect/table_query/TableQuery.java b/connectors/grok_connect/src/main/java/grok_connect/table_query/TableQuery.java index 8e9524f3ed..aeb5d28637 100644 --- a/connectors/grok_connect/src/main/java/grok_connect/table_query/TableQuery.java +++ b/connectors/grok_connect/src/main/java/grok_connect/table_query/TableQuery.java @@ -1,12 +1,13 @@ package grok_connect.table_query; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; +import java.util.*; import java.util.stream.Collectors; -import grok_connect.connectors_info.DataConnection; +import grok_connect.connectors_info.DataQuery; +import grok_connect.connectors_info.FuncParam; +import grok_connect.utils.GrokConnectUtil; +import serialization.Types; -public class TableQuery { +public class TableQuery extends DataQuery { public String whereOp = "and"; public String havingOp = "and"; public List fields = new ArrayList<>(); @@ -17,9 +18,9 @@ public class TableQuery { public List groupByFields = new ArrayList<>(); public List having = new ArrayList<>(); public List orderBy = new ArrayList<>(); + List joins = new ArrayList<>(); public String tableName; public String schema; - public DataConnection connection; public Integer limit; public TableQuery() { @@ -28,6 +29,7 @@ public TableQuery() { public String toSql(AggrToSql aggrToSql, PatternToSql patternToSql, LimitToSql limitToSql, AddBrackets addBrackets, boolean limitAtEnd) { StringBuilder sql = new StringBuilder(); + StringBuilder sqlHeader = new StringBuilder(); String table = tableName; if (table.contains(".")) { int idx = table.indexOf("."); @@ -47,17 +49,42 @@ public String toSql(AggrToSql aggrToSql, PatternToSql patternToSql, LimitToSql l sql.append("FROM"); sql.append(System.lineSeparator()); sql.append(table); + + if (!joins.isEmpty()) { + sql.append(System.lineSeparator()); + for (TableJoin joinTable: joins) { + sql.append(joinTable.joinType) + .append(" join ") + .append(addBrackets.convert(joinTable.rightTableName)); + if (GrokConnectUtil.isNotEmpty(joinTable.rightTableAlias)) + sql.append(" as ") + .append(addBrackets.convert(joinTable.rightTableAlias)); + sql.append(" on "); + for (int i = 0; i < joinTable.leftTableKeys.size(); i++) { + if (i > 0) { + sql.append(" AND "); + sql.append(System.lineSeparator()); + } + sql.append(addBrackets.convert(joinTable.leftTableName)) + .append('.') + .append(addBrackets.convert(joinTable.leftTableKeys.get(i))) + .append(" = ") + .append(addBrackets.convert(GrokConnectUtil.isNotEmpty(joinTable.rightTableAlias) ? joinTable.rightTableAlias : joinTable.rightTableName)) + .append(".") + .append(addBrackets.convert(joinTable.rightTableKeys.get(i))) + .append(System.lineSeparator()); + } + } + } + if (!whereClauses.isEmpty()) { sql.append(System.lineSeparator()); List clauses = new ArrayList<>(); sql.append("WHERE"); sql.append(System.lineSeparator()); for (FieldPredicate clause: whereClauses) - clauses.add(String.format(" (%s)", patternToSql.convert(clause))); - sql.append(clauses.stream() - .collect(Collectors - .joining(String.format(" %s%s", whereOp, System.lineSeparator()))) - ); + clauses.add(String.format(" (%s)", preparePredicate(clause, patternToSql, sqlHeader))); + sql.append(String.join(String.format(" %s%s", whereOp, System.lineSeparator()), clauses)); } if (!groupByFields.isEmpty()) { @@ -81,7 +108,7 @@ public String toSql(AggrToSql aggrToSql, PatternToSql patternToSql, LimitToSql l sql.append(System.lineSeparator()); List clauses = new ArrayList<>(); for (FieldPredicate clause: having) - clauses.add(String.format("\t(%s)", patternToSql.convert(clause))); + clauses.add(String.format("\t(%s)", preparePredicate(clause, patternToSql, sqlHeader))); sql.append(String.join(String.format(" %s%s", havingOp, System.lineSeparator()), clauses)); } @@ -103,15 +130,26 @@ public String toSql(AggrToSql aggrToSql, PatternToSql patternToSql, LimitToSql l result = limitToSql.convert(sql.toString(), limit); } else - result = sql.toString(); - return result; + result = sql.toString().trim(); + String header = sqlHeader.toString().trim(); + return header.isEmpty() ? result : header + System.lineSeparator() + result; } private String getSelectFields(AggrToSql aggrToSql, AddBrackets addBrackets) { + // use list to preserve order List preparedFields = new ArrayList<>(); + Set uniqueNames = new HashSet<>(); for (String field : fields) { + String[] splitField = field.split("\\."); + String fieldName = splitField[splitField.length - 1]; + String bracket; + if (uniqueNames.contains(fieldName) && splitField.length > 1 /* table alias in field */) + bracket = addBrackets.convert(field) + " as " + addBrackets.convert(field.replaceAll("\\.", "_")); + else + bracket = addBrackets.convert(field); + uniqueNames.add(fieldName); + // fallback for old code int num = 1; - String bracket = addBrackets.convert(field); while (true) { if (!preparedFields.contains(bracket)) { preparedFields.add(bracket); @@ -122,7 +160,7 @@ private String getSelectFields(AggrToSql aggrToSql, AddBrackets addBrackets) { } } preparedFields.addAll(getAggFuncs().stream().map(aggrToSql::convert).filter(Objects::nonNull).collect(Collectors.toList())); - return preparedFields.isEmpty() ? "*\n" : preparedFields.stream() + return preparedFields.isEmpty() ? "*" + System.lineSeparator() : preparedFields.stream() .collect(Collectors.joining(String.format(",%s", System.lineSeparator()), "", System.lineSeparator())); } @@ -138,4 +176,20 @@ private List pad(List strings) { strings.replaceAll(s -> " " + s); return strings; } + + private String preparePredicate(FieldPredicate clause, PatternToSql patternToSql, StringBuilder sqlHeader) { + String part = patternToSql.convert(clause); + String dataType = part.contains("=") ? clause.dataType : Types.STRING; + String paramName = clause.getParamName(); + sqlHeader.append(String.format("--input: %s %s", dataType, paramName)); + sqlHeader.append(System.lineSeparator()); + FuncParam param = new FuncParam(); + param.name = paramName; + param.propertyType = dataType; + param.options = new HashMap<>(); + param.options.put("default", clause.pattern); + param.options.put("pattern", part.contains("=") ? null : clause.dataType); + params.add(param); + return part; + } } diff --git a/connectors/grok_connect/src/test/java/grok_connect/table_query/MsSqlTableQueryTest.java b/connectors/grok_connect/src/test/java/grok_connect/table_query/MsSqlTableQueryTest.java new file mode 100644 index 0000000000..09d32ce971 --- /dev/null +++ b/connectors/grok_connect/src/test/java/grok_connect/table_query/MsSqlTableQueryTest.java @@ -0,0 +1,144 @@ +package grok_connect.table_query; + +import grok_connect.providers.MsSqlDataProvider; + +public class MsSqlTableQueryTest extends TableQueryTest { + public MsSqlTableQueryTest() { + super(new MsSqlDataProvider()); + } + + @Override + public String[] getExpectedEmptySchema() { + return new String[] {"SELECT", "*", "FROM", "[events]"}; + } + + @Override + public String[] getExpectedEmptySchemaTableWithDot() { + return new String[] {"SELECT", "*", "FROM", "[public].[events]"}; + } + + @Override + public String[] getExpectedEmptyFields() { + return new String[] {"SELECT", "*", "FROM", "[public].[events]"}; + } + + @Override + public String[] getExpectedEmptyFieldsLimit() { + return new String[] {"SELECT", "top 50", "*", "FROM", "[public].[events]"}; + } + + @Override + public String[] getExpectedFieldsWithoutDotLimit() { + return new String[] {"SELECT", "top 50", "[id],", "[friendly_name],", "[name],", "[source],", "[session_id],", + "[status],", "[description],", "[error_message],", "[error_stack_trace],", + "[event_type_id]", "FROM", "[public].[events]"}; + } + + @Override + public String[] getExpectedFieldsWithDotLimit() { + return new String[] {"SELECT", "top 50", "[events].[id],", "[events].[friendly_name],", "[events].[name],", "[events].[source],", + "[events].[session_id],", "[events].[status],", "[events].[description],", "[events].[error_message],", + "[events].[error_stack_trace],", + "[events].[event_type_id]", "FROM", "[public].[events]"}; + } + + @Override + public String[] getExpectedAggregationWithoutPattern() { + return new String[] {"SELECT", "count(*) as [count(*)]", "FROM", "[public].[events]"}; + } + + @Override + public String[] getExpectedAggregationWithPattern() { + return new String[] {"SELECT", "sum([session_id]) as [sum(session_id)]", "FROM", "[public].[events]"}; + } + + @Override + public String[] getAggregationAndGroupByLimitWithoutDot() { + return new String[] {"SELECT", "top 50", "[id],", "[friendly_name],", "[name],", "[source],", + "[status],", "[description],", "[error_message],", "[error_stack_trace],", + "[event_type_id],", "count([session_id]) as [count(session_id)]", "FROM", "[public].[events]", "GROUP BY", "[id], [friendly_name], [name], [source], " + + "[status], [description], [error_message], [error_stack_trace], [event_type_id]"}; + } + + @Override + public String[] getAggregationAndGroupByLimitWithDot() { + return new String[] {"SELECT", "top 50", "[events].[id],", "[events].[friendly_name],", "[events].[name],", "[events].[source],", + "[events].[status],", "[events].[description],", "[events].[error_message],", + "[events].[error_stack_trace],", "[events].[event_type_id],", "count([events].[session_id]) as [count(events.session_id)]", + "FROM", "[public].[events]", "GROUP BY", "[events].[id], [events].[friendly_name], [events].[name], [events].[source], " + + "[events].[status], [events].[description], [events].[error_message], [events].[error_stack_trace], [events].[event_type_id]"}; + } + + @Override + public String[] getAggregationAndGroupByAndHavingLimitWithoutDot() { + return new String[] {"--input: string source", "SELECT", "top 50", "[id],", "[friendly_name],", "[name],", "[source],", + "[status],", "[description],", "[error_message],", "[error_stack_trace],", + "[event_type_id],", "count([session_id]) as [count(session_id)]", "FROM", "[public].[events]", "GROUP BY", "[id], [friendly_name], [name], [source], " + + "[status], [description], [error_message], [error_stack_trace], [event_type_id]", + "HAVING", "\t(@source([source]))"}; + } + + @Override + public String[] getAggregationAndGroupByAndHavingLimitWithDot() { + return new String[] {"--input: string events_source", "SELECT", "top 50", "[events].[id],", "[events].[friendly_name],", "[events].[name],", "[events].[source],", + "[events].[status],", "[events].[description],", "[events].[error_message],", + "[events].[error_stack_trace],", "[events].[event_type_id],", "count([events].[session_id]) as [count(events.session_id)]", + "FROM", "[public].[events]", "GROUP BY", "[events].[id], [events].[friendly_name], [events].[name], [events].[source], " + + "[events].[status], [events].[description], [events].[error_message], [events].[error_stack_trace], [events].[event_type_id]", + "HAVING", "\t(@events_source([events].[source]))"}; + } + + @Override + public String[] getSimpleJoin() { + return new String[] {"SELECT", "[events].[id],", "[events].[friendly_name],", "[events].[name],", "[events].[source],", + "[events].[session_id],", "[events].[status],", "[events].[description],", "[events].[error_message],", + "[events].[error_stack_trace],", "[events].[event_type_id],", + "[event_types].[id] as [event_types_id],", "[event_types].[friendly_name] as [event_types_friendly_name],", + "[event_types].[name] as [event_types_name],", "[event_types].[description] as [event_types_description],", + "[event_types].[error_message] as [event_types_error_message],", "[event_types].[error_stack_trace] as [event_types_error_stack_trace]", + "FROM", "[public].[events]", "left join [event_types] on [events].[event_type_id] = [event_types].[id]"}; + } + + @Override + public String[] getSelfJoin() { + return new String[] {"SELECT", "[events].[id],", "[events].[friendly_name],", "[events].[name],", "[events].[source],", + "[events].[session_id],", "[events].[status],", "[events].[description],", "[events].[error_message],", + "[events].[error_stack_trace],", "[events].[event_type_id],", + "[events_1].[id] as [events_1_id],", "[events_1].[friendly_name] as [events_1_friendly_name],", + "[events_1].[name] as [events_1_name],", "[events_1].[source] as [events_1_source],", + "[events_1].[session_id] as [events_1_session_id],", "[events_1].[status] as [events_1_status],", + "[events_1].[description] as [events_1_description],", "[events_1].[error_message] as [events_1_error_message],", + "[events_1].[error_stack_trace] as [events_1_error_stack_trace],", "[events_1].[event_type_id] as [events_1_event_type_id]", + "FROM", "[public].[events]", "left join [events] as [events_1] on [events].[event_type_id] = [events_1].[event_type_id]"}; + } + + @Override + public String[] getSeveralJoins() { + return new String[] {"SELECT", "[events].[id],", "[events].[friendly_name],", "[events].[name],", "[events].[source],", + "[events].[session_id],", "[events].[status],", "[events].[description],", "[events].[error_message],", + "[events].[error_stack_trace],", "[events].[event_type_id],", + "[event_types].[id] as [event_types_id],", "[event_types].[friendly_name] as [event_types_friendly_name],", + "[event_types].[name] as [event_types_name],", "[event_types].[description] as [event_types_description],", + "[event_types].[error_message] as [event_types_error_message],", "[event_types].[error_stack_trace] as [event_types_error_stack_trace],", + "[sessions].[id] as [sessions_id],", "[sessions].[user_id],", "[sessions].[started],", "[sessions].[ended],", "[sessions].[token],", + "[u].[id] as [u_id],", "[u].[email],", "[u].[first_name],", "[u].[last_name],", "[u].[status] as [u_status]", + "FROM", "[public].[events]", + "left join [event_types] on [events].[event_type_id] = [event_types].[id]", + "right join [users_sessions] as [sessions] on [events].[session_id] = [sessions].[id]", + "inner join [users] as [u] on [sessions].[user_id] = [u].[id]" + }; + } + + @Override + public String[] getSeveralOnJoin() { + return new String[] {"SELECT", "[events].[id],", "[events].[friendly_name],", "[events].[name],", "[events].[source],", + "[events].[session_id],", "[events].[status],", "[events].[description],", "[events].[error_message],", + "[events].[error_stack_trace],", "[events].[event_type_id],", + "[t].[id] as [t_id],", "[t].[friendly_name] as [t_friendly_name],", + "[t].[name] as [t_name],", "[t].[description] as [t_description],", + "[t].[error_message] as [t_error_message],", "[t].[error_stack_trace] as [t_error_stack_trace]", + "FROM", "[public].[events]", "left join [event_types] as [t] on [events].[event_type_id] = [t].[id]", + " AND ", "[events].[name] = [t].[name]" + }; + } +} diff --git a/connectors/grok_connect/src/test/java/grok_connect/table_query/PostgresTableQueryTest.java b/connectors/grok_connect/src/test/java/grok_connect/table_query/PostgresTableQueryTest.java new file mode 100644 index 0000000000..5175dc3753 --- /dev/null +++ b/connectors/grok_connect/src/test/java/grok_connect/table_query/PostgresTableQueryTest.java @@ -0,0 +1,145 @@ +package grok_connect.table_query; + +import grok_connect.providers.PostgresDataProvider; + +public class PostgresTableQueryTest extends TableQueryTest { + + public PostgresTableQueryTest() { + super(new PostgresDataProvider()); + } + + @Override + public String[] getExpectedEmptySchema() { + return new String[] {"SELECT", "*", "FROM", "\"events\""}; + } + + @Override + public String[] getExpectedEmptySchemaTableWithDot() { + return new String[] {"SELECT", "*", "FROM", "\"public\".\"events\""}; + } + + @Override + public String[] getExpectedEmptyFields() { + return new String[] {"SELECT", "*", "FROM", "\"public\".\"events\""}; + } + + @Override + public String[] getExpectedEmptyFieldsLimit() { + return new String[] {"SELECT", "*", "FROM", "\"public\".\"events\"", "limit 50"}; + } + + @Override + public String[] getExpectedFieldsWithoutDotLimit() { + return new String[] {"SELECT", "\"id\",", "\"friendly_name\",", "\"name\",", "\"source\",", "\"session_id\",", + "\"status\",", "\"description\",", "\"error_message\",", "\"error_stack_trace\",", + "\"event_type_id\"", "FROM", "\"public\".\"events\"", "limit 50"}; + } + + @Override + public String[] getExpectedFieldsWithDotLimit() { + return new String[] {"SELECT", "\"events\".\"id\",", "\"events\".\"friendly_name\",", "\"events\".\"name\",", "\"events\".\"source\",", + "\"events\".\"session_id\",", "\"events\".\"status\",", "\"events\".\"description\",", "\"events\".\"error_message\",", + "\"events\".\"error_stack_trace\",", + "\"events\".\"event_type_id\"", "FROM", "\"public\".\"events\"", "limit 50"}; + } + + @Override + public String[] getExpectedAggregationWithoutPattern() { + return new String[] {"SELECT", "count(*) as \"count(*)\"", "FROM", "\"public\".\"events\""}; + } + + @Override + public String[] getExpectedAggregationWithPattern() { + return new String[] {"SELECT", "sum(\"session_id\") as \"sum(session_id)\"", "FROM", "\"public\".\"events\""}; + } + + @Override + public String[] getAggregationAndGroupByLimitWithoutDot() { + return new String[] {"SELECT", "\"id\",", "\"friendly_name\",", "\"name\",", "\"source\",", + "\"status\",", "\"description\",", "\"error_message\",", "\"error_stack_trace\",", + "\"event_type_id\",", "count(\"session_id\") as \"count(session_id)\"", "FROM", "\"public\".\"events\"", "GROUP BY", "\"id\", \"friendly_name\", \"name\", \"source\", " + + "\"status\", \"description\", \"error_message\", \"error_stack_trace\", \"event_type_id\"", "limit 50"}; + } + + @Override + public String[] getAggregationAndGroupByLimitWithDot() { + return new String[] {"SELECT", "\"events\".\"id\",", "\"events\".\"friendly_name\",", "\"events\".\"name\",", "\"events\".\"source\",", + "\"events\".\"status\",", "\"events\".\"description\",", "\"events\".\"error_message\",", + "\"events\".\"error_stack_trace\",", "\"events\".\"event_type_id\",", "count(\"events\".\"session_id\") as \"count(events.session_id)\"", + "FROM", "\"public\".\"events\"", "GROUP BY", "\"events\".\"id\", \"events\".\"friendly_name\", \"events\".\"name\", \"events\".\"source\", " + + "\"events\".\"status\", \"events\".\"description\", \"events\".\"error_message\", \"events\".\"error_stack_trace\", \"events\".\"event_type_id\"", "limit 50"}; + } + + @Override + public String[] getAggregationAndGroupByAndHavingLimitWithoutDot() { + return new String[] {"--input: string source", "SELECT", "\"id\",", "\"friendly_name\",", "\"name\",", "\"source\",", + "\"status\",", "\"description\",", "\"error_message\",", "\"error_stack_trace\",", + "\"event_type_id\",", "count(\"session_id\") as \"count(session_id)\"", "FROM", "\"public\".\"events\"", "GROUP BY", "\"id\", \"friendly_name\", \"name\", \"source\", " + + "\"status\", \"description\", \"error_message\", \"error_stack_trace\", \"event_type_id\"", + "HAVING", "\t(@source(\"source\"))", "limit 50"}; + } + + @Override + public String[] getAggregationAndGroupByAndHavingLimitWithDot() { + return new String[] {"--input: string events_source", "SELECT", "\"events\".\"id\",", "\"events\".\"friendly_name\",", "\"events\".\"name\",", "\"events\".\"source\",", + "\"events\".\"status\",", "\"events\".\"description\",", "\"events\".\"error_message\",", + "\"events\".\"error_stack_trace\",", "\"events\".\"event_type_id\",", "count(\"events\".\"session_id\") as \"count(events.session_id)\"", + "FROM", "\"public\".\"events\"", "GROUP BY", "\"events\".\"id\", \"events\".\"friendly_name\", \"events\".\"name\", \"events\".\"source\", " + + "\"events\".\"status\", \"events\".\"description\", \"events\".\"error_message\", \"events\".\"error_stack_trace\", \"events\".\"event_type_id\"", + "HAVING", "\t(@events_source(\"events\".\"source\"))", "limit 50"}; + } + + @Override + public String[] getSimpleJoin() { + return new String[] {"SELECT", "\"events\".\"id\",", "\"events\".\"friendly_name\",", "\"events\".\"name\",", "\"events\".\"source\",", + "\"events\".\"session_id\",", "\"events\".\"status\",", "\"events\".\"description\",", "\"events\".\"error_message\",", + "\"events\".\"error_stack_trace\",", "\"events\".\"event_type_id\",", + "\"event_types\".\"id\" as \"event_types_id\",", "\"event_types\".\"friendly_name\" as \"event_types_friendly_name\",", + "\"event_types\".\"name\" as \"event_types_name\",", "\"event_types\".\"description\" as \"event_types_description\",", + "\"event_types\".\"error_message\" as \"event_types_error_message\",", "\"event_types\".\"error_stack_trace\" as \"event_types_error_stack_trace\"", + "FROM", "\"public\".\"events\"", "left join \"event_types\" on \"events\".\"event_type_id\" = \"event_types\".\"id\""}; + } + + @Override + public String[] getSelfJoin() { + return new String[] {"SELECT", "\"events\".\"id\",", "\"events\".\"friendly_name\",", "\"events\".\"name\",", "\"events\".\"source\",", + "\"events\".\"session_id\",", "\"events\".\"status\",", "\"events\".\"description\",", "\"events\".\"error_message\",", + "\"events\".\"error_stack_trace\",", "\"events\".\"event_type_id\",", + "\"events_1\".\"id\" as \"events_1_id\",", "\"events_1\".\"friendly_name\" as \"events_1_friendly_name\",", + "\"events_1\".\"name\" as \"events_1_name\",", "\"events_1\".\"source\" as \"events_1_source\",", + "\"events_1\".\"session_id\" as \"events_1_session_id\",", "\"events_1\".\"status\" as \"events_1_status\",", + "\"events_1\".\"description\" as \"events_1_description\",", "\"events_1\".\"error_message\" as \"events_1_error_message\",", + "\"events_1\".\"error_stack_trace\" as \"events_1_error_stack_trace\",", "\"events_1\".\"event_type_id\" as \"events_1_event_type_id\"", + "FROM", "\"public\".\"events\"", "left join \"events\" as \"events_1\" on \"events\".\"event_type_id\" = \"events_1\".\"event_type_id\""}; + } + + @Override + public String[] getSeveralJoins() { + return new String[] {"SELECT", "\"events\".\"id\",", "\"events\".\"friendly_name\",", "\"events\".\"name\",", "\"events\".\"source\",", + "\"events\".\"session_id\",", "\"events\".\"status\",", "\"events\".\"description\",", "\"events\".\"error_message\",", + "\"events\".\"error_stack_trace\",", "\"events\".\"event_type_id\",", + "\"event_types\".\"id\" as \"event_types_id\",", "\"event_types\".\"friendly_name\" as \"event_types_friendly_name\",", + "\"event_types\".\"name\" as \"event_types_name\",", "\"event_types\".\"description\" as \"event_types_description\",", + "\"event_types\".\"error_message\" as \"event_types_error_message\",", "\"event_types\".\"error_stack_trace\" as \"event_types_error_stack_trace\",", + "\"sessions\".\"id\" as \"sessions_id\",", "\"sessions\".\"user_id\",", "\"sessions\".\"started\",", "\"sessions\".\"ended\",", "\"sessions\".\"token\",", + "\"u\".\"id\" as \"u_id\",", "\"u\".\"email\",", "\"u\".\"first_name\",", "\"u\".\"last_name\",", "\"u\".\"status\" as \"u_status\"", + "FROM", "\"public\".\"events\"", + "left join \"event_types\" on \"events\".\"event_type_id\" = \"event_types\".\"id\"", + "right join \"users_sessions\" as \"sessions\" on \"events\".\"session_id\" = \"sessions\".\"id\"", + "inner join \"users\" as \"u\" on \"sessions\".\"user_id\" = \"u\".\"id\"" + }; + } + + @Override + public String[] getSeveralOnJoin() { + return new String[] {"SELECT", "\"events\".\"id\",", "\"events\".\"friendly_name\",", "\"events\".\"name\",", "\"events\".\"source\",", + "\"events\".\"session_id\",", "\"events\".\"status\",", "\"events\".\"description\",", "\"events\".\"error_message\",", + "\"events\".\"error_stack_trace\",", "\"events\".\"event_type_id\",", + "\"t\".\"id\" as \"t_id\",", "\"t\".\"friendly_name\" as \"t_friendly_name\",", + "\"t\".\"name\" as \"t_name\",", "\"t\".\"description\" as \"t_description\",", + "\"t\".\"error_message\" as \"t_error_message\",", "\"t\".\"error_stack_trace\" as \"t_error_stack_trace\"", + "FROM", "\"public\".\"events\"", "left join \"event_types\" as \"t\" on \"events\".\"event_type_id\" = \"t\".\"id\"", + " AND ", "\"events\".\"name\" = \"t\".\"name\"" + }; + } +} diff --git a/connectors/grok_connect/src/test/java/grok_connect/table_query/TableQueryTest.java b/connectors/grok_connect/src/test/java/grok_connect/table_query/TableQueryTest.java index e36a0bb74e..2205fac934 100644 --- a/connectors/grok_connect/src/test/java/grok_connect/table_query/TableQueryTest.java +++ b/connectors/grok_connect/src/test/java/grok_connect/table_query/TableQueryTest.java @@ -3,9 +3,7 @@ import grok_connect.connectors_info.DataConnection; import grok_connect.providers.JdbcDataProvider; import grok_connect.utils.PatternMatcher; -import grok_connect.utils.ProviderManager; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -17,138 +15,291 @@ import java.util.stream.Collectors; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -class TableQueryTest { - private static final String TEST_TABLE = "users"; - private static final String TEST_SCHEME = "public"; - private JdbcDataProvider postgres; - private JdbcDataProvider mssql; - private TableQuery limitTest; - private TableQuery aggregationTest; +abstract class TableQueryTest { + public static final String TEST_SCHEME = "public"; + public static final String TEST_TABLE = "events"; + public static final String[] TEST_TABLE_FIELDS = { + "id", "friendly_name", "name", "source", "session_id", + "status", "description", "error_message", "error_stack_trace", + "event_type_id" + }; - @BeforeAll - public void init() { - ProviderManager providerManager = new ProviderManager(); - postgres = providerManager.getByName("Postgres"); - mssql = providerManager.getByName("MS SQL"); + public static final String JOIN_TEST_TABLE_1 = "event_types"; + public static final String[] JOIN_TEST_TABLE_1_FIELDS = { + "id", "friendly_name", "name", "description", + "error_message", "error_stack_trace" + }; + public static final String JOIN_TEST_TABLE_2 = "users_sessions"; + public static final String[] JOIN_TEST_TABLE_2_FIELDS = { + "id", "user_id", "started", "ended", "token" + }; + public static final String JOIN_TEST_TABLE_3 = "users"; + public static final String[] JOIN_TEST_TABLE_3_FIELDS = { + "id", "email", "first_name", "last_name", "status" + }; + + private final JdbcDataProvider provider; + private TableQuery testTableQuery; + + public TableQueryTest(JdbcDataProvider provider) { + this.provider = provider; } @BeforeEach public void restore() { - limitTest = new TableQuery(); - limitTest.tableName = TEST_TABLE; - limitTest.schema = TEST_SCHEME; - List fields = new ArrayList<>(); - fields.add("first_name"); - fields.add("last_name"); - fields.add("email"); - fields.add("address"); - limitTest.fields = fields; - limitTest.limit = 50; - limitTest.connection = new DataConnection(); - aggregationTest = new TableQuery(); - aggregationTest.connection = new DataConnection(); - aggregationTest.tableName = TEST_TABLE; - aggregationTest.schema = TEST_SCHEME; - } - - @Test - public void testLimitAtEnd() { - limitTest.connection.dataSource = postgres.descriptor.type; - String sqlQuery = postgres.queryTableSql(limitTest.connection, limitTest); - String[] expected = new String[] {"SELECT", "\"first_name\",", "\"last_name\",", "\"email\",", "\"address\"", "FROM", - "\"public\".\"users\"", "limit 50"}; - Assertions.assertEquals(Arrays.stream(expected).collect(Collectors.joining(System.lineSeparator())), - sqlQuery); - } - - @Test - public void testLimitAtStart() { - limitTest.connection.dataSource = mssql.descriptor.type; - String sqlQuery = mssql.queryTableSql(limitTest.connection, limitTest); - String[] expected = new String[] {"SELECT", "top 50", "[first_name],", "[last_name],", "[email],", "[address]", "FROM", - "[public].[users]"}; - Assertions.assertEquals(getExpectedQuery(expected), - sqlQuery); - } - - @Test - public void testAggregation() { - aggregationTest.connection.dataSource = postgres.descriptor.type; - List aggregations = new ArrayList<>(); - aggregations.add(new GroupAggregation(Stats.TOTAL_COUNT, "*", "count(*)")); - aggregationTest.aggregations = aggregations; - DataConnection connection = new DataConnection(); - connection.dataSource = postgres.descriptor.type; - aggregationTest.connection = connection; - String sqlQuery = postgres.queryTableSql(connection, aggregationTest); - String[] expected = new String[] {"SELECT", "count(*) as \"count(*)\"", "FROM", "\"public\".\"users\""}; - Assertions.assertEquals(getExpectedQuery(expected), sqlQuery); - } - - @Test - public void testAggregationAndGroupBy() { - aggregationTest.connection.dataSource = postgres.descriptor.type; - List aggregations = new ArrayList<>(); - aggregations.add(new GroupAggregation(Stats.TOTAL_COUNT, "*", "count(*)")); - aggregationTest.aggregations = aggregations; - List fields = new ArrayList<>(); - fields.add("country"); - fields.add("region"); - aggregationTest.fields = fields; - List groupByFields = new ArrayList<>(); - groupByFields.add("country"); - groupByFields.add("region"); - aggregationTest.groupByFields = groupByFields; - String sqlQuery = postgres.queryTableSql(aggregationTest.connection, aggregationTest); - String[] expected = new String[] {"SELECT", "\"country\",", "\"region\",", "count(*) as \"count(*)\"", "FROM", "\"public\".\"users\"", - "GROUP BY", "\"country\", \"region\""}; - Assertions.assertEquals(getExpectedQuery(expected), sqlQuery); - } - - @Test - public void testAggregationAndGroupByAndHavingLimitAtEnd() { - prepareComplexQuery(postgres.descriptor.type); - String sqlQuery = postgres.queryTableSql(aggregationTest.connection, aggregationTest); - String[] expected = new String[] {"SELECT", "\"country\",", "\"region\",", "count(*) as \"count(*)\"", "FROM", "\"public\".\"users\"", - "GROUP BY", "\"country\", \"region\"", "HAVING", "\t((LOWER(country) IN ('spain','ukraine','brazil')))", "limit 50"}; - Assertions.assertEquals(getExpectedQuery(expected), sqlQuery); - } - - @Test - public void testAggregationAndGroupByAndHavingLimitAtStart() { - prepareComplexQuery(mssql.descriptor.type); - String sqlQuery = mssql.queryTableSql(aggregationTest.connection, aggregationTest); - String[] expected = new String[] {"SELECT", "top 50", "[country],", "[region],", "count(*) as [count(*)]", "FROM", "[public].[users]", - "GROUP BY", "[country], [region]", "HAVING", "\t((LOWER(country) IN ('spain','ukraine','brazil')))"}; - Assertions.assertEquals(getExpectedQuery(expected), sqlQuery); - } - - private void prepareComplexQuery(String providerType) { - aggregationTest.connection.dataSource = providerType; - List aggregations = new ArrayList<>(); - aggregations.add(new GroupAggregation(Stats.TOTAL_COUNT, "*", "count(*)")); - aggregationTest.aggregations = aggregations; - List fields = new ArrayList<>(); - fields.add("country"); - fields.add("region"); - aggregationTest.fields = fields; - List groupByFields = new ArrayList<>(); - groupByFields.add("country"); - groupByFields.add("region"); - aggregationTest.groupByFields = groupByFields; - FieldPredicate fieldPredicate = new FieldPredicate("country", "in", "string"); + testTableQuery = new TableQuery(); + testTableQuery.tableName = TEST_TABLE; + testTableQuery.schema = TEST_SCHEME; + testTableQuery.connection = new DataConnection(); + testTableQuery.connection.dataSource = provider.descriptor.type; + testTableQuery.params = new ArrayList<>(); + } + + @Test + public void testEmptySchema() { + testTableQuery.schema = null; + testQuery(testTableQuery, getExpectedEmptySchema()); + } + + public abstract String[] getExpectedEmptySchema(); + + @Test + public void testEmptySchemaTableWithDot() { + testTableQuery.schema = null; + testTableQuery.tableName = TEST_SCHEME + "." + TEST_TABLE; + testQuery(testTableQuery, getExpectedEmptySchemaTableWithDot()); + } + + public abstract String[] getExpectedEmptySchemaTableWithDot(); + + @Test + public void testEmptyFields() { + testQuery(testTableQuery, getExpectedEmptyFields()); + } + + public abstract String[] getExpectedEmptyFields(); + + @Test + public void testEmptyFieldsLimit() { + testTableQuery.limit = 50; + testQuery(testTableQuery, getExpectedEmptyFieldsLimit()); + } + + public abstract String[] getExpectedEmptyFieldsLimit(); + + @Test + public void testFieldsWithoutDotLimit() { + testTableQuery.limit = 50; + testTableQuery.fields = Arrays.stream(TEST_TABLE_FIELDS).collect(Collectors.toList()); + testQuery(testTableQuery, getExpectedFieldsWithoutDotLimit()); + } + + public abstract String[] getExpectedFieldsWithoutDotLimit(); + + @Test + public void testFieldsWithDotLimit() { + testTableQuery.limit = 50; + testTableQuery.fields = Arrays.stream(TEST_TABLE_FIELDS) + .map(f -> TEST_TABLE + "." + f) + .collect(Collectors.toList()); + testQuery(testTableQuery, getExpectedFieldsWithDotLimit()); + } + + public abstract String[] getExpectedFieldsWithDotLimit(); + + @Test + public void testAggregationWithoutPattern() { + testTableQuery.aggregations.add(new GroupAggregation(Stats.TOTAL_COUNT, "*", "count(*)")); + testQuery(testTableQuery, getExpectedAggregationWithoutPattern()); + } + + public abstract String[] getExpectedAggregationWithoutPattern(); + + @Test + public void testAggregationWithPattern() { + testTableQuery.aggregations.add(new GroupAggregation(Stats.SUM, "session_id", "sum(session_id)")); + testQuery(testTableQuery, getExpectedAggregationWithPattern()); + } + + public abstract String[] getExpectedAggregationWithPattern(); + + @Test + public void testAggregationAndGroupByLimitWithoutDot() { + testTableQuery.aggregations.add(new GroupAggregation(Stats.VALUE_COUNT, "session_id", "count(session_id)")); + List fields = Arrays.stream(TEST_TABLE_FIELDS).filter(f -> !f.equals("session_id")).collect(Collectors.toList()); + testTableQuery.fields = fields; + testTableQuery.groupByFields = fields; + testTableQuery.limit = 50; + testQuery(testTableQuery, getAggregationAndGroupByLimitWithoutDot()); + } + + public abstract String[] getAggregationAndGroupByLimitWithoutDot(); + + @Test + public void testAggregationAndGroupByLimitWithDot() { + testTableQuery.aggregations.add(new GroupAggregation(Stats.VALUE_COUNT, "events.session_id", "count(events.session_id)")); + List fields = Arrays.stream(TEST_TABLE_FIELDS).filter(f -> !f.equals("session_id")) + .map(f -> TEST_TABLE + "." + f) + .collect(Collectors.toList()); + testTableQuery.fields = fields; + testTableQuery.groupByFields = fields; + testTableQuery.limit = 50; + testQuery(testTableQuery, getAggregationAndGroupByLimitWithDot()); + } + + public abstract String[] getAggregationAndGroupByLimitWithDot(); + + @Test + public void testAggregationAndGroupByAndHavingLimitWithoutDot() { + testTableQuery.aggregations.add(new GroupAggregation(Stats.VALUE_COUNT, "session_id", "count(session_id)")); + List fields = Arrays.stream(TEST_TABLE_FIELDS).filter(f -> !f.equals("session_id")).collect(Collectors.toList()); + testTableQuery.fields = fields; + testTableQuery.groupByFields = fields; + testTableQuery.limit = 50; + Map options = new HashMap<>(); + options.put("op", PatternMatcher.IN); + FieldPredicate fieldPredicate = new FieldPredicate("source", "in", "string"); + List values = new ArrayList<>(); + options.put("values", values); + values.add("func"); + values.add("query"); + values.add("script"); + fieldPredicate.matcher = new PatternMatcher(options, "source"); + testTableQuery.having.add(fieldPredicate); + testQuery(testTableQuery, getAggregationAndGroupByAndHavingLimitWithoutDot()); + } + + public abstract String[] getAggregationAndGroupByAndHavingLimitWithoutDot(); + + + @Test + public void testAggregationAndGroupByAndHavingLimitWithDot() { + testTableQuery.aggregations.add(new GroupAggregation(Stats.VALUE_COUNT, "events.session_id", "count(events.session_id)")); + List fields = Arrays.stream(TEST_TABLE_FIELDS).filter(f -> !f.equals("session_id")) + .map(f -> TEST_TABLE + "." + f) + .collect(Collectors.toList()); + testTableQuery.fields = fields; + testTableQuery.groupByFields = fields; + testTableQuery.limit = 50; Map options = new HashMap<>(); options.put("op", PatternMatcher.IN); + FieldPredicate fieldPredicate = new FieldPredicate("events.source", "in", "string"); List values = new ArrayList<>(); options.put("values", values); - values.add("Spain"); - values.add("Ukraine"); - values.add("Brazil"); - fieldPredicate.matcher = new PatternMatcher(options, "country"); - List having = new ArrayList<>(); - having.add(fieldPredicate); - aggregationTest.having = having; - aggregationTest.limit = 50; + values.add("func"); + values.add("query"); + values.add("script"); + fieldPredicate.matcher = new PatternMatcher(options, "events.source"); + testTableQuery.having.add(fieldPredicate); + testQuery(testTableQuery, getAggregationAndGroupByAndHavingLimitWithDot()); + } + + public abstract String[] getAggregationAndGroupByAndHavingLimitWithDot(); + + @Test + public void testSimpleJoin() { + testTableQuery.fields = Arrays.stream(TEST_TABLE_FIELDS).map(f -> TEST_TABLE + "." + f).collect(Collectors.toList()); + testTableQuery.fields.addAll(Arrays.stream(JOIN_TEST_TABLE_1_FIELDS).map(f -> JOIN_TEST_TABLE_1 + "." + f).collect(Collectors.toList())); + TableJoin tableJoin = new TableJoin(); + tableJoin.joinType = "left"; + tableJoin.leftTableName = TEST_TABLE; + tableJoin.rightTableName = JOIN_TEST_TABLE_1; + tableJoin.leftTableKeys = new ArrayList(){{ add("event_type_id"); }}; + tableJoin.rightTableKeys = new ArrayList(){{ add("id"); }}; + testTableQuery.joins = new ArrayList(){{ add(tableJoin); }}; + testQuery(testTableQuery, getSimpleJoin()); + } + + public abstract String[] getSimpleJoin(); + + + @Test + public void testSelfJoin() { + String selfJoinAlias = "events_1"; + testTableQuery.fields = Arrays.stream(TEST_TABLE_FIELDS).map(f -> TEST_TABLE + "." + f).collect(Collectors.toList()); + testTableQuery.fields.addAll(Arrays.stream(TEST_TABLE_FIELDS).map(f -> selfJoinAlias + "." + f).collect(Collectors.toList())); + TableJoin tableJoin = new TableJoin(); + tableJoin.joinType = "left"; + tableJoin.leftTableName = TEST_TABLE; + tableJoin.rightTableName = TEST_TABLE; + tableJoin.rightTableAlias = selfJoinAlias; + tableJoin.leftTableKeys = new ArrayList(){{ add("event_type_id"); }}; + tableJoin.rightTableKeys = new ArrayList(){{ add("event_type_id"); }}; + testTableQuery.joins = new ArrayList(){{ add(tableJoin); }}; + testQuery(testTableQuery, getSelfJoin()); + } + + public abstract String[] getSelfJoin(); + + @Test + public void testSeveralJoins() { + String sessionsAlias = "sessions"; + String usersAlias = "u"; + + testTableQuery.fields = Arrays.stream(TEST_TABLE_FIELDS).map(f -> TEST_TABLE + "." + f).collect(Collectors.toList()); + testTableQuery.fields.addAll(Arrays.stream(JOIN_TEST_TABLE_1_FIELDS).map(f -> JOIN_TEST_TABLE_1 + "." + f).collect(Collectors.toList())); + testTableQuery.fields.addAll(Arrays.stream(JOIN_TEST_TABLE_2_FIELDS).map(f -> sessionsAlias + "." + f).collect(Collectors.toList())); + testTableQuery.fields.addAll(Arrays.stream(JOIN_TEST_TABLE_3_FIELDS).map(f -> usersAlias + "." + f).collect(Collectors.toList())); + + TableJoin tableJoin1 = new TableJoin(); + tableJoin1.joinType = "left"; + tableJoin1.leftTableName = TEST_TABLE; + tableJoin1.rightTableName = JOIN_TEST_TABLE_1; + tableJoin1.leftTableKeys = new ArrayList(){{ add("event_type_id"); }}; + tableJoin1.rightTableKeys = new ArrayList(){{ add("id"); }}; + testTableQuery.joins = new ArrayList(){{ add(tableJoin1); }}; + + TableJoin tableJoin2 = new TableJoin(); + tableJoin2.joinType = "right"; + tableJoin2.leftTableName = TEST_TABLE; + tableJoin2.rightTableName = JOIN_TEST_TABLE_2; + tableJoin2.rightTableAlias = sessionsAlias; + tableJoin2.leftTableKeys = new ArrayList(){{ add("session_id"); }}; + tableJoin2.rightTableKeys = new ArrayList(){{ add("id"); }}; + testTableQuery.joins.add(tableJoin2); + + TableJoin tableJoin3 = new TableJoin(); + tableJoin3.joinType = "inner"; + tableJoin3.leftTableName = sessionsAlias; + tableJoin3.rightTableName = JOIN_TEST_TABLE_3; + tableJoin3.rightTableAlias = usersAlias; + tableJoin3.leftTableKeys = new ArrayList(){{ add("user_id"); }}; + tableJoin3.rightTableKeys = new ArrayList(){{ add("id"); }}; + testTableQuery.joins.add(tableJoin3); + + testQuery(testTableQuery, getSeveralJoins()); + } + + public abstract String[] getSeveralJoins(); + + @Test + public void testSeveralOnJoin() { + String rightTableAlias = "t"; + testTableQuery.fields = Arrays.stream(TEST_TABLE_FIELDS).map(f -> TEST_TABLE + "." + f).collect(Collectors.toList()); + testTableQuery.fields.addAll(Arrays.stream(JOIN_TEST_TABLE_1_FIELDS).map(f -> rightTableAlias + "." + f).collect(Collectors.toList())); + TableJoin tableJoin = new TableJoin(); + tableJoin.joinType = "left"; + tableJoin.leftTableName = TEST_TABLE; + tableJoin.rightTableName = JOIN_TEST_TABLE_1; + tableJoin.rightTableAlias = rightTableAlias; + tableJoin.leftTableKeys = new ArrayList(){{ + add("event_type_id"); + add("name"); + }}; + tableJoin.rightTableKeys = new ArrayList(){{ + add("id"); + add("name"); + }}; + testTableQuery.joins = new ArrayList(){{ add(tableJoin); }}; + testQuery(testTableQuery, getSeveralOnJoin()); + } + + public abstract String[] getSeveralOnJoin(); + + private void testQuery(TableQuery tableQuery, String[] expected) { + String actual = provider.queryTableSql(tableQuery.connection, tableQuery); + String expectedQuery = getExpectedQuery(expected); + Assertions.assertEquals(expectedQuery, actual); } private String getExpectedQuery(String[] expected) { diff --git a/docusaurus/docsearch.json b/docusaurus/docsearch.json index 72f75b2898..549fbe1d67 100644 --- a/docusaurus/docsearch.json +++ b/docusaurus/docsearch.json @@ -18,7 +18,7 @@ "global": true, "default_value": "Documentation" }, - "lvl1": "header h1", + "lvl1": "article h1, header h1", "lvl2": "article h2", "lvl3": "article h3", "lvl4": "article h4", diff --git a/docusaurus/docusaurus-debug.config.js b/docusaurus/docusaurus-debug.config.js index 346adf2549..7977dc06da 100644 --- a/docusaurus/docusaurus-debug.config.js +++ b/docusaurus/docusaurus-debug.config.js @@ -189,7 +189,7 @@ const config = { protocol: 'https', } ], - apiKey: 'Crm7kyT3PHDgImIroB9cpkE0SHqTKAJZ', + apiKey: 'hyoeYSu26w6jqpicj6C2A1Qy2X0nvv4l', }, // Optional: Typesense search parameters: https://typesense.org/docs/0.21.0/api/search.md#search-parameters diff --git a/docusaurus/docusaurus.config.js b/docusaurus/docusaurus.config.js index 74bcff8534..1d7b0b9b94 100644 --- a/docusaurus/docusaurus.config.js +++ b/docusaurus/docusaurus.config.js @@ -187,7 +187,7 @@ const config = { protocol: 'https', } ], - apiKey: 'Crm7kyT3PHDgImIroB9cpkE0SHqTKAJZ', + apiKey: 'hyoeYSu26w6jqpicj6C2A1Qy2X0nvv4l', }, // Optional: Typesense search parameters: https://typesense.org/docs/0.21.0/api/search.md#search-parameters diff --git a/docusaurus/static/versions/1.21.4.json b/docusaurus/static/versions/1.21.4.json new file mode 100644 index 0000000000..f109332cbd --- /dev/null +++ b/docusaurus/static/versions/1.21.4.json @@ -0,0 +1 @@ +{"UsageType":{},"ViewerEvent":{"create":{"type":"","params":"","result":"ViewerEvent","static":true},"get viewer":{"type":"get","result":"any"},"set viewer":{"type":"set","params":"x: any"},"get type":{"type":"get","result":"string"},"set type":{"type":"set","params":"x: string"},"get eventFlag":{"type":"get","result":"boolean"},"set eventFlag":{"type":"set","params":"x: boolean"},"get filters":{"type":"get","result":""},"get row":{"type":"get","result":"number"},"set row":{"type":"set","params":"x: number"},"get mouseEvent":{"type":"get","result":"any"},"set mouseEvent":{"type":"set","params":"x: any"},"get bitset":{"type":"get","result":"any"}},"InputType":{},"GridCellStyleEx":{"create":{"type":"","params":"","result":"GridCellStyleEx","static":true},"get defaultStyle":{"type":"get","result":"any","static":true},"set defaultStyle":{"type":"set","params":"x: any","static":true},"get textStyle":{"type":"get","result":"any","static":true},"set textStyle":{"type":"set","params":"x: any","static":true},"get numberStyle":{"type":"get","result":"any","static":true},"set numberStyle":{"type":"set","params":"x: any","static":true},"get styles":{"type":"get","result":"","static":true},"get font":{"type":"get","result":"string"},"set font":{"type":"set","params":"x: string"},"get horzAlign":{"type":"get","result":"string"},"set horzAlign":{"type":"set","params":"x: string"},"get vertAlign":{"type":"get","result":"string"},"set vertAlign":{"type":"set","params":"x: string"},"get tooltip":{"type":"get","result":"string"},"set tooltip":{"type":"set","params":"x: string"},"get cursor":{"type":"get","result":"string"},"set cursor":{"type":"set","params":"x: string"},"get textWrap":{"type":"get","result":"string"},"set textWrap":{"type":"set","params":"x: string"},"get marker":{"type":"get","result":"string"},"set marker":{"type":"set","params":"x: string"},"get textColor":{"type":"get","result":"number"},"set textColor":{"type":"set","params":"x: number"},"get backColor":{"type":"get","result":"number"},"set backColor":{"type":"set","params":"x: number"},"get marginLeft":{"type":"get","result":"number"},"set marginLeft":{"type":"set","params":"x: number"},"get marginRight":{"type":"get","result":"number"},"set marginRight":{"type":"set","params":"x: number"},"get marginTop":{"type":"get","result":"number"},"set marginTop":{"type":"set","params":"x: number"},"get marginBottom":{"type":"get","result":"number"},"set marginBottom":{"type":"set","params":"x: number"},"get textVertical":{"type":"get","result":"boolean"},"set textVertical":{"type":"set","params":"x: boolean"},"get imageScale":{"type":"get","result":"number"},"set imageScale":{"type":"set","params":"x: number"},"get opacity":{"type":"get","result":"number"},"set opacity":{"type":"set","params":"x: number"},"get clip":{"type":"get","result":"boolean"},"set clip":{"type":"set","params":"x: boolean"},"get element":{"type":"get","result":"any"},"set element":{"type":"set","params":"x: any"},"get choices":{"type":"get","result":"Array"},"set choices":{"type":"set","params":"x: Array"}},"Tags":{},"FuncOptions":{},"FuncParamOptions":{},"DataSourceType":{},"DockerImage":{"fromJson":{"type":"","params":"map: { [index: string]: any; }","result":"DockerImage","static":true},"get description":{"type":"get","result":"string"},"set description":{"type":"set","params":"x: string"},"get dockerfile":{"type":"get","result":"string"},"set dockerfile":{"type":"set","params":"x: string"},"get status":{"type":"get","result":"string"},"set status":{"type":"set","params":"x: string"},"get dockerName":{"type":"get","result":"string"},"set dockerName":{"type":"set","params":"x: string"},"get version":{"type":"get","result":"string"},"set version":{"type":"set","params":"x: string"},"get dockerfilePath":{"type":"get","result":"string"},"set dockerfilePath":{"type":"set","params":"x: string"},"get updatedBy":{"type":"get","result":"string"},"set updatedBy":{"type":"set","params":"x: string"},"get logs":{"type":"get","result":"string"},"set logs":{"type":"set","params":"x: string"},"get completed":{"type":"get","result":"boolean"},"get iconStatus":{"type":"get","result":"string"},"get dockerFullName":{"type":"get","result":"string"}},"chem":{"isMolBlock":{"type":"function","params":"s: string | null","result":"boolean"},"get smiles":{"type":"get","result":"string"},"set smiles":{"type":"set","params":"s: string"},"get molFile":{"type":"get","result":"string"},"get molV3000":{"type":"get","result":"string"},"set molV3000":{"type":"set","params":"s: string"},"set molFile":{"type":"set","params":"s: string"},"getSmarts":{"type":"","params":"","result":"Promise"},"set smarts":{"type":"set","params":"s: string"},"get isInitialized":{"type":"get","result":"boolean"},"get supportedExportFormats":{"type":"get","result":"string[]"},"get width":{"type":"get","result":"number"},"get height":{"type":"get","result":"number"},"init":{"type":"","params":"host: Sketcher","result":"Promise"},"refresh":{"type":"","params":"","result":"void"},"resize":{"type":"","params":"","result":"void"},"set sketcherType":{"type":"set","params":"type: string"},"get isResizing":{"type":"get","result":"boolean"},"get autoResized":{"type":"get","result":"boolean"},"get sketcherTypeChanged":{"type":"get","result":"boolean"},"get calculating":{"type":"get","result":"boolean"},"set calculating":{"type":"set","params":"value: boolean"},"get isSubstructureFilter":{"type":"get","result":"boolean"},"set isSubstructureFilter":{"type":"set","params":"value: boolean"},"get align":{"type":"get","result":"boolean"},"set align":{"type":"set","params":"value: boolean"},"get highlight":{"type":"get","result":"boolean"},"set highlight":{"type":"set","params":"value: boolean"},"get filterOptions":{"type":"get","result":"HTMLElement"},"getSmiles":{"type":"","params":"","result":"string"},"setSmiles":{"type":"","params":"x: string","result":"void"},"getMolFile":{"type":"","params":"","result":"string"},"setMolFile":{"type":"","params":"x: string","result":"void"},"setSmarts":{"type":"","params":"x: string","result":"void"},"isEmpty":{"type":"","params":"","result":"boolean"},"setMolecule":{"type":"","params":"molString: string, substructure?: boolean","result":"void"},"setValue":{"type":"","params":"x: string","result":"void"},"validate":{"type":"","params":"x: string","result":"void"},"setExternalModeForSubstrFilter":{"type":"","params":"","result":"void"},"createSketcher":{"type":"","params":"","result":"void"},"updateExtSketcherContent":{"type":"","params":"","result":"void"},"createMoleculeTooltip":{"type":"","params":"currentMolfile: string","result":"HTMLElement"},"createClearSketcherButton":{"type":"","params":"canvas: HTMLCanvasElement","result":"HTMLButtonElement"},"updateInvalidMoleculeWarning":{"type":"","params":"","result":"void"},"createExternalModeSketcher":{"type":"","params":"","result":"HTMLElement"},"createInplaceModeSketcher":{"type":"","params":"","result":"HTMLElement"},"getCollection":{"type":"","params":"key: string","result":"string[]","static":true},"addToCollection":{"type":"","params":"key: string, molecule: string","result":"void","static":true},"checkDuplicatesAndAddToStorage":{"type":"","params":"storage: string[], molecule: string, localStorageKey: string","result":"void","static":true},"isEmptyMolfile":{"type":"","params":"molFile: string","result":"boolean","static":true},"detach":{"type":"","params":"","result":"void"},"drawToCanvas":{"type":"","params":"w: number, h: number, molecule: string","result":"HTMLElement"},"getSimilarities":{"type":"function","params":"column: Column, molecule?: string, settings?: object","result":"Promise"},"findSimilar":{"type":"","params":"column: Column, molecule?: string, settings?: { limit: number; cutoff: number; }","result":"Promise"},"diversitySearch":{"type":"","params":"column: Column, settings?: { limit: number; }","result":"Promise"},"searchSubstructure":{"type":"","params":"column: Column, pattern?: string, settings?: { molBlockFailover?: string; }","result":"Promise"},"rGroup":{"type":"function","params":"table: DataFrame, column: string, core: string","result":"Promise"},"mcs":{"type":"function","params":"table: DataFrame, column: string, returnSmarts?: boolean, exactAtomSearch?: boolean, exactBondSearch?: boolean","result":"Promise"},"descriptors":{"type":"function","params":"table: DataFrame, column: string, descriptors: string[]","result":"Promise"},"descriptorsTree":{"type":"function","params":"","result":"Promise"},"svgMol":{"type":"","params":"smiles: string, width?: number, height?: number, options?: { [key: string]: boolean | number | string; }","result":"HTMLDivElement"},"canvasMol":{"type":"","params":"x: number, y: number, w: number, h: number, canvas: Object, molString: string, scaffoldMolString?: string | null, options?: { normalizeDepiction: boolean; straightenDepiction: boolean; }","result":"Promise"},"drawMolecule":{"type":"function","params":"molString: string, w?: number, h?: number, popupMenu?: boolean","result":"HTMLDivElement"},"sketcher":{"type":"function","params":"onChangedCallback: Function, smiles?: string","result":"HTMLElement"},"convert":{"type":"function","params":"s: string, sourceFormat: Notation, targetFormat: Notation","result":"string"},"checkSmiles":{"type":"function","params":"s: string","result":"boolean"},"isSmarts":{"type":"function","params":"s: string","result":"boolean"},"smilesFromSmartsWarning":{"type":"function","params":"","result":"string"}},"ComponentBuildInfo":{},"Dapi":{"get root":{"type":"get","result":"string"},"getEntities":{"type":"","params":"ids: string[]","result":"Promise"},"get entities":{"type":"get","result":"EntitiesDataSource"},"get queries":{"type":"get","result":"HttpDataSource"},"get functions":{"type":"get","result":"FuncsDataSource"},"get connections":{"type":"get","result":"DataConnectionsDataSource"},"get credentials":{"type":"get","result":"CredentialsDataSource"},"get jobs":{"type":"get","result":"HttpDataSource"},"get notebooks":{"type":"get","result":"HttpDataSource"},"get models":{"type":"get","result":"HttpDataSource"},"get packages":{"type":"get","result":"HttpDataSource"},"get layouts":{"type":"get","result":"LayoutsDataSource"},"get views":{"type":"get","result":"ViewsDataSource"},"get tables":{"type":"get","result":"TablesDataSource"},"get users":{"type":"get","result":"UsersDataSource"},"get groups":{"type":"get","result":"GroupsDataSource"},"get permissions":{"type":"get","result":"PermissionsDataSource"},"get scripts":{"type":"get","result":"HttpDataSource