Updated the test generator + added allowed-warnings feature

elastic · Nov 3, 2020 · 85fadc2 · 85fadc2
1 parent 2e56ea9
commit 85fadc2
Show file tree

Hide file tree

Showing 8 changed files with 572 additions and 62 deletions.
diff --git a/tests/Elasticsearch/Tests/Utility.php b/tests/Elasticsearch/Tests/Utility.php
diff --git a/util/ActionTest.php b/util/ActionTest.php
@@ -47,12 +47,22 @@ class ActionTest
     const TEMPLATE_SET_VARIABLE       = __DIR__ . '/template/test/set-variable';
     const TEMPLATE_TRANSFORM_AND_SET  = __DIR__ . '/template/test/transform-and-set';
     const TEMPLATE_WARNINGS           = __DIR__ . '/template/test/warnings';
+    const TEMPLATE_ALLOWED_WARNINGS   = __DIR__ . '/template/test/allowed-warnings';
     const TEMPLATE_GT                 = __DIR__ . '/template/test/gt';
     const TEMPLATE_GTE                = __DIR__ . '/template/test/gte';
     const TEMPLATE_LT                 = __DIR__ . '/template/test/lt';
     const TEMPLATE_LTE                = __DIR__ . '/template/test/lte';
     const TAB14                 = '              ';
-    const SUPPORTED_FEATURES    = ['xpack', 'no_xpack', 'headers', 'node_selector', 'warnings', 'catch_unauthorized', 'transform_and_set'];
+    const SUPPORTED_FEATURES    = [
+        'xpack', 
+        'no_xpack', 
+        'headers', 
+        'node_selector', 
+        'warnings', 
+        'catch_unauthorized', 
+        'transform_and_set',
+        'allowed_warnings'
+    ];
 
     private $headers = [];
     private $variables = [];
@@ -146,6 +156,13 @@ private function warnings(array $action, array &$vars)
         ]);
     }
 
+    private function allowed_warnings(array $action, array &$vars)
+    {
+        $vars[':response-check'] .= YamlTests::render(self::TEMPLATE_ALLOWED_WARNINGS, [
+            ':expected' => $action
+        ]);
+    }
+
     private function catch(string $action, array &$vars)
     {
         switch ($action) {

diff --git a/util/YamlTests.php b/util/YamlTests.php
@@ -28,11 +28,12 @@
 
 class YamlTests
 {
-    const TEMPLATE_UNIT_TEST_OSS    = __DIR__ . '/template/test/unit-test-oss';
-    const TEMPLATE_UNIT_TEST_XPACK  = __DIR__ . '/template/test/unit-test-xpack';
-    const TEMPLATE_FUNCTION_TEST    = __DIR__ . '/template/test/function-test';
-    const TEMPLATE_FUNCTION_SKIPPED = __DIR__ . '/template/test/function-skipped';
-    const ELASTICSEARCH_GIT_URL     = 'https://github.com/elastic/elasticsearch/tree/%s/rest-api-spec/src/main/resources/rest-api-spec/test/%s';
+    const TEMPLATE_UNIT_TEST_OSS     = __DIR__ . '/template/test/unit-test-oss';
+    const TEMPLATE_UNIT_TEST_XPACK   = __DIR__ . '/template/test/unit-test-xpack';
+    const TEMPLATE_UNIT_TEST_SKIPPED = __DIR__ . '/template/test/unit-test-skipped';
+    const TEMPLATE_FUNCTION_TEST     = __DIR__ . '/template/test/function-test';
+    const TEMPLATE_FUNCTION_SKIPPED  = __DIR__ . '/template/test/function-skipped';
+    const ELASTICSEARCH_GIT_URL      = 'https://github.com/elastic/elasticsearch/tree/%s/rest-api-spec/src/main/resources/rest-api-spec/test/%s';
 
     const SKIPPED_TEST_OSS = [
         'Cat\Nodeattrs\_10_BasicTest::TestCatNodesAttrsOutput' => 'Regexp error, it seems not compatible with PHP',
@@ -41,12 +42,31 @@ class YamlTests
     ];
 
     const SKIPPED_TEST_XPACK = [
+        'DataStream\_80_Resolve_Index_Data_StreamsTest::*' => 'Skipped all tests',
         'License\_20_Put_LicenseTest::CurrentLicenseIsTrialMeansNotEligleToStartTrial' => 'License issue',
         'License\_20_Put_LicenseTest::MustAcknowledgeToStartBasic' => 'License issue',
         'License\_20_Put_LicenseTest::InstallingAndGettingLicenseWorks' => 'Invalid license',
         'License\_20_Put_LicenseTest::ShouldInstallAFeatureTypeLicense' => 'Invalid license',
         'License\_20_Put_LicenseTest::CanStartBasicLicenseIfDoNotAlreadyHaveBasic' => 'Invalid license',
-        'License\_30_Enterprise_LicenseTest::InstallingEnterpriseLicense' => 'Invalid license'
+        'License\_30_Enterprise_LicenseTest::InstallingEnterpriseLicense' => 'Invalid license',
+        'Ml\_Jobs_CrudTest::TestPutJobWithModel_memory_limitAsStringAndLazyOpen' => 'Memory limit',
+        'Ml\_Data_Frame_Analytics_CrudTest::TestPutClassificationGivenNum_top_classesIsLessThanZero' => 'No error catched',
+        'Ml\_Set_Upgrade_ModeTest::*' => 'Skipped all tests',
+        'Ml\_Filter_CrudTest::*' => 'Skipped all tests',
+        'Ml\_Inference_CrudTest::*' => 'Skipped all tests',
+        'Ml\_Inference_Stats_CrudTest::*' => 'Skipped all tests',
+        'Ml\_Ml_InfoTest::TestMlInfo' => 'response[\'limits\'][\'max_model_memory_limit\'] is not empty',
+        'Ml\_Delete_Expired_DataTest::TestDeleteExpiredDataWithJobId' => 'Substring mismatch',
+        'Ml\_Explain_Data_Frame_AnalyticsTest::TestNonemptyDataFrameGivenBody' => 'Expected a different value',
+        'Rollup\_Put_JobTest::TestPutJobWithTemplates' => 'version not converted from variable',
+        'Snapshot\_10_BasicTest::CreateASourceOnlySnapshotAndThenRestoreIt' => 'spanshop name already exists',
+        'Ssl\_10_BasicTest::TestGetSSLCertificates' => 'Mismatch values',
+        'Transform\_Transforms_CrudTest::TestDeleteTransformWhenItDoesNotExist' => 'Invalid version format: TRANSFORM HTTP/1.1',
+        'UnsignedLong\_10_BasicTest::*' => 'Skipped all tests',
+        'UnsignedLong\_20_Null_ValueTest::*' => 'Skipped all tests',
+        'UnsignedLong\_30_Multi_FieldsTest::*' => 'Skipped all tests',
+        'UnsignedLong\_50_Script_ValuesTest::*' => 'Skipped all tests',
+        'Vectors\_30_Sparse_Vector_BasicTest::DeprecatedFunctionSignature' => 'Failed asserting contains string',
     ];
 
     const PHP_RESERVED_WORDS     = [
@@ -134,6 +154,7 @@ public function build(): array
             $setup = '';
             $teardown = '';
             $alreadyAssignedNames = [];
+            $allSkipped = false;
             foreach ($value as $test) {
                 if (!is_array($test)) {
                     continue;
@@ -151,10 +172,20 @@ public function build(): array
                             $alreadyAssignedNames[] = $functionName;
 
                             $skippedTest = sprintf("%s\\%s::%s", $namespace, $testName, $functionName);
+                            $skippedAllTest = sprintf("%s\\%s::*", $namespace, $testName);
                             $skip = strtolower(self::$testSuite) === 'oss' 
                                 ? self::SKIPPED_TEST_OSS 
                                 : self::SKIPPED_TEST_XPACK;
-                            if (isset($skip[$skippedTest])) {
+                            if (isset($skip[$skippedAllTest])) {
+                                $allSkipped = true;
+                                $functions .= self::render(
+                                    self::TEMPLATE_FUNCTION_SKIPPED,
+                                    [ 
+                                        ':name' => $functionName,
+                                        ':skipped_msg'  => $skip[$skippedAllTest] 
+                                    ]
+                                );
+                            } elseif (isset($skip[$skippedTest])) {
                                 $functions .= self::render(
                                     self::TEMPLATE_FUNCTION_SKIPPED,
                                     [ 
@@ -175,20 +206,33 @@ public function build(): array
                     }
                 }
             }
-            $test = self::render(
-                strtolower(self::$testSuite) === 'oss'
-                    ? self::TEMPLATE_UNIT_TEST_OSS
-                    : self::TEMPLATE_UNIT_TEST_XPACK,
-                [
-                    ':namespace' => sprintf("Elasticsearch\Tests\Yaml\%s\%s", self::$testSuite, $namespace),
-                    ':test-name' => $testName,
-                    ':tests'     => $functions,
-                    ':setup'     => $setup,
-                    ':teardown'  => $teardown,
-                    ':yamlfile'  => sprintf(self::ELASTICSEARCH_GIT_URL, self::$minorEsVersion, $yamlFileName),
-                    ':group'     => strtolower(self::$testSuite)
-                ]
-            );
+            if ($allSkipped) {
+                $test = self::render(
+                    self::TEMPLATE_UNIT_TEST_SKIPPED,
+                    [
+                        ':namespace' => sprintf("Elasticsearch\Tests\Yaml\%s\%s", self::$testSuite, $namespace),
+                        ':test-name' => $testName,
+                        ':tests'     => $functions,
+                        ':yamlfile'  => sprintf(self::ELASTICSEARCH_GIT_URL, self::$minorEsVersion, $yamlFileName),
+                        ':group'     => strtolower(self::$testSuite)
+                    ]
+                );
+            } else {
+                $test = self::render(
+                    strtolower(self::$testSuite) === 'oss'
+                        ? self::TEMPLATE_UNIT_TEST_OSS
+                        : self::TEMPLATE_UNIT_TEST_XPACK,
+                    [
+                        ':namespace' => sprintf("Elasticsearch\Tests\Yaml\%s\%s", self::$testSuite, $namespace),
+                        ':test-name' => $testName,
+                        ':tests'     => $functions,
+                        ':setup'     => $setup,
+                        ':teardown'  => $teardown,
+                        ':yamlfile'  => sprintf(self::ELASTICSEARCH_GIT_URL, self::$minorEsVersion, $yamlFileName),
+                        ':group'     => strtolower(self::$testSuite)
+                    ]
+                );
+            }
             file_put_contents($testDirName . '/' . $testName . '.php', $test);
             try {
                 eval(substr($test, 5)); // remove <?php header

diff --git a/util/template/test/allowed-warnings b/util/template/test/allowed-warnings
@@ -0,0 +1,10 @@
+        // check for allowed_warnings
+        $headers = $this->client->transport->getLastConnection()->getLastRequestInfo()['response']['headers'];
+        if (isset($headers['Warning'])) {
+            $expectedHeaders = :expected;
+            $contains = false;
+            foreach ($expectedHeaders as $expected) {
+                $contains = $contains || (false !== strpos($headers['Warning'][0], $expected));
+            }
+            $this->assertTrue($contains);
+        }
diff --git a/util/template/test/unit-test-oss b/util/template/test/unit-test-oss
@@ -38,7 +38,7 @@ class :test-name extends TestCase
     protected function tearDown(): void
     {
         $this->yamlTearDown();
-        Utility::cleanYamlOssTest($this->client);
+        Utility::cleanUpCluster($this->client);
     }
 
     private function yamlSetup()

diff --git a/util/template/test/unit-test-skipped b/util/template/test/unit-test-skipped
@@ -0,0 +1,33 @@
+<?php
+/**
+ * Elasticsearch PHP client
+ *
+ * @link      https://github.com/elastic/elasticsearch-php/
+ * @copyright Copyright (c) Elasticsearch B.V (https://www.elastic.co)
+ * @license   http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0
+ * @license   https://www.gnu.org/licenses/lgpl-2.1.html GNU Lesser General Public License, Version 2.1 
+ * 
+ * Licensed to Elasticsearch B.V under one or more agreements.
+ * Elasticsearch B.V licenses this file to you under the Apache 2.0 License or
+ * the GNU Lesser General Public License, Version 2.1, at your option.
+ * See the LICENSE file in the project root for more information.
+ */
+declare(strict_types = 1);
+
+namespace :namespace;
+
+use PHPUnit\Framework\TestCase;
+use Elasticsearch\Tests\Utility;
+use stdClass;
+
+/**
+ * YAML test for Elasticsearch
+ * @see :yamlfile
+ * @group :group
+ *
+ * NOTE: This file is autogenerated using util/build_tests.php. Please, don't change it manually.
+ */
+class :test-name extends TestCase
+{   
+:tests
+}
diff --git a/util/template/test/unit-test-xpack b/util/template/test/unit-test-xpack
@@ -40,8 +40,7 @@ class :test-name extends TestCase
     {
         $this->yamlTearDown();
         Utility::removeYamlXPackUsers($this->client);
-        Utility::cleanYamlOssTest($this->client);
-        Utility::cleanYamlXpackTest($this->client);
+        Utility::cleanUpCluster($this->client);
     }
 
     private function yamlSetup()

diff --git a/util/template/test/warnings b/util/template/test/warnings
@@ -1,3 +1,4 @@
+        // check for warnings
         $headers = $this->client->transport->getLastConnection()->getLastRequestInfo()['response']['headers'];
         $this->assertTrue(isset($headers['Warning']));
         $expectedHeaders = :expected;