Selenium Test Release 2 Changes (#499)

* seltest rel2 final2

* added test case for policy changes dashboard

* typo changes on policy changes dashboard

* updates on failed api & sel tests

* Update cluster.yml

* test changes to pass cicd

* Update cluster.yml

testing/tests/api_tests/cluster/test_server.py

@@ -43,11 +43,11 @@ def test_host_search(es_host, es_port, username, password):
     for key in data:
         rootKey = key
 
+
     assert (data[rootKey]["total"]["value"] > 0)
     assert ".ds-metrics-system.cpu-default" in data[rootKey]["hits"][0]["_index"]    
     assert (data[rootKey]["hits"][0]["_source"]["agent"]["name"] == "ubuntu-vm")    
-    assert (data[rootKey]["hits"][0]["_source"]["agent"]["version"] == "8.15.3") 
-    assert (data[rootKey]["hits"][0]["_source"]["cloud"]["instance"]["name"] == "ubuntu")  
+    assert (data[rootKey]["hits"][0]["_source"]["agent"]["version"] == "8.15.3")   
     assert (data[rootKey]["hits"][0]["_source"]["data_stream"]["dataset"] == "system.cpu") 
     assert (data[rootKey]["hits"][0]["_source"]["ecs"]["version"] == "8.0.0") 
     assert (data[rootKey]["hits"][0]["_source"]["elastic_agent"]["version"] == "8.15.3") 

testing/tests/api_tests/linux_only/test_server.py

@@ -49,7 +49,8 @@ def test_elastic_root(es_host, es_port, username, password):
     ), f"Expected 'docker', got {body['version']['build_type']}"
     assert (
             body["version"]["lucene_version"] == "9.11.1"
-    ), f"Expected '9.9.2', got {body['version']['lucene_version']}"
+    ), f"Expected '9.11.1', got {body['version']['lucene_version']}"
+
     assert (
             body["version"]["minimum_wire_compatibility_version"] == "7.17.0"
     ), f"Expected '7.17.0', got {body['version']['minimum_wire_compatibility_version']}"

testing/tests/selenium_tests/cluster/test_computer_software_overview_dashboard.py

@@ -7,36 +7,38 @@
 from .lib import dashboard_test_function
 
 class TestComputerSoftwareOverviewDashboard:
-    dashboard_id = "33f0d3b0-8b8a-11ea-b1c6-a5bf39283f12"
-
+    #dashboard_id = "33f0d3b0-8b8a-11ea-b1c6-a5bf39283f12"
+    #dashboard_id = "new dashboard"
+    dashboard_id = "ce98c19b-587f-4d76-9c49-2e9acee257d5"
+
     @pytest.fixture(scope="class")
     def setup_login(self, driver, login):
         login()
         yield driver
 
-    @pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
     def test_application_crashing_and_hanging(self, setup_login, kibana_url, timeout):
         driver = setup_login
         dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Application Crashing and Hanging", ".echChart",".xyChart__empty")
 
-    @pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
     def test_application_crashing_and_hanging_count(self, setup_login, kibana_url, timeout):
         driver = setup_login
         dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Application Crashing and Hanging Count", ".tbvChart",".visError")
 
-    @pytest.mark.skip(reason="Skipping this test")
+    #@pytest.mark.skip(reason="Skipping this test")
     def test_create_remote_threat_events(self, setup_login, kibana_url, timeout):
         driver = setup_login
-        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "CreateRemoteThread events", ".tbvChart",".visError")
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "CreateRemoteThread events", ".euiFlexGroup",".visError")
 
-    @pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
     def test_filter_hosts(self, setup_login, kibana_url, timeout):
         driver = setup_login
         dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Filter Hosts", ".tbvChart",".visError")
 
 
-    @pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
     def test_processes(self, setup_login, kibana_url, timeout):
         driver = setup_login
-        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Processes", ".tbvChart",".visError")
-
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Processes", ".euiDataGrid__focusWrap",".euiText")
+        

testing/tests/selenium_tests/cluster/test_credentials_access_logs.py

@@ -0,0 +1,59 @@
+import pytest
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.by import By
+from selenium.common.exceptions import NoSuchElementException
+
+from .lib import dashboard_test_function
+class TestCredentialsAccessLogsDashboard:
+    dashboard_id = "e4d7b207-99aa-4410-8a2e-03487222bda1"
+
+    @pytest.fixture(scope="class")
+    def setup_login(self, driver, login):
+        login()
+        yield driver
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_audit_logons(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Audit logons", ".echChart",".euiText")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_kerberos_ticket_failed_attempts(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Kerberos ticket - Failed attempts", ".expExpressionRenderer",".xyChart__empty")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_special_logon_attempts(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Special logon-attempts", ".echChart",".dummyval")    
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_account_lockout_attempts(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Account lockout -attempts", ".euiDataGrid",".euiSpacer")    
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_other_logon_logoff_disconnection_attempts(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Other logon /logoff-Disconnection attempts", ".expExpressionRenderer",".euiSpacer")  
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_kerberos_auth_request(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Kerberos auth request", ".echChart",".euiText")        
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_logon_attempts_by_host(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Logon attempts by hosts", ".echChart",".euiText")  
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_credential_validation_attempts(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Credential validation- attempts", ".echChart",".euiText")   
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_logon_using_explicit_credential_attempts(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Logon-using explicit credential attempts", ".echChart",".euiText")                     

testing/tests/selenium_tests/cluster/test_health_check_dashboard.py

@@ -7,14 +7,15 @@
 from .lib import dashboard_test_function
 
 class TestHealthCheckDashboard:
-    dashboard_id = "51fe1470-fa59-11e9-bf25-8f92ffa3e3ec"
-
+    #dashboard_id = "51fe1470-fa59-11e9-bf25-8f92ffa3e3ec"
+    dashboard_id = "fff78bfe-2758-4fa1-939f-362380fc607d"
+
     @pytest.fixture(scope="class")
     def setup_login(self, driver, login):
         login()
         yield driver
 
-    @pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
     def test_number_of_admins(self, setup_login, kibana_url, timeout):
         driver = setup_login
         dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Number of Admins", ".expExpressionRenderer",".dummyval")
@@ -23,24 +24,22 @@ def test_number_of_admins(self, setup_login, kibana_url, timeout):
         # If there is no visualization rendered or "No Results found" message is displayed for this panel on dashboard, this test should fail which is correct behavior
 
 
-    @pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
     def test_total_hosts(self, setup_login, kibana_url, timeout):
         driver = setup_login
         dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Total Hosts", ".visualization",".dummyval")
 
-    @pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
     def test_events_by_machine(self, setup_login, kibana_url, timeout):
         driver = setup_login
         dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Events by machine", ".echChart",".euiText")
 
-    @pytest.mark.skip(reason="Skipping this test")
+    #@pytest.mark.skip(reason="Skipping this test")
     def test_unexpected_shutdowns(self, setup_login, kibana_url, timeout):
         driver = setup_login
-        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Unexpected shutdowns", ".echChart",".visError")
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Unexpected shutdowns", ".tbvChart",".visError")
 
-    @pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
     def test_users_seen(self, setup_login, kibana_url, timeout):
         driver = setup_login
-        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Users seen", ".visualization",".dummyval")
-
-
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Users seen", ".visualization",".dummyval")

testing/tests/selenium_tests/cluster/test_identity_access_management.py

@@ -0,0 +1,49 @@
+import pytest
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.by import By
+from selenium.common.exceptions import NoSuchElementException
+from .lib import dashboard_test_function
+
+class TestCredentialsAccessLogsDashboard:
+    dashboard_id = "32ed7a33-b22e-4c4b-b4bd-a55c2cf4c0d0"
+
+    @pytest.fixture(scope="class")
+    def setup_login(self, driver, login):
+        login()
+        yield driver
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_registry_object_access(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Registry Object Access", ".echChart",".dummyval")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_updated_scheduler_jobs(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Updated Scheduler Jobs", ".visualization",".dummyval")
+
+    @pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_new_scheduler_jobs(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "New Scheduler Jobs", ".visualization",".dummyval")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_password_resets_changes(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Password Resets and Changes", ".echChart",".euiDataGrid__noResults")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_user_lockouts(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "User Lockouts", ".echChart",".euiDataGrid__noResults")  
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_password_hash_access(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Password Hash Access", ".echChart",".euiDataGrid__noResults")  
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_changes_to_default_domain_policy(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Changes to Default Domain Policy", ".euiFlexGroup",".euiDataGrid__noResults")                                                  

testing/tests/selenium_tests/cluster/test_policy_changes_system_activity.py

@@ -0,0 +1,81 @@
+import pytest
+import os
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.by import By
+from selenium.common.exceptions import NoSuchElementException
+from .lib import dashboard_test_function
+
+class TestPolicyChangesSystemActivityDashboard:
+
+    dashboard_id = "614a8392-17b5-49c4-9397-bc3cac526c61"
+
+    @pytest.fixture(scope="class")
+    def setup_login(self, driver, login):
+        login()
+        yield driver
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_rpc_connection_attempts(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "RPC Connection Attempts", ".expExpressionRenderer",".dummyval")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_exception_firewall_rules(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Added or Updated Exception Firewall Rules", ".expExpressionRenderer",".dummyval")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_rpc_connections(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "RPC Connections", ".echChart",".dummyval")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_firewall_setting_changes(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Firewall Setting Changes", ".euiDataGrid",".euiDataGrid__noResults")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_firewall_policy_changes(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Firewall Policy Changes", ".euiDataGrid",".euiDataGrid__noResults")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_firewall_turned_on(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Firewall Turned On", ".euiDataGrid",".euiDataGrid__noResults")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_firewall_turned_off(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Firewall Turned Off", ".euiDataGrid",".euiDataGrid__noResults")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_audit_policy_changes(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Audit Policy Changes", ".euiDataGrid",".euiDataGrid__noResults")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_kerberos_policy_changes(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "Kerberos Policy Changes", ".euiDataGrid",".euiDataGrid__noResults")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_pc_start_up(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "PC Start Up", ".echChart",".dummyval")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_pc_shut_down(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "PC Shut Down", ".echChart",".dummyval")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_pc_startups(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "PC Startups", ".expExpressionRenderer",".euiText")
+
+    #@pytest.mark.skip(reason="This test is for reference to use in 2.0")
+    def test_pc_shutdowns(self, setup_login, kibana_url, timeout):
+        driver = setup_login
+        dashboard_test_function(driver, kibana_url, timeout, self.dashboard_id, "PC Shutdowns", ".expExpressionRenderer",".euiText")