Flag aws_ssm as unstable rather than disabled (ansible-collections#295)

* Flag aws_ssm as unstable rather than disabled

* Try tweaking the policy

* no security_token

* Use module_defaults rather than fact hack

* Fix session token env var

* Move bucket creation to avoid aws-terminator whacking it.

* Change the order of Instance creation, Windows takes longer to start.

* Add reset call to aws_ssm

* Add a wait_for_connection call at the start of the connection tests.
Reduce the delay waiting on the EC2 instances

* Explicitly call community.aws.aws_ssm - looks like 2.9 isn't redirecting properly

* Try running SSM connection tests in parallel.

* Delay a little more

* Turn file names into a variable, and make sure they're unique to the host.

* Try bumping up the instance size

* More delays...

* Remove unused imports

* Move botocore import into HAS_BOTO3 try block

* Make sure that path_unescaped initial string is unicode because out_path can be

* Don't wait for the Windows VM before starting the Linux one

* Add a note why the test is 'unstable'

* Use EBS optimized volumes to improve Windows boot speed.

* Try disabling the Windows connection test - Windows is very slow to start

* instances sometimes isn't there if the host took to long to start, use instance_ids

* changelog

* Remove duplicate wait

* Move to more modern instance type

* Don't try installing boto, it's already available in the images

* remove debugging

* Remove old (unused) inventory templates

* Add comment

changelogs/fragments/295-connection-aws_ssm.yml

@@ -0,0 +1,2 @@
+bugfixes:
+- aws_ssm - fixed ``UnicodeEncodeError`` error when using unicode file names (https://github.com/ansible-collections/community.aws/pull/295).

plugins/connection/aws_ssm.py

@@ -172,17 +172,15 @@
 
 try:
     import boto3
+    from botocore.client import Config
     HAS_BOTO_3 = True
 except ImportError as e:
     HAS_BOTO_3_ERROR = str(e)
     HAS_BOTO_3 = False
-from botocore.client import Config
 
 from functools import wraps
-from ansible import constants as C
 from ansible.errors import AnsibleConnectionFailure, AnsibleError, AnsibleFileNotFound
 from ansible.module_utils.basic import missing_required_lib
-from ansible.module_utils.six import PY3
 from ansible.module_utils.six.moves import xrange
 from ansible.module_utils._text import to_bytes, to_native, to_text
 from ansible.plugins.connection import ConnectionBase
@@ -286,6 +284,11 @@ def _connect(self):
             self.start_session()
         return self
 
+    def reset(self):
+        ''' start a fresh ssm session '''
+        display.vvvv('reset called on ssm connection')
+        return self.start_session()
+
     def start_session(self):
         ''' start ssm session '''
 
@@ -525,7 +528,7 @@ def _get_boto_client(self, service, region_name=None):
     def _file_transport_command(self, in_path, out_path, ssm_action):
         ''' transfer a file from using an intermediate S3 bucket '''
 
-        path_unescaped = "{0}/{1}".format(self.instance_id, out_path)
+        path_unescaped = u"{0}/{1}".format(self.instance_id, out_path)
         s3_path = path_unescaped.replace('\\', '/')
         bucket_url = 's3://%s/%s' % (self.get_option('bucket_name'), s3_path)
 

tests/integration/targets/connection/test_connection.yml

@@ -1,8 +1,16 @@
 - hosts: "{{ target_hosts }}"
   gather_facts: no
-  serial: 1
+  strategy: free
+  vars:
+    local_dir: '{{ local_tmp }}-{{ inventory_hostname }}-汉语'
+    local_file: '{{ local_dir }}/汉语.txt'
+    remote_dir: '{{ remote_tmp }}-汉语'
+    remote_file: '{{ remote_dir }}/汉语.txt'
   tasks:
 
+  ### test wait_for_connection plugin
+  - wait_for_connection:
+
   ### raw with unicode arg and output
 
   - name: raw with unicode arg and output
@@ -17,27 +25,24 @@
   ### copy local file with unicode filename and content
 
   - name: create local file with unicode filename and content
-    local_action: lineinfile dest={{ local_tmp }}-汉语/汉语.txt create=true line=汉语
+    local_action: lineinfile dest={{ local_file }} create=true line=汉语
   - name: remove remote file with unicode filename and content
-    action: "{{ action_prefix }}file path={{ remote_tmp }}-汉语/汉语.txt state=absent"
+    action: "{{ action_prefix }}file path={{ remote_file }} state=absent"
   - name: create remote directory with unicode name
-    action: "{{ action_prefix }}file path={{ remote_tmp }}-汉语 state=directory"
+    action: "{{ action_prefix }}file path={{ remote_dir }} state=directory"
   - name: copy local file with unicode filename and content
-    action: "{{ action_prefix }}copy src={{ local_tmp }}-汉语/汉语.txt dest={{ remote_tmp }}-汉语/汉语.txt"
+    action: "{{ action_prefix }}copy src={{ local_file }} dest={{ remote_file }}"
 
   ### fetch remote file with unicode filename and content
 
   - name: remove local file with unicode filename and content
-    local_action: file path={{ local_tmp }}-汉语/汉语.txt state=absent
+    local_action: file path={{ local_file }} state=absent
   - name: fetch remote file with unicode filename and content
-    fetch: src={{ remote_tmp }}-汉语/汉语.txt dest={{ local_tmp }}-汉语/汉语.txt fail_on_missing=true validate_checksum=true flat=true
+    fetch: src={{ remote_file }} dest={{ local_file }} fail_on_missing=true validate_checksum=true flat=true
 
   ### remove local and remote temp files
 
   - name: remove local temp file
-    local_action: file path={{ local_tmp }}-汉语 state=absent
+    local_action: file path={{ local_file }} state=absent
   - name: remove remote temp file
-    action: "{{ action_prefix }}file path={{ remote_tmp }}-汉语 state=absent"
-
-  ### test wait_for_connection plugin
-  - wait_for_connection:
+    action: "{{ action_prefix }}file path={{ remote_file }} state=absent"

tests/integration/targets/connection_aws_ssm/aliases

@@ -1,6 +1,6 @@
-# reason: unstable
 # reason: slow
-disabled
+# This test suite can take almost 25 minutes (on a good day)
+unstable
 
 cloud/aws
 shippable/aws/group4

tests/integration/targets/connection_aws_ssm/aws_ssm_integration_test_setup/defaults/main.yml

@@ -1,5 +1,5 @@
 ---
-instance_type: t2.micro
-linux_ami_name: amzn-ami-hvm-2018.03.0.20190611-x86_64-ebs
+instance_type: t3.micro
+linux_ami_name: amzn-ami-hvm-2018.03*x86_64-ebs
 # Windows AMIs get replaced every few months, don't be too specific
 windows_ami_name: Windows_Server-2019-English-Full-Base-*

tests/integration/targets/connection_aws_ssm/aws_ssm_integration_test_setup/tasks/main.yml

@@ -1,4 +1,5 @@
 ---
+## Task file for setup/teardown AWS resources for aws_ssm integration testing
 - name: 'aws_ssm connection plugin integration test resource creation'
   collections:
     - amazon.aws
@@ -9,7 +10,6 @@
       security_token: '{{ security_token | default(omit) }}'
       region: '{{ aws_region }}'
   block:
-
     - name: AMI Lookup
       ec2_ami_info:
         owners: 'amazon'
@@ -45,51 +45,26 @@
         - install_plugin_debian is skipped
         - install_plugin_redhat is skipped
 
-    - name: Install Boto3
-      pip:
-        name: boto3
-
-    - name: Install Boto
-      pip:
-        name: boto
-
     - name: Ensure IAM instance role exists
       iam_role:
         name: "ansible-test-{{resource_prefix}}-aws-ssm-role"
         assume_role_policy_document: "{{ lookup('file','ec2-trust-policy.json') }}"
         state: present
         create_instance_profile: yes
         managed_policy:
-        - AmazonEC2RoleforSSM
+        - AmazonSSMManagedInstanceCore
       register: role_output
 
-    - name: Create S3 bucket
-      s3_bucket:
-        name: "{{resource_prefix}}-aws-ssm-s3"
-      register: s3_output
-
     - name: Wait for IAM Role getting created
       pause:
         seconds: 10
 
-    - name: Create Linux EC2 instance
-      ec2_instance:
-        instance_type: "{{instance_type}}"
-        image_id: "{{linux_ami_id}}"
-        wait: "yes"
-        instance_role: "{{role_output.iam_role.role_name}}"
-        name: "{{resource_prefix}}-integration-test-aws-ssm-linux"
-        user_data: |
-                    #!/bin/sh
-                    sudo systemctl start amazon-ssm-agent
-        state: present
-      register: linux_output
-
     - name: Create Windows EC2 instance
       ec2_instance:
         instance_type: "{{instance_type}}"
+        ebs_optimized: True
         image_id: "{{windows_ami_id}}"
-        wait: "yes"
+        wait: no
         instance_role: "{{role_output.iam_role.role_name}}"
         name: "{{resource_prefix}}-integration-test-aws-ssm-windows"
         user_data: |
@@ -99,21 +74,40 @@
                     Restart-Service AmazonSSMAgent
                     </powershell>
         state: present
+        tags:
+          TestPrefix: '{{ resource_prefix }}'
       register: windows_output
 
+    - name: Create Linux EC2 instance
+      ec2_instance:
+        instance_type: "{{instance_type}}"
+        ebs_optimized: True
+        image_id: "{{linux_ami_id}}"
+        wait: "yes"
+        instance_role: "{{role_output.iam_role.role_name}}"
+        name: "{{resource_prefix}}-integration-test-aws-ssm-linux"
+        user_data: |
+                    #!/bin/sh
+                    sudo systemctl start amazon-ssm-agent
+        state: present
+        tags:
+          TestPrefix: '{{ resource_prefix }}'
+      register: linux_output
+
+    # This is just a delay, current host is localhost
     - name: Wait for EC2 to be available
       wait_for_connection:
-        delay: 300
+        delay: 360
 
-    - name: Create Inventory file for Linux host
-      template:
-        dest: "{{playbook_dir}}/inventory-linux.aws_ssm"
-        src: inventory-linux.aws_ssm.j2
+    - name: Create S3 bucket
+      s3_bucket:
+        name: "{{resource_prefix}}-aws-ssm-s3"
+      register: s3_output
 
-    - name: Create Inventory file for Windows host
+    - name: Create Inventory file
       template:
-        dest: "{{playbook_dir}}/inventory-windows.aws_ssm"
-        src: inventory-windows.aws_ssm.j2
+        dest: "{{playbook_dir}}/ssm_inventory"
+        src: inventory-combined.aws_ssm.j2
 
     - name: Create AWS Keys Environement
       template:

tests/integration/targets/connection_aws_ssm/aws_ssm_integration_test_setup/templates/aws-env-vars.j2

@@ -1,4 +1,6 @@
 export AWS_ACCESS_KEY_ID={{aws_access_key}}
 export AWS_SECRET_ACCESS_KEY={{aws_secret_key}}
-export AWS_SECURITY_TOKEN={{security_token}}
+{% if security_token is defined %}
+export AWS_SESSION_TOKEN={{security_token}}
+{% endif %}
 export AWS_REGION={{aws_region}}

tests/integration/targets/connection_aws_ssm/aws_ssm_integration_test_setup/templates/ec2_linux_vars_to_delete.yml.j2

@@ -1,2 +1,2 @@
 ---
-linux_instance_id: {{ linux_output.instances[0].instance_id }}
+linux_instance_id: {{ linux_output.instance_ids[0] }}

tests/integration/targets/connection_aws_ssm/aws_ssm_integration_test_setup/templates/ec2_windows_vars_to_delete.yml.j2

@@ -1,2 +1,2 @@
 ---
-windows_instance_id: {{ windows_output.instances[0].instance_id }}
+windows_instance_id: {{ windows_output.instance_ids[0] }}

tests/integration/targets/connection_aws_ssm/aws_ssm_integration_test_setup/templates/inventory-combined.aws_ssm.j2

@@ -0,0 +1,30 @@
+[aws_ssm_linux]
+linux_{{linux_output.instance_ids[0]}} ansible_aws_ssm_instance_id={{linux_output.instance_ids[0]}} ansible_aws_ssm_region={{aws_region}}
+
+[aws_ssm_linux:vars]
+remote_tmp=/tmp/ansible-remote
+action_prefix=
+
+[aws_ssm_windows]
+windows_{{windows_output.instance_ids[0]}} ansible_aws_ssm_instance_id={{windows_output.instance_ids[0]}} ansible_aws_ssm_region={{aws_region}}
+
+[aws_ssm_windows:vars]
+ansible_shell_type=powershell
+remote_tmp=c:/windows/temp/ansible-remote
+action_prefix=win_
+
+[aws_ssm:children]
+aws_ssm_linux
+## To run the connection test uncomment here
+# aws_ssm_windows
+
+[aws_ssm:vars]
+ansible_connection=community.aws.aws_ssm
+ansible_aws_ssm_bucket_name={{s3_output.name}}
+ansible_aws_ssm_plugin=/usr/local/sessionmanagerplugin/bin/session-manager-plugin
+ansible_python_interpreter=/usr/bin/env python
+local_tmp=/tmp/ansible-local-
+
+# support tests that target testhost
+[testhost:children]
+aws_ssm

tests/integration/targets/connection_aws_ssm/runme.sh

@@ -25,18 +25,7 @@ set -x
 
 cd ../connection
 
-# Execute Integration tests for Linux
-INVENTORY=../connection_aws_ssm/inventory-linux.aws_ssm ./test.sh \
+# Execute Integration tests
+INVENTORY=../connection_aws_ssm/ssm_inventory ./test.sh \
     -e target_hosts=aws_ssm \
-    -e local_tmp=/tmp/ansible-local \
-    -e remote_tmp=/tmp/ansible-remote \
-    -e action_prefix= \
-    "$@"
-
-# Execute Integration tests for Windows
-INVENTORY=../connection_aws_ssm/inventory-windows.aws_ssm ./test.sh \
-    -e target_hosts=aws_ssm \
-    -e local_tmp=/tmp/ansible-local \
-    -e remote_tmp=c:/windows/temp/ansible-remote \
-    -e action_prefix=win_ \
     "$@"

tests/unit/plugins/connection/test_aws_ssm.py

@@ -5,15 +5,9 @@
 from io import StringIO
 import pytest
 import sys
-from ansible import constants as C
-from ansible.compat.selectors import SelectorKey, EVENT_READ
 from ansible_collections.community.aws.tests.unit.compat import unittest
-from ansible_collections.community.aws.tests.unit.compat.mock import patch, MagicMock, PropertyMock
-from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
-from ansible.module_utils.six.moves import shlex_quote
-from ansible.module_utils._text import to_bytes
+from ansible_collections.community.aws.tests.unit.compat.mock import patch, MagicMock
 from ansible.playbook.play_context import PlayContext
-from ansible_collections.community.aws.plugins.connection import aws_ssm
 from ansible.plugins.loader import connection_loader