files handling refactoring #20

README.rst

@@ -42,7 +42,6 @@ The signature of request is the following::
           data=None,
           headers=None,
           cookies=None,
-          files=None,
           auth=None,
           allow_redirects=True,
           max_redirects=10,

aiohttp/client.py

@@ -12,7 +12,6 @@
 import itertools
 import random
 import time
-import uuid
 import urllib.parse
 import weakref
 import warnings
@@ -60,8 +59,6 @@ def request(method, url, *,
     :param headers: (optional) Dictionary of HTTP Headers to send with
       the request
     :param cookies: (optional) Dict object to send with the request
-    :param files: (optional) Dictionary of 'name': file-like-objects
-       for multipart encoding upload
     :param auth: (optional) BasicAuth named tuple represent HTTP Basic Auth
     :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE
        redirect following is allowed.
@@ -193,11 +190,18 @@ def __init__(self, method, url, *,
         self.update_content_encoding()
         self.update_auth(auth)
 
-        if data and not files:
-            if self.method not in self.GET_METHODS:
-                self.update_body_from_data(data)
-        elif files:
-            self.update_body_from_files(files, data)
+        if files:
+            warnings.warn(
+                'files parameter is deprecated. use data instead',
+                DeprecationWarning)
+            if data:
+                raise ValueError(
+                    'data and files parameters are '
+                    'not supported at the same time.')
+            data = files
+
+        if data:
+            self.update_body_from_data(data)
 
         self.update_transfer_encoding()
         self.update_expect_continue(expect100)
@@ -361,78 +365,35 @@ def update_auth(self, auth):
             raise ValueError("HTTP Auth login or password is missing")
 
     def update_body_from_data(self, data):
-        if (hasattr(data, '__iter__') and not isinstance(
-                data, (bytes, bytearray, str, list, dict))):
-            self.body = data
-            if 'CONTENT-LENGTH' not in self.headers and self.chunked is None:
-                self.chunked = True
-        else:
-            if isinstance(data, (bytes, bytearray)):
-                self.body = data
-                if 'CONTENT-TYPE' not in self.headers:
-                    self.headers['CONTENT-TYPE'] = 'application/octet-stream'
-            else:
-                # form data (x-www-form-urlencoded)
-                if isinstance(data, dict):
-                    data = list(data.items())
-
-                if not isinstance(data, str):
-                    data = urllib.parse.urlencode(data, doseq=True)
-
-                self.body = data.encode(self.encoding)
-
-                if 'CONTENT-TYPE' not in self.headers:
-                    self.headers['CONTENT-TYPE'] = (
-                        'application/x-www-form-urlencoded')
+        if isinstance(data, str):
+            data = data.encode(self.encoding)
 
+        if isinstance(data, (bytes, bytearray)):
+            self.body = data
+            if 'CONTENT-TYPE' not in self.headers:
+                self.headers['CONTENT-TYPE'] = 'application/octet-stream'
             if 'CONTENT-LENGTH' not in self.headers and not self.chunked:
                 self.headers['CONTENT-LENGTH'] = str(len(self.body))
 
-    def update_body_from_files(self, files, data):
-        """Generate multipart/form-data body."""
-        fields = []
-
-        if data:
-            if not isinstance(data, (list, dict)):
-                raise NotImplementedError(
-                    'Streamed body is not compatible with files.')
-
-            if isinstance(data, dict):
-                data = data.items()
-
-            for field, val in data:
-                fields.append((field, helpers.str_to_bytes(val)))
-
-        if isinstance(files, dict):
-            files = list(files.items())
-
-        for rec in files:
-            if not isinstance(rec, (tuple, list)):
-                rec = (rec,)
+        elif (hasattr(data, '__iter__') and not
+              isinstance(data, (tuple, list, dict, io.IOBase))):
+            self.body = data
+            if 'CONTENT-LENGTH' not in self.headers and self.chunked is None:
+                self.chunked = True
+        else:
+            if not isinstance(data, helpers.FormData):
+                data = helpers.FormData(data)
 
-            ft = None
-            if len(rec) == 1:
-                k = helpers.guess_filename(rec[0], 'unknown')
-                fields.append((k, k, rec[0]))
+            self.body = data(self.encoding)
 
-            elif len(rec) == 2:
-                k, fp = rec
-                fn = helpers.guess_filename(fp, k)
-                fields.append((k, fn, fp))
+            if 'CONTENT-TYPE' not in self.headers:
+                self.headers['CONTENT-TYPE'] = data.contenttype
 
+            if data.is_form_data():
+                self.chunked = self.chunked or 8196
             else:
-                k, fp, ft = rec
-                fn = helpers.guess_filename(fp, k)
-                fields.append((k, fn, fp, ft))
-
-        self.chunked = self.chunked or 8192
-        boundary = uuid.uuid4().hex
-
-        self.body = helpers.encode_multipart_data(
-            fields, bytes(boundary, 'latin1'))
-
-        self.headers['CONTENT-TYPE'] = (
-            'multipart/form-data; boundary=%s' % boundary)
+                if 'CONTENT-LENGTH' not in self.headers and not self.chunked:
+                    self.headers['CONTENT-LENGTH'] = str(len(self.body))
 
     def update_transfer_encoding(self):
         """Analyze transfer-encoding header."""

aiohttp/helpers.py

@@ -1,7 +1,118 @@
 """Various helper functions"""
 import io
-import mimetypes
 import os
+import uuid
+import urllib.parse
+
+
+class FormData:
+    """Generate multipart/form-data body."""
+
+    def __init__(self, fields):
+        self._fields = []
+        self._has_io = False
+        self._boundary = uuid.uuid4().hex
+
+        if isinstance(fields, dict):
+            fields = list(fields.items())
+        elif not isinstance(fields, (list, tuple)):
+            fields = (fields,)
+        self.add_fields(*fields)
+
+    def is_form_data(self):
+        return self._has_io
+
+    @property
+    def contenttype(self):
+        if self._has_io:
+            return 'multipart/form-data; boundary=%s' % self._boundary
+        else:
+            return 'application/x-www-form-urlencoded'
+
+    def add_field(self, name, value, contenttype=None, filename=None):
+        if filename is None and isinstance(value, io.IOBase):
+            filename = name
+
+        self._fields.append((name, value, contenttype, filename))
+
+    def add_fields(self, *fields):
+        for rec in fields:
+            if isinstance(rec, io.IOBase):
+                k = guess_filename(rec, 'unknown')
+                self.add_field(k, rec)
+                self._has_io = True
+
+            elif len(rec) == 1:
+                k = guess_filename(rec[0], 'unknown')
+                self.add_field(k, rec[0])
+                if isinstance(rec[0], io.IOBase):
+                    self._has_io = True
+
+            elif len(rec) == 2:
+                k, fp = rec
+                fn = guess_filename(fp)
+                self.add_field(k, fp, filename=fn)
+                if isinstance(fp, io.IOBase):
+                    self._has_io = True
+
+            else:
+                k, fp, ft = rec
+                fn = guess_filename(fp, k)
+                self.add_field(k, fp, contenttype=ft, filename=fn)
+                self._has_io = True
+
+    def gen_form_urlencoded(self, encoding):
+        # form data (x-www-form-urlencoded)
+        data = []
+        for name, value, contenttype, filename in self._fields:
+            data.append((name, value))
+
+        data = urllib.parse.urlencode(data, doseq=True)
+        return data.encode(encoding)
+
+    def gen_form_data(self, encoding='utf-8', chunk_size=8196):
+        """Encode a list of fields using the multipart/form-data MIME format"""
+        boundary = self._boundary.encode('latin1')
+
+        for name, value, ctype, fname in self._fields:
+            yield b'--' + boundary + b'\r\n'
+
+            headers = []
+            if fname:
+                headers.append(
+                    ('Content-Disposition: form-data; name="%s"; '
+                     'filename="%s"\r\n' % (name, fname)).encode(encoding))
+            else:
+                headers.append(
+                    ('Content-Disposition: form-data; name="%s"\r\n\r\n' %
+                     name).encode(encoding))
+            if ctype:
+                headers.append(
+                    ('Content-Type: %s\r\n\r\n' % ctype).encode(encoding))
+
+            yield b''.join(headers)
+
+            if isinstance(value, str):
+                yield value.encode(encoding)
+            else:
+                if isinstance(value, (bytes, bytearray)):
+                    value = io.BytesIO(value)
+
+                while True:
+                    chunk = value.read(chunk_size)
+                    if not chunk:
+                        break
+                    yield str_to_bytes(chunk, encoding)
+
+            yield b'\r\n'
+
+        yield b'--' + boundary + b'--\r\n'
+
+    def __call__(self, encoding):
+        if self._has_io:
+            return self.gen_form_data(encoding)
+        else:
+            return self.gen_form_urlencoded(encoding)
 
 
 def parse_mimetype(mimetype):
@@ -47,51 +158,3 @@ def guess_filename(obj, default=None):
     if name and name[0] != '<' and name[-1] != '>':
         return os.path.split(name)[-1]
     return default
-
-
-def encode_multipart_data(fields, boundary, encoding='utf-8', chunk_size=8196):
-    """
-    Encode a list of fields using the multipart/form-data MIME format.
-
-    fields:
-        List of (name, value) or (name, filename, io) or
-        (name, filename, io, MIME type) field tuples.
-    """
-    for rec in fields:
-        yield b'--' + boundary + b'\r\n'
-
-        field, *rec = rec
-
-        if len(rec) == 1:
-            data = rec[0]
-            yield (('Content-Disposition: form-data; name="%s"\r\n\r\n' %
-                    (field,)).encode(encoding))
-            yield data + b'\r\n'
-
-        else:
-            if len(rec) == 3:
-                fn, fp, ct = rec
-            else:
-                fn, fp = rec
-                ct = (mimetypes.guess_type(fn)[0] or
-                      'application/octet-stream')
-
-            yield ('Content-Disposition: form-data; name="%s"; '
-                   'filename="%s"\r\n' % (field, fn)).encode(encoding)
-            yield ('Content-Type: %s\r\n\r\n' % (ct,)).encode(encoding)
-
-            if isinstance(fp, str):
-                fp = fp.encode(encoding)
-
-            if isinstance(fp, bytes):
-                fp = io.BytesIO(fp)
-
-            while True:
-                chunk = fp.read(chunk_size)
-                if not chunk:
-                    break
-                yield str_to_bytes(chunk)
-
-            yield b'\r\n'
-
-    yield b'--' + boundary + b'--\r\n'

aiohttp/test_utils.py

@@ -238,6 +238,8 @@ def _response(self, response, body=None,
         }
         if body:  # pragma: no cover
             resp['content'] = body
+        else:
+            resp['content'] = self._body.decode('utf-8')
 
         ct = self._headers.get('content-type', '').lower()
 

tests/test_client.py

@@ -418,7 +418,7 @@ def test_bytes_data(self):
 
     def test_files_and_bytes_data(self):
         self.assertRaises(
-            NotImplementedError, ClientRequest,
+            ValueError, ClientRequest,
             'POST', 'http://python.org/',
             data=b'binary data', files={'file': b'file data'})