Merge pull request #28 from lennier1/master

Retry 64 byte files (from Lord_Nigh)

yahoogroupsapi.py

@@ -33,6 +33,9 @@ class Unrecoverable(YGAException):
     """An error that can not be resolved by retrying the request."""
     pass
 
+class BadSize(YGAException):
+    """The filesize is between 60 and 68 bytes, which could be in error"""
+    pass
 
 class AuthenticationError(Unrecoverable):
     pass
@@ -135,6 +138,14 @@ def download_file(self, url, f=None, **args):
                         self.logger.warning("Giving up, too many failed attempts at downloading %s", url)
                 elif r.status_code != 200:
                     self.logger.error("Unknown %d error for %s, giving up on this download", r.status_code, url)
+                elif len(r.content) in range(60, 69):
+                    self.logger.info("Got potentially invalid size of %d for %s, will sleep and retry", len(r.content), url)
+                    if attempt < self.retries-1:
+                        delay = self.backoff_time(attempt)
+                        self.logger.info("Attempt %d, delaying for %.2f seconds", attempt+1, delay)
+                        time.sleep(delay)
+                        continue
+                    self.logger.warning("Giving up, too many potentially failed attempts at downloading %s", url)
                 r.raise_for_status()
                 break
 
@@ -166,12 +177,14 @@ def get_json(self, target, *parts, **opts):
                         raise Unauthorized()
                     elif code == 404:
                         raise NotFound()
+                    elif len(r.content) in range(60, 69):
+                        raise BadSize()
                     elif code != 200:
                         # TODO: Test ygError response?
                         raise Recoverable()
 
                     return r.json()['ygData']
-                except (ConnectionError, Timeout, Recoverable) as e:
+                except (ConnectionError, Timeout, Recoverable, BadSize) as e:
                     self.logger.info("API query failed for '%s': %s", uri, e)
                     self.logger.debug("Exception detail:", exc_info=e)