Fixed Nonetype error if Domain object if domain doesn't exist or last…

…_crawled is Null when checking robots.txt compliance.

src/crawler/robots.py

@@ -32,15 +32,16 @@ def does_page_follow_robots_rules(
         request_delay = parser.request_rate(crawler_options.ua)
 
         now = datetime.datetime.now()
-        if crawl_delay and (now - domain.last_crawled).total_seconds() < int(
-            crawl_delay
-        ):
-            raise WaitBeforeRetryException()
-
-        if request_delay and (now - domain.last_crawled).total_seconds() < int(
-            request_delay.seconds
-        ):
-            raise WaitBeforeRetryException()
+        if domain and domain.last_crawled:
+            if crawl_delay and (now - domain.last_crawled).total_seconds() < int(
+                crawl_delay
+            ):
+                raise WaitBeforeRetryException()
+
+            if request_delay and (now - domain.last_crawled).total_seconds() < int(
+                request_delay.seconds
+            ):
+                raise WaitBeforeRetryException()
     except ValueError:
         pass