AlienVault OTX API

intelmq/bots/BOTS

@@ -216,6 +216,15 @@
                 "url": "http://dshield.org/asdetailsascii.html?as=<AS Number>"
             }
         },
+        "AlienVault OTX": {
+            "module": "intelmq.bots.collectors.alienvault-otx.collector",
+            "description": "AlienVault OTX Collector is the bot responsible to get the report through the API. Report could vary according to subscriptions.",
+            "parameters": {
+                "api_key": "<insert your api key>",
+                "feed": "AlienVault OTX",
+                "rate_limit": "3600"
+            }
+        },
         "DShield Block": {
             "description": "DShield Block Collector is the bot responsible to get the report from source of information.",
             "module": "intelmq.bots.collectors.http.collector_http",
@@ -634,6 +643,11 @@
             "module": "intelmq.bots.parsers.alienvault.parser",
             "parameters": {}
         },
+        "AlienVault OTX": {
+            "description": "AlienVault Parser is the bot responsible to parse the report and sanitize the information.",
+            "module": "intelmq.bots.parsers.alienvault.parser-otx",
+            "parameters": {}
+        },
         "Arbor": {
             "description": "Arbor Parser is the bot responsible to parse the report and sanitize the information.",
             "module": "intelmq.bots.parsers.arbor.parser",

intelmq/bots/collectors/alienvault-otx/OTXv2.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+
+import httplib
+import urlparse
+import urllib
+import urllib2
+import simplejson as json
+import time
+import re
+import logging
+import datetime
+
+logger = logging.getLogger("OTXv2")
+
+
+class InvalidAPIKey(Exception):
+
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return repr(self.value)
+
+
+class BadRequest(Exception):
+
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return repr(self.value)
+
+
+class OTXv2(object):
+
+    def __init__(self, key, server="http://otx.alienvault.com"):
+        self.key = key
+        self.server = server
+
+    def get(self, url):
+        request = urllib2.build_opener()
+        request.addheaders = [('X-OTX-API-KEY', self.key)]
+        response = None
+        try:
+            response = request.open(url)
+        except urllib2.URLError as e:
+            if e.code == 403:
+                raise InvalidAPIKey("Invalid API Key")
+            elif e.code == 400:
+                raise BadRequest("Bad Request")
+        data = response.read()
+        json_data = json.loads(data)
+        return json_data
+
+    def getall(self, limit=20):
+        pulses = []
+        next = "%s/api/v1/pulses/subscribed?limit=%d" % (self.server, limit)
+        while next:
+            json_data = self.get(next)
+            for r in json_data["results"]:
+                pulses.append(r)
+            next = json_data["next"]
+        return pulses
+
+    def getall_iter(self, limit=20):
+        pulses = []
+        next = "%s/api/v1/pulses/subscribed?limit=%d" % (self.server, limit)
+        while next:
+            json_data = self.get(next)
+            for r in json_data["results"]:
+                yield r
+            next = json_data["next"]
+
+    def getsince(self, mytimestamp, limit=20):
+        pulses = []
+        next = "%s/api/v1/pulses/subscribed?limit=%d&modified_since=%s" % (
+            self.server, limit, mytimestamp)
+        while next:
+            json_data = self.get(next)
+            for r in json_data["results"]:
+                pulses.append(r)
+            next = json_data["next"]
+        return pulses
+
+    def getsince_iter(self, mytimestamp, limit=20):
+        pulses = []
+        next = "%s/api/v1/pulses/subscribed?limit=%d&modified_since=%s" % (
+            self.server, limit, mytimestamp)
+        while next:
+            json_data = self.get(next)
+            for r in json_data["results"]:
+                yield r
+            next = json_data["next"]
+
+    def getevents_since(self, mytimestamp, limit=20):
+        events = []
+        next = "%s/api/v1/pulses/events?limit=%d&since=%s" % (
+            self.server, limit, mytimestamp)
+        while next:
+            json_data = self.get(next)
+            for r in json_data["results"]:
+                events.append(r)
+            next = json_data["next"]
+        return events

intelmq/bots/collectors/alienvault-otx/README.md

@@ -0,0 +1,4 @@
+- Collector for: https://otx.alienvault.com
+- Needs this script to be run: https://github.com/AlienVault-Labs/OTX-Python-SDK/blob/master/OTXv2.py
+- The runtime.conf parameter "api_key" has to be set (register on the website to get one)
+

intelmq/bots/collectors/alienvault-otx/collector.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import sys
+from OTXv2 import OTXv2
+import json
+
+from intelmq.bots.collectors.http.lib import fetch_url
+from intelmq.lib.bot import Bot
+from intelmq.lib.message import Report
+
+
+class AlienVaultOTXCollectorBot(Bot):
+
+    def process(self):
+        self.logger.info("Downloading report through API")
+        otx = OTXv2(self.parameters.api_key)
+        pulses = otx.getall()
+        self.logger.info("Report downloaded.")
+
+        report = Report()
+        report.add("raw", json.dumps(pulses), sanitize=True)
+        report.add("feed.name", self.parameters.feed, sanitize=True)
+        self.send_message(report)
+
+
+if __name__ == "__main__":
+    bot = AlienVaultOTXCollectorBot(sys.argv[1])
+    bot.start()

intelmq/bots/parsers/alienvault/parser-otx.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import sys
+import json
+
+from intelmq.lib import utils
+from intelmq.lib.bot import Bot
+from intelmq.lib.harmonization import DateTime
+from intelmq.lib.message import Event
+
+HASHES = {
+    'FileHash-SHA256': 'SHA-256',
+    'FileHash-SHA1': 'SHA-1',
+    'FileHash-MD5': 'MD5'
+}
+
+
+class AlienVaultOTXParserBot(Bot):
+
+    def process(self):
+        report = self.receive_message()
+        if (report is None or not report.contains("raw"):
+            self.acknowledge_message()
+            return
+
+        time_observation = DateTime().generate_datetime_now()
+        raw_report = utils.base64_decode(report.value("raw"))
+
+        for pulse in json.loads(raw_report):
+            additional_information = json.dumps(
+                {'author':pulse['author_name'],
+                'pulse':pulse['name']})
+            for indicator in pulse["indicators"]:
+                event = Event()
+                #hashes
+                if indicator["type"] in ['FileHash-SHA256', 'FileHash-SHA1', 'FileHash-MD5']:
+                    event.add('malware.hash', indicator["indicator"], sanitize = True)
+                #    event.add('malware.hash_type', HASHES[indicator["type"]], sanitize = True)
+                # fqdn
+                if indicator["type"] in ['hostname', 'domain']:
+                    event.add(
+                        'source.fqdn',
+                        indicator["indicator"],
+                        sanitize=True)
+                # IP addresses
+                elif indicator["type"] in ['IPv4', 'IPv6']:
+                    event.add(
+                        'source.ip',
+                        indicator["indicator"],
+                        sanitize=True)
+                # emails
+                elif indicator["type"] == 'email':
+                    event.add(
+                        'source.email_address',
+                        indicator["indicator"],
+                        sanitize=True)
+                # URLs
+                elif indicator["type"] in ['URL', 'URI']:
+                    event.add(
+                        'source.url',
+                        indicator["indicator"],
+                        sanitize=True)
+                #CIDR
+                elif indicator["type"] in ['CIDR']:
+                    event.add(
+                        'source.network',
+                        indicator["indicator"],
+                        sanitize=True)
+                elif indicator["type"]
+                #FilePath, Mutex, CVE, hashes - TODO: process these IoCs as well
+                else:
+                    continue
+
+                event.add('comment', pulse['description'])
+                event.add('additional_information', additional_information)
+                event.add('classification.type', 'blacklist', sanitize=True)
+                event.add('time.observation', time_observation, sanitize=True)
+                event.add('time.source', indicator["created"], sanitize=True)
+                event.add('feed.name', report.value("feed.name"))
+                event.add("raw", json.dumps(indicator), sanitize=True)
+                self.send_message(event)
+        self.acknowledge_message()
+
+if __name__ == "__main__":
+    bot = AlienVaultOTXParserBot(sys.argv[1])
+    bot.start()