Merge pull request #298 from robcza/alienvault-otx

AlienVault OTX API

intelmq/bots/BOTS

@@ -216,6 +216,15 @@
                 "url": "https://dshield.org/asdetailsascii.html?as=<AS Number>"
             }
         },
+        "AlienVault OTX": {
+            "module": "intelmq.bots.collectors.alienvault_otx.collector",
+            "description": "AlienVault OTX Collector is the bot responsible to get the report through the API. Report could vary according to subscriptions.",
+            "parameters": {
+                "api_key": "<insert your api key>",
+                "feed": "AlienVault OTX",
+                "rate_limit": "3600"
+            }
+        },
         "DShield Block": {
             "description": "DShield Block Collector is the bot responsible to get the report from source of information.",
             "module": "intelmq.bots.collectors.http.collector_http",
@@ -654,6 +663,11 @@
             "module": "intelmq.bots.parsers.alienvault.parser",
             "parameters": {}
         },
+        "AlienVault OTX": {
+            "description": "AlienVault Parser is the bot responsible to parse the report and sanitize the information.",
+            "module": "intelmq.bots.parsers.alienvault.parser_otx",
+            "parameters": {}
+        },
         "Arbor": {
             "description": "Arbor Parser is the bot responsible to parse the report and sanitize the information.",
             "module": "intelmq.bots.parsers.arbor.parser",

intelmq/bots/collectors/alienvault_otx/COPYRIGHT.md

@@ -0,0 +1,2 @@
+Contents of the file "OTXv2.py" are licensed under the Apache license and originally taken from AlienVault Labs: https://github.com/AlienVault-Labs/OTX-Python-SDK
+Complete license TERMS AND CONDITIONS: https://github.com/AlienVault-Labs/OTX-Python-SDK/blob/master/LICENSE

intelmq/bots/collectors/alienvault_otx/OTXv2.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+
+import httplib
+import urlparse
+import urllib
+import urllib2
+import simplejson as json
+import time
+import re
+import logging
+import datetime
+
+logger = logging.getLogger("OTXv2")
+
+
+class InvalidAPIKey(Exception):
+
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return repr(self.value)
+
+
+class BadRequest(Exception):
+
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return repr(self.value)
+
+
+class OTXv2(object):
+
+    def __init__(self, key, server="http://otx.alienvault.com"):
+        self.key = key
+        self.server = server
+
+    def get(self, url):
+        request = urllib2.build_opener()
+        request.addheaders = [('X-OTX-API-KEY', self.key)]
+        response = None
+        try:
+            response = request.open(url)
+        except urllib2.URLError as e:
+            if e.code == 403:
+                raise InvalidAPIKey("Invalid API Key")
+            elif e.code == 400:
+                raise BadRequest("Bad Request")
+        data = response.read()
+        json_data = json.loads(data)
+        return json_data
+
+    def getall(self, limit=20):
+        pulses = []
+        next = "%s/api/v1/pulses/subscribed?limit=%d" % (self.server, limit)
+        while next:
+            json_data = self.get(next)
+            for r in json_data["results"]:
+                pulses.append(r)
+            next = json_data["next"]
+        return pulses
+
+    def getall_iter(self, limit=20):
+        pulses = []
+        next = "%s/api/v1/pulses/subscribed?limit=%d" % (self.server, limit)
+        while next:
+            json_data = self.get(next)
+            for r in json_data["results"]:
+                yield r
+            next = json_data["next"]
+
+    def getsince(self, mytimestamp, limit=20):
+        pulses = []
+        next = "%s/api/v1/pulses/subscribed?limit=%d&modified_since=%s" % (
+            self.server, limit, mytimestamp)
+        while next:
+            json_data = self.get(next)
+            for r in json_data["results"]:
+                pulses.append(r)
+            next = json_data["next"]
+        return pulses
+
+    def getsince_iter(self, mytimestamp, limit=20):
+        pulses = []
+        next = "%s/api/v1/pulses/subscribed?limit=%d&modified_since=%s" % (
+            self.server, limit, mytimestamp)
+        while next:
+            json_data = self.get(next)
+            for r in json_data["results"]:
+                yield r
+            next = json_data["next"]
+
+    def getevents_since(self, mytimestamp, limit=20):
+        events = []
+        next = "%s/api/v1/pulses/events?limit=%d&since=%s" % (
+            self.server, limit, mytimestamp)
+        while next:
+            json_data = self.get(next)
+            for r in json_data["results"]:
+                events.append(r)
+            next = json_data["next"]
+        return events

intelmq/bots/collectors/alienvault_otx/README.md

@@ -0,0 +1,4 @@
+- Collector for: https://otx.alienvault.com
+- Needs this script to be run: https://github.com/AlienVault-Labs/OTX-Python-SDK/blob/master/OTXv2.py
+- The runtime.conf parameter "api_key" has to be set (register on the website to get one)
+

intelmq/bots/collectors/alienvault_otx/collector.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import sys
+from OTXv2 import OTXv2
+import json
+
+from intelmq.lib.bot import Bot
+from intelmq.lib.message import Report
+from intelmq.lib.harmonization import DateTime
+
+
+class AlienVaultOTXCollectorBot(Bot):
+
+    def process(self):
+        self.logger.info("Downloading report through API")
+        otx = OTXv2(self.parameters.api_key)
+        pulses = otx.getall()
+        self.logger.info("Report downloaded.")
+
+        report = Report()
+        report.add("raw", json.dumps(pulses), sanitize=True)
+        report.add("feed.name", self.parameters.feed, sanitize=True)
+        time_observation = DateTime().generate_datetime_now()
+        report.add('time.observation', time_observation, sanitize=True)
+        self.send_message(report)
+
+
+if __name__ == "__main__":
+    bot = AlienVaultOTXCollectorBot(sys.argv[1])
+    bot.start()

intelmq/bots/parsers/alienvault/parser_otx.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+"""
+Events are gathered based on user subscriptions in AlienVault OTX
+The data structure is described in detail here:
+https://github.com/AlienVault-Labs/OTX-Python-SDK/blob/master/
+howto_use_python_otx_api.ipynb
+"""
+from __future__ import unicode_literals
+
+import json
+import sys
+
+from intelmq.lib import utils
+from intelmq.lib.bot import Bot
+from intelmq.lib.message import Event
+
+HASHES = {
+    'FileHash-SHA256': '$5$',
+    'FileHash-SHA1': '$sha1$',
+    'FileHash-MD5': '$1$'
+}
+
+
+class AlienVaultOTXParserBot(Bot):
+
+    def process(self):
+        report = self.receive_message()
+        if report is None or not report.contains("raw"):
+            self.acknowledge_message()
+            return
+
+        raw_report = utils.base64_decode(report.value("raw"))
+
+        for pulse in json.loads(raw_report):
+            additional = json.dumps({"author": pulse['author_name'],
+                                     "pulse": pulse['name']},
+                                    sort_keys=True)
+            for indicator in pulse["indicators"]:
+                event = Event(report)
+                # hashes
+                if indicator["type"] in HASHES.keys():
+                    event.add('malware.hash', HASHES[indicator["type"]] +
+                              indicator["indicator"])
+                # fqdn
+                if indicator["type"] in ['hostname', 'domain']:
+                    event.add('source.fqdn',
+                              indicator["indicator"], sanitize=True)
+                # IP addresses
+                elif indicator["type"] in ['IPv4', 'IPv6']:
+                    event.add('source.ip',
+                              indicator["indicator"], sanitize=True)
+                # emails
+                elif indicator["type"] == 'email':
+                    event.add('source.account',
+                              indicator["indicator"], sanitize=True)
+                # URLs
+                elif indicator["type"] in ['URL', 'URI']:
+                    event.add('source.url',
+                              indicator["indicator"], sanitize=True)
+                # CIDR
+                elif indicator["type"] in ['CIDR']:
+                    event.add('source.network',
+                              indicator["indicator"], sanitize=True)
+                # FilePath, Mutex, CVE - TODO: process these IoCs as well
+                else:
+                    continue
+
+                event.add('comment', pulse['description'])
+                event.add('extra', additional, sanitize=True)
+                event.add('classification.type', 'blacklist', sanitize=True)
+                event.add('time.source', indicator["created"][:-4] + "+00:00",
+                          sanitize=True)
+                event.add("raw", json.dumps(indicator, sort_keys=True),
+                          sanitize=True)
+                self.send_message(event)
+        self.acknowledge_message()
+
+if __name__ == "__main__":
+    bot = AlienVaultOTXParserBot(sys.argv[1])
+    bot.start()

intelmq/conf/harmonization.conf

@@ -148,7 +148,7 @@
             "type": "URL"
         },
         "malware.hash": {
-            "description": "A string depicting a checksum for a file, be it a malware sample for example.",
+            "description": "A string depicting a checksum for a file, be it a malware sample for example. Includes hash type according to https://en.wikipedia.org/wiki/Crypt_%28C%29",
             "regex": "^[a-fA-F0-9]+$",
             "type": "String"
         },