datajson: unix commands to add/remove

Ticket: OISF#7372

python/suricata/sc/specs.py

@@ -194,6 +194,38 @@
             "required": 1,
         },
     ],
+    "datajson-add": [
+        {
+            "name": "setname",
+            "required": 1,
+        },
+        {
+            "name": "settype",
+            "required": 1,
+        },
+        {
+            "name": "datavalue",
+            "required": 1,
+        },
+        {
+            "name": "datajson",
+            "required": 1,
+        },
+    ],
+    "datajson-remove": [
+        {
+            "name": "setname",
+            "required": 1,
+        },
+        {
+            "name": "settype",
+            "required": 1,
+        },
+        {
+            "name": "datavalue",
+            "required": 1,
+        },
+    ],
     "get-flow-stats-by-id": [
         {
             "name": "flow_id",

python/suricata/sc/suricatasc.py

@@ -113,6 +113,8 @@ def __init__(self, sck_path, verbose=False):
                 "memcap-show",
                 "dataset-add",
                 "dataset-remove",
+                "datajson-add",
+                "datajson-remove",
                 "get-flow-stats-by-id",
                 "dataset-clear",
                 "dataset-lookup",

src/datasets.c

@@ -2303,3 +2303,188 @@ int DatasetRemove(Dataset *set, const uint8_t *data, const uint32_t data_len)
     }
     return -1;
 }
+
+typedef int (*DatajsonOpFunc)(
+        Dataset *set, const uint8_t *data, const uint32_t data_len, const DataJsonType *json);
+
+static int DatajsonOpSerialized(Dataset *set, const char *string, const char *json,
+        DatajsonOpFunc DatasetOpString, DatajsonOpFunc DatasetOpMd5, DatajsonOpFunc DatasetOpSha256,
+        DatajsonOpFunc DatasetOpIPv4, DatajsonOpFunc DatasetOpIPv6)
+{
+    int ret;
+
+    if (set == NULL)
+        return -1;
+    if (strlen(string) == 0)
+        return -1;
+
+    DataJsonType jvalue = { .value = NULL, .len = 0 };
+    if (json) {
+        if (ParseJsonLine(json, strlen(json), &jvalue) < 0) {
+            SCLogNotice("bad json value for dataset %s/%s", set->name, set->load);
+            return -1;
+        }
+    }
+
+    switch (set->type) {
+        case DATASET_TYPE_STRING: {
+            uint32_t decoded_size = Base64DecodeBufferSize(strlen(string));
+            uint8_t decoded[decoded_size];
+            uint32_t num_decoded = Base64Decode(
+                    (const uint8_t *)string, strlen(string), Base64ModeStrict, decoded);
+            if (num_decoded == 0)
+                goto operror;
+            ret = DatasetOpString(set, decoded, num_decoded, &jvalue);
+            if (ret <= 0) {
+                SCFree(jvalue.value);
+            }
+            return ret;
+        }
+        case DATASET_TYPE_MD5: {
+            if (strlen(string) != 32)
+                goto operror;
+            uint8_t hash[16];
+            if (HexToRaw((const uint8_t *)string, 32, hash, sizeof(hash)) < 0)
+                goto operror;
+            ret = DatasetOpMd5(set, hash, 16, &jvalue);
+            if (ret <= 0) {
+                SCFree(jvalue.value);
+            }
+            return ret;
+        }
+        case DATASET_TYPE_SHA256: {
+            if (strlen(string) != 64)
+                goto operror;
+            uint8_t hash[32];
+            if (HexToRaw((const uint8_t *)string, 64, hash, sizeof(hash)) < 0)
+                goto operror;
+            ret = DatasetOpSha256(set, hash, 32, &jvalue);
+            if (ret <= 0) {
+                SCFree(jvalue.value);
+            }
+            return ret;
+        }
+        case DATASET_TYPE_IPV4: {
+            struct in_addr in;
+            if (inet_pton(AF_INET, string, &in) != 1)
+                goto operror;
+            ret = DatasetOpIPv4(set, (uint8_t *)&in.s_addr, 4, &jvalue);
+            if (ret <= 0) {
+                SCFree(jvalue.value);
+            }
+            return ret;
+        }
+        case DATASET_TYPE_IPV6: {
+            struct in6_addr in6;
+            if (ParseIpv6String(set, string, &in6) != 0) {
+                SCLogError("Dataset failed to import %s as IPv6", string);
+                goto operror;
+            }
+            ret = DatasetOpIPv6(set, (uint8_t *)&in6.s6_addr, 16, &jvalue);
+            if (ret <= 0) {
+                SCFree(jvalue.value);
+            }
+            return ret;
+        }
+    }
+    return -1;
+operror:
+    SCFree(jvalue.value);
+    return -2;
+}
+
+/** \brief add serialized data to json set
+ *  \retval int 1 added
+ *  \retval int 0 already in hash
+ *  \retval int -1 API error (not added)
+ *  \retval int -2 DATA error
+ */
+int DatajsonAddSerialized(Dataset *set, const char *value, const char *json)
+{
+    return DatajsonOpSerialized(set, value, json, DatasetAddStringwJson, DatasetAddMd5wJson,
+            DatasetAddSha256wJson, DatasetAddIPv4wJson, DatasetAddIPv6wJson);
+}
+
+/**
+ *  \retval 1 data was removed from the hash
+ *  \retval 0 data not removed (busy)
+ *  \retval -1 data not found
+ */
+static int DatajsonRemoveString(
+        Dataset *set, const uint8_t *data, const uint32_t data_len, const DataJsonType *json)
+{
+    if (set == NULL)
+        return -1;
+
+    StringTypeJson lookup = {
+        .ptr = (uint8_t *)data, .len = data_len, .json.value = NULL, .json.len = 0
+    };
+    return THashRemoveFromHash(set->hash, &lookup);
+}
+
+static int DatajsonRemoveIPv4(
+        Dataset *set, const uint8_t *data, const uint32_t data_len, const DataJsonType *json)
+{
+    if (set == NULL)
+        return -1;
+
+    if (data_len != 4)
+        return -2;
+
+    IPv4TypeJson lookup = { .json.value = NULL, .json.len = 0 };
+    memcpy(lookup.ipv4, data, 4);
+    return THashRemoveFromHash(set->hash, &lookup);
+}
+
+static int DatajsonRemoveIPv6(
+        Dataset *set, const uint8_t *data, const uint32_t data_len, const DataJsonType *json)
+{
+    if (set == NULL)
+        return -1;
+
+    if (data_len != 16)
+        return -2;
+
+    IPv6TypeJson lookup = { .json.value = NULL, .json.len = 0 };
+    memcpy(lookup.ipv6, data, 16);
+    return THashRemoveFromHash(set->hash, &lookup);
+}
+
+static int DatajsonRemoveMd5(
+        Dataset *set, const uint8_t *data, const uint32_t data_len, const DataJsonType *json)
+{
+    if (set == NULL)
+        return -1;
+
+    if (data_len != 16)
+        return -2;
+
+    Md5TypeJson lookup = { .json.value = NULL, .json.len = 0 };
+    memcpy(lookup.md5, data, 16);
+    return THashRemoveFromHash(set->hash, &lookup);
+}
+
+static int DatajsonRemoveSha256(
+        Dataset *set, const uint8_t *data, const uint32_t data_len, const DataJsonType *json)
+{
+    if (set == NULL)
+        return -1;
+
+    if (data_len != 32)
+        return -2;
+
+    Sha256TypeJson lookup = { .json.value = NULL, .json.len = 0 };
+    memcpy(lookup.sha256, data, 32);
+    return THashRemoveFromHash(set->hash, &lookup);
+}
+
+/** \brief remove serialized data from set
+ *  \retval int 1 removed
+ *  \retval int 0 found but busy (not removed)
+ *  \retval int -1 API error (not removed)
+ *  \retval int -2 DATA error */
+int DatajsonRemoveSerialized(Dataset *set, const char *string)
+{
+    return DatajsonOpSerialized(set, string, NULL, DatajsonRemoveString, DatajsonRemoveMd5,
+            DatajsonRemoveSha256, DatajsonRemoveIPv4, DatajsonRemoveIPv6);
+}

src/datasets.h

@@ -69,4 +69,7 @@ int DatasetAddSerialized(Dataset *set, const char *string);
 int DatasetRemoveSerialized(Dataset *set, const char *string);
 int DatasetLookupSerialized(Dataset *set, const char *string);
 
+int DatajsonAddSerialized(Dataset *set, const char *string, const char *json);
+int DatajsonRemoveSerialized(Dataset *set, const char *string);
+
 #endif /* SURICATA_DATASETS_H */

src/runmode-unix-socket.c

@@ -806,6 +806,127 @@ TmEcode UnixSocketDatasetLookup(json_t *cmd, json_t *answer, void *data)
     }
 }
 
+/**
+ * \brief Command to add data to a datajson
+ *
+ * \param cmd the content of command Arguments as a json_t object
+ * \param answer the json_t object that has to be used to answer
+ * \param data pointer to data defining the context here a PcapCommand::
+ */
+TmEcode UnixSocketDatajsonAdd(json_t *cmd, json_t *answer, void *data)
+{
+    /* 1 get dataset name */
+    json_t *narg = json_object_get(cmd, "setname");
+    if (!json_is_string(narg)) {
+        json_object_set_new(answer, "message", json_string("setname is not a string"));
+        return TM_ECODE_FAILED;
+    }
+    const char *set_name = json_string_value(narg);
+
+    /* 2 get the data type */
+    json_t *targ = json_object_get(cmd, "settype");
+    if (!json_is_string(targ)) {
+        json_object_set_new(answer, "message", json_string("settype is not a string"));
+        return TM_ECODE_FAILED;
+    }
+    const char *type = json_string_value(targ);
+
+    /* 3 get value */
+    json_t *varg = json_object_get(cmd, "datavalue");
+    if (!json_is_string(varg)) {
+        json_object_set_new(answer, "message", json_string("datavalue is not string"));
+        return TM_ECODE_FAILED;
+    }
+    const char *value = json_string_value(varg);
+
+    /* 4 get json */
+    json_t *jarg = json_object_get(cmd, "datajson");
+    if (!json_is_string(varg)) {
+        json_object_set_new(answer, "message", json_string("datajson is not string"));
+        return TM_ECODE_FAILED;
+    }
+    const char *json = json_string_value(jarg);
+
+    SCLogDebug("datajson-add: %s type %s value %s json %s", set_name, type, value, json);
+
+    enum DatasetTypes t = DatasetGetTypeFromString(type);
+    if (t == DATASET_TYPE_NOTSET) {
+        json_object_set_new(answer, "message", json_string("unknown settype"));
+        return TM_ECODE_FAILED;
+    }
+
+    Dataset *set = DatasetFind(set_name, t);
+    if (set == NULL) {
+        json_object_set_new(answer, "message", json_string("set not found or wrong type"));
+        return TM_ECODE_FAILED;
+    }
+
+    int r = DatajsonAddSerialized(set, value, json);
+    if (r == 1) {
+        json_object_set_new(answer, "message", json_string("data added"));
+        return TM_ECODE_OK;
+    } else if (r == 0) {
+        json_object_set_new(answer, "message", json_string("data already in set"));
+        return TM_ECODE_OK;
+    } else {
+        json_object_set_new(answer, "message", json_string("failed to add data"));
+        return TM_ECODE_FAILED;
+    }
+}
+
+TmEcode UnixSocketDatajsonRemove(json_t *cmd, json_t *answer, void *data)
+{
+    /* 1 get dataset name */
+    json_t *narg = json_object_get(cmd, "setname");
+    if (!json_is_string(narg)) {
+        json_object_set_new(answer, "message", json_string("setname is not a string"));
+        return TM_ECODE_FAILED;
+    }
+    const char *set_name = json_string_value(narg);
+
+    /* 2 get the data type */
+    json_t *targ = json_object_get(cmd, "settype");
+    if (!json_is_string(targ)) {
+        json_object_set_new(answer, "message", json_string("settype is not a string"));
+        return TM_ECODE_FAILED;
+    }
+    const char *type = json_string_value(targ);
+
+    /* 3 get value */
+    json_t *varg = json_object_get(cmd, "datavalue");
+    if (!json_is_string(varg)) {
+        json_object_set_new(answer, "message", json_string("datavalue is not string"));
+        return TM_ECODE_FAILED;
+    }
+    const char *value = json_string_value(varg);
+
+    SCLogDebug("datajson-remove: %s type %s value %s", set_name, type, value);
+
+    enum DatasetTypes t = DatasetGetTypeFromString(type);
+    if (t == DATASET_TYPE_NOTSET) {
+        json_object_set_new(answer, "message", json_string("unknown settype"));
+        return TM_ECODE_FAILED;
+    }
+
+    Dataset *set = DatasetFind(set_name, t);
+    if (set == NULL) {
+        json_object_set_new(answer, "message", json_string("set not found or wrong type"));
+        return TM_ECODE_FAILED;
+    }
+
+    int r = DatajsonRemoveSerialized(set, value);
+    if (r == 1) {
+        json_object_set_new(answer, "message", json_string("data removed"));
+        return TM_ECODE_OK;
+    } else if (r == 0) {
+        json_object_set_new(answer, "message", json_string("data is busy, try again"));
+        return TM_ECODE_OK;
+    } else {
+        json_object_set_new(answer, "message", json_string("failed to remove data"));
+        return TM_ECODE_FAILED;
+    }
+}
+
 static bool JsonU32Value(json_t *jarg, uint32_t *ret)
 {
     int64_t r = json_integer_value(jarg);

src/runmode-unix-socket.h

@@ -38,6 +38,9 @@ TmEcode UnixSocketDatasetRemove(json_t *cmd, json_t* answer, void *data);
 TmEcode UnixSocketDatasetDump(json_t *cmd, json_t *answer, void *data);
 TmEcode UnixSocketDatasetClear(json_t *cmd, json_t *answer, void *data);
 TmEcode UnixSocketDatasetLookup(json_t *cmd, json_t *answer, void *data);
+TmEcode UnixSocketDatajsonAdd(json_t *cmd, json_t *answer, void *data);
+TmEcode UnixSocketDatajsonRemove(json_t *cmd, json_t *answer, void *data);
+TmEcode UnixSocketDatajsonReplace(json_t *cmd, json_t *answer, void *data);
 TmEcode UnixSocketRegisterTenantHandler(json_t *cmd, json_t* answer, void *data);
 TmEcode UnixSocketUnregisterTenantHandler(json_t *cmd, json_t* answer, void *data);
 TmEcode UnixSocketRegisterTenant(json_t *cmd, json_t* answer, void *data);

src/unix-manager.c

@@ -1111,6 +1111,9 @@ int UnixManagerInit(void)
 
     UnixManagerRegisterCommand("dataset-add", UnixSocketDatasetAdd, &command, UNIX_CMD_TAKE_ARGS);
     UnixManagerRegisterCommand("dataset-remove", UnixSocketDatasetRemove, &command, UNIX_CMD_TAKE_ARGS);
+    UnixManagerRegisterCommand("datajson-add", UnixSocketDatajsonAdd, &command, UNIX_CMD_TAKE_ARGS);
+    UnixManagerRegisterCommand(
+            "datajson-remove", UnixSocketDatajsonRemove, &command, UNIX_CMD_TAKE_ARGS);
     UnixManagerRegisterCommand(
             "get-flow-stats-by-id", UnixSocketGetFlowStatsById, &command, UNIX_CMD_TAKE_ARGS);
     UnixManagerRegisterCommand("dataset-dump", UnixSocketDatasetDump, NULL, 0);