Merge pull request #409 from vEpiphyte/epiphyte_synprop_req

 Epiphyte synprop req

synapse/cores/common.py

@@ -1839,6 +1839,9 @@ def addTufoEvents(self, form, propss):
 
                     self._addDefProps(form, fulls)
 
+                    # Ensure we have ALL the required props
+                    self._reqProps(form, fulls)
+
                     fulls[form] = iden
                     fulls['tufo:form'] = form
 
@@ -1878,6 +1881,26 @@ def _runAutoAdd(self, toadd):
                 continue
             self.formTufoByProp(form, valu)
 
+    def _reqProps(self, form, fulls):
+        if not self.enforce:
+            return
+
+        props = self.getFormReqs(form)
+
+        # Return fast for perf
+        if not props:
+            return
+
+        # Special case for handling syn:prop:glob=1 on will not have a ptype
+        # despite the model requiring a ptype to be present.
+        if fulls.get('syn:prop:glob'):
+            props.pop('syn:prop:ptype', None)
+
+        missing = set(props) - set(fulls)
+        if missing:
+            raise s_common.PropNotFound(mesg='Node is missing required a prop during formation',
+                                        prop=list(missing)[0], form=form)
+
     def formTufoByTufo(self, tufo):
         '''
         Form an (iden,info) tufo by extracting information from an existing one.
@@ -1972,6 +1995,9 @@ def formTufoByProp(self, prop, valu, **props):
             # create a "full" props dict which includes defaults
             self._addDefProps(prop, fulls)
 
+            # Ensure we have ALL the required props
+            self._reqProps(prop, fulls)
+
             fulls[prop] = valu
             fulls['tufo:form'] = prop
 
@@ -2441,7 +2467,7 @@ def setTufoProps(self, tufo, **props):
                 xact.fire('node:prop:set', form=form, valu=valu, prop=p, newv=v, oldv=oldv, node=tufo)
 
                 # fire the splice event
-                xact.spliced('node:prop:set', form=form, valu=valu, prop=p[len(form)+1:], newv=v, oldv=oldv, node=tufo)
+                xact.spliced('node:prop:set', form=form, valu=valu, prop=p[len(form) + 1:], newv=v, oldv=oldv, node=tufo)
 
             if self.autoadd:
                 self._runAutoAdd(toadd)

synapse/datamodel.py

@@ -100,6 +100,7 @@ def __init__(self, load=True):
         self.props = {}
         self.forms = set()
 
+        self.reqprops = collections.defaultdict(list)
         self.defvals = collections.defaultdict(list)
         self.subprops = collections.defaultdict(list)
         self.propsbytype = collections.defaultdict(list)
@@ -125,10 +126,11 @@ def __init__(self, load=True):
         self.addTufoProp('syn:form', 'ptype', ptype='syn:type', req=1, doc='Synapse type for this form')
 
         self.addTufoForm('syn:prop', ptype='syn:prop')
-        self.addTufoProp('syn:prop', 'doc', ptype='str', req=1, doc='Description of the property definition')
+        # TODO - Re-enable syn:prop:doc req = 1 after cleaning up property docstrings.
+        self.addTufoProp('syn:prop', 'doc', ptype='str', req=0, doc='Description of the property definition')
         self.addTufoProp('syn:prop', 'form', ptype='syn:prop', req=1, doc='Synapse form which contains this property')
         self.addTufoProp('syn:prop', 'ptype', ptype='syn:type', req=1, doc='Synapse type for this field')
-        self.addTufoProp('syn:prop', 'req', ptype='bool', defval=0, doc='Set to 1 if this property is required')
+        self.addTufoProp('syn:prop', 'req', ptype='bool', defval=0, doc='Set to 1 if this property is required to form the node.')
         self.addTufoProp('syn:prop', 'glob', ptype='bool', defval=0, doc='Set to 1 if this property defines a glob')
         self.addTufoProp('syn:prop', 'defval', doc='Set to the default value for this property')
 
@@ -264,6 +266,7 @@ def addPropDef(self, prop, **info):
             raise s_common.DupPropName(name=prop)
 
         info.setdefault('doc', None)
+        info.setdefault('req', False)
         info.setdefault('uniq', False)
         info.setdefault('ptype', None)
         info.setdefault('title', None)
@@ -279,6 +282,10 @@ def addPropDef(self, prop, **info):
         if defval is not None:
             self.defvals[form].append((prop, defval))
 
+        req = info.get('req')
+        if req:
+            self.reqprops[form].append(prop)
+
         pdef = (prop, info)
 
         ptype = info.get('ptype')
@@ -299,6 +306,18 @@ def getFormDefs(self, form):
         '''
         return self.defvals.get(form, ())
 
+    def getFormReqs(self, form):
+        '''
+        Return a list of prop values which are required form a form.
+
+        Args:
+            form (str): Form to request values for.
+
+        Returns:
+            list: List of required properties needed for making the given form.
+        '''
+        return self.reqprops.get(form, ())
+
     def _addSubRefs(self, pdef):
         name = pdef[0]
         for prop in s_tags.iterTagUp(pdef[0], div=':'):

synapse/exc.py

@@ -121,7 +121,7 @@ class NoInitCore(Exception): pass # API disabled because no cortex
 class NoCurrSess(Exception): pass # API requires a current session
 
 class SidNotFound(Exception): pass
-class PropNotFound(Exception): pass
+class PropNotFound(SynErr): pass
 
 class HitMaxTime(Exception): pass
 class HitMaxRetry(Exception): pass

synapse/lib/ingest.py

@@ -560,23 +560,19 @@ def _ingFormInfo(self, core, data, info, scope):
                 if valu is None:
                     return
 
-                tufo = core.formTufoByProp(form, valu)
-                if tufo is None:
-                    return
-
-                self.fire('gest:prog', act='form')
-
                 props = {}
                 for prop, pnfo in info.get('props', {}).items():
-                    valu = self._get_prop(core, data, pnfo, scope)
-                    if valu is None:
+                    pvalu = self._get_prop(core, data, pnfo, scope)
+                    if pvalu is None:
                         continue
 
-                    props[prop] = valu
+                    props[prop] = pvalu
 
-                if props:
-                    core.setTufoProps(tufo, **props)
-                    self.fire('gest:prog', act='set')
+                tufo = core.formTufoByProp(form, valu, **props)
+                if tufo is None:
+                    return
+
+                self.fire('gest:prog', act='form')
 
                 for tag in scope.iter('tags'):
                     core.addTufoTag(tufo, tag)

synapse/models/crypto.py

@@ -1,5 +1,11 @@
+import logging
+
+import synapse.common as s_common
+
 from synapse.lib.module import CoreModule, modelrev
 
+logger = logging.getLogger(__name__)
+
 ex_md5 = 'd41d8cd98f00b204e9800998ecf8427e'
 ex_sha1 = 'da39a3ee5e6b4b0d3255bfef95601890afd80709'
 ex_sha256 = 'ad9f4fe922b61e674a09530831759843b1880381de686a43460a76864ca0340c'
@@ -8,6 +14,19 @@
 
 class CryptoMod(CoreModule):
 
+    @modelrev('crypto', 201708231712)
+    def _revModl201708231712(self):
+        node = self.core.getTufoByProp('syn:prop', 'rsa:key:mod')
+        if not node:  # pragma: no cover
+            # Its possible someone deleted their syn:prop=rsa:key:mod node :(
+            mesg = 'No syn:prop="rsa:key:mod" node found during model revision.'
+            logger.warning(mesg)
+            self.core.log(logging.WARNING, mesg=mesg)
+            return
+        self.core.delRowsByIdProp(node[0], 'syn:prop:pytpe')
+        if 'syn:prop:ptype' not in node[1]:
+            self.core.addRows([(node[0], 'syn:prop:ptype', 'str:hex', s_common.now())])
+
     @staticmethod
     def getBaseModels():
         modl = {
@@ -33,7 +52,7 @@ def getBaseModels():
                 ('hash:sha384', {'ptype': 'hash:sha384'}, []),
                 ('hash:sha512', {'ptype': 'hash:sha512'}, []),
                 ('rsa:key', {'ptype': 'rsa:key'}, [
-                    ('mod', {'pytpe': 'str:hex', 'doc': 'The modulus'}),
+                    ('mod', {'ptype': 'str:hex', 'doc': 'The modulus'}),
                     ('bits', {'ptype': 'int', 'doc': 'The length of the modulus in bits'}),
                     ('pub:exp', {'ptype': 'str:hex', 'doc': 'The public exponent'}),
                     ('priv:exp', {'ptype': 'str:hex', 'doc': 'The private exponent'}),

synapse/models/inet.py

@@ -398,11 +398,11 @@ def _revModl201706121318(self):
 
     @modelrev('inet', 201708141516)
     def _revModl201708141516(self):
-        node = self.core.formTufoByProp('syn:type', 'inet:urlfile')  # type: list
-        if not node:
+        node = self.core.getTufoByProp('syn:type', 'inet:urlfile')
+        if not node:  # pragma: no cover
             # Its possible someone deleted their syn:type=inet:urlfile node :(
             mesg = 'No syn:type="inet:urlfile" node found during model revision.'
-            logger.warn(mesg)
+            logger.warning(mesg)
             self.core.log(logging.WARNING, mesg=mesg)
             return
         self.core.delRowsByIdProp(node[0], 'syn:type:names')
@@ -411,6 +411,18 @@ def _revModl201708141516(self):
         if 'syn:type:fields' not in node[1]:
             self.core.addRows([(node[0], 'syn:type:fields', 'url=inet:url,file=file:bytes', s_common.now())])
 
+    @modelrev('inet', 201708231646)
+    def _revModl201708231646(self):
+        node = self.core.getTufoByProp('syn:prop', 'inet:ipv4:type')
+        if not node:  # pragma: no cover
+            # Its possible someone deleted their syn:prop=inet:ipv4:type node :(
+            mesg = 'No syn:prop="inet:ipv4:type" node found during model revision.'
+            logger.warning(mesg)
+            self.core.log(logging.WARNING, mesg=mesg)
+            return
+        if 'syn:prop:ptype' not in node[1]:
+            self.core.addRows([(node[0], 'syn:prop:ptype', 'str', s_common.now())])
+
     @staticmethod
     def getBaseModels():
         modl = {
@@ -500,7 +512,8 @@ def getBaseModels():
 
                 ('inet:ipv4', {'ptype': 'inet:ipv4'}, [
                     ('cc', {'ptype': 'pol:iso2', 'defval': '??'}),
-                    ('type', {'defval': '??', 'doc': 'what type of ipv4 address ( uni, multi, priv )'}),
+                    ('type', {'ptype': 'str', 'defval': '??',
+                              'doc': 'what type of ipv4 address ( uni, multi, priv )'}),
                     ('asn', {'ptype': 'inet:asn', 'defval': -1}),
                 ]),
 

synapse/tests/test_cortex.py

@@ -2562,6 +2562,27 @@ def test_cortex_formtufosbyprops(self):
                     for s in ['NoSuchForm', 'name=', 'bad']:
                         self.isin(s, actual[2][1]['syn:err:errmsg'])
 
+    def test_cortex_reqprops(self):
+
+        with s_cortex.openurl('ram:///') as core:
+            # TODO - We will have to invert the logic here slightly when enforce=1 is merged.
+
+            # Required prop "time" not provided but enforce=0.
+            t0 = core.addTufoEvent('inet:dns:look', a='WOOT.com/1.2.3.4')
+            self.nn(t0)
+
+            # enable enforce
+            core.setConfOpt('enforce', 1)
+
+            # fails without required prop present
+            self.raises(PropNotFound, core.addTufoEvent, 'inet:dns:look', a='WOOT.com/1.2.3.5', )
+
+            # Works with required prop present
+            tick = now()
+            t1 = core.addTufoEvent('inet:dns:look', a='WOOT.com/1.2.3.4', time=tick)
+            self.eq(t1[1].get('inet:dns:look:time'), tick)
+            self.ne(t0[0], t1[0])
+
 class StorageTest(SynTest):
 
     def test_nonexist_ctor(self):

synapse/tests/test_datamodel.py

@@ -255,7 +255,7 @@ def test_datamodel_getPropDef(self):
         model.addTufoForm('foo')
         model.addTufoProp('foo', 'meow', ptype='int')
 
-        self.eq(model.getPropDef('foo:meow'), ('foo:meow', {'doc': None, 'title': None, 'defval': None, 'form': 'foo', 'base': 'meow', 'uniq': False, 'ptype': 'int'}))
+        self.eq(model.getPropDef('foo:meow'), ('foo:meow', {'doc': None, 'title': None, 'defval': None, 'form': 'foo', 'base': 'meow', 'uniq': False, 'ptype': 'int', 'req': False}))
         self.eq(model.getPropDef('foo:meow:nonexistent'), None)
         self.eq(model.getPropDef('foo:meow:nonexistent', glob=False), None)