Skip to content

Commit

Permalink
add validation for assignment - workin progress
Browse files Browse the repository at this point in the history
  • Loading branch information
@ticoann
ticoann committed Aug 7, 2013
1 parent 957e6a3 commit 4468319
Show file tree
Hide file tree
Showing 7 changed files with 271 additions and 199 deletions.
11 changes: 10 additions & 1 deletion src/python/WMCore/ReqMgr/DataStructs/RequestStatus.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,4 +105,13 @@

# each item from STATUS_TRANSITION is a dictionary with 1 item, the key
# is name of the status
REQUEST_STATUS_LIST = [s.keys()[0] for s in REQUEST_STATUS_TRANSITION]
REQUEST_STATUS_LIST = [s.keys()[0] for s in REQUEST_STATUS_TRANSITION]

def check_allowed_transition(preStatus, postStatus):
statusList = REQUEST_STATUS_TRANSITION.get(preStatus, [])
if postStatus in statusList:
return True
else:
return False


280 changes: 91 additions & 189 deletions src/python/WMCore/ReqMgr/Service/Request.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,14 @@
from datetime import datetime, timedelta

import WMCore.Lexicon
from WMCore.Database.CMSCouch import Database, CouchError
from WMCore.Database.CMSCouch import CouchError
from WMCore.WMSpec.WMWorkload import WMWorkloadHelper
from WMCore.WMSpec.StdSpecs.StdBase import WMSpecFactoryException
from WMCore.WMSpec.WMWorkloadTools import loadSpecByType
from WMCore.Wrappers import JsonWrapper

from WMCore.REST.Server import RESTEntity, restcall, rows
from WMCore.REST.Auth import authz_match
from WMCore.REST.Tools import tools
from WMCore.REST.Validation import validate_str, validate_strlist

Expand Down Expand Up @@ -40,7 +43,7 @@ def validate(self, apiobj, method, api, param, safe):
# make param empty
# other wise raise the error

if method in ['GET', 'PUT', 'POST']:
if method in ['GET']:
for prop in param.kwargs:
safe.kwargs[prop] = param.kwargs[prop]

Expand All @@ -59,7 +62,60 @@ def validate(self, apiobj, method, api, param, safe):
# validate_str("campaign", param, safe, "*", optional=True)
# validate_str("workqueue", param, safe, "*", optional=True)
# validate_str("team", param, safe, "*", optional=True)
if method == 'PUT':
# need to validate the post arguemtn and pass as argument
self.validate_request_update_args(param, safe)

if method == 'POST':
# need to validate the post arguemtn and pass as argument
self.validate_request_create_args(safe)

def validate_request_update_args(self, param, safe):
"""
validate post request
1. read data from body
2. validate the permission
3. validate state transition
2. validate using workload validation
3. convert data from body to arguments (spec instance, argument with default setting)
"""
#convert request.body to json (python dict)
request_args = JsonWrapper.loads(cherrypy.request.body.read())

print param

couchurl = '%s/%s' % (self.config.couch_host, self.config.couch_reqmgr_db)
helper = WMWorkloadHelper()
helper.loadSpecFromCouch(couchurl, param.args[0])

#TODO need to validates kwargs (add WorkloadHelper validation)
#helper.updateAssignment(request_args)
safe.args.append(param.args[0])
param.args.pop()
safe.kwargs['RequestStatus'] = request_args['RequestStatus']


def validate_request_create_args(self, safe):
"""
validate post request
1. read data from body
2. validate using spec validation
3. convert data from body to arguments (spec instance, argument with default setting)
"""
request_args = JsonWrapper.loads(cherrypy.request.body.read())

#TODO: this fuction need to be update in validateSchema in each Spec
#request.validate_automatic_args_empty()
self.request_initialize(request_args)

# get the spec type and validate arguments
spec = loadSpecByType(request_args["RequestType"])
workload = spec.factoryWorkloadConstruction(request_args["RequestName"],
request_args)
safe.kwargs['workload'] = workload
safe.kwargs['schema'] = request_args


@restcall
def get(self, **kwargs):
"""
Expand Down Expand Up @@ -172,8 +228,9 @@ def _combine_request(self, request_info, requestAgentUrl, cache):

return requestAgentUrlList;

@restcall
def put(self, name, **kwargs):

cherrypy.log("INFO: '%s -- %s' ..." % (name, kwargs))
return self.reqmgr_db.updateDocument(name, "ReqMgr", "updaterequest",
fields=kwargs)

Expand All @@ -193,17 +250,12 @@ def delete(self, request_name):


@restcall
def post(self):
def post(self, workload, schema):
"""
Create / inject a new request. Request input schema is specified in
the body of the request as JSON encoded data.
ReqMgr related request arguments validation to happen in
DataStructs.Request.validate(), the rest in spec.
ReqMgr related arguments manipulation to happen in the .request_initialize(),
before the spec is instantiated.
Create and update couchDB with a new request.
request argument is passed from validation
(validation convert cherrypy.request.body data to argument)
TODO:
this method will have some parts factored out so that e.g. clone call
can share functionality.
Expand All @@ -216,132 +268,15 @@ def post(self):
(from ReqMgrRESTModel.putRequest)
"""
json_input_request_args = cherrypy.request.body.read()
request_input_dict = JsonWrapper.loads(json_input_request_args)

cherrypy.log("INFO: Create request, input args: %s ..." % request_input_dict)

request = RequestData() # this returns a new request dictionary
request.update(request_input_dict)

try:
request.validate_automatic_args_empty()
# fill in automatic request arguments and further request args meddling
self.request_initialize(request)
self.request_validate(request)
except RequestDataError, ex:
cherrypy.log(ex.message)
raise cherrypy.HTTPError(400, ex.message)

cherrypy.log("INFO: Request initialization and validation succeeded."
" Instantiating spec/workload ...")
# TODO
# watch the above instantiation, it seems to take rather long time ...

# will be stored under request["WorkloadSpec"]
self.create_workload_attach_to_request(request, request_input_dict)

cherrypy.log("INFO: Request corresponding workload instantiated, storing ...")

helper = WMWorkloadHelper(request["WorkloadSpec"])
# TODO
# this should be revised for ReqMgr2
#4378 - ACDC (Resubmission) requests should inherit the Campaign ...
# for Resubmission request, there already is previous Campaign set
# this call would override it with initial request arguments where
# it is not specified, so would become ''
# TODO
# these kind of calls should go into some workload initialization
if not helper.getCampaign():
helper.setCampaign(request["Campaign"])

if request.has_key("RunWhitelist"):
helper.setRunWhitelist(request["RunWhitelist"])
cherrypy.log("INFO: Create request, input args: %s ..." % schema)

# storing the request document into Couch

# can't save Request object directly, because it makes it hard to retrieve
# the _rev
# TODO
# don't understand this. may just be possible to keep dealing with
# 'request' and not create this metadata
metadata = {}
metadata.update(request)

# TODO
# this should be verified and straighten up in ReqMgr2, should not need this
# Add the output datasets if necessary
# for some bizarre reason OutpuDatasets is list of lists, when cloning
# [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
# #3743
#if not clone:
# for ds in helper.listOutputDatasets():
# if ds not in request['OutputDatasets']:
# request['OutputDatasets'].append(ds)

# Store new request into Couch
try:
# don't want to JSONify the whole workflow
del metadata["WorkloadSpec"]
workload_url = helper.saveCouch(request["CouchURL"],
request["CouchWorkloadDBName"],
metadata=metadata)
# TODO
# this will have to be updated now, when the Couch url is known. The question
# is whether this request argument is necessary at all since it should
# always be CouchUrl/DbName/RequestName/spec so it can easily be derived
# if this not necessary, this below step of updating the document is
# not necessary unlike it was the case in ReqMgr1
request["RequestWorkflow"] = workload_url
params_to_update = ["RequestWorkflow"]
fields = {}
for key in params_to_update:
fields[key] = request[key]
self.reqmgr_db.updateDocument(request["RequestName"], "ReqMgr", "updaterequest",
fields=fields)
except CouchError, ex:
# TODO simulate exception here to see how much gets exposed to the client
# and how much gets logged when it's like this
msg = "ERROR: Storing into Couch failed, reason: %s" % ex.reason
cherrypy.log(msg)
raise cherrypy.HTTPError(500, msg)

cherrypy.log("INFO: Request '%s' created and stored." % request["RequestName"])
# do not want to return to client spec data
del request["WorkloadSpec"]
return rows([request])


def create_workload_attach_to_request(self, request, request_input_dict):
try:
factory_name = "%sWorkloadFactory" % request["RequestType"]
mod = __import__("WMCore.WMSpec.StdSpecs.%s" % request["RequestType"],
globals(), locals(), [factory_name])
Factory = getattr(mod, factory_name)
except ImportError:
msg = "ERROR: Spec type '%s' not found in WMCore.WMSpec.StdSpecs" % request["RequestType"]
cherrypy.log(msg)
raise RuntimeError, msg
except AttributeError, ex:
msg = "ERROR: Factory not found in Spec for type '%s'" % request["RequestType"]
cherrypy.log(msg)
raise RuntimeError, msg

try:
factory = Factory()
# TODO
# this method is only used by ReqMgr1, once ReqMgr1 is gone,
# there can be only 1 argument to this method
workload = factory.factoryWorkloadConstruction(workloadName=request["RequestName"],
arguments=request)
self.request_initilize_attach_input_to_workload(workload, request_input_dict)
except WMSpecFactoryException, ex:
msg = "ERROR: Error in spec/workload validation: %s" % ex._message
cherrypy.log(msg)
raise cherrypy.HTTPError(400, msg)

# make instantiated spec part of the request instance
request["WorkloadSpec"] = workload.data
workload.saveCouch(schema["CouchURL"], schema["CouchWorkloadDBName"],
metadata=schema)

#TODO should return something else instead on whole schema
return schema


def request_validate(self, request):
Expand Down Expand Up @@ -413,77 +348,44 @@ def request_initialize(self, request):
request is changed here.
"""
#update the information from config
request["CouchURL"] = self.config.couch_host
request["CouchWorkloadDBName"] = self.config.couch_reqmgr_db
request["CouchDBName"] = self.config.couch_config_cache_db

#user information for cert. (which is converted to cherry py log in)
request["Requestor"] = cherrypy.request.user["login"]
request["RequestorDN"] = cherrypy.request.user.get("dn", "unknown")
# assign first starting status, should be 'new'
request["RequestStatus"] = REQUEST_STATUS_LIST[0]
request["RequestTransition"] = [{"Status": request["RequestStatus"], "UpdateTime": int(time.time())}]
current_time = time.strftime('%y%m%d_%H%M%S', time.localtime(time.time()))
seconds = int(10000 * (time.time() % 1.0))
request_string = request.get("RequestString", "")
if request_string != "":
request["RequestName"] = "%s_%s" % (request["Requestor"], request_string)
else:
request["RequestName"] = request["Requestor"]
request["RequestName"] += "_%s_%s" % (current_time, seconds)

#TODO: generate this automatically from the spec
# generate request name using request
self._generateRequestName(request)

request["RequestDate"] = list(time.gmtime()[:6])

if request["CMSSWVersion"] and request["CMSSWVersion"] not in request["SoftwareVersions"]:
request.setdefault("SoftwareVersions", [])
if request["CMSSWVersion"] and (request["CMSSWVersion"] not in request["SoftwareVersions"]):
request["SoftwareVersions"].append(request["CMSSWVersion"])

# TODO
# do we need InputDataset and InputDatasets? when one is just a list
# containing the other? ; could be related to #3743 problem
if request.has_key("InputDataset"):
request["InputDatasets"] = [request["InputDataset"]]


def request_initilize_attach_input_to_workload(self, workload, request_input_dict):
"""
request_input_dict are input arguments for request injection
workload is a corresponding newly created workload instance and
request_input_dict is attached to workload under 'schema'
ReqMgr1 does this in RequestMaker.Registry.loadRequestSchema(), and
this method is only a slight modification of it.
Storing this original injection time information is probably not
crucially necessary but is definitely practical, keep that.

def _generateRequestName(self, request):

"""
wl = workload
rid = request_input_dict
schema = wl.data.request.section_("schema")
for key, value in rid.iteritems():
try:
setattr(schema, key, value)
except Exception, ex:
# attach TaskChain tasks
# TODO this may be a good example where recursion would be practical
if (type(value) == dict and rid["RequestType"] == 'TaskChain' and
"Task" in key):
new_section = schema.section_(key)
for k, v in rid[key].iteritems():
try:
setattr(new_section, k, v)
except Exception, ex:
# what does this mean then?
pass
else:
pass
schema.timeStamp = int(time.time())
wl.data.owner.Group = schema.Group
# TODO
# ReqMgr2 does not allow Requestor request argument specified in the
# injection input so it can't be set here based on request_input_dict.
# If it's absolutely necessary, it can be passed to his method from the
# caller, where it's known, and set on the workload. But I doubt there
# needs to be so much data duplication yet again between stuff stored
# in Couch and on workload.
#wl.data.owner.Requestor = schema.Requestor
current_time = time.strftime('%y%m%d_%H%M%S', time.localtime(time.time()))
seconds = int(10000 * (time.time() % 1.0))
request_string = request.get("RequestString", "")
if request_string != "":
request["RequestName"] = "%s_%s" % (request["Requestor"], request_string)
else:
request["RequestName"] = request["Requestor"]
request["RequestName"] += "_%s_%s" % (current_time, seconds)

class RequestStatus(RESTEntity):
def __init__(self, app, api, config, mount):
Expand Down
Loading

0 comments on commit 4468319

Please sign in to comment.