From 5d3468dec1428ec418e85e8a0a5695ae15db2d72 Mon Sep 17 00:00:00 2001 From: srinandan Date: Sun, 12 Apr 2020 16:51:52 -0700 Subject: [PATCH] import oas doc to proxy and validate policy --- cmd/apis/apiproxydef/apiproxydef.go | 64 +++++---- cmd/apis/crtapi.go | 23 +++- cmd/apis/generateapi.go | 160 ++++++++++++++++++---- cmd/apis/policies/policies.go | 36 +++++ cmd/apis/proxies/proxies.go | 35 +++-- cmd/apis/proxybundle/proxybundle.go | 53 +++++-- cmd/apis/targetendpoint/targetendpoint.go | 15 +- 7 files changed, 298 insertions(+), 88 deletions(-) create mode 100644 cmd/apis/policies/policies.go diff --git a/cmd/apis/apiproxydef/apiproxydef.go b/cmd/apis/apiproxydef/apiproxydef.go index 1cc43ab3..2de2c23f 100644 --- a/cmd/apis/apiproxydef/apiproxydef.go +++ b/cmd/apis/apiproxydef/apiproxydef.go @@ -20,47 +20,50 @@ import ( "time" ) -type policiesType struct { - XMLName xml.Name `xml:"Policies"` - Policy []string `xml:"Policy,omitempty"` +type policiesDef struct { + Policy []string `xml:"Policy,omitempty"` } -type proxyEndpointsType struct { +type proxyEndpointsDef struct { XMLName xml.Name `xml:"ProxyEndpoints"` ProxyEndpoint []string `xml:"ProxyEndpoint,omitempty"` } -type targetEndpointsType struct { +type targetEndpointsDef struct { XMLName xml.Name `xml:"TargetEndpoints"` TargetEndpoint []string `xml:"TargetEndpoint,omitempty"` } -type configurationVersionType struct { +type configurationVersionDef struct { XMLName xml.Name `xml:"ConfigurationVersion,omitempty"` MajorVersion string `xml:"majorVersion,attr"` MinorVersion string `xml:"minorVersion,attr"` } -type apiProxyType struct { - XMLName xml.Name `xml:"APIProxy"` - Name string `xml:"name,attr"` - Revision string `xml:"revision,attr"` - BasePaths string `xml:"Basepaths,omitempty"` - ConfigurationVersion configurationVersionType `xml:"ConfigurationVersion,omitempty"` - CreatedAt string `xml:"CreatedAt,omitempty"` - Description string `xml:"Description,omitempty"` - DisplayName string `xml:"DisplayName,omitempty"` - LastModifiedAt string `xml:"LastModifiedAt,omitempty"` - Policies policiesType `xml:"Policies,omitempty"` - ProxyEndpoints proxyEndpointsType `xml:"ProxyEndpoints,omitempty"` - Resources string `xml:"Resources,omitempty"` - Spec string `xml:"Spec,omitempty"` - TargetServers string `xml:"TargetServers,omitempty"` - TargetEndpoints targetEndpointsType `xml:"TargetEndpoints,omitempty"` - Validate string `xml:"validate,omitempty"` +type resourcesDef struct { + Resource []string `xml:"Resource,omitempty"` } -var apiProxy apiProxyType +type apiProxyDef struct { + XMLName xml.Name `xml:"APIProxy"` + Name string `xml:"name,attr"` + Revision string `xml:"revision,attr"` + BasePaths string `xml:"Basepaths,omitempty"` + ConfigurationVersion configurationVersionDef `xml:"ConfigurationVersion,omitempty"` + CreatedAt string `xml:"CreatedAt,omitempty"` + Description string `xml:"Description,omitempty"` + DisplayName string `xml:"DisplayName,omitempty"` + LastModifiedAt string `xml:"LastModifiedAt,omitempty"` + Policies policiesDef `xml:"Policies,omitempty"` + ProxyEndpoints proxyEndpointsDef `xml:"ProxyEndpoints,omitempty"` + Resources resourcesDef `xml:"Resources,omitempty"` + Spec string `xml:"Spec,omitempty"` + TargetServers string `xml:"TargetServers,omitempty"` + TargetEndpoints targetEndpointsDef `xml:"TargetEndpoints,omitempty"` + Validate string `xml:"validate,omitempty"` +} + +var apiProxy apiProxyDef func SetDisplayName(name string) { apiProxy.DisplayName = name @@ -99,12 +102,19 @@ func SetDescription(description string) { apiProxy.Description = description } -func GetAPIProxy() string { - proxyBody, _ := xml.MarshalIndent(apiProxy, "", " ") - return string(proxyBody) +func GetAPIProxy() (string, error) { + proxyBody, err := xml.MarshalIndent(apiProxy, "", " ") + if err != nil { + return "", err + } + return string(proxyBody), nil } func SetConfigurationVersion() { apiProxy.ConfigurationVersion.MajorVersion = "4" apiProxy.ConfigurationVersion.MinorVersion = "0" } + +func AddResource(name string) { + apiProxy.Resources.Resource = append(apiProxy.Resources.Resource, "oas://"+name) +} diff --git a/cmd/apis/crtapi.go b/cmd/apis/crtapi.go index 5613d49d..30cbdabc 100644 --- a/cmd/apis/crtapi.go +++ b/cmd/apis/crtapi.go @@ -33,13 +33,24 @@ var CreateCmd = &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) (err error) { if proxy != "" { _, err = apis.CreateProxy(name, proxy) - } else if oasDoc != "" { - err = GenerateAPIProxyDefFromOAS(name, oasDoc) + } else if oasFile != "" || oasURI != "" { + var content []byte + var oasDocName string + if oasFile != "" { + oasDocName, content, err = LoadDocumentFromFile(oasFile) + } else { + oasDocName, content, err = LoadDocumentFromURI(oasURI) + } + if err != nil { + return err + } + + err = GenerateAPIProxyDefFromOAS(name, oasDocName) if err != nil { return err } - err = proxybundle.GenerateAPIProxyBundle(name) + err = proxybundle.GenerateAPIProxyBundle(name, string(content), oasDocName) if err != nil { return err } @@ -56,7 +67,7 @@ var CreateCmd = &cobra.Command{ }, } -var proxy, oasDoc string +var proxy, oasFile, oasURI string var importProxy bool func init() { @@ -65,8 +76,10 @@ func init() { "", "API Proxy name") CreateCmd.Flags().StringVarP(&proxy, "proxy", "p", "", "API Proxy Bundle path") - CreateCmd.Flags().StringVarP(&oasDoc, "oas", "f", + CreateCmd.Flags().StringVarP(&oasFile, "oasfile", "f", "", "Open API 3.0 Specification file") + CreateCmd.Flags().StringVarP(&oasURI, "oasuri", "u", + "", "Open API 3.0 Specification URI location") CreateCmd.Flags().BoolVarP(&importProxy, "import", "", true, "Import API Proxy after generation from spec") diff --git a/cmd/apis/generateapi.go b/cmd/apis/generateapi.go index 1c51c809..60248bcc 100644 --- a/cmd/apis/generateapi.go +++ b/cmd/apis/generateapi.go @@ -17,24 +17,91 @@ package apis import ( "fmt" "net/url" + "path" + "path/filepath" "regexp" "strings" "github.com/getkin/kin-openapi/openapi3" + "github.com/ghodss/yaml" apiproxy "github.com/srinandan/apigeecli/cmd/apis/apiproxydef" proxies "github.com/srinandan/apigeecli/cmd/apis/proxies" target "github.com/srinandan/apigeecli/cmd/apis/targetendpoint" ) -func LoadDocument(filePath string) (doc *openapi3.Swagger, err error) { +type pathDetailDef struct { + OperationID string + Description string +} + +var doc *openapi3.Swagger + +func LoadDocumentFromFile(filePath string) (string, []byte, error) { + var err error + var jsonContent []byte + doc, err = openapi3.NewSwaggerLoader().LoadSwaggerFromFile(filePath) - return doc, err + if err != nil { + return "", nil, err + } + + if err = doc.Validate(openapi3.NewSwaggerLoader().Context); err != nil { + return "", nil, err + } + + if jsonContent, err = doc.MarshalJSON(); err != nil { + return "", nil, err + } + + if isFileYaml(filePath) { + yamlContent, err := yaml.JSONToYAML(jsonContent) + return filepath.Base(filePath), yamlContent, err + } else { + return filepath.Base(filePath), jsonContent, err + } } -func GenerateAPIProxyDefFromOAS(name string, filePath string) (err error) { - doc, err := LoadDocument(filePath) +func LoadDocumentFromURI(uri string) (string, []byte, error) { + var err error + var jsonContent []byte + + u, err := url.Parse(uri) if err != nil { - return err + return "", nil, err + } + + doc, err = openapi3.NewSwaggerLoader().LoadSwaggerFromURI(u) + if err != nil { + return "", nil, err + } + + if err = doc.Validate(openapi3.NewSwaggerLoader().Context); err != nil { + return "", nil, err + } + + if jsonContent, err = doc.MarshalJSON(); err != nil { + return "", nil, err + } + + if isFileYaml(uri) { + yamlContent, err := yaml.JSONToYAML(jsonContent) + return path.Base(u.Path), yamlContent, err + } else { + return path.Base(u.Path), jsonContent, err + } +} + +func isFileYaml(name string) bool { + if strings.Contains(name, ".yaml") || strings.Contains(name, ".yml") { + return true + } + return false +} + +func GenerateAPIProxyDefFromOAS(name string, oasDocName string) (err error) { + + if doc == nil { + return fmt.Errorf("Open API document not loaded") } apiproxy.SetDisplayName(name) @@ -49,6 +116,8 @@ func GenerateAPIProxyDefFromOAS(name string, filePath string) (err error) { apiproxy.SetConfigurationVersion() apiproxy.AddTargetEndpoint("default") apiproxy.AddProxyEndpoint("default") + apiproxy.AddResource(oasDocName) + apiproxy.AddPolicy("Validate-" + name + "-Schema") u, err := GetEndpoint(doc) if err != nil { @@ -60,6 +129,7 @@ func GenerateAPIProxyDefFromOAS(name string, filePath string) (err error) { target.NewTargetEndpoint(u.Scheme + "://" + u.Hostname()) proxies.NewProxyEndpoint(u.Path) + proxies.AddStepToPreFlowRequest("OpenAPI-Spec-Validation-1") GenerateFlows(doc.Paths) @@ -74,73 +144,113 @@ func GetEndpoint(doc *openapi3.Swagger) (u *url.URL, err error) { return url.Parse(doc.Servers[0].URL) } -func GetHTTPMethod(pathItem *openapi3.PathItem, keyPath string) map[string]string { +func GetHTTPMethod(pathItem *openapi3.PathItem, keyPath string) map[string]pathDetailDef { - pathMap := make(map[string]string) + pathMap := make(map[string]pathDetailDef) alternateOperationId := strings.ReplaceAll(keyPath, "\\", "_") if pathItem.Get != nil { + getPathDetail := pathDetailDef{} if pathItem.Get.OperationID != "" { - pathMap["get"] = pathItem.Get.OperationID + getPathDetail.OperationID = pathItem.Get.OperationID } else { - pathMap["get"] = "get_" + alternateOperationId + getPathDetail.OperationID = "get_" + alternateOperationId + } + if pathItem.Get.Description != "" { + getPathDetail.Description = pathItem.Get.Description } + pathMap["get"] = getPathDetail } if pathItem.Post != nil { + postPathDetail := pathDetailDef{} if pathItem.Post.OperationID != "" { - pathMap["post"] = pathItem.Post.OperationID + postPathDetail.OperationID = pathItem.Post.OperationID } else { - pathMap["post"] = "post_" + alternateOperationId + postPathDetail.OperationID = "post_" + alternateOperationId } + if pathItem.Post.Description != "" { + postPathDetail.Description = pathItem.Post.Description + } + pathMap["post"] = postPathDetail } if pathItem.Put != nil { + putPathDetail := pathDetailDef{} if pathItem.Put.OperationID != "" { - pathMap["put"] = pathItem.Put.OperationID + putPathDetail.OperationID = pathItem.Put.OperationID } else { - pathMap["put"] = "put_" + alternateOperationId + putPathDetail.OperationID = "put_" + alternateOperationId + } + if pathItem.Put.Description != "" { + putPathDetail.Description = pathItem.Put.Description } + pathMap["put"] = putPathDetail } if pathItem.Patch != nil { + patchPathDetail := pathDetailDef{} if pathItem.Patch.OperationID != "" { - pathMap["patch"] = pathItem.Patch.OperationID + patchPathDetail.OperationID = pathItem.Patch.OperationID } else { - pathMap["patch"] = "patch_" + alternateOperationId + patchPathDetail.OperationID = "patch_" + alternateOperationId } + if pathItem.Patch.Description != "" { + patchPathDetail.Description = pathItem.Patch.Description + } + pathMap["patch"] = patchPathDetail } if pathItem.Delete != nil { + deletePathDetail := pathDetailDef{} if pathItem.Delete.OperationID != "" { - pathMap["delete"] = pathItem.Delete.OperationID + deletePathDetail.OperationID = pathItem.Delete.OperationID } else { - pathMap["delete"] = "delete_" + alternateOperationId + deletePathDetail.OperationID = "delete_" + alternateOperationId + } + if pathItem.Delete.Description != "" { + deletePathDetail.Description = pathItem.Delete.Description } + pathMap["delete"] = deletePathDetail } if pathItem.Options != nil { + optionsPathDetail := pathDetailDef{} if pathItem.Options.OperationID != "" { - pathMap["options"] = pathItem.Options.OperationID + optionsPathDetail.OperationID = pathItem.Options.OperationID } else { - pathMap["options"] = "options_" + alternateOperationId + optionsPathDetail.OperationID = "options_" + alternateOperationId + } + if pathItem.Options.Description != "" { + optionsPathDetail.Description = pathItem.Options.Description } + pathMap["options"] = optionsPathDetail } if pathItem.Trace != nil { + tracePathDetail := pathDetailDef{} if pathItem.Trace.OperationID != "" { - pathMap["trace"] = pathItem.Trace.OperationID + tracePathDetail.OperationID = pathItem.Trace.OperationID } else { - pathMap["trace"] = "trace_" + alternateOperationId + tracePathDetail.OperationID = "trace_" + alternateOperationId } + if pathItem.Trace.Description != "" { + tracePathDetail.Description = pathItem.Trace.Description + } + pathMap["trace"] = tracePathDetail } if pathItem.Head != nil { + headPathDetail := pathDetailDef{} if pathItem.Head.OperationID != "" { - pathMap["head"] = pathItem.Head.OperationID + headPathDetail.OperationID = pathItem.Head.OperationID } else { - pathMap["head"] = "head_" + alternateOperationId + headPathDetail.OperationID = "head_" + alternateOperationId + } + if pathItem.Head.Description != "" { + headPathDetail.Description = pathItem.Head.Description } + pathMap["head"] = headPathDetail } return pathMap @@ -149,8 +259,8 @@ func GetHTTPMethod(pathItem *openapi3.PathItem, keyPath string) map[string]strin func GenerateFlows(paths openapi3.Paths) { for keyPath := range paths { pathMap := GetHTTPMethod(paths[keyPath], keyPath) - for method, operationId := range pathMap { - proxies.AddFlow(operationId, replacePathWithWildCard(keyPath), method) + for method, pathDetail := range pathMap { + proxies.AddFlow(pathDetail.OperationID, replacePathWithWildCard(keyPath), method, pathDetail.Description) } } } diff --git a/cmd/apis/policies/policies.go b/cmd/apis/policies/policies.go new file mode 100644 index 00000000..9655b06b --- /dev/null +++ b/cmd/apis/policies/policies.go @@ -0,0 +1,36 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policies + +import ( + "regexp" +) + +var oasPolicyTemplate = ` + + OpenAPI Spec Validation-1 + + request + oas://{PolicyName} +` + +func AddOpenAPIValidatePolicy(name string) string { + return replaceTemplateWithPolicy(name) +} + +func replaceTemplateWithPolicy(name string) string { + re := regexp.MustCompile(`{(.*?)}`) + return re.ReplaceAllLiteralString(oasPolicyTemplate, name) +} diff --git a/cmd/apis/proxies/proxies.go b/cmd/apis/proxies/proxies.go index 50494ddb..cd311559 100644 --- a/cmd/apis/proxies/proxies.go +++ b/cmd/apis/proxies/proxies.go @@ -45,15 +45,15 @@ type postFlowDef struct { } type requestFlowDef struct { - Step stepDef `xml:"Step"` + Step []*stepDef `xml:"Step"` } type responseFlowDef struct { - Step stepDef `xml:"Step"` + Step []*stepDef `xml:"Step"` } type stepDef struct { - Name string `xml:"name"` + Name string `xml:"Name"` } type routeRuleDef struct { @@ -68,11 +68,12 @@ type flowsDef struct { } type flowDef struct { - XMLName xml.Name `xml:"Flow"` - Name string `xml:"name,attr"` - Request requestFlowDef `xml:"Request"` - Response responseFlowDef `xml:"Response"` - Condition conditionDef `xml:"Condition"` + XMLName xml.Name `xml:"Flow"` + Name string `xml:"name,attr"` + Description string `xml:"Description,omitempty"` + Request requestFlowDef `xml:"Request"` + Response responseFlowDef `xml:"Response"` + Condition conditionDef `xml:"Condition"` } type conditionDef struct { @@ -88,9 +89,12 @@ type httpProxyConnectionDef struct { var proxyEndpoint proxyEndpointDef -func GetProxyEndpoint() string { - proxyBody, _ := xml.MarshalIndent(proxyEndpoint, "", " ") - return string(proxyBody) +func GetProxyEndpoint() (string, error) { + proxyBody, err := xml.MarshalIndent(proxyEndpoint, "", " ") + if err != nil { + return "", nil + } + return string(proxyBody), nil } func NewProxyEndpoint(basePath string) { @@ -102,9 +106,16 @@ func NewProxyEndpoint(basePath string) { proxyEndpoint.RouteRule.TargetEndpoint = "default" } -func AddFlow(operationId string, keyPath string, method string) { +func AddFlow(operationId string, keyPath string, method string, description string) { flow := flowDef{} flow.Name = operationId + flow.Description = description flow.Condition.ConditionData = "(proxy.pathsuffix MatchesPath \"" + keyPath + "\") and (request.verb = \"" + method + "\")" proxyEndpoint.Flows.Flow = append(proxyEndpoint.Flows.Flow, flow) } + +func AddStepToPreFlowRequest(name string) { + step := stepDef{} + step.Name = name + proxyEndpoint.PreFlow.Request.Step = append(proxyEndpoint.PreFlow.Request.Step, &step) +} diff --git a/cmd/apis/proxybundle/proxybundle.go b/cmd/apis/proxybundle/proxybundle.go index 9a49dfad..534738a7 100644 --- a/cmd/apis/proxybundle/proxybundle.go +++ b/cmd/apis/proxybundle/proxybundle.go @@ -22,49 +22,76 @@ import ( "strings" apiproxy "github.com/srinandan/apigeecli/cmd/apis/apiproxydef" + policies "github.com/srinandan/apigeecli/cmd/apis/policies" proxies "github.com/srinandan/apigeecli/cmd/apis/proxies" target "github.com/srinandan/apigeecli/cmd/apis/targetendpoint" ) -func GenerateAPIProxyBundle(name string) (err error) { +func GenerateAPIProxyBundle(name string, content string, fileName string) (err error) { const rootDir = "apiproxy" + var apiProxyData, proxyEndpointData, targetEndpointData string - err = os.Mkdir(rootDir, os.ModePerm) - if err != nil { + if err = os.Mkdir(rootDir, os.ModePerm); err != nil { return err } // write API Proxy file - err = writeXMLData(rootDir+string(os.PathSeparator)+name+".xml", apiproxy.GetAPIProxy()) + if apiProxyData, err = apiproxy.GetAPIProxy(); err != nil { + return err + } + + err = writeXMLData(rootDir+string(os.PathSeparator)+name+".xml", apiProxyData) if err != nil { return err } proxiesDirPath := rootDir + string(os.PathSeparator) + "proxies" + policiesDirPath := rootDir + string(os.PathSeparator) + "policies" targetDirPath := rootDir + string(os.PathSeparator) + "targets" + oasDirPath := rootDir + string(os.PathSeparator) + "resources" + string(os.PathSeparator) + "oas" - err = os.Mkdir(proxiesDirPath, os.ModePerm) - if err != nil { + if err = os.Mkdir(proxiesDirPath, os.ModePerm); err != nil { return err } - err = writeXMLData(proxiesDirPath+string(os.PathSeparator)+"default.xml", proxies.GetProxyEndpoint()) - if err != nil { + if proxyEndpointData, err = proxies.GetProxyEndpoint(); err != nil { return err } - err = os.Mkdir(targetDirPath, os.ModePerm) + err = writeXMLData(proxiesDirPath+string(os.PathSeparator)+"default.xml", proxyEndpointData) if err != nil { return err } - err = writeXMLData(targetDirPath+string(os.PathSeparator)+"default.xml", target.GetTargetEndpoint()) - if err != nil { + if err = os.Mkdir(targetDirPath, os.ModePerm); err != nil { return err } - err = archiveBundle(rootDir, name+".zip") - if err != nil { + if targetEndpointData, err = target.GetTargetEndpoint(); err != nil { + return err + } + + if err = writeXMLData(targetDirPath+string(os.PathSeparator)+"default.xml", targetEndpointData); err != nil { + return err + } + + if err = os.MkdirAll(oasDirPath, os.ModePerm); err != nil { + return err + } + + if err = writeXMLData(oasDirPath+string(os.PathSeparator)+fileName, content); err != nil { + return err + } + + if err = os.Mkdir(policiesDirPath, os.ModePerm); err != nil { + return err + } + + if err = writeXMLData(policiesDirPath+string(os.PathSeparator)+"OpenAPI-Spec-Validation-1.xml", policies.AddOpenAPIValidatePolicy(fileName)); err != nil { + return err + } + + if err = archiveBundle(rootDir, name+".zip"); err != nil { return err } diff --git a/cmd/apis/targetendpoint/targetendpoint.go b/cmd/apis/targetendpoint/targetendpoint.go index c2b7e8e5..dcfde2f6 100644 --- a/cmd/apis/targetendpoint/targetendpoint.go +++ b/cmd/apis/targetendpoint/targetendpoint.go @@ -37,15 +37,15 @@ type httpTargetConnectionDef struct { } type requestFlowDef struct { - Step stepDef `xml:"Step,omitempty"` + Step []*stepDef `xml:"Step,omitempty"` } type responseFlowDef struct { - Step stepDef `xml:"Step,omitempty"` + Step []*stepDef `xml:"Step,omitempty"` } type stepDef struct { - Name string `xml:"name"` + Name string `xml:"Name"` } type targetEndpointDef struct { @@ -58,9 +58,12 @@ type targetEndpointDef struct { var targetEndpoint targetEndpointDef -func GetTargetEndpoint() string { - targetBody, _ := xml.MarshalIndent(targetEndpoint, "", " ") - return string(targetBody) +func GetTargetEndpoint() (string, error) { + targetBody, err := xml.MarshalIndent(targetEndpoint, "", " ") + if err != nil { + return "", nil + } + return string(targetBody), nil } func NewTargetEndpoint(endpoint string) {