Skip to content

Commit

Permalink
WIP: File Watcher gateway as core gateway, notification filters to st…
Browse files Browse the repository at this point in the history
…orage grid gateway, k8 configmap as trigger artifact location. (argoproj#117)

* Added notification filters to storage grid gateway

* Added file watcher as core gateway. Added readiness probe to gateway transformer to ensure correct ordering of gateway containers startup

* Added K8 configmaps as artifact location for triggers

* Started adding validation for gateway configurations

* Adding validation for gateways, adding gateway type openapi specs

* Validation continued
  • Loading branch information
VaibhavPage authored and magaldima committed Nov 1, 2018
1 parent c9d7d0c commit 02780f9
Show file tree
Hide file tree
Showing 47 changed files with 1,918 additions and 662 deletions.
24 changes: 18 additions & 6 deletions Gopkg.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions Gopkg.toml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ required = [
name = "github.com/stretchr/testify"
version = "1.1.4"

[[constraint]]
name = "github.com/fsnotify/fsnotify"
branch = "master"

[[override]]
branch = "release-1.10"
name = "k8s.io/api"
Expand Down
19 changes: 15 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,13 @@ endif

# Build the project images
.DELETE_ON_ERROR:
all: sensor-linux sensor-controller-linux gateway-controller-linux gateway-http-transformer-linux webhook-linux calendar-linux resource-linux artifact-linux nats-linux kafka-linux amqp-linux mqtt-linux gateway-processor-grpc-client-linux calendar-grpc-linux gateway-processor-http-client-linux calendar-http-linux storage-grid-linux
all: sensor-linux sensor-controller-linux gateway-controller-linux gateway-http-transformer-linux webhook-linux calendar-linux resource-linux artifact-linux file-linux nats-linux kafka-linux amqp-linux mqtt-linux gateway-processor-grpc-client-linux calendar-grpc-linux gateway-processor-http-client-linux calendar-http-linux storage-grid-linux

all-images: sensor-image sensor-controller-image gateway-controller-image gateway-http-transformer-image webhook-image calendar-image resource-image artifact-image nats-image kafka-image amqp-image mqtt-image gateway-processor-grpc-client-image calendar-grpc-image gateway-processor-http-client-image calendar-http-image storage-grid-image
all-images: sensor-image sensor-controller-image gateway-controller-image gateway-http-transformer-image webhook-image calendar-image resource-image artifact-image file-image nats-image kafka-image amqp-image mqtt-image gateway-processor-grpc-client-image calendar-grpc-image gateway-processor-http-client-image calendar-http-image storage-grid-image

all-controller-images: sensor-controller-image gateway-controller-image

all-core-gateway-images: webhook-image calendar-image artifact-image nats-image kafka-image amqp-image mqtt-image resource-image
all-core-gateway-images: webhook-image calendar-image artifact-image file-image nats-image kafka-image amqp-image mqtt-image resource-image

.PHONY: all clean test

Expand Down Expand Up @@ -126,7 +126,6 @@ resource-image: resource-linux
@if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)resource-gateway:$(IMAGE_TAG) ; fi



artifact:
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/artifact-gateway ./gateways/core/artifact/

Expand All @@ -138,6 +137,18 @@ artifact-image: artifact-linux
@if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)artifact-gateway:$(IMAGE_TAG) ; fi


file:
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/file-gateway ./gateways/core/file/

file-linux:
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make file

file-image: file-linux
docker build -t $(IMAGE_PREFIX)file-gateway:$(IMAGE_TAG) -f ./gateways/core/file/Dockerfile .
@if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)file-gateway:$(IMAGE_TAG) ; fi


#Stream gateways
nats:
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/nats-gateway ./gateways/core/stream/nats/

Expand Down
1 change: 1 addition & 0 deletions cmd/controllers/gateway/transform/http/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,5 +110,6 @@ func main() {

// endpoint to listen events
http.HandleFunc("/", toc.TransformRequest)
http.HandleFunc("/readiness", toc.ReadinessProbe)
log.Fatal(http.ListenAndServe(":"+fmt.Sprintf("%s", common.GatewayTransformerPort), nil))
}
5 changes: 5 additions & 0 deletions controllers/gateway/transform/transformer.go
Original file line number Diff line number Diff line change
Expand Up @@ -286,3 +286,8 @@ func (toc *tOperationCtx) TransformRequest(w http.ResponseWriter, r *http.Reques
}
common.SendSuccessResponse(w)
}

// ReadinessProbe is probe to check whether server is running or not
func (toc *tOperationCtx) ReadinessProbe(w http.ResponseWriter, r *http.Request) {
common.SendSuccessResponse(w)
}
9 changes: 9 additions & 0 deletions examples/gateways/file-gateway-configmap.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: file-watcher-gateway-configmap
data:
fileWatcher.defaultConfig: |-
directory: "/bin/"
type: CREATE
path: x.txt
22 changes: 22 additions & 0 deletions examples/gateways/file.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
apiVersion: argoproj.io/v1alpha1
kind: Gateway
metadata:
name: file-watcher-gateway
labels:
gateways.argoproj.io/gateway-controller-instanceid: argo-events
gateway-name: "file-watcher-gateway"
spec:
deploySpec:
containers:
- name: "file-watcher-events"
image: "argoproj/file-watcher-gateway"
imagePullPolicy: "Always"
command: ["/bin/file-watcher-gateway"]
serviceAccountName: "argo-events-sa"
configMap: "file-watcher-gateway-configmap"
version: "1.0"
type: "file-watcher"
dispatchMechanism: "HTTP"
watchers:
sensors:
- name: "file-watcher-sensor"
13 changes: 13 additions & 0 deletions examples/gateways/storage-grid-configmap.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: storage-grid-gateway-configmap
data:
storage-grid.defaultConfig: |-
port: "8080"
endpoint: "/"
events:
- "ObjectCreated:Put"
filter:
suffix: ".txt"
prefix: "hello-"
28 changes: 28 additions & 0 deletions examples/gateways/storage-grid.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
apiVersion: argoproj.io/v1alpha1
kind: Gateway
metadata:
name: storage-grid-gateway
labels:
gateways.argoproj.io/gateway-controller-instanceid: argo-events
gateway-name: "storage-grid-gateway"
spec:
configMap: "storage-grid-gateway-configmap"
type: "storage-grid"
dispatchMechanism: "HTTP"
version: "1.0"
deploySpec:
containers:
- name: "storage-grid-events"
image: "argoproj/storage-grid-gateway"
imagePullPolicy: "Always"
serviceAccountName: "argo-events-sa"
serviceSpec:
selector:
gateway-name: "storage-grid-gateway"
ports:
- port: 8080
targetPort: 8080
type: LoadBalancer
watchers:
sensors:
- name: "storage-grid-watcher-sensor"
35 changes: 35 additions & 0 deletions examples/sensors/file.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
apiVersion: argoproj.io/v1alpha1
kind: Sensor
metadata:
name: file-watcher-sensor
labels:
sensors.argoproj.io/sensor-controller-instanceid: argo-events
spec:
serviceAccountName: argo-events-sa
repeat: true
signals:
- name: file-watcher-gateway/fileWatcher.defaultConfig
triggers:
- name: file-watcher-workflow-trigger
resource:
namespace: argo-events
group: argoproj.io
version: v1alpha1
kind: Workflow
source:
inline: |
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
generateName: file-watcher-
spec:
entrypoint: whalesay
templates:
-
container:
args:
- "hello world"
command:
- cowsay
image: "docker/whalesay:latest"
name: whalesay
38 changes: 38 additions & 0 deletions examples/sensors/storage-grid-sensor.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
apiVersion: argoproj.io/v1alpha1
kind: Sensor
metadata:
name: storage-grid-watcher-sensor
labels:
sensors.argoproj.io/sensor-controller-instanceid: argo-events
spec:
repeat: true
serviceAccountName: argo-events-sa
signals:
- name: storage-grid-gateway/storage-grid.defaultConfig
triggers:
- name: argo-workflow
resource:
namespace: argo-events
group: argoproj.io
version: v1alpha1
kind: Workflow
parameters:
- src:
signal: storage-grid-gateway/storage-grid.defaultConfig
path:
value: hello world
dest: spec.templates.0.container.args.0
source:
inline: |
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
generateName: hello-world-
spec:
entrypoint: whalesay
templates:
- name: whalesay
container:
args:
- "hello world"
image: "metalgearsolid/args-printer:latest"
61 changes: 61 additions & 0 deletions gateways/core/artifact/artifact.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
{
"swagger": "2.0",
"info": {
"title": "Integration Test",
"version": "1.0"
},
"paths": {},
"definitions": {
"argo-events.S3EventConfig": {
"description": "S3EventConfig contains configuration for bucket notification",
"properties": {
"bucket": {
"type": "string"
},
"endpoint": {
"type": "string"
},
"event": {
"type": "string"
},
"filter": {
"$ref": "#/definitions/argo-events.S3Filter"
},
"region": {
"type": "string"
}
}
},
"argo-events.S3Filter": {
"description": "S3Filter represents filters to apply to bucket nofifications for specifying constraints on objects",
"required": [
"prefix",
"suffix"
],
"properties": {
"prefix": {
"type": "string"
},
"suffix": {
"type": "string"
}
}
},
"argo-events.s3Artifact": {
"description": "S3Artifact contains information about an artifact in S3",
"required": [
"s3EventConfig"
],
"properties": {
"insecure": {
"description": "Mode of operation for s3 client",
"type": "boolean"
},
"s3EventConfig": {
"description": "S3EventConfig contains configuration for bucket notification",
"$ref": "#/definitions/argo-events.S3EventConfig"
}
}
}
}
}
Loading

0 comments on commit 02780f9

Please sign in to comment.