Merge pull request #3108 from Joystream/giza

Giza Release

.dockerignore

@@ -7,3 +7,4 @@ query-node/lib
 cli/
 tests/
 devops/
+metadata-protobuf/lib

.env

@@ -2,40 +2,64 @@ COMPOSE_PROJECT_NAME=joystream
 PROJECT_NAME=query_node
 
 # We will use a single postgres service with multiple databases
-# The env variables below are by default used by all services and should be 
+# The env variables below are by default used by all services and should be
 # overriden in local env files
 # DB config
 INDEXER_DB_NAME=query_node_indexer
 DB_NAME=query_node_processor
 DB_USER=postgres
 DB_PASS=postgres
+# This value will not be used by query-node docker containers.
+# When running query-node with docker these services will always use the db service
 DB_HOST=localhost
 DB_PORT=5432
 DEBUG=index-builder:*
 TYPEORM_LOGGING=error
 
-DEBUG=index-builder:*
-TYPEORM_LOGGING=error
-
-###########################
-#    Indexer options      #
-###########################
-
+## Indexer options
 # Block height to start indexing from.
 # Note, that if there are already some indexed events, this setting is ignored
 BLOCK_HEIGHT=0
 
-###############################
-#    Processor GraphQL API    #
-###############################
+# Query node GraphQL server port
+GRAPHQL_SERVER_PORT=8081
 
-GRAPHQL_SERVER_PORT=4002
+# Query node playground subscription endpoint
+GRAPHQL_PLAYGROUND_SUBSCRIPTION_ENDPOINT=ws://localhost:8081/graphql
+
+# Hydra indexer gateway GraphQL server port
+HYDRA_INDEXER_GATEWAY_PORT=4000
+
+# Default GraphQL server host. It is required during "query-node config:dev"
 GRAPHQL_SERVER_HOST=localhost
-WARTHOG_APP_PORT=4002
-WARTHOG_APP_HOST=localhost
 
-# Default configuration is to use the docker container
-WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944/
+# Websocket RPC endpoint containers will use.
+JOYSTREAM_NODE_WS=ws://joystream-node:9944/
+
+# Query node which colossus will use
+COLOSSUS_QUERY_NODE_URL=http://graphql-server:${GRAPHQL_SERVER_PORT}/graphql
+
+# Query node which distributor will use
+DISTRIBUTOR_QUERY_NODE_URL=http://graphql-server:${GRAPHQL_SERVER_PORT}/graphql
+
+# Indexer gateway used by processor. If you don't use the local indexer set this to a remote gateway
+PROCESSOR_INDEXER_GATEWAY=http://hydra-indexer-gateway:${HYDRA_INDEXER_GATEWAY_PORT}/graphql
+
+# Colossus services identities
+COLOSSUS_1_WORKER_ID=0
+COLOSSUS_1_WORKER_URI=//testing//worker//Storage//${COLOSSUS_1_WORKER_ID}
+COLOSSUS_1_TRANSACTOR_URI=//Colossus1
+
+COLOSSUS_2_WORKER_ID=1
+COLOSSUS_2_WORKER_URI=//testing//worker//Storage//${COLOSSUS_2_WORKER_ID}
+COLOSSUS_2_TRANSACTOR_URI=//Colossus2
+
+# Distributor node services identities
+DISTRIBUTOR_1_WORKER_ID=0
+DISTRIBUTOR_1_ACCOUNT_URI=//testing//worker//Distribution//${DISTRIBUTOR_1_WORKER_ID}
+
+DISTRIBUTOR_2_WORKER_ID=1
+DISTRIBUTOR_2_ACCOUNT_URI=//testing//worker//Distribution//${DISTRIBUTOR_2_WORKER_ID}
 
-# If running joystream-node on host machine you can use following address to reach it instead
-# WS_PROVIDER_ENDPOINT_URI=ws://host.docker.internal:9944/
+# joystream/node docker image tag
+JOYSTREAM_NODE_TAG=latest

.github/workflows/deploy-playground.yml

@@ -19,7 +19,15 @@ on:
         description: 'AWS EC2 instance type (t2.micro, t2.large)'
         required: false
         default: 't2.micro'
-
+      stackNamePrefix:
+        description: 'Additional identifier to include in stack name'
+        required: false
+        default: 'playground'
+      # TODO: customDomain instead of ip_address.nip.io
+      # customDomain:
+      #   description: 'DNS hostname to use for deployment'
+      #   required: false
+      #   default: ''
 defaults:
   run:
     working-directory: devops/aws
@@ -29,7 +37,40 @@ jobs:
     name: Create an EC2 instance and configure docker-compose stack
     runs-on: ubuntu-latest
     env:
-      STACK_NAME: joystream-playground-{{ github.event.inputs.branchName }}-${{ github.run_number }}
+      STACK_NAME: ${{ github.event.inputs.stackNamePrefix }}-${{ github.event.inputs.branchName }}-${{ github.run_number }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+
+      - name: Install Ansible dependencies
+        run: pipx inject ansible-core boto3 botocore
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: Deploy to AWS CloudFormation
+        uses: aws-actions/aws-cloudformation-github-deploy@v1
+        id: deploy_stack
+        with:
+          name: ${{ env.STACK_NAME }}
+          template: devops/aws/cloudformation/single-instance-docker.yml
+          no-fail-on-empty-changeset: '1'
+          parameter-overrides: 'KeyName=${{ github.event.inputs.keyName }},EC2InstanceType=${{ github.event.inputs.instanceType }}'
+
+      - name: Run playbook
+        uses: dawidd6/action-ansible-playbook@v2
+        with:
+          playbook: deploy-playground-playbook.yml
+          directory: devops/aws
+          requirements: requirements.yml
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          inventory: |
+            [all]
+            ${{ steps.deploy_stack.outputs.PublicIp }}
+          options: |
+            --extra-vars "git_repo=${{ github.event.inputs.gitRepo }} \
+                          branch_name=${{ github.event.inputs.branchName }}"

.github/workflows/joystream-cli.yml

@@ -9,22 +9,22 @@ jobs:
       matrix:
         node-version: [14.x]
     steps:
-    - uses: actions/checkout@v1
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: checks
-      run: |
-        yarn install --frozen-lockfile
-        yarn workspace @joystream/types build
-        yarn workspace @joystream/content-metadata-protobuf build:ts
-        yarn workspace @joystream/cli checks --quiet
-    - name: yarn pack test
-      run: |
-        yarn workspace @joystream/cli pack --filename cli-pack-test.tgz
-        tar zxvf ./cli/cli-pack-test.tgz -C cli
-        cd ./cli/package && yarn link
+      - uses: actions/checkout@v1
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: checks
+        run: |
+          yarn install --frozen-lockfile
+          yarn workspace @joystream/types build
+          yarn workspace @joystream/metadata-protobuf build
+          yarn workspace @joystream/cli checks --quiet
+      - name: yarn pack test
+        run: |
+          yarn workspace @joystream/cli pack --filename cli-pack-test.tgz
+          tar zxvf ./cli/cli-pack-test.tgz -C cli
+          cd ./cli/package && yarn link
 
   cli_build_osx:
     name: MacOS Checks
@@ -33,19 +33,19 @@ jobs:
       matrix:
         node-version: [14.x]
     steps:
-    - uses: actions/checkout@v1
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: checks
-      run: |
-        yarn install --frozen-lockfile --network-timeout 120000
-        yarn workspace @joystream/types build
-        yarn workspace @joystream/content-metadata-protobuf build:ts
-        yarn workspace @joystream/cli checks --quiet
-    - name: yarn pack test
-      run: |
-        yarn workspace @joystream/cli pack --filename cli-pack-test.tgz
-        tar zxvf ./cli/cli-pack-test.tgz -C cli
-        cd ./cli/package && yarn link
+      - uses: actions/checkout@v1
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: checks
+        run: |
+          yarn install --frozen-lockfile --network-timeout 120000
+          yarn workspace @joystream/types build
+          yarn workspace @joystream/metadata-protobuf build
+          yarn workspace @joystream/cli checks --quiet
+      - name: yarn pack test
+        run: |
+          yarn workspace @joystream/cli pack --filename cli-pack-test.tgz
+          tar zxvf ./cli/cli-pack-test.tgz -C cli
+          cd ./cli/package && yarn link

.github/workflows/joystream-node-checks.yml

@@ -34,4 +34,4 @@ jobs:
       #     ./setup.sh
       #     yarn cargo-checks
       #     yarn cargo-build
-      #   if: env.GIT_DIFF
+      #   if: env.GIT_DIFF

.github/workflows/metadata-protobuf.yml

@@ -0,0 +1,22 @@
+name: metadata-protobuf
+on: [pull_request, push]
+
+jobs:
+  schemas_checks:
+    name: Checks
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [14.x]
+    steps:
+    - uses: actions/checkout@v1
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: test protobuf
+      run: |
+        yarn install --frozen-lockfile
+        yarn workspace @joystream/metadata-protobuf build
+        yarn workspace @joystream/metadata-protobuf checks --quiet
+        yarn workspace @joystream/metadata-protobuf test

.github/workflows/network-tests.yml

@@ -18,7 +18,8 @@ jobs:
       run: |
         yarn install --frozen-lockfile
         yarn workspace @joystream/types build
-        yarn workspace @joystream/content-metadata-protobuf build:ts
+        yarn workspace @joystream/metadata-protobuf build
+        yarn workspace @joystream/cli build
         yarn workspace network-tests checks --quiet
 
   network_build_osx:
@@ -37,5 +38,6 @@ jobs:
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
         yarn workspace @joystream/types build
-        yarn workspace @joystream/content-metadata-protobuf build:ts
+        yarn workspace @joystream/metadata-protobuf build
+        yarn workspace @joystream/cli build
         yarn workspace network-tests checks --quiet

.github/workflows/query-node.yml

@@ -18,8 +18,8 @@ jobs:
       run: |
         yarn install --frozen-lockfile
         yarn workspace @joystream/types build
-        yarn workspace @joystream/content-metadata-protobuf build:ts
-        ./query-node/build.sh
+        yarn workspace @joystream/metadata-protobuf build
+        yarn workspace query-node-root build
         yarn workspace query-node-mappings checks --quiet
 
   query_node_build_osx:
@@ -38,6 +38,6 @@ jobs:
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
         yarn workspace @joystream/types build
-        yarn workspace @joystream/content-metadata-protobuf build:ts
-        ./query-node/build.sh
+        yarn workspace @joystream/metadata-protobuf build
+        yarn workspace query-node-root build
         yarn workspace query-node-mappings checks --quiet