Add Runtime Dataflow Viewer

.github/workflows/test.yml

@@ -1,6 +1,10 @@
 name: Test
 
-on: [push]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
 
 jobs:
   publish:

package.json

@@ -5,8 +5,10 @@
   "license": "BSD-3-Clause",
   "devDependencies": {
     "@types/react-select": "^5.0.1",
-    "autoprefixer": "^10.3.6",
+    "autoprefixer": "^10.3.4",
+    "copy-webpack-plugin": "^9.0.1",
     "css-loader": "^6.3.0",
+    "eslint-plugin-prettier": "4.0.0",
     "gh-pages": "^3.2.3",
     "html-webpack-plugin": "^5.3.2",
     "husky": "^7.0.2",
@@ -25,19 +27,26 @@
     "webpack-dev-server": "^4.3.0"
   },
   "dependencies": {
+    "@reduxjs/toolkit": "^1.6.1",
+    "@tippyjs/react": "^4.2.5",
+    "@types/cytoscape": "^3.14.17",
     "@types/d3": "^7.0.0",
     "@types/react": "^17.0.26",
     "@types/react-redux": "^7.1.18",
     "@types/semver": "^7.3.8",
     "ajv": "^8.6.3",
     "ajv-formats": "^2.1.1",
+    "cytoscape": "^3.19.0",
+    "cytoscape-popper": "^2.0.0",
     "d3-array": "^3.1.0",
     "d3-geo-projection": "^4.0.0",
     "d3-scale": "^4.0.2",
+    "elkjs": "^0.7.1",
     "history": "^5.0.1",
     "json-stringify-pretty-compact": "^3.0.0",
     "lz-string": "^1.4.4",
     "monaco-editor": "^0.27.0",
+    "prettier": "^2.1.x",
     "prop-types": "^15.7.2",
     "rc-resize-observer": "^1.0.1",
     "react": "^17.0.2",
@@ -51,18 +60,19 @@
     "react-router-dom": "^5.3.0",
     "react-select": "^4.3.1",
     "react-split-pane": "^0.1.92",
-    "react-tooltip": "^4.2.21",
     "redux": "^4.1.1",
     "redux-localstorage": "^0.4.1",
     "redux-thunk": "^2.3.0",
+    "tippy.js": "^6.3.1",
     "tslib": "^2.3.1",
     "vega-cli": "5.21.0",
     "vega-datasets": "^2.2.0",
     "vega-embed": "^6.19.1",
     "vega-lite": "5.1.1",
     "vega-schema-url-parser": "^2.2.0",
     "vega-themes": "^2.10.0",
-    "vega-tooltip": "^0.27.0"
+    "vega-tooltip": "^0.27.0",
+    "vega-typings": "^0.22.1"
   },
   "lint-staged": {
     "*.{ts,tsx}": [

src/actions/editor.ts

@@ -1,6 +1,5 @@
 import * as Monaco from 'monaco-editor/esm/vs/editor/editor.api';
-import {Renderers, Spec} from 'vega';
-import {TopLevelSpec} from 'vega-lite/src/spec';
+import {Renderers, Runtime} from 'vega';
 import {Mode, View} from '../constants';
 
 export const RECEIVE_CURRENT_USER = 'RECEIVE_CURRENT_USER' as const;

src/components/input-panel/compiled-spec-header/index.tsx

@@ -5,8 +5,8 @@ import {connect} from 'react-redux';
 import {RouteComponentProps, withRouter} from 'react-router-dom';
 import {bindActionCreators, Dispatch} from 'redux';
 import * as EditorActions from '../../../actions/editor';
-import {State} from '../../../constants/default-state';
 import {COMPILEDPANE} from '../../../constants';
+import {State} from '../../../constants/default-state';
 
 const toggleStyle = {
   cursor: 'pointer',

src/components/input-panel/index.tsx

@@ -8,9 +8,9 @@ import {State} from '../../constants/default-state';
 import ConfigEditor from '../config-editor';
 import CompiledSpecDisplay from './compiled-spec-display';
 import CompiledSpecHeader from './compiled-spec-header';
+import './index.css';
 import SpecEditor from './spec-editor';
 import SpecEditorHeader from './spec-editor-header';
-import './index.css';
 
 type Props = ReturnType<typeof mapStateToProps> & ReturnType<typeof mapDispatchToProps>;
 

src/components/input-panel/spec-editor-header/index.tsx

@@ -1,12 +1,12 @@
 import * as React from 'react';
 import {connect} from 'react-redux';
-import {withRouter, RouteComponentProps} from 'react-router-dom';
+import {RouteComponentProps, withRouter} from 'react-router-dom';
 import {bindActionCreators, Dispatch} from 'redux';
 import * as EditorActions from '../../../actions/editor';
 import {SIDEPANE} from '../../../constants';
+import {State} from '../../../constants/default-state';
 import ConfigEditorHeader from '../../config-editor/config-editor-header';
 import './index.css';
-import {State} from '../../../constants/default-state';
 
 type Props = ReturnType<typeof mapStateToProps> & ReturnType<typeof mapDispatchToProps> & RouteComponentProps;
 

src/components/popup/index.tsx

@@ -0,0 +1,9 @@
+import * as React from 'react';
+
+import 'tippy.js/dist/tippy.css';
+import 'tippy.js/themes/light-border.css';
+import Tippy, {TippyProps} from '@tippyjs/react';
+
+export function Popup(props: TippyProps) {
+  return <Tippy theme="light-border" {...props} />;
+}

src/components/renderer/index.tsx

@@ -2,6 +2,8 @@ import {connect} from 'react-redux';
 import {bindActionCreators, Dispatch} from 'redux';
 import * as EditorActions from '../../actions/editor';
 import {State} from '../../constants/default-state';
+import {recordPulse} from '../../features/dataflow/pulsesSlice';
+import {setRuntime} from '../../features/dataflow/runtimeSlice';
 import Renderer from './renderer';
 
 export function mapStateToProps(state: State) {
@@ -26,6 +28,8 @@ export function mapDispatchToProps(dispatch: Dispatch<EditorActions.Action>) {
   return bindActionCreators(
     {
       setView: EditorActions.setView,
+      setRuntime: setRuntime,
+      recordPulse: recordPulse,
     },
     dispatch
   );

src/components/renderer/renderer.tsx

@@ -2,7 +2,6 @@ import * as React from 'react';
 import {Maximize} from 'react-feather';
 import {Portal} from 'react-portal';
 import {RouteComponentProps, withRouter} from 'react-router-dom';
-import ReactTooltip from 'react-tooltip';
 import * as vega from 'vega';
 import {Config as VgConfig} from 'vega';
 import {deepEqual} from 'vega-lite';
@@ -11,6 +10,7 @@ import {mapDispatchToProps, mapStateToProps} from '.';
 import {KEYCODES, Mode} from '../../constants';
 import addProjections from '../../utils/addProjections';
 import {dispatchingLogger} from '../../utils/logger';
+import {Popup} from '../popup';
 import './index.css';
 
 // Add additional projections
@@ -30,6 +30,7 @@ class Editor extends React.PureComponent<Props, State> {
     this.handleKeydown = this.handleKeydown.bind(this);
     this.onOpenPortal = this.onOpenPortal.bind(this);
     this.onClosePortal = this.onClosePortal.bind(this);
+    this.runAfter = this.runAfter.bind(this);
   }
 
   public onOpenPortal() {
@@ -125,7 +126,17 @@ class Editor extends React.PureComponent<Props, State> {
   }
 
   public initView() {
-    const {vegaSpec, vegaLiteSpec, normalizedVegaLiteSpec, config, baseURL, mode, setView, hoverEnable} = this.props;
+    const {
+      vegaSpec,
+      vegaLiteSpec,
+      normalizedVegaLiteSpec,
+      config,
+      baseURL,
+      mode,
+      setView,
+      setRuntime,
+      hoverEnable,
+    } = this.props;
 
     let runtime: vega.Runtime;
     if (mode === Mode.VegaLite) {
@@ -134,6 +145,7 @@ class Editor extends React.PureComponent<Props, State> {
     } else {
       runtime = vega.parse(vegaSpec, config as VgConfig);
     }
+
     const loader = vega.loader();
     const originalLoad = loader.load.bind(loader);
 
@@ -154,6 +166,10 @@ class Editor extends React.PureComponent<Props, State> {
 
     // Finalize previous view so that memory can be freed
     if (this.props.view) {
+      // Remove run after callback for the previous view
+      // to disable logging from an evaluate that will run after we finalize it
+      (this.props.view as any)._postrun = [];
+
       this.props.view.finalize();
     }
 
@@ -163,6 +179,7 @@ class Editor extends React.PureComponent<Props, State> {
       loader,
     });
 
+    view.runAfter(this.runAfter, true);
     (view as any).logger(dispatchingLogger);
 
     const debug = (window as any).VEGA_DEBUG;
@@ -176,10 +193,38 @@ class Editor extends React.PureComponent<Props, State> {
     } else {
       debug.vegaLiteSpec = debug.normalizedVegaLiteSpec = undefined;
     }
-
+    setRuntime(runtime);
     setView(view);
   }
 
+  private runAfter(df: any) {
+    const clock = df._clock;
+
+    // Mapping from ID to value
+    const values: Record<string, unknown> = {};
+    const toProcessContexts = new Set<any>([df._runtime]);
+    const processedContexts = new Set<any>();
+    while (toProcessContexts.size > 0) {
+      const context = toProcessContexts.values().next().value;
+      toProcessContexts.delete(context);
+      processedContexts.add(context);
+      for (const [id, operator] of Object.entries(context.nodes as Record<string, any>)) {
+        if (operator.stamp === clock) {
+          values[id] = operator.value;
+        }
+      }
+      for (const subContext of (context.subcontext as undefined | any[]) ?? []) {
+        if (!processedContexts.has(subContext)) {
+          toProcessContexts.add(subContext);
+        }
+      }
+    }
+    this.props.recordPulse(clock, values);
+
+    // Set it up to run again
+    df.runAfter(this.runAfter, true, 10);
+  }
+
   public async renderVega() {
     // Selecting chart for rendering vega
     const chart = this.state.fullscreen ? (this.refs.fchart as any) : (this.refs.chart as any);
@@ -292,21 +337,25 @@ class Editor extends React.PureComponent<Props, State> {
     return (
       <div>
         <div className="chart" style={{backgroundColor: this.props.backgroundColor}}>
-          <div
-            data-tip={`Click on "Continue Recording" to make the chart interactive`}
-            data-place="right"
-            className="chart-overlay"
-          ></div>
+          <Popup
+            content={`Click on "Continue Recording" to make the chart interactive`}
+            placement="right"
+            // Make skinnier so it fits on the right side of the chart
+            maxWidth={200}
+          >
+            <div className="chart-overlay"></div>
+          </Popup>
           <div aria-label="visualization" ref="chart" style={chartStyle} />
           {this.renderResizeHandle()}
         </div>
         <div className="fullscreen-open">
-          <Maximize
-            data-tip="Fullscreen"
-            onClick={() => {
-              this.setState({fullscreen: true}, this.onOpenPortal);
-            }}
-          />
+          <Popup content="Fullscreen" placement="left">
+            <Maximize
+              onClick={() => {
+                this.setState({fullscreen: true}, this.onOpenPortal);
+              }}
+            />
+          </Popup>
         </div>
         {this.state.fullscreen && (
           <Portal>
@@ -326,7 +375,6 @@ class Editor extends React.PureComponent<Props, State> {
             </div>
           </Portal>
         )}
-        <ReactTooltip place="left" type="dark" effect="solid" />
       </div>
     );
   }

src/components/viz-pane/debug-pane-header/index.tsx

@@ -71,6 +71,20 @@ class DebugPaneHeader extends React.PureComponent<Props> {
               Signal Viewer
             </li>
           )}
+          {error === null && (
+            <li
+              className={navItem === NAVBAR.DataflowViewer ? 'active-tab' : undefined}
+              onClick={(e) => {
+                if (debugPane) {
+                  e.stopPropagation();
+                }
+                showLogs(false);
+                toggleNavbar(NAVBAR.DataflowViewer);
+              }}
+            >
+              Dataflow Viewer
+            </li>
+          )}
         </ul>
         {debugPane ? <ChevronDown /> : <ChevronUp />}
       </div>

src/components/viz-pane/renderer.tsx

@@ -13,6 +13,7 @@ import {version as VG_VERSION} from 'vega';
 import {version as VL_VERSION} from 'vega-lite';
 import {version as TOOLTIP_VERSION} from 'vega-tooltip';
 import {COMMIT_HASH} from '../header/help-modal';
+import {DataflowViewer} from '../../features/dataflow/DataflowViewer';
 
 const defaultState = {
   header: '',
@@ -98,6 +99,8 @@ export default class VizPane extends React.PureComponent<Props, State> {
           return <DataViewer onClickHandler={(header) => this.onClickHandler(header)} />;
         case NAVBAR.SignalViewer:
           return <SignalViewer onClickHandler={(header) => this.onClickHandler(header)} />;
+        case NAVBAR.DataflowViewer:
+          return <DataflowViewer />;
         default:
           return null;
       }

src/constants/consts.ts

@@ -47,6 +47,7 @@ export const NAVBAR = {
   DataViewer: 'DataViewer',
   Logs: 'Logs',
   SignalViewer: 'SignalViewer',
+  DataflowViewer: 'DataflowViewer',
 } as const;
 
 export const SIDEPANE = {

src/constants/default-state.ts

@@ -2,19 +2,21 @@ import * as Monaco from 'monaco-editor/esm/vs/editor/editor.api';
 import {Renderers, Spec} from 'vega';
 import {Config, vega} from 'vega-embed';
 import {TopLevelSpec as VlSpec} from 'vega-lite';
+import {dataflowInitialState} from '../features/dataflow';
+
 import {
+  COMPILEDPANE,
   EDITOR_FOCUS,
+  GistPrivacy,
   LAYOUT,
   Mode,
   NAVBAR,
   SIDEPANE,
   VEGA_LITE_START_SPEC,
   View,
-  GistPrivacy,
-  COMPILEDPANE,
 } from './consts';
 
-export interface State {
+export type State = {
   isAuthenticated: boolean;
   baseURL: string;
   compiledVegaSpec: boolean;
@@ -60,7 +62,7 @@ export interface State {
   debugs: string[];
   themeName: string;
   backgroundColor: string;
-}
+} & typeof dataflowInitialState;
 
 export const DEFAULT_STATE: State = {
   baseURL: null,
@@ -108,4 +110,5 @@ export const DEFAULT_STATE: State = {
   debugs: [],
   infos: [],
   backgroundColor: '#ffffff',
+  ...dataflowInitialState,
 };