Get critical chains: pure js version (#310)

scripts/netdep_graph_json.py

@@ -46,6 +46,7 @@ def main():
   with open('clovis-trace.log') as f:
     graph = get_network_dependency_graph(json.load(f))
 
+
   output_file = "dependency-graph.json"
   with open(output_file, 'w') as f:
     json.dump(graph.deps_graph.ToJsonDict(), f)

scripts/process_artifacts.py

@@ -35,18 +35,25 @@ def create_tracing_track(trace_events):
             or event['cat'] == '__metadata']}
 
 def create_page_track(frame_load_events):
-  events = [{'frame_id': e['frameId'], 'method': e['method']}
-            for e in frame_load_events]
+  events = []
+  for event in frame_load_events:
+    clovis_event = {
+      'frame_id': event['frameId'],
+      'method': event['method'],
+      'parent_frame_id': event.get('parentFrameId', None)
+    }
+    events.append(clovis_event)
   return {'events': events}
 
 def create_request_track(raw_network_events):
   request_track = RequestTrack(None)
   for event in raw_network_events:
-    request_track.Handle(event['method'], event)
+    if event['method'] in RequestTrack._METHOD_TO_HANDLER:  # pylint: disable=protected-access
+      request_track.Handle(event['method'], event)
   return request_track.ToJsonDict()
 
 def main():
-  with open('artifacts.log', 'r') as f:
+  with open('clovisData.json', 'r') as f:
     artifacts = json.load(f)
 
   clovis_trace = {}

src/gatherers/critical-network-chains.js

@@ -0,0 +1,190 @@
+/**
+ * @license
+ * Copyright 2016 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const Gather = require('./gather');
+const log = require('../lib/log.js');
+
+/**
+ * @param {!Array<Array<T>>} arr
+ * @return {!Array<T>}
+ */
+const flatten = arr => arr.reduce((a, b) => a.concat(b), []);
+
+const includes = (arr, elm) => arr.indexOf(elm) > -1;
+
+class RequestNode {
+  /** @return string */
+  get requestId() {
+    return this.request.requestId;
+  }
+
+  /**
+   * @param {!NetworkRequest} request
+   * @param {RequestNode} parent
+   */
+  constructor(request, parent) {
+    // The children of a RequestNode are the requests initiated by it
+    this.children = [];
+    // The parent of a RequestNode is the request that initiated it
+    this.parent = parent;
+
+    this.request = request;
+  }
+
+  setParent(parentNode) {
+    this.parent = parentNode;
+  }
+
+  addChild(childNode) {
+    this.children.push(childNode);
+  }
+
+  toJSON() {
+    // Prevents circular reference so we can print nodes when needed
+    return `{
+      id: ${this.requestId},
+      parent: ${this.parent ? this.parent.requestId : null},
+      children: ${JSON.stringify(this.children.map(child => child.requestId))}
+    }`;
+  }
+
+}
+
+class CriticalNetworkChains extends Gather {
+  /**
+   * A sequential chain of RequestNodes
+   * @typedef {!Array<RequestNode>} RequestNodeChain
+   */
+
+  /**
+   * A sequential chain of WebInspector.NetworkRequest
+   * @typedef {!Array<NetworkRequest>} NetworkRequestChain
+   */
+
+  /** @return {String} */
+  get criticalPriorities() {
+    // For now, critical request == render blocking request (as decided by
+    // blink). Blink treats requests with the following priority levels as
+    // render blocking.
+    // See https://docs.google.com/document/d/1bCDuq9H1ih9iNjgzyAL0gpwNFiEP4TZS-YLRp_RuMlc
+    return ['VeryHigh', 'High', 'Medium'];
+  }
+
+  /**
+   * @param {!Array<NetworkRequest>} networkRecords
+   * @return {!Array<NetworkRequestChain>}
+  */
+  getCriticalChains(networkRecords) {
+    // Drop the first request because it's uninteresting - it's the page html
+    // and always critical. No point including it in every request
+    /** @type {!Array<NetworkRequest>} */
+    const criticalRequests = networkRecords.slice(1).filter(
+      req => includes(this.criticalPriorities, req.initialPriority()));
+
+    // Build a map of requestID -> Node.
+    /** @type {!Map<string, RequestNode} */
+    const requestIdToNodes = new Map();
+    for (let request of criticalRequests) {
+      /** @type {RequestNode} */
+      const requestNode = new RequestNode(request, null);
+      requestIdToNodes.set(requestNode.requestId, requestNode);
+    }
+
+    // Connect the parents and children
+    for (let request of criticalRequests) {
+      if (request.initiatorRequest()) {
+        /** @type {!string} */
+        const parentRequestId = request.initiatorRequest().requestId;
+        /** @type {?RequestNode} */
+        const childNode = requestIdToNodes.get(request.requestId);
+        /** @type {?RequestNode} */
+        const parentNode = requestIdToNodes.get(parentRequestId);
+        if (childNode && parentNode) {
+          // Both child and parent must be critical
+          // TODO: We may need handle redirects carefully. Investigate
+          childNode.setParent(parentNode);
+          parentNode.addChild(childNode);
+        }
+      }
+    }
+
+    /** @type {!Array<RequestNode>} */
+    const nodesList = [...requestIdToNodes.values()];
+    /** @type {!Array<RequestNode>} */
+    const orphanNodes = nodesList.filter(node => node.parent === null);
+    /** @type {!Array<Array<RequestNodeChain>>} */
+    const orphanNodeChains = orphanNodes.map(node => this._getChainsDFS(node));
+    /** @type {!Array<RequestNodeChain>} */
+    const nodeChains = flatten(orphanNodeChains);
+    /** @type {!Array<NetworkRequestChain>} */
+    const requestChains = nodeChains.map(chain => chain.map(
+      node => node.request));
+    return requestChains;
+  }
+
+  postProfiling(options, tracingData) {
+    const chains = this.getCriticalChains(tracingData.networkRecords);
+
+    if (options.flags.useNetDepGraph) {
+      // There logs are here so we can test this gatherer
+      // Will be removed when we have a way to surface them in the report
+      const nonTrivialChains = chains.filter(chain => chain.length > 1);
+
+      // Note: Approximately,
+      // startTime: time when request was dispatched
+      // responseReceivedTime: either time to first byte, or time of receiving
+      //  the end of response headers
+      // endTime: time when response loading finished
+      const debuggingData = nonTrivialChains.map(chain => ({
+        urls: chain.map(request => request._url),
+        totalRequests: chain.length,
+        times: chain.map(request => ({
+          startTime: request.startTime,
+          endTime: request.endTime,
+          responseReceivedTime: request.responseReceivedTime
+        })),
+        totalTimeBetweenBeginAndEnd:
+          (chain[chain.length - 1].endTime - chain[0].startTime) * 1000,
+        totalLoadingTime: (chain.reduce((acc, req) =>
+          acc + (req.endTime - req.responseReceivedTime), 0)) * 1000
+      })).sort((a, b) =>
+        b.totalTimeBetweenBeginAndEnd - a.totalTimeBetweenBeginAndEnd);
+      log.log('info', 'cricital chains', JSON.stringify(debuggingData));
+      log.log('info', 'lengths of critical chains', debuggingData.map(d => d.totalRequests));
+    }
+
+    this.artifacts = {CriticalNetworkChains: chains};
+  }
+
+  /**
+   * @param {!RequestNode} startNode
+   * @return {!Array<RequestNodeChain>}
+   */
+  _getChainsDFS(startNode) {
+    if (startNode.children.length === 0) {
+      return [[startNode]];
+    }
+
+    const childrenChains = flatten(startNode.children.map(child =>
+      this._getChainsDFS(child)));
+    return childrenChains.map(chain => [startNode].concat(chain));
+  }
+}
+
+module.exports = CriticalNetworkChains;

src/lib/drivers/driver.js

@@ -280,6 +280,7 @@ class DriverBase {
       this.on('Network.dataReceived', this._networkRecorder.onDataReceived);
       this.on('Network.loadingFinished', this._networkRecorder.onLoadingFinished);
       this.on('Network.loadingFailed', this._networkRecorder.onLoadingFailed);
+      this.on('Network.resourceChangedPriority', this._networkRecorder.onResourceChangedPriority);
 
       this.sendCommand('Network.enable').then(_ => {
         resolve();
@@ -295,6 +296,7 @@ class DriverBase {
       this.off('Network.dataReceived', this._networkRecorder.onDataReceived);
       this.off('Network.loadingFinished', this._networkRecorder.onLoadingFinished);
       this.off('Network.loadingFailed', this._networkRecorder.onLoadingFailed);
+      this.off('Network.resourceChangedPriority', this._networkRecorder.onResourceChangedPriority);
 
       resolve({
         networkRecords: this._networkRecords,

src/lib/frame-load-recorder.js

@@ -38,7 +38,10 @@ class FrameLoadRecorder {
   }
 
   onFrameAttached(data) {
-    this._events.push({frameId: data.frameId, method: 'Page.frameAttached'});
+    this._events.push({
+      frameId: data.frameId,
+      method: 'Page.frameAttached',
+      parentFrameId: data.parentFrameId});
   }
 
 }

src/lib/network-recorder.js

@@ -38,6 +38,7 @@ class NetworkRecorder {
     this.onDataReceived = this.onDataReceived.bind(this);
     this.onLoadingFinished = this.onLoadingFinished.bind(this);
     this.onLoadingFailed = this.onLoadingFailed.bind(this);
+    this.onResourceChangedPriority = this.onResourceChangedPriority.bind(this);
   }
 
   // There are a few differences between the debugging protocol naming and
@@ -86,6 +87,10 @@ class NetworkRecorder {
         data.timestamp, data.type, data.errorText, data.canceled,
         data.blockedReason);
   }
+
+  onResourceChangedPriority(data) {
+    this._rawEvents.push({method: 'Network.resourceChangedPriority', params: data});
+  }
 }
 
 module.exports = NetworkRecorder;

src/lib/web-inspector.js

@@ -94,8 +94,15 @@ global.NetworkAgent = {
     Origin: 'origin',
     Inspector: 'inspector',
     Other: 'other'
+  },
+  InitiatorType: {
+    Other: 'other',
+    Parser: 'parser',
+    Redirect: 'redirect',
+    Script: 'script'
   }
 };
+
 // Enum from SecurityState enum in protocol's Security domain
 global.SecurityAgent = {
   SecurityState: {
@@ -195,6 +202,24 @@ WebInspector.ConsoleMessage.MessageType = {
   Log: 'log'
 };
 
+// Mock NetworkLog
+WebInspector.NetworkLog = function(target) {
+  this._requests = new Map();
+  target.networkManager.addEventListener(
+    WebInspector.NetworkManager.EventTypes.RequestStarted, this._onRequestStarted, this);
+};
+
+WebInspector.NetworkLog.prototype = {
+  requestForURL: function(url) {
+    return this._requests.get(url) || null;
+  },
+
+  _onRequestStarted: function(event) {
+    var request = event.data;
+    this._requests.set(request.url, request);
+  }
+};
+
 // Dependencies for color parsing.
 require('chrome-devtools-frontend/front_end/common/Color.js');
 
@@ -222,7 +247,10 @@ WebInspector.NetworkManager.createWithFakeTarget = function() {
     registerNetworkDispatcher() {}
   };
 
-  return new WebInspector.NetworkManager(fakeTarget);
+  fakeTarget.networkManager = new WebInspector.NetworkManager(fakeTarget);
+  fakeTarget.networkLog = new WebInspector.NetworkLog(fakeTarget);
+
+  return fakeTarget.networkManager;
 };
 
 module.exports = WebInspector;

src/lighthouse.js

@@ -30,7 +30,8 @@ const gathererClasses = [
   require('./gatherers/html'),
   require('./gatherers/manifest'),
   require('./gatherers/accessibility'),
-  require('./gatherers/offline')
+  require('./gatherers/offline'),
+  require('./gatherers/critical-network-chains')
 ];
 
 const audits = [

src/scheduler.js

@@ -17,7 +17,6 @@
 'use strict';
 
 const fs = require('fs');
-
 const log = require('./lib/log.js');
 
 function loadPage(driver, gatherers, options) {