92% price accuracy

src/extraction/fathom/index.js

@@ -32,8 +32,22 @@ const FEATURE_DEFAULTS = {
 const PRODUCT_FEATURES = {
   image: {
     ...FEATURE_DEFAULTS,
+    /**
+     * @return the URL of an image resource
+     */
     getValueFromElement(element) {
-      return element.src;
+      /**
+       * Given a CSS url() declaration 'url("http://foo")', return 'http://foo'.
+       */
+      function urlFromCssDeclaration(declaration) {
+        return declaration.substring(5, declaration.length - 2);
+      }
+      if (element.tagName === 'IMG') {
+        return element.src;
+      }
+      // The other thing the ruleset can return is an arbitrary element with
+      // a CSS background image.
+      return urlFromCssDeclaration(getComputedStyle(element)['background-image']);
     },
   },
   title: {

src/extraction/fathom/ruleset_factory.js

@@ -38,9 +38,7 @@ export default class RulesetFactory {
     ] = coefficients;
   }
 
-  /**
-   * Scores fnode in direct proportion to its size
-   */
+  /** Scores fnode in direct proportion to its size */
   isBig(fnode) {
     const domRect = fnode.element.getBoundingClientRect();
     const area = domRect.width * domRect.height;
@@ -50,74 +48,79 @@ export default class RulesetFactory {
     // (though we should have distinct penalties for that sort of thing if we
     // care). More importantly, clamp the upper bound of the score so we don't
     // overcome other bonuses and penalties.
-    return trapezoid(area, 80 ** 2, 1000 ** 2) ** this.bigImageCoeff;
+    return linearScale(area, 80 ** 2, 1000 ** 2) ** this.bigImageCoeff;
   }
 
   /** Return whether the computed font size of an element is big. */
   fontIsBig(fnode) {
     const size = parseInt(getComputedStyle(fnode.element).fontSize, 10);
-    return trapezoid(size, 14, 50) ** this.bigFontCoeff;
+    return linearScale(size, 14, 50) ** this.bigFontCoeff;
   }
 
-  /**
-   * Scores fnode with a '$' in its innerText
-   */
+  /** Scores fnode with a '$' in its innerText */
   hasDollarSign(fnode) {
     return (fnode.element.innerText.includes('$') ? ONEISH : ZEROISH) ** this.hasDollarSignCoeff;
   }
 
-  /** Return whether some substring is within a given string, case insensitively. */
-  doesContain(haystack, needle) {
+  /**
+   * Return whether some substring is within a given string, case
+   * insensitively.
+   */
+  caselessIncludes(haystack, needle) {
     return haystack.toLowerCase().includes(needle);
   }
 
-  /** Return a weighted confidence of whether a substring is within a given
-   *  string, case insensitively.
+  /**
+   * Return a weighted confidence of whether a substring is within a given
+   * string, case insensitively.
    */
-  contains(haystack, needle, coeff) {
-    return (this.doesContain(haystack, needle) ? ONEISH : ZEROISH) ** coeff;
+  weightedIncludes(haystack, needle, coeff) {
+    return (this.caselessIncludes(haystack, needle) ? ONEISH : ZEROISH) ** coeff;
   }
 
   /**
-   * Scores fnode with 'price' in its id
+   * Punish elements with "background" in their ID. Do nothing to those without.
    */
+  hasBackgroundInID(fnode) {
+    return this.caselessIncludes(fnode.element.id, 'background') ? (ZEROISH ** this.backgroundIdImageCoeff) : 1;
+  }
+
+  /** Scores fnode with 'price' in its id */
   hasPriceInID(fnode) {
-    return this.contains(fnode.element.id, 'price', this.hasPriceInIDCoeff);
+    return this.weightedIncludes(fnode.element.id, 'price', this.hasPriceInIDCoeff);
   }
 
   hasPriceInParentID(fnode) {
-    return this.contains(fnode.element.parentElement.id, 'price', this.hasPriceInParentIDCoeff);
+    return this.weightedIncludes(fnode.element.parentElement.id, 'price', this.hasPriceInParentIDCoeff);
   }
 
   /** Scores fnode with 'price' in its class name */
   hasPriceInClassName(fnode) {
-    return this.contains(fnode.element.className, 'price', this.hasPriceInClassNameCoeff);
+    return this.weightedIncludes(fnode.element.className, 'price', this.hasPriceInClassNameCoeff);
   }
 
-  /** Scores fnode with 'price' in its class name */
+  /** Scores fnode with 'price' in its parent's class name */
   hasPriceInParentClassName(fnode) {
-    return this.contains(fnode.element.parentElement.className, 'price', this.hasPriceInParentClassNameCoeff);
+    return this.weightedIncludes(fnode.element.parentElement.className, 'price', this.hasPriceInParentClassNameCoeff);
   }
 
-  /**
-   * Scores fnode by its vertical location relative to the fold
-   */
+  /** Scores fnode by its vertical location relative to the fold */
   isAboveTheFold(fnode, featureCoeff) {
     const viewportHeight = 950;
     const imageTop = fnode.element.getBoundingClientRect().top;
 
     // Stop giving additional bonus for anything closer than 200px to the top
     // of the viewport. Those are probably usually headers.
-    return trapezoid(imageTop, viewportHeight * 2, 200) ** featureCoeff;
+    return linearScale(imageTop, viewportHeight * 2, 200) ** featureCoeff;
   }
 
   /**
    * Return whether the centerpoint of the element is near that of the highest-
    * scoring image.
    */
   isNearImage(fnode) {
-    const image = this.getHighestScoringImage(fnode);
-    return trapezoid(euclidean(fnode, image), 1000, 0) ** this.isNearImageCoeff;
+    const imageFnode = this.getHighestScoringImage(fnode);
+    return linearScale(euclidean(fnode, imageFnode), 1000, 0) ** this.isNearImageCoeff;
   }
 
   /**
@@ -128,8 +131,8 @@ export default class RulesetFactory {
    * bottom" one.
    */
   isNearImageTopOrBottom(fnode) {
-    const image = this.getHighestScoringImage(fnode).element;
-    const imageRect = image.getBoundingClientRect();
+    const imageElement = this.getHighestScoringImage(fnode).element;
+    const imageRect = imageElement.getBoundingClientRect();
     const nodeRect = fnode.element.getBoundingClientRect();
 
     // Should cover title above image and title in a column next to image.
@@ -141,7 +144,7 @@ export default class RulesetFactory {
     const bottomDistance = Math.abs(imageRect.bottom - nodeRect.top);
 
     const shortestDistance = Math.min(topDistance, bottomDistance);
-    return trapezoid(shortestDistance, 200, 0) ** this.isNearbyImageYAxisTitleCoeff;
+    return linearScale(shortestDistance, 200, 0) ** this.isNearbyImageYAxisTitleCoeff;
   }
 
   /**
@@ -158,9 +161,7 @@ export default class RulesetFactory {
     return (regExp.test(text) ? ONEISH : ZEROISH) ** this.hasPriceishPatternCoeff;
   }
 
-  /**
-   * Checks to see if a 'priceish' fnode is eligible for scoring
-   */
+  /** Checks to see if a 'priceish' fnode is eligible for scoring */
   isEligiblePrice(fnode) {
     return (
       this.isVisible(fnode)
@@ -169,9 +170,7 @@ export default class RulesetFactory {
     );
   }
 
-  /**
-   * Checks to see if a 'titleish' fnode is eligible for scoring
-   */
+  /** Checks to see if a 'titleish' fnode is eligible for scoring */
   isEligibleTitle(fnode) {
     return (
       this.isVisible(fnode)
@@ -237,7 +236,9 @@ export default class RulesetFactory {
 
   hasBackgroundImage(fnode) {
     const bgImage = getComputedStyle(fnode.element)['background-image'];
-    return !!bgImage && bgImage !== 'none';
+    return bgImage && bgImage.startsWith('url("') && bgImage.endsWith('")');
+    // The app doesn't yet know how to deal with non-url() types of background
+    // images.
   }
 
   /**
@@ -249,6 +250,11 @@ export default class RulesetFactory {
     return (rect.width > rect.height) ? (rect.width / rect.height) : (rect.height / rect.width);
   }
 
+  /** Give a bonus for elements that have a non-extreme aspect ratio. */
+  hasSquareAspectRatio(fnode) {
+    return linearScale(this.aspectRatio(fnode.element), 10, 5) ** this.extremeAspectCoeff;
+  }
+
   /**
   * Using coefficients passed into the constructor method, returns a weighted
   * ruleset used to score elements in an HTML document.
@@ -267,13 +273,14 @@ export default class RulesetFactory {
       rule(type('imageish'), score(fnode => this.isAboveTheFold(fnode, this.isAboveTheFoldImageCoeff))),
       // better score for larger images
       rule(type('imageish'), score(this.isBig.bind(this))),
-      // punishment for extreme aspect ratios, to filter out banners or nav elements
-      rule(type('imageish'), score(fnode => trapezoid(this.aspectRatio(fnode.element), 10, 5)
-                                            ** this.extremeAspectCoeff)),
+      // bonus for non-extreme aspect ratios, to filter out banners or nav elements
+      // TODO: Meant to make this a penalty, but it turns out to work as is.
+      // Try as a penalty.
+      rule(type('imageish'), score(this.hasSquareAspectRatio.bind(this))),
       // no background images, even ones that have reasonable aspect ratios
       // TODO: If necessary, also look at parents. I've seen them say
       // "background" in their IDs as well.
-      rule(type('imageish'), score(fnode => (this.doesContain(fnode.element.id, 'background') ? (ZEROISH ** this.backgroundIdImageCoeff) : 1))),
+      rule(type('imageish'), score(this.hasBackgroundInID.bind(this))),
       // return image element(s) with max score
       rule(type('imageish').max(), out('image')),
 
@@ -337,12 +344,12 @@ export default class RulesetFactory {
 /**
  * Scale a number to the range [ZEROISH, ONEISH].
  *
- * For a rising trapezoid, the result is ZEROISH until the input reaches
+ * For a rising line, the result is ZEROISH until the input reaches
  * zeroAt, then increases linearly until oneAt, at which it becomes ONEISH. To
- * make a falling trapezoid, where the result is ONEISH to the left and ZEROISH
+ * make a falling line, where the result is ONEISH to the left and ZEROISH
  * to the right, use a zeroAt greater than oneAt.
  */
-function trapezoid(number, zeroAt, oneAt) {
+function linearScale(number, zeroAt, oneAt) {
   const isRising = zeroAt < oneAt;
   if (isRising) {
     if (number <= zeroAt) {