Simplify finding handling, add finding fingerprints

lib/mix/tasks/sobelow.ex

@@ -90,6 +90,8 @@ defmodule Mix.Tasks.Sobelow do
     private: :boolean,
     diff: :string,
     skip: :boolean,
+    mark_skip_all: :boolean,
+    clear_skip: :boolean,
     router: :string,
     exit: :string,
     format: :string,
@@ -126,8 +128,8 @@ defmodule Mix.Tasks.Sobelow do
         opts
       end
 
-    {verbose, diff, details, private, skip, router, exit_on, format, ignored, ignored_files,
-     all_details, out, threshold} = get_opts(opts, root, conf_file?)
+    {verbose, diff, details, private, skip, mark_skip_all, clear_skip, router, exit_on, format,
+     ignored, ignored_files, all_details, out, threshold} = get_opts(opts, root, conf_file?)
 
     set_env(:verbose, verbose)
 
@@ -140,6 +142,8 @@ defmodule Mix.Tasks.Sobelow do
     set_env(:details, details)
     set_env(:private, private)
     set_env(:skip, skip)
+    set_env(:mark_skip_all, mark_skip_all)
+    set_env(:clear_skip, clear_skip)
     set_env(:router, router)
     set_env(:exit_on, exit_on)
     set_env(:format, format)
@@ -191,6 +195,8 @@ defmodule Mix.Tasks.Sobelow do
     private = Keyword.get(opts, :private, false)
     diff = Keyword.get(opts, :diff, false)
     skip = Keyword.get(opts, :skip, false)
+    mark_skip_all = Keyword.get(opts, :mark_skip_all, false)
+    clear_skip = Keyword.get(opts, :clear_skip, false)
     router = Keyword.get(opts, :router)
     out = Keyword.get(opts, :out)
 
@@ -236,8 +242,8 @@ defmodule Mix.Tasks.Sobelow do
         _ -> :low
       end
 
-    {verbose, diff, details, private, skip, router, exit_on, format, ignored, ignored_files,
-     all_details, out, threshold}
+    {verbose, diff, details, private, skip, mark_skip_all, clear_skip, router, exit_on, format,
+     ignored, ignored_files, all_details, out, threshold}
   end
 
   # Future updates will include format hinting based on the outfile name. Additional output

lib/sobelow.ex

@@ -20,8 +20,10 @@ defmodule Sobelow do
   alias Sobelow.Config
   alias Sobelow.Parse
   alias Sobelow.Vuln
+  alias Sobelow.Finding
   alias Sobelow.FindingLog
   alias Sobelow.MetaLog
+  alias Sobelow.Fingerprint
   alias Mix.Shell.IO, as: MixIO
 
   def run() do
@@ -66,10 +68,9 @@ defmodule Sobelow do
 
     if Enum.empty?(routers), do: no_router()
 
-    FindingLog.start_link()
-    MetaLog.start_link()
+    init_state(project_root, template_meta_files)
 
-    MetaLog.add_templates(template_meta_files)
+    if get_env(:clear_skip), do: clear_skip(project_root)
 
     # This is where the core testing-pipeline starts.
     #
@@ -98,8 +99,6 @@ defmodule Sobelow do
       |> Enum.each(&get_fun_vulns(&1, meta_file, "", allowed))
     end)
 
-    template_meta_files = MetaLog.get_templates()
-
     Enum.each(template_meta_files, fn {_, meta_file} ->
       if Sobelow.XSS in allowed, do: Sobelow.XSS.get_template_vulns(meta_file)
     end)
@@ -114,9 +113,19 @@ defmodule Sobelow do
       IO.puts(:stderr, "... SCAN COMPLETE ...\n")
     end
 
+    if get_env(:mark_skip_all), do: mark_skip_all(project_root)
+
     exit_with_status()
   end
 
+  defp init_state(project_root, template_meta_files) do
+    FindingLog.start_link()
+    MetaLog.start_link()
+    Fingerprint.start_link()
+    load_ignored_fingerprints(project_root)
+    MetaLog.add_templates(template_meta_files)
+  end
+
   defp print_output() do
     details =
       case format() do
@@ -183,12 +192,22 @@ defmodule Sobelow do
     end
   end
 
-  def log_finding(finding, severity) do
-    if Sobelow.meets_threshold?(severity) do
-      FindingLog.add(finding, severity)
+  def log_finding(%Finding{} = finding) do
+    log_finding(finding.type, finding)
+  end
+
+  def log_finding(details, %Finding{} = finding) do
+    if loggable?(finding.fingerprint, finding.confidence) do
+      Fingerprint.put(finding.fingerprint)
+      FindingLog.add(details, finding.confidence)
     end
   end
 
+  def loggable?(fingerprint, severity) do
+    !(get_env(:skip) && Fingerprint.member?(fingerprint)) &&
+      meets_threshold?(severity)
+  end
+
   def all_details() do
     @submodules
     |> Enum.each(&apply(&1, :details, []))
@@ -425,17 +444,68 @@ defmodule Sobelow do
     System.halt(0)
   end
 
+  defp clear_skip(project_root) do
+    cfile = project_root <> ".sobelow"
+
+    if File.exists?(cfile) do
+      {:ok, iofile} = :file.open(cfile, [:read, :write])
+      {:ok, timestamp} = :file.read_line(iofile)
+      :file.close(iofile)
+
+      :file.write_file(cfile, timestamp)
+    end
+
+    System.halt(0)
+  end
+
+  defp mark_skip_all(project_root) do
+    cfile = project_root <> ".sobelow"
+
+    if File.exists?(cfile) do
+      {:ok, iofile} = :file.open(cfile, [:append])
+      fingerprints = Fingerprint.new_skips() |> Enum.join("\n")
+      :file.write(iofile, ["\n", fingerprints])
+      :file.close(iofile)
+    end
+  end
+
+  defp load_ignored_fingerprints(project_root) do
+    cfile = project_root <> ".sobelow"
+
+    if File.exists?(cfile) do
+      {:ok, iofile} = :file.open(cfile, [:read])
+
+      # Skip timestamp
+      :file.read_line(iofile)
+
+      :file.read_line(iofile) |> load_ignored_fingerprints(iofile)
+      :file.close(iofile)
+    end
+  end
+
+  defp load_ignored_fingerprints({:ok, fingerprint}, iofile) do
+    to_string(fingerprint) |> String.trim() |> Fingerprint.put_ignore()
+    :file.read_line(iofile) |> load_ignored_fingerprints(iofile)
+  end
+
+  defp load_ignored_fingerprints(:eof, _), do: nil
+  defp load_ignored_fingerprints(_, _), do: nil
+
   defp version_check(project_root) do
     cfile = project_root <> ".sobelow"
     time = DateTime.utc_now() |> DateTime.to_unix()
 
     if File.exists?(cfile) do
+      {:ok, iofile} = :file.open(cfile, [:read])
+
       {timestamp, _} =
-        case File.read!(cfile) do
-          "sobelow-" <> timestamp -> Integer.parse(timestamp)
+        case :file.read_line(iofile) do
+          {:ok, 'sobelow-' ++ timestamp} -> to_string(timestamp) |> Integer.parse()
           _ -> file_error()
         end
 
+      :file.close(iofile)
+
       if time - 12 * 60 * 60 > timestamp do
         maybe_prompt_update(time, cfile)
       end
@@ -487,7 +557,9 @@ defmodule Sobelow do
     end
 
     timestamp = "sobelow-" <> to_string(time)
-    File.write(cfile, timestamp)
+    {:ok, iofile} = :file.open(cfile, [:write, :read])
+    :ok = :file.pwrite(iofile, 0, timestamp)
+    :ok = :file.close(iofile)
   end
 
   def get_mod(mod_string) do

lib/sobelow/config/csp.ex

@@ -104,6 +104,7 @@ defmodule Sobelow.Config.CSP do
   defp maybe_add_finding(_, _), do: nil
 
   defp add_finding(%Finding{} = finding) do
+    finding = Finding.fetch_fingerprint(finding)
     file_header = "File: #{finding.filename}"
     pipeline_header = "Pipeline: #{finding.fun_name}"
     line_header = "Line: #{finding.vuln_line_no}"
@@ -113,25 +114,26 @@ defmodule Sobelow.Config.CSP do
         json_finding = [
           type: finding.type,
           file: finding.filename,
+          fingerprint: finding.fingerprint,
           pipeline: finding.fun_name,
           line: finding.vuln_line_no
         ]
 
-        Sobelow.log_finding(json_finding, finding.confidence)
+        Sobelow.log_finding(json_finding, finding)
 
       "txt" ->
-        Sobelow.log_finding(finding.type, finding.confidence)
+        Sobelow.log_finding(finding)
 
         Print.print_custom_finding_metadata(
           finding,
           [file_header, pipeline_header, line_header]
         )
 
       "compact" ->
-        Print.log_compact_finding(finding.type, finding.confidence)
+        Print.log_compact_finding(finding)
 
       _ ->
-        Sobelow.log_finding(finding.type, finding.confidence)
+        Sobelow.log_finding(finding)
     end
   end
 end

lib/sobelow/config/csrf.ex

@@ -46,6 +46,7 @@ defmodule Sobelow.Config.CSRF do
   end
 
   defp add_finding(%Finding{} = finding) do
+    finding = Finding.fetch_fingerprint(finding)
     file_header = "File: #{finding.filename}"
     pipeline_header = "Pipeline: #{finding.fun_name}"
     line_header = "Line: #{finding.vuln_line_no}"
@@ -55,25 +56,26 @@ defmodule Sobelow.Config.CSRF do
         json_finding = [
           type: finding.type,
           file: finding.filename,
+          fingerprint: finding.fingerprint,
           pipeline: finding.fun_name,
           line: finding.vuln_line_no
         ]
 
-        Sobelow.log_finding(json_finding, :high)
+        Sobelow.log_finding(json_finding, finding)
 
       "txt" ->
-        Sobelow.log_finding(finding.type, :high)
+        Sobelow.log_finding(finding)
 
         Print.print_custom_finding_metadata(
           finding,
           [file_header, pipeline_header, line_header]
         )
 
       "compact" ->
-        Print.log_compact_finding(finding.type, :high)
+        Print.log_compact_finding(finding)
 
       _ ->
-        Sobelow.log_finding(finding.type, :high)
+        Sobelow.log_finding(finding)
     end
   end
 end

lib/sobelow/config/csrf_route.ex

@@ -37,6 +37,8 @@ defmodule Sobelow.Config.CSRFRoute do
     |> Parse.get_top_level_funs_of_type(:scope)
     |> combine_scopes()
     |> Stream.flat_map(&route_findings(&1, finding))
+    # Sort for deterministic txt-format output
+    |> Enum.sort()
     |> Enum.each(&add_finding/1)
   end
 
@@ -73,6 +75,7 @@ defmodule Sobelow.Config.CSRFRoute do
   defp put_finding_details(_, acc, _), do: acc
 
   defp add_finding(%Finding{} = finding) do
+    finding = Finding.fetch_fingerprint(finding)
     file_header = "File: #{finding.filename}"
     action_header = "Action: #{finding.fun_name}"
     line_header = "Line: #{finding.vuln_line_no}"
@@ -82,25 +85,26 @@ defmodule Sobelow.Config.CSRFRoute do
         json_finding = [
           type: finding.type,
           file: finding.filename,
+          fingerprint: finding.fingerprint,
           route: finding.fun_name,
           line: finding.vuln_line_no
         ]
 
-        Sobelow.log_finding(json_finding, finding.confidence)
+        Sobelow.log_finding(json_finding, finding)
 
       "txt" ->
-        Sobelow.log_finding(finding.type, finding.confidence)
+        Sobelow.log_finding(finding)
 
         Print.print_custom_finding_metadata(
           finding,
           [file_header, action_header, line_header]
         )
 
       "compact" ->
-        Print.log_compact_finding(finding.type, finding.confidence)
+        Print.log_compact_finding(finding)
 
       _ ->
-        Sobelow.log_finding(finding.type, finding.confidence)
+        Sobelow.log_finding(finding)
     end
   end
 

lib/sobelow/config/cswh.ex

@@ -51,12 +51,14 @@ defmodule Sobelow.Config.CSWH do
   defp add_finding({true, confidence}, socket, endpoint) do
     finding = Finding.init(@finding_type, Utils.normalize_path(endpoint), confidence)
 
-    finding = %{
-      finding
-      | vuln_source: :highlight_all,
-        vuln_line_no: Parse.get_fun_line(socket),
-        fun_source: socket
-    }
+    finding =
+      %{
+        finding
+        | vuln_source: :highlight_all,
+          vuln_line_no: Parse.get_fun_line(socket),
+          fun_source: socket
+      }
+      |> Finding.fetch_fingerprint()
 
     file_header = "File: #{finding.filename}"
     line_header = "Line: #{finding.vuln_line_no}"
@@ -66,24 +68,25 @@ defmodule Sobelow.Config.CSWH do
         json_finding = [
           type: finding.type,
           file: finding.filename,
+          fingerprint: finding.fingerprint,
           line: finding.vuln_line_no
         ]
 
-        Sobelow.log_finding(json_finding, finding.confidence)
+        Sobelow.log_finding(json_finding, finding)
 
       "txt" ->
-        Sobelow.log_finding(finding.type, finding.confidence)
+        Sobelow.log_finding(finding)
 
         Print.print_custom_finding_metadata(
           finding,
           [file_header, line_header]
         )
 
       "compact" ->
-        Print.log_compact_finding(finding.type, finding.confidence)
+        Print.log_compact_finding(finding)
 
       _ ->
-        Sobelow.log_finding(finding.type, finding.confidence)
+        Sobelow.log_finding(finding)
     end
   end
 end