1.89b: Save file of discovered URLS

Skipfish now saves all discovered URLs in a single file for third-party tools: pivots.txt.

ChangeLog

@@ -1,3 +1,9 @@
+Version 1.89b:
+--------------
+
+  - Skipfish now saves all discovered URLs in a single file for third-party
+    tools: pivots.txt.
+
 Version 1.88b:
 --------------
 

Makefile

@@ -20,7 +20,7 @@
 #
 
 PROGNAME   = skipfish
-VERSION    = 1.88b
+VERSION    = 1.89b
 
 OBJFILES   = http_client.c database.c crawler.c analysis.c report.c
 INCFILES   = alloc-inl.h string-inl.h debug.h types.h http_client.h \

README

@@ -250,7 +250,8 @@ your favorite browser (JavaScript must be enabled; and because of recent
 file:/// security improvements in certain browsers, you might need to access 
 results over HTTP). The index.html file is static; actual results are stored 
 as a hierarchy of JSON files, suitable for machine processing or different 
-presentation frontends if needs be.
+presentation frontends if needs be. In addition, a list of all the discovered
+URLs will be saved to a single file, pivots.txt, for easy postprocessing.
 
 A simple companion script, sfscandiff, can be used to compute a delta for
 two scans executed against the same target with the same flags. The newer

dictionaries/complete.wl

@@ -1157,6 +1157,7 @@ w 1 1 1 location
 w 1 1 1 lock
 w 1 1 1 locked
 w 1 1 1 log4j
+w 1 1 1 log4net
 w 1 1 1 logfile
 w 1 1 1 logger
 w 1 1 1 logging

dictionaries/medium.wl

@@ -26,6 +26,7 @@ e 1 1 1 htm
 e 1 1 1 html
 e 1 1 1 inc
 e 1 1 1 ini
+e 1 1 1 jar
 e 1 1 1 java
 e 1 1 1 jhtml
 e 1 1 1 js
@@ -1051,7 +1052,6 @@ w 1 1 1 j2ee
 w 1 1 1 j2me
 w 1 1 1 jacob
 w 1 1 1 jakarta
-w 1 1 1 jar
 w 1 1 1 java-plugin
 w 1 1 1 javadoc
 w 1 1 1 javascript
@@ -1134,6 +1134,7 @@ w 1 1 1 location
 w 1 1 1 lock
 w 1 1 1 locked
 w 1 1 1 log4j
+w 1 1 1 log4net
 w 1 1 1 logfile
 w 1 1 1 logger
 w 1 1 1 logging

dictionaries/minimal.wl

@@ -1114,6 +1114,7 @@ w 1 1 1 location
 w 1 1 1 lock
 w 1 1 1 locked
 w 1 1 1 log4j
+w 1 1 1 log4net
 w 1 1 1 logfile
 w 1 1 1 logger
 w 1 1 1 logging

http_client.c

@@ -720,7 +720,7 @@ u8* serialize_path(struct http_request* req, u8 with_host, u8 with_post) {
       if (POST_SUBTYPE(req->par.t[i])) {
 
       if (!got_search) {
-        ASD(" POST: ");
+        ASD(" DATA:");
         got_search = 1;
       } else ASD("&");
 

report.c

@@ -753,6 +753,63 @@ static void copy_static_code(u8* out_dir) {
 }
 
 
+/* Saves all pivots for use by third-party tools. */
+
+static void save_pivots(FILE* f, struct pivot_desc* cur) {
+
+  u32 i;
+
+  if (cur->req) {
+    u8* url = serialize_path(cur->req, 1, 1);
+
+    fprintf(f, "%s %s ", cur->req->method ? cur->req->method : (u8*)"GET",
+           js_escape(url));
+
+    ck_free(url);
+
+    switch (cur->type) {
+      case PIVOT_SERV:     fprintf(f, "type=serv"); break;
+      case PIVOT_DIR:      fprintf(f, "type=dir"); break;
+      case PIVOT_FILE:     fprintf(f, "type=file"); break;
+      case PIVOT_PATHINFO: fprintf(f, "type=pathinfo"); break;
+      case PIVOT_VALUE:    fprintf(f, "type=value"); break;
+      case PIVOT_UNKNOWN:  fprintf(f, "type=unknown"); break;
+      case PIVOT_PARAM:    fprintf(f, "type=param"); break;
+      default:             fprintf(f, "type=???");
+    }
+
+    switch (cur->linked) {
+      case 0:  fprintf(f, "linked=no"); break;
+      case 1:  fprintf(f, "linked=maybe"); break;
+      default: fprintf(f, "linked=yes");
+    }
+
+    if (cur->res)
+      fprintf(f, " dup=%u %scode=%u len=%u notes=%u\n", cur->dupe,
+             cur->missing ? "returns_404 " : "",
+             cur->res->code, cur->res->pay_len, cur->issue_cnt);
+    else
+      fprintf(f, " not_fetched\n");
+
+  }
+
+  for (i=0;i<cur->child_cnt;i++) save_pivots(f, cur->child[i]);
+
+}
+
+
+static void save_all_pivots(void) {
+  FILE *f = fopen("pivots.txt", "w");
+
+  if (!f) PFATAL("Cannot create 'pivots.txt'.");
+
+  save_pivots(f, &root_pivot);
+
+  fclose(f);
+}
+
+
+
 /* Writes report to index.html in the current directory. Will create
    subdirectories, helper files, etc. */
 
@@ -772,6 +829,9 @@ void write_report(u8* out_dir, u64 scan_time, u32 seed) {
   compute_counts(&root_pivot);
   SAY("\n");
 
+  SAY(cLGN "[+] " cNOR "Saving pivot data for third-party tools...\n");
+  save_all_pivots();
+
   SAY(cLGN "[+] " cNOR "Writing scan description...\n");
   output_scan_info(scan_time, seed);