1.92b: Reading starting URLs from file now supported (@ prefix).

ChangeLog

@@ -1,3 +1,8 @@
+Version 1.92b:
+--------------
+
+  - Reading starting URLs from file is now supported (@ prefix).
+
 Version 1.90b / 1.91b:
 ----------------------
 

Makefile

@@ -4,7 +4,7 @@
 #
 # Author: Michal Zalewski <[email protected]>
 #
-# Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+# Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,15 +20,15 @@
 #
 
 PROGNAME   = skipfish
-VERSION    = 1.91b
+VERSION    = 1.92b
 
 OBJFILES   = http_client.c database.c crawler.c analysis.c report.c
 INCFILES   = alloc-inl.h string-inl.h debug.h types.h http_client.h \
              database.h crawler.h analysis.h config.h report.h
 
 CFLAGS_GEN = -Wall -funsigned-char -g -ggdb -I/usr/local/include/ \
              -I/opt/local/include/ $(CFLAGS) -DVERSION=\"$(VERSION)\"
-CFLAGS_DBG = -DLOG_STDERR=1 -DDEBUG_ALLOCATOR=1 $(CFLAGS_GEN) 
+CFLAGS_DBG = -DLOG_STDERR=1 -DDEBUG_ALLOCATOR=1 $(CFLAGS_GEN)
 CFLAGS_OPT = -O3 -Wno-format $(CFLAGS_GEN)
 
 LDFLAGS   += -L/usr/local/lib/ -L/opt/local/lib

README

@@ -5,7 +5,7 @@ skipfish - web application security scanner
   http://code.google.com/p/skipfish/
 
   * Written and maintained by Michal Zalewski <[email protected]>.
-  * Copyright 2009, 2010 Google Inc, rights reserved.
+  * Copyright 2009, 2010, 2011 Google Inc, rights reserved.
   * Released under terms and conditions of the Apache License, version 2.0.
 
 --------------------
@@ -237,8 +237,11 @@ Once you have the dictionary selected, you can try:
 
 $ ./skipfish -o output_dir http://www.example.com/some/starting/path.txt
 
-Note that you can provide more than one starting URL if so desired; all of 
-them will be crawled.
+Note that you can provide more than one starting URL if so desired; all of
+them will be crawled. It is also possible to read URLs from file, using
+the following syntax:
+
+$ ./skipfish -o output_dir @../path/to/url_list.txt
 
 The tool will display some helpful stats while the scan is in progress. You 
 can also switch to a list of in-flight HTTP requests by pressing return.

alloc-inl.h

@@ -7,7 +7,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

analysis.c

@@ -4,7 +4,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

analysis.h

@@ -4,7 +4,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

assets/index.html

@@ -7,7 +7,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

config.h

@@ -4,7 +4,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

crawler.c

@@ -6,7 +6,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

crawler.h

@@ -4,7 +4,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

database.c

@@ -4,7 +4,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

database.h

@@ -4,7 +4,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

debug.h

@@ -5,7 +5,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

http_client.c

@@ -4,7 +4,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

http_client.h

@@ -4,7 +4,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

report.c

@@ -4,7 +4,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

report.h

@@ -4,7 +4,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

same_test.c

@@ -4,7 +4,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

skipfish.c

@@ -4,7 +4,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -181,6 +181,54 @@ void splash_screen(void) {
 #endif /* SHOW_SPLASH */
 
 
+/* Load URLs from file. */
+
+static void read_urls(u8* fn) {
+  FILE* f = fopen((char*)fn, "r");
+  u8    tmp[MAX_URL_LEN];
+  u32   loaded = 0;
+
+  if (!f) FATAL("Unable to open '%s'.", fn);
+
+  while (fgets((char*)tmp, MAX_URL_LEN, f)) {
+    struct http_request *req;
+    u8* url = tmp;
+    u32 l;
+
+    while (isspace(*url)) url++;
+
+    l = strlen((char*)url);
+    while (l && isspace(url[l-1])) l--;
+    url[l] = 0;
+
+    if (*url == '#' || !*url) continue;
+
+    req = ck_alloc(sizeof(struct http_request));
+
+    if (parse_url(url, req, NULL))
+      FATAL("Scan target '%s' in file '%s' is not a valid absolute URL.", url, fn);
+
+    if (!url_allowed_host(req))
+      APPEND_FILTER(allow_domains, num_allow_domains,
+                    __DFL_ck_strdup(req->host));
+
+    if (!url_allowed(req))
+      FATAL("URL '%s' in file '%s' explicitly excluded by -I / -X rules.",
+            url, fn);
+
+    maybe_add_pivot(req, NULL, 2);
+    destroy_request(req);
+    loaded++;
+
+  }
+
+  fclose(f);
+
+  if (!loaded) FATAL("No valid URLs found in '%s'.", fn);
+
+}
+
+
 /* Main entry point */
 
 int main(int argc, char** argv) {
@@ -458,14 +506,23 @@ int main(int argc, char** argv) {
 
   load_keywords((u8*)wordlist, purge_age);
 
-  /* Schedule all URLs in the command line for scanning */
+  /* Schedule all URLs in the command line for scanning. */
 
   while (optind < argc) {
 
-    struct http_request *req = ck_alloc(sizeof(struct http_request));
+    struct http_request *req;
+
+    /* Support @ notation for reading URL lists from files. */
+
+    if (argv[optind][0] == '@') {
+      read_urls((u8*)argv[optind++] + 1);
+      continue;
+    }
+
+    req = ck_alloc(sizeof(struct http_request));
 
     if (parse_url((u8*)argv[optind], req, NULL))
-      FATAL("One of specified scan targets is not a valid absolute URL.");
+      FATAL("Scan target '%s' is not a valid absolute URL.", argv[optind]);
 
     if (!url_allowed_host(req))
       APPEND_FILTER(allow_domains, num_allow_domains,

types.h

@@ -4,7 +4,7 @@
 
    Author: Michal Zalewski <[email protected]>
 
-   Copyright 2009, 2010 by Google Inc. All Rights Reserved.
+   Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.