2.10b: Many updates, see changelog

ChangeLog

@@ -1,3 +1,44 @@
+Version 2.10b:
+  - Updated HTML tags and attributes that are checked for URL XSS
+    injections to also include a few HTML5 specific ones
+
+  - Updated test and description for semi-colon injection in HTML meta
+    refresh tags (this is IE6 specific)
+
+  - Relaxed HTML parsing a bit to allow spaces between HTML tag attributes
+    and their values (e.g. "foo =bar").
+
+  - Major update of LFI tests by adding more dynamic tests (double
+    encoding, dynamic amount of ../'s for web.xml). The total amount of
+    tests for this vulnerability is now 40 per injection point.
+
+  - The RFI test is now a separate test and no longer requires special
+    compile options. The default RFI URL and it's payload check are
+    still defined in src/config.h.
+
+  - Using the --flush-to-disk flag will cause requests and responses
+    to be flushed to disk which reduces the memory footprint. (especially
+    noticable in large scans)
+
+  - Fixed a bug where in some conditions (e.g. a page looks similar to
+    another) links were not scraped from responses which lead to links
+    to be missed (thanks to Anurag Chaurasia for reporting)
+
+  - Added configuration file support with the --config flag. In
+    config/example.conf you can find flags and examples.
+
+  - Several signature keyword enhancements have been made. Most
+    significant are the "header" keyword, which allows header matching
+    and the "depend" keyword which allows signature chaining.
+
+  - Fixed basic authentication which was broken per 2.08b. Cheers to
+    Michael Stevens for reporting.
+
+  - Fixed -k scheduling where 1:0:0 would count as a second in stead of
+    an hour (also visa versa). Cheers to Claudio Criscione for reporting.
+
+  - Small fix to compile time warnings
+
 Version 2.09b:
 
   - Fixed a crash that could be triggered during 404 fingerprint failures

Makefile

@@ -2,7 +2,8 @@
 # skipfish - Makefile
 # -------------------
 #
-# Author: Michal Zalewski <[email protected]>
+# Author: Michal Zalewski <[email protected]>,
+#         Niels Heinen <[email protected]>
 #
 # Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved.
 #
@@ -20,21 +21,22 @@
 #
 
 PROGNAME   = skipfish
-VERSION    = 2.09b
+VERSION    = 2.10b
 
 SRCDIR     = src
 SFILES     = http_client.c database.c crawler.c analysis.c report.c \
-             checks.c signatures.c auth.c
+             checks.c signatures.c auth.c options.c
 IFILES     = alloc-inl.h string-inl.h debug.h types.h http_client.h \
              database.h crawler.h analysis.h config.h report.h \
-             checks.h signatures.h auth.h
+             checks.h signatures.h auth.h options.h
 
 OBJFILES   = $(patsubst %,$(SRCDIR)/%,$(SFILES))
 INCFILES   = $(patsubst %,$(SRCDIR)/%,$(IFILES))
 
 CFLAGS_GEN = -Wall -funsigned-char -g -ggdb -I/usr/local/include/ \
              -I/opt/local/include/ $(CFLAGS) -DVERSION=\"$(VERSION)\"
-CFLAGS_DBG = -DLOG_STDERR=1 -DDEBUG_ALLOCATOR=1 $(CFLAGS_GEN)
+CFLAGS_DBG = -DLOG_STDERR=1 -DDEBUG_ALLOCATOR=1 \
+             $(CFLAGS_GEN)
 CFLAGS_OPT =  -O3 -Wno-format $(CFLAGS_GEN)
 
 LDFLAGS   += -L/usr/local/lib/ -L/opt/local/lib
@@ -55,6 +57,12 @@ $(PROGNAME): $(SRCDIR)/$(PROGNAME).c $(OBJFILES) $(INCFILES)
 debug: $(SRCDIR)/$(PROGNAME).c $(OBJFILES) $(INCFILES)
 	$(CC) $(LDFLAGS) $(SRCDIR)/$(PROGNAME).c -o $(PROGNAME) \
         $(CFLAGS_DBG) $(OBJFILES) $(LIBS)
+	@echo
+	@echo "The debug build prints runtime information to stderr. You"
+	@echo "probably want to redirect this output to a file. like:"
+	@echo
+	@echo " $ ./skipfish [.option.] 2> debug.log"
+	@echo
 
 clean:
 	rm -f $(PROGNAME) *.exe *.o *~ a.out core core.[1-9][0-9]* *.stackdump \
@@ -66,6 +74,7 @@ same_test: $(SRCDIR)/same_test.c $(OBJFILES) $(INCFILES)
 	      $(LIBS)
 
 publish: clean
-	cd ..; rm -rf skipfish-$(VERSION); cp -pr skipfish skipfish-$(VERSION); \
-	  tar cfvz ~/www/skipfish.tgz skipfish-$(VERSION)
+	cd ..; rm -rf skipfish-$(VERSION); \
+	cp -pr skipfish-release skipfish-$(VERSION); \
+	tar cfvz ~/www/skipfish.tgz skipfish-$(VERSION); \
 	chmod 644 ~/www/skipfish.tgz

README

@@ -238,7 +238,7 @@ behavior there.
 To compile it, simply unpack the archive and try make. Chances are, you will
 need to install libidn first.
 
-Next, you need to read the instructions provided in dictionaries/README-FIRST
+Next, you need to read the instructions provided in doc/dictionaries.txt
 to select the right dictionary file and configure it correctly. This step has a 
 profound impact on the quality of scan results later on, so don't skip it.
 
@@ -278,7 +278,7 @@ new or changed nodes; and blue background to all new or changed issues
 found.
 
 Some sites may require authentication for which our support is described
-in docs/authentication.txt. In most cases, you'll be wanting to use the
+in doc/authentication.txt. In most cases, you'll be wanting to use the
 form authentication method which is capable of detecting broken sessions
 in order to re-authenticate.
 
@@ -398,7 +398,7 @@ HTTP links seen, even if they have no immediate security impact. Use the -U
 option to have these logged.
 
 Dictionary management is a special topic, and - as mentioned - is covered in
-more detail in dictionaries/README-FIRST. Please read that file before
+more detail in doc/dictionaries.txt. Please read that file before
 proceeding. Some of the relevant options include -S and -W (covered earlier),
 -L to suppress auto-learning, -G to limit the keyword guess jar size, -R to
 drop old dictionary entries, and -Y to inhibit expensive $keyword.$extension
@@ -531,12 +531,8 @@ know:
 
   * Scan resume option, better runtime info.
 
-  * Option to limit document sampling or save samples directly to disk.
-
   * Standalone installation (make install) support.
 
-  * Config file support.
-
   * Scheduling and management web UI.
 
 -------------------------------------

assets/index.html

@@ -261,7 +261,7 @@
   "10402": "HTTP authentication required",
   "10403": "Server error triggered",
   "10404": "Directory listing enabled",
-  "10405": "Discovered files / directories",
+  "10405": "Hidden files / directories",
 
   "10501": "All external links",
   "10502": "External URL redirector",
@@ -304,6 +304,7 @@
   "30503": "HTTPS form submitting to a HTTP URL",
   "30601": "HTML form with no apparent XSRF protection",
   "30602": "JSON response with no apparent XSSI protection",
+  "30603": "Auth form leaks credentials via HTTP GET",
   "30701": "Incorrect caching directives (lower risk)",
   "30801": "User-controlled response prefix (BOM / plugin attacks)",
   "30901": "HTTP header injection vector",
@@ -333,6 +334,7 @@
   "50104": "Format string vector",
   "50105": "Integer overflow vector",
   "50106": "File inclusion",
+  "50107": "Remote file inclusion",
   "50201": "SQL query or similar syntax in parameters",
   "50301": "PUT request accepted",
   "50909": "Signature match detected (high risk)"

config/example.conf

@@ -0,0 +1,188 @@
+
+######################################
+## Reporting options
+##################################
+
+# Output to this directory
+output = CHANGEME
+
+# Toggle mixed content reporting
+log-mixed-content = false
+
+# Toggle logging of all external URLs
+log-external-urls = false
+
+# Enable extra cache related logging
+log-cache-mismatches = false
+
+# Turn off console statistics reporting
+#quiet = false
+
+# Increase verbosity of runtime reporting
+#verbose = false
+
+######################################
+## Crawler user agent options
+##################################
+
+# Pretend that 'domain' resolves to 'IP'
+#host = domain=IP
+
+# Specify header values that will be send with every request
+#header = headername=value
+#header = X-Scanner=skipfish
+
+# Specify which one of the pre-defined user agents to use (i|p|f).
+user-agent = i
+
+# Set cookie value and send it with every request
+#cookie = name1=value1
+#cookie = name2=value3
+
+# Reject any new cookies
+reject-cookies = false
+
+######################################
+## Authentication options
+##################################
+
+# Specify the location of the login form
+#auth-form = http://example.org/login.php
+
+# Specify the username and password that you want to authenticate
+# with. It's advised to use throw away (test) accounts.
+#auth-user = myuser
+#auth-pass = mypass
+
+# Specify the credential field names when not detected by skipfish.
+#auth-user-field = user-field-name
+#auth-pass-field = pass-field-name
+
+# The URL to test is the scan is authenticated.
+#auth-verify-url = http://example.org/show-profile.php
+
+# In some cases, you might have to specify the location to which the
+# form data has to be submitted.
+#auth-form-target
+
+# Specify credentials for basic HTTP authentication
+#auth = user:pass
+
+
+######################################
+## Crawler scope / depth options
+##################################
+
+# Maximum crawl tree depth
+max-crawl-depth = 16
+
+# Maximum children to index per node
+max-crawl-child = 512
+
+# Maximum descendants to index per branch
+max-crawl-descendants = 8192
+
+# Max total number of requests to send
+max-request-total = 100000000
+
+# Max requests per second
+#max-request-rate = 200
+
+# Node and link crawl probability
+crawl-probability = 100
+
+# Repeat probabilistic scan with given seed
+#seed = 0xXXXXXX
+
+# Only follow URLs matching 'string'
+#include-string = /want/
+
+# Exclude URLs matching 'string'
+#exclude-string = /want-not/
+
+# Crawl cross-site links to another domain
+#include-domain = scan.also.example.org
+
+# Trust, but do not crawl, another domain
+#trust-domain = .google-analytics.com
+
+# Do not parse HTML, etc, to find new links
+#no-html-parsing = false
+
+# Do not descend into 5xx locations
+skip-error-pages = false
+
+# Add new form auto-fill rule
+#form-value = field=value
+
+######################################
+## Dictionary management
+##################################
+
+# The read-only wordlist that is used for bruteforcing
+wordlist = dictionaries/medium.wl
+
+# The read-write wordlist and where learned keywords will be written
+# for future scans.
+#rw-wordlist = my-wordlist.wl
+
+# Disable extension fuzzing
+no-extension-brute = false
+
+# Disable keyword learning
+no-keyword-learning = false
+
+######################################
+## Performance options
+##################################
+
+# Max simultaneous TCP connections, global
+max-connections = 40
+
+# Max simultaneous connections, per target IP
+max-host-connections = 10
+
+# Max number of consecutive HTTP errors
+max-failed-requests = 100
+
+# Total request response timeout
+request-timeout = 20
+
+# Individual network I/O timeout
+network-timeout = 10
+
+# Timeout on idle HTTP connections
+idle-timeout = 10
+
+# Response size limit in bytes
+response-size = 400000
+
+# Do not keep binary responses for reporting
+discard-binary = true
+
+# Flush request / response data immediately to disk
+flush-to-disk = false
+
+# Stop scanning after the given duration h:m:s
+#scan-timeout = h:m:s
+
+######################################
+## Detection / inject options
+##################################
+
+# Specify the signatures file location. To disable signatures, specify /dev/null.
+signatures = signatures/signatures.conf
+
+# Enable or disable specific injection tests
+#checks-toggle
+
+# Disable all injection tests which means the scan will focus on crawling,
+# bruteforcing and passively detect security issues via signatures.
+no-injection-tests = false
+
+# Ignore this parameter in the scan
+#skip-parameter = search
+
+# Do not submit forms
+no-form-submits = false
+