Land rapid7#16668, HTTP Crawler: don't expect page object for msg

Author: gwillcox-r7

lib/msf/core/auxiliary/http_crawler.rb

@@ -243,7 +243,8 @@ def crawl_target(t)
   # Specific module implementations should redefine this method
   # with whatever is meaningful to them.
   def crawler_process_page(t, page, cnt)
-    msg = "[#{"%.5d" % cnt}/#{"%.5d" % max_page_count}]    #{page.code || "ERR"} - #{@current_site.vhost} - #{page.url}"
+    return if page.nil? # Skip over pages that don't contain any info aka page is nil. We can't process these types of pages since there is no data to process.
+    msg = "[#{"%.5d" % cnt}/#{"%.5d" % max_page_count}]    #{page ? page.code || "ERR" : "ERR"} - #{@current_site.vhost} - #{page.url}"
     case page.code
       when 301,302
         if page.headers and page.headers["location"]

modules/auxiliary/scanner/http/crawler.rb

@@ -63,7 +63,8 @@ def for_each_page( &block )
   # - The occurence of any form (web.form :path, :type (get|post|path_info), :params)
   #
   def crawler_process_page(t, page, cnt)
-    msg = "[#{"%.5d" % cnt}/#{"%.5d" % max_page_count}]    #{page.code || "ERR"} - #{t[:vhost]} - #{page.url}"
+    return if page.nil? # Skip over pages that don't contain any info aka page is nil. We can't process these types of pages since there is no data to process.
+    msg = "[#{"%.5d" % cnt}/#{"%.5d" % max_page_count}]    #{page ? page.code || "ERR" : "ERR"} - #{t[:vhost]} - #{page.url}"
     if page.error
       print_error("Error accessing page #{page.error.to_s}")
       elog(page.error)