Implement experimental --http2 (#4402)

Author: stamparm

data/txt/sha256sums.txt

@@ -180,15 +180,15 @@ ec8d94fb704c0a40c88f5f283624cda025e2ea0e8b68722fe156c2b5676f53ac  lib/core/dicts
 93c256111dc753967169988e1289a0ea10ec77bfb8e2cbd1f6725e939bfbc235  lib/core/gui.py
 1d6e741e19e467650dce2ca84aa824d6df68ff74aedbe4afa8dbdb0193d94918  lib/core/__init__.py
 53499dc202a036289e3b2b9699d19568e794d077e16fd3a5c91771983de45451  lib/core/log.py
-eb1890d111e6187cac4cf81c3a525e95e7061607847d4f05ec23f9dba8febdcd  lib/core/optiondict.py
+bcb54f1813b3757fe717d7b4f3429fbcd08ff416af1100b716708955702e66d6  lib/core/optiondict.py
 ceea031ce1a49a20af689d750d33d057e38a7c631f008872b04f380e2de39bb9  lib/core/option.py
 81275fdbd463d89a2bfd8c00417a17a872aad74f34c18e44be79c0503e67dfa5  lib/core/patch.py
 e79df3790f16f67988e46f94b0a516d7ee725967f7698c8e17f210e4052203a7  lib/core/profiling.py
 c6a182f6b7d3b0ad6f0888ea2a4de4148f0770549038d7de8bc3267b4c6635f7  lib/core/readlineng.py
 63ae69713c6ea9abfa10e71dfab8f2dcf42432177a38d2c1e98785bf1468674c  lib/core/replication.py
 5bad5bc7115051cef7b84efa73fbafbf5e1db46eef32a445056b56cda750b66f  lib/core/revision.py
 0dcb52c9c76a4b0acf2e9038f7d8f08c14543cef3cf7032831c6c0a99376ad24  lib/core/session.py
-167941c1f7c279d31a377a80915de0cae31f06ba39bf802571a9980bb5ffbfff  lib/core/settings.py
+2511201edc299a8efca0f9f5b55423503ef5e5982c16c4938ec4b0be842abb6f  lib/core/settings.py
 a1e4f2860bffc73bbf2e5db293fa49dcb600ea35f950cda43dc953b3160ab3db  lib/core/shell.py
 841716e87b90a3b598515910841f7cf8d33bb87c24a27fba1a80e36a831cbcd7  lib/core/subprocessng.py
 9731092f195e346716929323ea3c93247b23b9b92b0f32d3fd0acc3adf9876cc  lib/core/target.py
@@ -199,7 +199,7 @@ b1071f449a66b4ceacd4b84b33a73d9e0a3197d271d72daaa406ba473a8bb625  lib/core/testi
 12cbead4e9e563b970fafb891127927445bd53bada1fac323b9cd27da551ba30  lib/core/wordlist.py
 1d6e741e19e467650dce2ca84aa824d6df68ff74aedbe4afa8dbdb0193d94918  lib/__init__.py
 a027f4c44811cb74aa367525f353706de3d3fc719e6c6162f7a61dc838acf0c2  lib/parse/banner.py
-9c7f95948cb6ee20b2b5bff7b36c23179c44303d3c8ad555247f65f12f30e0a9  lib/parse/cmdline.py
+f8d1701df33a31920e2ebf9a23fa7b6f4ccd2aff22b4ae1e14b495e51e5939fe  lib/parse/cmdline.py
 3907765df08c31f8d59350a287e826bd315a7714dc0e87496f67c8a0879c86ac  lib/parse/configfile.py
 ced03337edd5a16b56a379c9ac47775895e1053003c25f6ba5bec721b6e3aa64  lib/parse/handler.py
 3704a02dcf00b0988b101e30b2e0d48acdd20227e46d8b552e46c55d7e9bf28c  lib/parse/headers.py
@@ -211,7 +211,7 @@ cbabdde72df4bd8d6961d589f1721dd938d8f653aa6af8900a31af6e2586405d  lib/parse/site
 89417568d7f19e48d39a8a9a4227d3d2b71d1c9f61139a41b1835fb5266fcab8  lib/request/basic.py
 6139b926a3462d14ddd50acdb8575ae442b8fab089db222721535092b9af3ea1  lib/request/chunkedhandler.py
 ad661a075c6df0624747722d77ca3b1f69f36e54708e33673a33cfdef1ed5075  lib/request/comparison.py
-2dfe357dfa62f40d711e6809a93ce46d7c0478118155da4fc35ac081d4a43ec7  lib/request/connect.py
+40543c462d261c8cec1ec1f68033bd3d7b4e72688aa9eb564b2c474947a449da  lib/request/connect.py
 0649a39c5cc2fc0f4c062b100ced17e3e6934a7e578247dfc65b650edc29825e  lib/request/direct.py
 5283754cf387ce4e645ee50834ee387cde29a768aaada1a6a07c338da216c94d  lib/request/dns.py
 2dd88e1f75c0ee54c335d5d0d9199216194aa299bd8ce99dca333c2e4f9ea38b  lib/request/httpshandler.py
@@ -244,7 +244,7 @@ b781403433a2ad9a18fa9b1cc291165f04f734942268b4eba004a53afe8abe49  lib/techniques
 c09927bccdbdb9714865c9a72d2a739da745375702a935349ddb9edc1d50de70  lib/utils/api.py
 1d72a586358c5f6f0b44b48135229742d2e598d40cefbeeabcb40a1c2e0b70b2  lib/utils/brute.py
 dd0b67fc2bdf65a4c22a029b056698672a6409eff9a9e55da6250907e8995728  lib/utils/crawler.py
-41a037169ca0b595781d70d6af40e2b47c9a2732fd08378029502bbe6f522960  lib/utils/deps.py
+eac125d270256eff54e39736a423dde866bac3b2bb4c76d3cbc32fc53b3bbb99  lib/utils/deps.py
 0b83cc8657d5bea117c02facde2b1426c8fe35d9372d996c644d67575d8b755f  lib/utils/getch.py
 c2a2fa68d2c575ab35f472d50b8d52dd6fc5e1b4d6c86a06ac06365650fec321  lib/utils/har.py
 e6376fb0c3d001b6be0ef0f23e99a47734cfe3a3d271521dbe6d624d32f19953  lib/utils/hashdb.py
@@ -476,7 +476,7 @@ b3d9d0644197ecb864e899c04ee9c7cd63891ecf2a0d3c333aad563eef735294  plugins/generi
 5a473c60853f54f1a4b14d79b8237f659278fe8a6b42e935ed573bf22b6d5b2c  README.md
 8c4fd81d84598535643cf0ef1b2d350cd92977cb55287e23993b76eaa2215c30  sqlmapapi.py
 168309215af7dd5b0b71070e1770e72f1cbb29a3d8025143fb8aa0b88cd56b62  sqlmapapi.yaml
-6da15963699aa8916118f92c8838013bc02c84e4d7b9f33d971324c2ff348728  sqlmap.conf
+4037f1c78180550c1896543581c0c2423e970086bae46f175397f2b4c54b7323  sqlmap.conf
 3795c6d03bc341a0e3aef3d7990ea8c272d91a4c307e1498e850594375af39f7  sqlmap.py
 9d408612a6780f7f50a7f7887f923ff3f40be5bfa09a951c6dc273ded05b56c0  tamper/0eunion.py
 c1c2eaa7df016cc7786ccee0ae4f4f363b1dce139c61fb3e658937cb0d18fc54  tamper/apostrophemask.py

lib/core/optiondict.py

@@ -30,6 +30,7 @@
         "liveCookies": "string",
         "loadCookies": "string",
         "dropSetCookie": "boolean",
+        "http2": "boolean",
         "agent": "string",
         "mobile": "boolean",
         "randomAgent": "boolean",

lib/core/settings.py

@@ -19,7 +19,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.9.2.9"
+VERSION = "1.9.2.10"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/parse/cmdline.py

@@ -177,6 +177,9 @@ def cmdLineParser(argv=None):
         request.add_argument("--drop-set-cookie", dest="dropSetCookie", action="store_true",
             help="Ignore Set-Cookie header from response")
 
+        request.add_argument("--http2", dest="http2", action="store_true",
+            help="Use HTTP version 2 (experimental)")
+
         request.add_argument("--mobile", dest="mobile", action="store_true",
             help="Imitate smartphone through HTTP User-Agent header")
 

lib/request/connect.py

@@ -90,6 +90,7 @@ class WebSocketException(Exception):
 from lib.core.exception import SqlmapCompressionException
 from lib.core.exception import SqlmapConnectionException
 from lib.core.exception import SqlmapGenericException
+from lib.core.exception import SqlmapMissingDependence
 from lib.core.exception import SqlmapSkipTargetException
 from lib.core.exception import SqlmapSyntaxException
 from lib.core.exception import SqlmapTokenException
@@ -603,11 +604,6 @@ class _(dict):
                 if not chunked:
                     requestMsg += "\r\n"
 
-                if not multipart:
-                    threadData.lastRequestMsg = requestMsg
-
-                    logger.log(CUSTOM_LOGGING.TRAFFIC_OUT, requestMsg)
-
                 if conf.cj:
                     for cookie in conf.cj:
                         if cookie.value is None:
@@ -616,7 +612,46 @@ class _(dict):
                             for char in (r"\r", r"\n"):
                                 cookie.value = re.sub(r"(%s)([^ \t])" % char, r"\g<1>\t\g<2>", cookie.value)
 
-                conn = _urllib.request.urlopen(req)
+                if conf.http2:
+                    try:
+                        import httpx
+                        with httpx.Client(verify=False, http2=True, timeout=timeout, follow_redirects=True, cookies=conf.cj) as client:
+                            conn = client.request(method or (HTTPMETHOD.POST if post is not None else HTTPMETHOD.GET), url, headers=headers, data=post)
+                    except ImportError:
+                        raise SqlmapMissingDependence("httpx[http2] not available (e.g. 'pip%s install httpx[http2]')" % ('3' if six.PY3 else ""))
+                    else:
+                        conn.code = conn.status_code
+                        conn.msg = conn.reason_phrase
+                        conn.info = lambda c=conn: c.headers
+
+                        conn._read_buffer = conn.read()
+                        conn._read_offset = 0
+
+                        requestMsg = re.sub(" HTTP/[0-9.]+\r\n", " %s\r\n" % conn.http_version, requestMsg, count=1)
+
+                        if not multipart:
+                            threadData.lastRequestMsg = requestMsg
+
+                            logger.log(CUSTOM_LOGGING.TRAFFIC_OUT, requestMsg)
+
+                        def _read(count=None):
+                            offset = conn._read_offset
+                            if count is None:
+                                result = conn._read_buffer[offset:]
+                                conn._read_offset = len(conn._read_buffer)
+                            else:
+                                result = conn._read_buffer[offset: offset + count]
+                                conn._read_offset += len(result)
+                            return result
+
+                        conn.read = _read
+                else:
+                    if not multipart:
+                        threadData.lastRequestMsg = requestMsg
+
+                        logger.log(CUSTOM_LOGGING.TRAFFIC_OUT, requestMsg)
+
+                    conn = _urllib.request.urlopen(req)
 
                 if not kb.authHeader and getRequestHeader(req, HTTP_HEADER.AUTHORIZATION) and (conf.authType or "").lower() == AUTH_TYPE.BASIC.lower():
                     kb.authHeader = getUnicode(getRequestHeader(req, HTTP_HEADER.AUTHORIZATION))
@@ -699,7 +734,7 @@ class _(dict):
             # Explicit closing of connection object
             if conn and not conf.keepAlive:
                 try:
-                    if hasattr(conn.fp, '_sock'):
+                    if hasattr(conn, "fp") and hasattr(conn.fp, '_sock'):
                         conn.fp._sock.close()
                     conn.close()
                 except Exception as ex:

lib/utils/deps.py

@@ -94,6 +94,16 @@ def checkDependencies():
         logger.warning(warnMsg)
         missing_libraries.add('python-ntlm')
 
+    try:
+        __import__("httpx")
+        debugMsg = "'httpx[http2]' third-party library is found"
+        logger.debug(debugMsg)
+    except ImportError:
+        warnMsg = "sqlmap requires 'httpx[http2]' third-party library "
+        warnMsg += "if you plan to use HTTP version 2"
+        logger.warning(warnMsg)
+        missing_libraries.add('httpx[http2]')
+
     try:
         __import__("websocket._abnf")
         debugMsg = "'websocket-client' library is found"

sqlmap.conf

@@ -61,6 +61,10 @@ loadCookies =
 # Valid: True or False
 dropSetCookie = False
 
+# Use HTTP version 2 (experimental).
+# Valid: True or False
+http2 = False
+
 # HTTP User-Agent header value. Useful to fake the HTTP User-Agent header value
 # at each HTTP request.
 # sqlmap will also test for SQL injection on the HTTP User-Agent value.