[pull] master from sqlmapproject:master

data/xml/queries.xml

@@ -1050,7 +1050,7 @@
         <hostname query="SELECT MIN(node_name) FROM v_catalog.nodes"/>
         <table_comment query="SELECT comment FROM v_catalog.comments WHERE object_type='TABLE' AND object_schema='%s' AND object_name='%s'"/>
         <!-- NOTE: Vertica uses "projection columns" in case of column comments (e.g. testusers_super.surname) -->
-        <column_comment query="SELECT comment FROM v_catalog.comments WHERE object_type='COLUMN' AND object_schema='%s' AND object_name LIKE '%.%s'"/>
+        <column_comment query="SELECT comment FROM v_catalog.comments WHERE object_type='COLUMN' AND object_schema='%s' AND object_name LIKE '%s%%.%s'"/>
         <is_dba query="(SELECT is_super_user FROM v_catalog.users WHERE user_name=CURRENT_USER OFFSET 0 LIMIT 1)"/>
         <check_udf query="(SELECT procedure_name='%s' FROM v_catalog.user_procedures WHERE procedure_name='%s' OFFSET 0 LIMIT 1)"/>
         <users>

lib/core/common.py

@@ -4300,19 +4300,19 @@ def decodeStringEscape(value):
 
     >>> decodeStringEscape("a" + chr(92) + "tb") == "a" + chr(9) + "b"
     True
+    >>> decodeStringEscape(chr(92) + chr(0)) == chr(92) + chr(0)    # a NUL in the data must be preserved, not rewritten to a backslash
+    True
     """
 
     retVal = value
 
     if value and '\\' in value:
-        # Note: shield an escaped backslash ('\\\\') behind a marker BEFORE decoding the whitespace
-        # escapes, then restore it - otherwise decoding '\\\\' -> '\\' first turns a literal '\\n'
-        # into a newline (i.e. the round-trip with encodeStringEscape was not lossless)
-        _marker = "\x00"
-        retVal = retVal.replace("\\\\", _marker)
-        for _ in string.whitespace.replace(" ", ""):
-            retVal = retVal.replace(repr(_).strip("'"), _)
-        retVal = retVal.replace(_marker, "\\")
+        # Note: single left-to-right pass so an escaped backslash ('\\\\') shields the next char
+        # (a literal '\\n' stays '\\n', not a newline) WITHOUT a sentinel that could collide with a
+        # pre-existing byte (e.g. a NUL in the data) and get rewritten on restore
+        _mapping = dict((repr(_).strip("'"), _) for _ in string.whitespace.replace(" ", ""))
+        _mapping["\\\\"] = "\\"
+        retVal = re.sub("|".join(re.escape(_) for _ in ["\\\\"] + list(_mapping)), lambda match: _mapping[match.group(0)], retVal)
 
     return retVal
 

lib/core/dump.py

@@ -48,7 +48,6 @@
 from lib.core.exception import SqlmapSystemException
 from lib.core.exception import SqlmapValueException
 from lib.core.replication import Replication
-from lib.core.settings import CHECK_SQLITE_TYPE_THRESHOLD
 from lib.core.settings import DUMP_FILE_BUFFER_SIZE
 from lib.core.settings import HTML_DUMP_CSS_STYLE
 from lib.core.settings import IS_WIN
@@ -554,7 +553,11 @@ def dbTableValues(self, tableValues):
                 if column != "__infos__":
                     colType = Replication.INTEGER
 
-                    for i in xrange(min(CHECK_SQLITE_TYPE_THRESHOLD, len(tableValues[column]['values']))):
+                    # Note: the type must hold for EVERY value that will be inserted - sampling only a
+                    # prefix would type the column INTEGER/REAL while a later leading-zero/signed/overflow
+                    # value gets silently rewritten by SQLite's affinity (the INTEGER scan breaks early on
+                    # the first non-conforming value, so a genuine TEXT column costs almost nothing)
+                    for i in xrange(len(tableValues[column]['values'])):
                         value = tableValues[column]['values'][i]
                         try:
                             if not value or value == " ":  # NULL
@@ -571,7 +574,7 @@ def dbTableValues(self, tableValues):
                     if colType is None:
                         colType = Replication.REAL
 
-                        for i in xrange(min(CHECK_SQLITE_TYPE_THRESHOLD, len(tableValues[column]['values']))):
+                        for i in xrange(len(tableValues[column]['values'])):
                             value = tableValues[column]['values'][i]
                             try:
                                 if not value or value == " ":  # NULL

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.253"
+VERSION = "1.10.8.1"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -1407,9 +1407,6 @@
 # Check for empty columns only if table is sufficiently large
 CHECK_ZERO_COLUMNS_THRESHOLD = 10
 
-# Threshold for checking types of columns in case of SQLite dump format
-CHECK_SQLITE_TYPE_THRESHOLD = 100
-
 # Boldify all logger messages containing these "patterns"
 BOLD_PATTERNS = ("' injectable", "provided empty", "leftover chars", "might be injectable", "' is vulnerable", "is not injectable", "does not seem to be", "test failed", "test passed", "live test final result", "test shows that", "the back-end DBMS is", "created Github", "blocked by the target server", "protection is involved", "CAPTCHA", "specific response", "NULL connection is supported", "PASSED", "FAILED", "for more than", "connection to ", "will be trimmed", "counterpart to database")
 

lib/request/comparison.py

@@ -230,16 +230,20 @@ def _comparison(page, headers, code, getRatioValue, pageLength):
             else:
                 key = (hash(seq1), hash(seq2))
 
-            try:
-                seqMatcher.set_seq1(seq1)
-                seqMatcher.set_seq2(seq2)
-            except:
-                seqMatcher.set_seq1(repr(seq1))
-                seqMatcher.set_seq2(repr(seq2))
-
             ratio = kb.cache.comparison.get(key) if key else None
 
             if ratio is None:
+                # Note: populate the matcher only on a cache MISS - set_seq2() eagerly builds difflib's
+                # O(len(page)) b2j index, and since each response is a fresh string that whole build was
+                # thrown away on every cache hit (the common case after warmup: responses cluster into a
+                # few distinct pages). seqMatcher carries no state across calls that a hit would read.
+                try:
+                    seqMatcher.set_seq1(seq1)
+                    seqMatcher.set_seq2(seq2)
+                except:
+                    seqMatcher.set_seq1(repr(seq1))
+                    seqMatcher.set_seq2(repr(seq2))
+
                 try:
                     try:
                         ratio = seqMatcher.quick_ratio() if not kb.heavilyDynamic else seqMatcher.ratio()

lib/request/redirecthandler.py

@@ -157,8 +157,18 @@ def http_error_302(self, req, fp, code, msg, headers):
                     elif last:
                         cookies[last] += "%s%s" % (delimiter, part)
 
-                if HTTP_HEADER.SET_COOKIE in headers:
-                    for match in re.finditer(r"(?:^|,\s*)([^=;,]+)=([^;,]+)", headers[HTTP_HEADER.SET_COOKIE]):
+                # Note: multiple cookies arrive as SEPARATE Set-Cookie headers (RFC-6265 forbids folding
+                # them into one comma-joined value), and __getitem__ returns only the FIRST - iterate all
+                # values so 2nd+ cookies (e.g. a CSRF token) are not silently dropped across the redirect
+                # (get_all on py3 email.message.Message, getheaders on py2 mimetools.Message)
+                if hasattr(headers, "get_all"):
+                    setCookies = headers.get_all(HTTP_HEADER.SET_COOKIE)
+                elif hasattr(headers, "getheaders"):
+                    setCookies = headers.getheaders(HTTP_HEADER.SET_COOKIE)
+                else:
+                    setCookies = [headers[HTTP_HEADER.SET_COOKIE]]
+                for setCookie in setCookies:
+                    for match in re.finditer(r"(?:^|,\s*)([^=;,]+)=([^;,]+)", setCookie):
                         key = match.group(1).strip()
                         if key.lower() not in ("expires", "path", "domain", "max-age", "secure", "httponly", "samesite"):
                             cookies[key] = match.group(2).strip()

lib/techniques/nosql/inject.py

@@ -990,7 +990,7 @@ def _resolve(place, parameter, key):
         falseModel = _reproduced(lambda: _fetch(place, parameter, "$in", NOSQL_SENTINEL, isArray=True))   # matches nothing
         return Vector(_fingerprintMongo(place, parameter),
                       lambda value: _fetch(place, parameter, "$regex", value),
-                      lambda n: "^.{%d,}$" % n,
+                      lambda n: "(?s)^.{%d,}$" % n,   # (?s): a value containing '\n' must still match its own length (else the $-anchored probe fails for every n -> empty result)
                       lambda known, klass: "^%s%s" % (re.escape(known), klass),
                       template=template, bypass='{"$ne": null}', falseModel=falseModel)
 

lib/utils/har.py

@@ -125,11 +125,17 @@ def toDict(self):
         }
 
         if self.postBody:
-            contentType = self.headers.get("Content-Type")
-            out["postData"] = {
-                "mimeType": contentType,
-                "text": getText(self.postBody).rstrip("\r\n"),
-            }
+            out["postData"] = {"mimeType": self.headers.get("Content-Type")}
+
+            # HAR text must be UTF-8: a binary POST body (e.g. a file upload) that does not decode is
+            # base64-encoded losslessly rather than mangled through a lossy text decode - mirroring the
+            # Response.toDict() contract below (otherwise the exported HAR cannot reproduce the request)
+            raw = self.postBody if isinstance(self.postBody, bytes) else getBytes(self.postBody)
+            try:
+                out["postData"]["text"] = raw.decode("utf-8").rstrip("\r\n")
+            except UnicodeDecodeError:
+                out["postData"]["encoding"] = "base64"
+                out["postData"]["text"] = getText(base64.b64encode(raw))
 
         return out
 

plugins/generic/users.py

@@ -405,7 +405,7 @@ def getPrivileges(self, query2=False):
         # Set containing the list of DBMS administrators
         areAdmins = set()
 
-        if not kb.data.cachedUsersPrivileges and any(isTechniqueAvailable(_) for _ in (PAYLOAD.TECHNIQUE.UNION, PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY)) or conf.direct:
+        if not kb.data.cachedUsersPrivileges and (any(isTechniqueAvailable(_) for _ in (PAYLOAD.TECHNIQUE.UNION, PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY)) or conf.direct):
             if Backend.isDbms(DBMS.MYSQL) and not kb.data.has_information_schema:
                 query = rootQuery.inband.query2
                 condition = rootQuery.inband.condition2

tests/test_datafiles.py

@@ -62,6 +62,22 @@ def test_every_dbms_has_core_tags(self):
             missing = [t for t in self.CORE_TAGS if t not in present]
             self.assertEqual(missing, [], msg="%s missing core tags: %s" % (dbms.get("value"), missing))
 
+    def test_column_comment_queries_format_with_three_args(self):
+        # Regression: getColumns() formats every column_comment query with exactly (db, tbl, name)
+        # via '%'-formatting (plugins/generic/databases.py). A literal LIKE wildcard that is not
+        # escaped to '%%' (or a wrong placeholder count) raises at format time and aborts
+        # '--columns --comments' before any request. Vertica's entry had 'LIKE '%.%s'' (ValueError).
+        tree = ET.parse(os.path.join(ROOT, "data", "xml", "queries.xml"))
+        for dbms in tree.findall(".//dbms"):
+            for node in dbms.iter("column_comment"):
+                query = node.get("query")
+                if query:
+                    try:
+                        query % ("db", "tbl", "col")
+                    except (ValueError, TypeError) as ex:
+                        self.fail("%s column_comment query cannot be formatted with (db, tbl, name): %r (%s)"
+                                  % (dbms.get("value"), query, ex))
+
 
 class TestErrorsXmlCompile(unittest.TestCase):
     def test_all_error_regexes_compile(self):

tests/test_dump_format.py

@@ -389,6 +389,30 @@ def test_non_roundtrip_numbers_stay_text(self):
         finally:
             conn.close()
 
+    def test_type_breaking_value_past_sampling_prefix_stays_text(self):
+        # Regression: type inference once sampled only the first 100 values, so a leading-zero /
+        # signed / overflow value at index >= 100 was missed and the column got typed INTEGER,
+        # silently corrupting that value via SQLite affinity on insert. The whole column must be scanned.
+        values = [str(i) for i in range(1, 101)] + ["007"]   # 100 clean ints, then a leading-zero at index 100
+        tv = _PlainOrderedDict([
+            ("__infos__", {"count": len(values), "db": "testdb", "table": "big"}),
+            ("code", {"length": 3, "values": values}),
+        ])
+        conf.dumpFormat = DUMP_FORMAT.SQLITE
+        self.d.dbTableValues(tv)
+
+        import sqlite3
+        conn = sqlite3.connect(os.path.join(self.tmp, "testdb.sqlite3"))
+        try:
+            cur = conn.cursor()
+            cur.execute("PRAGMA table_info(big)")
+            types = {name: ctype for (_cid, name, ctype, _nn, _dv, _pk) in cur.fetchall()}
+            self.assertEqual(types["code"], "TEXT")   # a single non-round-trip value anywhere forces TEXT
+            cur.execute("SELECT code FROM big WHERE code = '007'")
+            self.assertEqual(cur.fetchone(), ("007",))   # stored verbatim, not rewritten to integer 7
+        finally:
+            conn.close()
+
 
 # --- replication backend tests (pure sqlite3, no network/DBMS) -----------------------------------
 

tests/test_har.py

@@ -114,6 +114,19 @@ def test_toDict_with_postbody(self):
         self.assertEqual(d["postData"]["mimeType"], "application/json")
         self.assertIn('{"a":1}', d["postData"]["text"])
 
+    def test_toDict_binary_postbody_base64(self):
+        # Regression: a non-UTF-8 (binary) POST body - e.g. a raw file upload - must be base64-encoded
+        # losslessly, not mangled through a lossy text decode, so the exported HAR reproduces the request.
+        # Mirrors the Response.toDict() contract (see TestResponse.test_toDict_binary_content_encoded).
+        import base64 as _b64
+        payload = b"\xff\xd8\xff\xe0\x00\x10JFIF"      # JPEG header: invalid UTF-8, contains a NUL
+        req = H.Request("POST", "/upload", "HTTP/1.1",
+                        {"Host": "test.com", "Content-Type": "application/octet-stream"},
+                        postBody=payload)
+        d = req.toDict()
+        self.assertEqual(d["postData"]["encoding"], "base64")
+        self.assertEqual(_b64.b64decode(d["postData"]["text"]), payload)   # losslessly reconstructable
+
     def test_url_property(self):
         req = H.Request("GET", "/path?q=1", "HTTP/1.0",
                         {"Host": "example.com"})

tests/test_nosql.py

@@ -88,6 +88,34 @@ def test_extract(self):
                             lambda known, klass: "^" + re.escape(known) + klass)
         self.assertEqual(value, SECRET)
 
+    def test_extract_value_with_newline_not_truncated(self):
+        # Regression: the length probe once used '.{n,}', and PCRE '.' does not match '\n', so a
+        # value with an embedded newline made the $-anchored '^.{n,}$' probe fail for every n ->
+        # empty result (total data loss). The '(?s)' DOTALL fix counts the newline toward the length,
+        # so the recovered value is length-correct (the unreadable newline itself shows as '?', but
+        # extraction no longer collapses to "").
+        secret = "ab\ncd"
+
+        def mongo_nl(place, parameter, op, value, isArray=False):
+            if op == "$ne":
+                return MATCH
+            if op == "$in":
+                return NOMATCH
+            if op == "$regex":
+                try:
+                    return MATCH if re.match(value, secret) is not None else NOMATCH
+                except re.error:
+                    return "<html><body>error</body></html>"
+            return ""
+
+        ni._fetch = mongo_nl
+        vector = ni._resolve("GET", "password", "password")
+        template = ni._fetch("GET", "password", "$ne", ni.NOSQL_SENTINEL)
+        value = ni._extract(template, vector.fetch, vector.lengthValue, vector.charValue, falseModel=vector.falseModel)
+        self.assertIsNotNone(value)
+        self.assertEqual(len(value), len(secret))         # honest length, not truncated / not empty
+        self.assertTrue(value.startswith("ab"))
+
     def test_not_injectable(self):
         ni._fetch = lambda *args, **kwargs: MATCH
         self.assertIsNone(ni._detectMongo("GET", "password"))

tests/test_request_basic.py

@@ -84,6 +84,61 @@ def test_empty_page(self):
         self.assertEqual(getText(decodePage(b"", None, "text/html")), "")
 
 
+class TestRedirectSetCookieMerge(unittest.TestCase):
+    """A 302 that sets more than one cookie sends SEPARATE Set-Cookie headers (RFC-6265 forbids
+    comma-folding them). The handler must merge ALL of them into the follow-up request's Cookie
+    header; a __getitem__ read returns only the first, silently dropping the 2nd+ (e.g. a CSRF token)."""
+
+    _CONF = ("cookieDel", "scope")
+
+    def setUp(self):
+        self._c = dict((k, conf.get(k)) for k in self._CONF)
+        self._redirect = kb.choices.get("redirect") if kb.get("choices") else None
+
+    def tearDown(self):
+        for k, v in self._c.items():
+            conf[k] = v
+        if kb.get("choices"):
+            kb.choices.redirect = self._redirect
+
+    def test_all_set_cookies_merged_across_redirect(self):
+        from lib.core.enums import HTTP_HEADER, REDIRECTION
+        from thirdparty import six
+        from thirdparty.six.moves import urllib as _urllib
+        import lib.request.redirecthandler as rh
+
+        conf.cookieDel = None
+        conf.scope = None
+        kb.choices.redirect = REDIRECTION.YES
+
+        raw = ("Location: http://example.com/home\r\n"
+               "Set-Cookie: sid=NEW; Path=/; HttpOnly\r\n"
+               "Set-Cookie: csrf=XYZ; Path=/\r\n\r\n")
+        # build the same headers object type the redirect handler receives on this interpreter:
+        # py3 http.client.HTTPMessage (email.message.Message, get_all) / py2 mimetools.Message (getheaders)
+        if six.PY2:
+            import mimetools
+            headers = mimetools.Message(six.StringIO(raw))
+        else:
+            from email import message_from_string
+            from thirdparty.six.moves.http_client import HTTPMessage
+            headers = message_from_string(raw, _class=HTTPMessage)
+
+        # stub the network-following parent so the test touches no socket
+        saved = _urllib.request.HTTPRedirectHandler.http_error_302
+        _urllib.request.HTTPRedirectHandler.http_error_302 = lambda self, req, fp, code, msg, headers: fp
+        try:
+            req = _urllib.request.Request("http://example.com/login", headers={"Cookie": "sid=OLD"})
+            fp = _urllib.response.addinfourl(six.BytesIO(b""), headers, req.get_full_url())
+            rh.SmartRedirectHandler().http_error_302(req, fp, 302, "Found", headers)
+        finally:
+            _urllib.request.HTTPRedirectHandler.http_error_302 = saved
+
+        merged = req.headers.get(HTTP_HEADER.COOKIE) or req.headers.get("Cookie") or ""
+        self.assertIn("sid=NEW", merged)
+        self.assertIn("csrf=XYZ", merged)   # the 2nd Set-Cookie must survive the redirect (order-independent)
+
+
 class TestForgeHeadersCookieMerge(unittest.TestCase):
     """A domain-scoped jar cookie (Domain=example.com -> '.example.com') must merge into the
     request for the apex host, not be dropped by a naive endswith() domain check."""