Populate the DRIVER_CONFIG report - phase 2

CHANGELOG.rst

@@ -14,12 +14,38 @@ Features
   ``Cluster(driver_config_reporting_enabled=False)``; ``SESSION_ID`` is unaffected by
   that setting. Reporting is best effort and never prevents a connection from being
   established.
+* ``DRIVER_CONFIG`` now describes the configuration itself rather than only the schema
+  version it follows (DRIVER-379). The report covers connection settings (timeouts,
+  request capacity, shard awareness, socket options, reconnection policy, TLS hostname
+  verification), the driver's own control-plane query timeouts, and the query defaults
+  and policies a statement gets when it overrides none of them. It follows the JSON
+  schema shared with the other ScyllaDB drivers, so the same document describes a
+  client whichever driver wrote it. Custom policies are reported by type name only and
+  never by their attributes, so a policy holding a credential does not leak it into the
+  clients table.
+* ``Cluster.sockopts`` is now materialized at construction, so a one-shot iterable is
+  applied to every connection the cluster opens rather than only to the first one.
 * Negotiate and implement the ``SCYLLA_USE_METADATA_ID`` protocol extension: prepared
   statements skip re-sending result metadata on EXECUTE, and the driver automatically
   refreshes cached metadata when the server detects a schema change (DRIVER-153)
 
 Others
 ------
+* ``DCAwareRoundRobinPolicy.local_dc`` is now read-only. It is set by the constructor,
+  and filled in by the policy itself when the constructor was given none, from the first
+  host to come up. Assigning it afterwards was indistinguishable from that inference,
+  and the two mean different things: a datacenter the application chose against one the
+  driver guessed. Code that assigned it should pass ``local_dc`` to the constructor
+  instead.
+* ``Connection.max_request_id`` and ``Connection.orphaned_threshold`` are now derived
+  again for a subclass that lowers ``max_in_flight``. Both are computed from it in the
+  class body, which runs once, so a subclass previously inherited values derived from the
+  base class -- leaving, for example, a ``max_in_flight`` of 256 with a threshold of
+  24576, which a connection holding at most 256 orphaned stream ids can never reach.
+  Orphan-based connection replacement therefore never happened for such a subclass. A
+  subclass that sets either itself keeps it. ``max_request_id`` also moves from the
+  instance to the class, so that it can be read before a connection exists; its value is
+  unchanged.
 * The ``STARTUP`` options that describe the driver itself are no longer the
   application's to set. An ``ApplicationInfoBase.add_startup_options`` that sets
   ``DRIVER_NAME``, ``DRIVER_VERSION``, ``SESSION_ID`` or ``DRIVER_CONFIG`` now has that

cassandra/cluster.py

@@ -1468,7 +1468,10 @@ def __init__(self,
 
         self.ssl_options = ssl_options
         self.ssl_context = ssl_context
-        self.sockopts = sockopts
+        # Materialized once: these are applied to every socket the cluster opens
+        # and are read again to build the configuration report, so a one-shot
+        # iterable would leave whichever consumer ran second with nothing at all.
+        self.sockopts = list(sockopts) if sockopts is not None else None
         self.cql_version = cql_version
         self.max_schema_agreement_wait = max_schema_agreement_wait
         self.control_connection_timeout = control_connection_timeout
@@ -1520,7 +1523,7 @@ def __init__(self,
         # Built whatever the flag says, so that the flag is the only thing that
         # decides whether a connection reports: see _make_connection_kwargs. The
         # reporter holds no state, so an unused one costs nothing.
-        self._driver_config_reporter = DriverConfigReporter()
+        self._driver_config_reporter = DriverConfigReporter(self)
 
         self.control_connection = ControlConnection(
             self, self.control_connection_timeout,

cassandra/connection.py

@@ -846,9 +846,37 @@ class Connection(object):
 
     # If the number of orphaned streams reaches this threshold, this connection
     # will become marked and will be replaced with a new connection by the
-    # owning pool (currently, only HostConnection supports this)
+    # owning pool (currently, only HostConnection supports this). The default
+    # for this class's max_in_flight; a connection derives its own in __init__.
     orphaned_threshold = 3  * max_in_flight // 4
 
+    @staticmethod
+    def max_request_id_for(max_in_flight):
+        """
+        The highest request id a connection with this limit will hand out.
+
+        Request ids run from zero to this inclusive, and borrow_connection
+        admits a request only while in_flight is below it. Capped at the CQL
+        stream id range, which is all the protocol can address however high
+        max_in_flight is set.
+        """
+        return min(max_in_flight - 1, (2 ** 15) - 1)
+
+    @staticmethod
+    def orphaned_threshold_for(max_in_flight):
+        """
+        The orphaned stream count at which a connection with this limit is
+        marked for replacement.
+        """
+        return 3 * max_in_flight // 4
+
+    # Both limits are derived, and both are asked for rather than stored on the
+    # class, because max_in_flight is tuned at runtime -- assigned on the class,
+    # or patched in a test -- and a value derived once does not follow it. A
+    # connection derives both in __init__ from the limit in force when it is
+    # built, and the configuration report, which has to describe them before any
+    # connection exists, asks with the class's current limit.
+
     is_defunct = False
     is_closed = False
     lock = None
@@ -944,7 +972,9 @@ def __init__(self, host='127.0.0.1', port=9042, authenticator=None,
         if not self.ssl_context and self.ssl_options:
             self.ssl_context = self._build_ssl_context_from_options()
 
-        self.max_request_id = min(self.max_in_flight - 1, (2 ** 15) - 1)
+        self.max_request_id = self.max_request_id_for(self.max_in_flight)
+        self.orphaned_threshold = self.orphaned_threshold_for(self.max_in_flight)
+
         # Don't fill the deque with 2**15 items right away. Start with some and add
         # more if needed.
         initial_size = min(300, self.max_in_flight)
@@ -1563,7 +1593,13 @@ def _handle_options_response(self, options_response):
         # only the control connection reports it. A reporter left as None means
         # the cluster has configuration reporting disabled.
         if self.is_control_connection and self._driver_config_reporter is not None:
-            self._driver_config_reporter.add_startup_options(options)
+            # Whether this is a ScyllaDB node is already known: the features
+            # above were parsed from the SUPPORTED response, and sharding info
+            # is what the driver itself keys ScyllaDB-only behaviour off (see
+            # ControlConnection._try_connect), so the report describes what the
+            # driver will actually do rather than only what it was configured to.
+            self._driver_config_reporter.add_startup_options(
+                options, is_scylla=self.features.sharding_info is not None)
 
         if self.cql_version:
             if self.cql_version not in supported_cql_versions: