From 686fa7536053cef5fd913904aa59c4c6d477d434 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Wed, 2 Sep 2026 17:54:08 +0500 Subject: [PATCH] Add a pluggable quorum layer. Spock decides WAL retention from local catalogs alone, so a single unreachable node pins WAL on every survivor indefinitely, and there is no notion of a majority to decide otherwise. Closing that needs agreement between nodes, but not a consensus implementation of Spock's own, and not a permanent marriage to somebody else's. An external system is consulted through one uniform interface and asked only three things: whether this node is in a quorum, which members the cluster considers live, and whether this node should act for the cluster. It is never asked to store anything. Spock keeps its durable state in its own crash-safe catalogs, and dropping storage from the interface is what lets a system with nothing but leader election sit behind the same seven entry points as one with a replicated key space. Providers for etcd, pgraft, and pgBully are included, selected by spock.quorum_provider; with the default of none, nothing is consulted and behaviour is unchanged. Every answer is three-valued, and an unusable one is always resolved conservatively: an error, a timeout, or an unreachable provider yields exactly the behaviour of having no provider at all. Unknown is kept distinct from no so that a cluster which lost quorum can be told apart from a provider that stopped answering, which matters to whoever is reading the status view during an incident. Providers are consulted only from a background worker's timer, never from a path a client waits on, and one reading is taken per tick and decided against, so that a tick cannot reason about a cluster state that never existed at any single instant. The objects ship in a 6.0.0 to 6.1.0 upgrade script rather than the released base script. Nothing consumes the layer yet: spock.quorum_status() reports what it sees, and the tests concentrate on the paths where no answer can be obtained, since that is the behaviour the rest of Spock has to be able to rely on. --- Makefile | 18 + .../specs/spock-quorum-layer-design.md | 262 ++++++ include/spock.h | 4 +- include/spock_quorum.h | 162 ++++ sql/spock--6.0.0--6.1.0.sql | 26 + src/spock.c | 64 ++ src/spock_quorum.c | 558 +++++++++++++ src/spock_quorum_cluster.c | 512 ++++++++++++ src/spock_quorum_etcd.c | 758 ++++++++++++++++++ tests/tap/schedule | 1 + tests/tap/t/106_quorum_layer.pl | 270 +++++++ 11 files changed, 2633 insertions(+), 2 deletions(-) create mode 100644 docs/internals-doc/specs/spock-quorum-layer-design.md create mode 100644 include/spock_quorum.h create mode 100644 sql/spock--6.0.0--6.1.0.sql create mode 100644 src/spock_quorum.c create mode 100644 src/spock_quorum_cluster.c create mode 100644 src/spock_quorum_etcd.c create mode 100644 tests/tap/t/106_quorum_layer.pl diff --git a/Makefile b/Makefile index 1105a4fb4..ad271c126 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,24 @@ ifdef SPOCK_RANDOM_DELAYS PG_CPPFLAGS += -DSPOCK_RANDOM_DELAYS endif SHLIB_LINK += $(libpq) $(filter -lintl, $(LIBS)) + +# ----------------------------------------------------------------------------- +# Optional HTTP client, for the etcd quorum provider +# ----------------------------------------------------------------------------- +# etcd runs as its own daemon and is reached over its v3 HTTP/JSON gateway, so +# that provider needs an HTTP client. libcurl is used when curl-config is on +# PATH and skipped otherwise: a dependency that only one optional provider +# needs must not be one everybody has to satisfy. Without it the provider is +# still built and still selectable; it reports why it cannot be used. +# +# Set NO_LIBCURL=1 to force it off even where libcurl is available. +ifndef NO_LIBCURL +CURL_CONFIG := $(shell command -v curl-config 2>/dev/null) +ifneq ($(CURL_CONFIG),) +PG_CPPFLAGS += -DSPOCK_HAVE_LIBCURL $(shell $(CURL_CONFIG) --cflags) +SHLIB_LINK += $(shell $(CURL_CONFIG) --libs) +endif +endif ifdef NO_LOG_OLD_VALUE PG_CPPFLAGS += -DNO_LOG_OLD_VALUE endif diff --git a/docs/internals-doc/specs/spock-quorum-layer-design.md b/docs/internals-doc/specs/spock-quorum-layer-design.md new file mode 100644 index 000000000..d5ae0cb27 --- /dev/null +++ b/docs/internals-doc/specs/spock-quorum-layer-design.md @@ -0,0 +1,262 @@ +# A pluggable quorum layer for Spock + +**Date:** 2026-09-01 (revised after implementing all three providers) +**Status:** Implemented — layer and providers; no consumer yet +**Branch:** `QURAM` (based on `main`) + +> **Revised from the original proposal.** Three things the design got wrong, +> corrected below and worth stating up front because they were the load-bearing +> assumptions: +> +> 1. **Capability tiers were dropped.** The interface is uniform. Tiering was +> solving a problem that disappeared once the interface stopped asking for +> storage. +> 2. **pgBully turned out to be the *most* capable provider for liveness, and +> pgraft the least** — the opposite of what was predicted. No upstream change +> to pgBully was needed. +> 3. **A per-tick snapshot was added** after the first implementation showed the +> layer could make decisions against a cluster state that never existed. + +## Problem + +Spock decides everything about WAL retention from local catalogs. +`group_slot_evaluate()` refuses to advance the group slot when any required +member has not reported progress recently, and `spock.progress` only records what +*this* node received *from* a member, never what that member has confirmed. + +So a single unreachable node pins WAL on every survivor, indefinitely — during an +incident, exactly when disk headroom matters most. There is also no notion of a +majority, so a topology change needs every node reachable. + +Closing that needs agreement between nodes. It does not need Spock to implement +consensus, or to marry itself to one implementation. + +## Goals + +- One interface, several backends, none privileged. +- With no provider configured, behaviour is exactly as today, bit for bit. +- Uncertainty never releases WAL. A broken quorum layer degrades to today's + conservative behaviour, never past it. +- Attaching a system requires no new build dependency in Spock core. + +## Non-goals + +- Spock does not implement consensus and does not arbitrate elections. +- No Raft vocabulary in the interface. No terms, no log indexes — those are one + implementation's concepts and would leak into an interface meant to outlast it. +- Not a general cluster manager. The scope is replication decisions. + +## Why the interface is uniform + +The original design tiered the interface, because the backends are not equally +capable and pgBully in particular has no replicated storage. That was the wrong +cut. The interface asks only for **judgements**, never for **storage**, and once +that is true there is nothing left to negotiate: Spock already keeps its durable +state in its own crash-safe catalogs, and what it lacks is not a place to write +things down but a second opinion about who is alive. + +So every provider implements the same seven entry points, and a provider with +nothing to renew supplies a `refresh()` that returns true. That is the whole +contract, in `include/spock_quorum.h`. + +Every answer is three-valued: + +```c +typedef enum SpockQuorumAnswer +{ + SPOCK_QUORUM_NO = 0, + SPOCK_QUORUM_YES, + SPOCK_QUORUM_UNKNOWN +} SpockQuorumAnswer; +``` + +`UNKNOWN` is deliberately not folded into `NO`. Callers treat them identically — +that is the fail-safe rule — but keeping them apart is what lets the status view +distinguish *a cluster that lost quorum* from *a provider that stopped +answering*. During an incident those demand different responses, and a boolean +cannot tell them apart. This distinction earned its keep immediately: with pgraft +selected, "pgraft is not installed" and "pgraft reports no leader" are both +failures to proceed, and an operator needs to know which one they have. + +## The fail-safe contract + +A layer that can release WAL can destroy a node's ability to catch up. These +rules make that impossible by construction: + +1. **Uncertainty means no.** Any error, timeout, or `UNKNOWN` produces today's + conservative behaviour. No configuration inverts this. +2. **Off the hot path.** Never from an apply worker, a walsender, or anything a + client waits on. Today the only consult is `spock.quorum_status()`, which + invalidates first. A consuming worker (group-slot eviction) is not built yet. +3. **Deadlines, not hope.** Every call is bounded by `spock.quorum_timeout` + (default 2s). Overrunning is `UNKNOWN`. etcd applies it as an HTTP deadline; + pgraft and pgBully apply it as `SET LOCAL statement_timeout`. +4. **Providers may not throw.** A callback that `ereport(ERROR)`s would abort the + very tick deciding whether releasing WAL is safe. Callbacks return a status + and an `errdetail` string. +5. **One reading per tick.** Added after implementation, see below. +6. **Releasing a member is bounded and loud.** See *Group-slot integration*. + +### Rule 5, and why it was added + +The first implementation consulted the provider per question. Deciding about five +members cost ten round trips — and, far worse, nothing stopped quorum from +answering YES to the first question and NO to the fourth. Decisions inside one +tick could rest on a cluster state that never existed at any instant. + +The layer now takes **one reading per tick** and decides against it. Efficiency +is the lesser benefit; self-consistency is the point. Leadership and membership +are not even asked for unless quorum is held, since a partitioned minority can +still believe it leads and still see some peers, and acting on either is the +failure this layer exists to prevent. + +`spock.quorum_status()` explicitly invalidates first, because an operator running +it is asking about now, not about whatever the last tick saw. + +## Where providers live + +Spock core ships all three providers and depends on nothing new. etcd's HTTP +client is the only external dependency, and it is detected at build time via +`curl-config` rather than required: without it the provider still compiles and is +still selectable, and reports why it cannot be used. `NO_LIBCURL=1` forces it +off. Selecting a provider you did not build is a configuration mistake, not a +reason to fail to start. + +Selection is one GUC: `spock.quorum_provider = none | etcd | pgraft | pgbully`. + +## What the three providers actually turned out to be + +The original table was a prediction. This one is measured, and it changed again +once pgraft reached 2.0 and pgBully grew a cluster-manager API. + +| | etcd | pgraft 2.0 | pgBully | +|---|---|---|---| +| Transport | HTTP/JSON (daemon) | SPI (in-database) | SPI (in-database) | +| API shape | etcd v3 | `pgraft.*` cluster manager | `pgbully.*`, **the same one** | +| Quorum | native | `leader_id` set | `leader_id` set | +| Leadership | leased key | native | native | +| **Per-member liveness** | **yes** (lease TTL) | **no** | **yes** (`peers().reachable`) | +| Name mapping | own registration | replicated KV | replicated KV | + +**pgraft and pgBully share one implementation.** They remain two providers, selected by distinct values of `spock.quorum_provider`, but they expose the identical +in-database interface — `get_cluster_status()`, `get_nodes()`, `is_leader()`, +`kv_put`/`kv_get` — differing only in the schema it lives under. Keeping two +near-identical files would have guaranteed drift, so `spock_quorum_cluster.c` +implements both provider tables, parameterised by schema name. + +That collapse also retired a piece of scaffolding: pgBully previously had no +replicated storage, so node names had to be recovered by joining connection +strings between `pgbully.peers()` and `spock.node_interface`. With a KV +available it registers its id-to-name mapping the same way pgraft does, and the +conninfo join is gone. + +**One real difference remains, and it is not cosmetic.** pgBully publishes +`peers().reachable`; pgraft still exposes no reachability at all. So the shared +implementation takes a per-backend liveness expression: pgBully joins its peer +table, pgraft reports every configured member live. + +Reporting all-live is the safe reading rather than a pretence that nothing has +failed: under rule 1 a live member is never evicted, so nothing can be released +on the strength of a backend with no opinion. The practical consequence is that +pgraft users get quorum and leadership from this layer but not eviction, until a +last-contact column exists upstream. Raft already tracks it internally to drive +heartbeats; it is simply not published. + +The lesson worth keeping: **consensus strength and observability are independent +axes.** The most sophisticated backend is the least useful here, because the +interface needs a fact it happens not to expose. + +## A pgraft bug found on the way + +`pgraft_shmem_startup_hook()` never called `prev_shmem_startup_hook`, though +`_PG_init` saves it and the matching *request* hook chains correctly. With +`shared_preload_libraries = 'spock,pgraft'`, pgraft's hook became head of the +chain and silently dropped spock's, so spock's shared memory was never +initialised and its supervisor segfaulted in a restart loop. This breaks any +extension loaded before pgraft. Fixed in `pgraft/src/pgraft.c`. + +## Known limitations + +**A leader id is weaker evidence than it looks.** Both in-database backends +report quorum from a non-zero `leader_id`. A follower keeps the last leader it +knew about until its election timeout expires, and pgraft leaves `CheckQuorum` +disabled, so an isolated leader can go on publishing its own id. Either can +therefore answer YES from a minority for a bounded window. + +Spock cannot close this from its side: neither backend publishes a majority +signal to ask for instead. The etcd provider avoids the problem entirely by +using a linearizable read, which only a member inside a majority can complete. +Until an equivalent exists in-database, the in-database backends should not be +trusted for anything that releases WAL — which is also why pgraft cannot +support eviction. + +**pgraft must carry the shared-memory hook fix.** `pgraft_shmem_startup_hook()` +historically failed to chain `prev_shmem_startup_hook`, which silently dropped +the shared-memory initialisation of every extension loaded before it and +segfaulted Spock's supervisor in a restart loop. Deployments must use a pgraft +build containing that fix; the extension carries no version constraint that +would enforce it, so it has to be checked by hand. + +## Group-slot integration + +Not yet built. This is where the correctness risk lives, so it is last and starts +disabled. + +`group_slot_evaluate()`'s `stale_progress` branch becomes conditional: + +| Provider | Quorum | Member | Behaviour | +|---|---|---|---| +| none | — | — | Exactly today: block. | +| any | no / unknown | — | Block. | +| any | yes | live | Block — it is up, just behind. A real backlog. | +| any | yes | not live | Eligible for release, subject to the rule below. | + +The rule matters more than the table. **Advancing past what a down node still +needs means it can never resume by replication and will require a full resync.** +PGD accepts that trade under majority; so should we, but only explicitly: + +- A member is released only after being continuously non-live for + `spock.quorum_member_eviction_timeout` (generous by default — minutes, not + seconds). A node rebooting must not lose its place. +- Release is logged at `LOG` naming the member and the horizon moved past, and + recorded in `group_slot_state`. An operator must be able to answer "why does n3 + need a resync?" from the log. +- All of it sits behind `spock.quorum_advance`, **default off**. + +Note that with pgraft this branch is unreachable, since it never reports a member +as non-live. That is correct, not a gap. + +## Testing + +Backend-specific tests only need to prove the mapping is right. Spock's own logic +should be proven against a driveable mock provider, which does not exist yet and +is the main gap in the current work — it is what would make the awkward states +(quorum lost mid-tick, leadership changing under a decision, a provider that +times out) deterministic in CI without an etcd daemon or a Raft cluster. + +The `none` provider carries a standing regression obligation: with no provider +configured, behaviour must be byte-identical to today. + +## Status and remaining work + +Done: the layer, all three providers, `spock.quorum_status()`, GUCs, build +wiring. All three verified against live daemons. The existing pg_regress suite +still passes; there are no quorum-specific regress or TAP tests yet. + +Remaining, in order: + +1. **Mock provider and TAP coverage.** The correctness of everything above rests + on states that are currently only reachable by hand. +2. **Group-slot integration** behind `spock.quorum_advance = off`. Not a + QURAM-layer remaining item — it is the consumer, on a separate branch. +3. Optional: the additive pgraft last-contact column, which would make its + provider a peer of the other two. + +## Open questions + +- Should `members()` reconcile against `spock.node`, or report the provider's + view raw and let the caller intersect? Currently the latter, since a quorum + system may legitimately govern more nodes than Spock does. +- Does anything besides the group slot want this layer? Read-only mode and + failover-slot promotion are plausible second consumers, and if either is + likely, nothing here should be named after group slots. diff --git a/include/spock.h b/include/spock.h index 61e150556..a26a7f969 100644 --- a/include/spock.h +++ b/include/spock.h @@ -24,8 +24,8 @@ #include "spock_fe.h" #include "spock_node.h" -#define SPOCK_VERSION "6.0.0" -#define SPOCK_VERSION_NUM 60000 +#define SPOCK_VERSION "6.1.0" +#define SPOCK_VERSION_NUM 60100 #define EXTENSION_NAME "spock" diff --git a/include/spock_quorum.h b/include/spock_quorum.h new file mode 100644 index 000000000..6d0040d11 --- /dev/null +++ b/include/spock_quorum.h @@ -0,0 +1,162 @@ +/*------------------------------------------------------------------------- + * + * spock_quorum.h + * Pluggable quorum provider interface. + * + * Spock does not implement consensus. It asks an external system a small + * number of questions and stays conservative when it cannot get an answer. + * This header is the whole contract. + * + * Every provider implements the same interface -- there are no optional + * entry points and no capability negotiation. That is affordable because + * the interface asks only for judgements, never for storage: Spock already + * keeps its durable state in its own crash-safe catalogs. Keeping shared + * storage out is what lets a leader-election-only system such as pgBully sit + * behind the identical interface as etcd, which has a replicated key space. + * + * Copyright (c) 2022-2026, pgEdge, Inc. + * + *------------------------------------------------------------------------- + */ +#ifndef SPOCK_QUORUM_H +#define SPOCK_QUORUM_H + +#include "postgres.h" +#include "nodes/pg_list.h" +#include "storage/latch.h" +#include "utils/timestamp.h" + +/* + * Which provider is active. Selected by spock.quorum_provider; the order + * here is the order of the GUC's enum table. + */ +typedef enum SpockQuorumProviderId +{ + SPOCK_QUORUM_PROVIDER_NONE = 0, + SPOCK_QUORUM_PROVIDER_ETCD, + SPOCK_QUORUM_PROVIDER_PGRAFT, + SPOCK_QUORUM_PROVIDER_PGBULLY +} SpockQuorumProviderId; + +/* + * Every answer is three-valued. UNKNOWN is not an error code: it is the + * honest reply when the provider is unreachable, slow, or partitioned. + * Callers treat it exactly as they treat NO -- that is the fail-safe rule -- + * but the two are kept apart so the status view and the log can distinguish + * a cluster that lost quorum from a provider that stopped answering. During + * an incident those call for different responses, and a boolean cannot tell + * them apart. + */ +typedef enum SpockQuorumAnswer +{ + SPOCK_QUORUM_NO = 0, + SPOCK_QUORUM_YES, + SPOCK_QUORUM_UNKNOWN +} SpockQuorumAnswer; + +/* One member, as the quorum system sees it -- not as spock.node sees it. */ +typedef struct SpockQuorumMember +{ + char *name; /* matches spock.node.node_name */ + bool live; /* reachable, in the provider's judgement */ + bool voting; /* counts toward a majority */ + TimestampTz last_seen; /* 0 when the provider does not track it */ +} SpockQuorumMember; + +/* + * Provider callbacks. All of them are mandatory: a provider with nothing to + * do in refresh() supplies a function that returns true. + * + * Contract for every entry point: + * + * - Never from an apply worker, a walsender, or any path a client waits + * on. A wedged provider must not be able to stall replication. Today + * the only consult is spock.quorum_status(), which invalidates first. + * A consuming worker (group-slot eviction) is not built yet. + * - Must respect spock.quorum_timeout. Overrunning it is UNKNOWN, not a + * reason to keep waiting. etcd applies it as an HTTP deadline; the + * in-database providers apply it as SET LOCAL statement_timeout. + * - Must not ereport(ERROR). Return UNKNOWN/false and put a human-readable + * reason in *errdetail (palloc'd in the caller's context) instead. An + * error thrown here would abort the very tick that is deciding whether it + * is safe to release WAL. + * - Must be free of side effects, with the deliberate exception of + * refresh(), which is where a provider renews whatever registration it + * needs to stay visible to its peers. + */ +typedef struct SpockQuorumProvider +{ + const char *name; /* shown in spock.quorum_status() */ + + /* Called once when the worker starts, and once when it stops. */ + bool (*startup) (char **errdetail); + void (*shutdown) (void); + + /* + * Called at the top of every worker tick. This is where a provider + * renews a lease or heartbeat. Providers whose peers track liveness for + * them return true without doing anything. + */ + bool (*refresh) (char **errdetail); + + /* Does this node currently belong to a quorum? */ + SpockQuorumAnswer (*have_quorum) (char **errdetail); + + /* + * The cluster's view of its members: a List of SpockQuorumMember *, or + * NIL with *errdetail set. Names with no matching spock.node row are + * ignored by the caller, since the quorum system may govern more than + * Spock does. + */ + List *(*members) (char **errdetail); + + /* Is this node the one that should act for the cluster? */ + SpockQuorumAnswer (*is_leader) (char **errdetail); + + /* Name of the current leader, or NULL when unknown. */ + char *(*leader) (char **errdetail); +} SpockQuorumProvider; + +/* --- GUCs (defined in spock.c) ----------------------------------------- */ + +extern int spock_quorum_provider; /* SpockQuorumProviderId */ +extern int spock_quorum_timeout; /* milliseconds */ +extern char *spock_quorum_etcd_endpoints; +extern char *spock_quorum_cluster_id; + +/* --- Consumed by the rest of Spock ------------------------------------- */ + +/* + * These wrap the active provider and apply the fail-safe rules, so callers + * never touch a provider directly and cannot forget to handle UNKNOWN. + */ +extern void spock_quorum_startup(void); +extern void spock_quorum_shutdown(void); +extern void spock_quorum_refresh(void); + +/* True only for an unambiguous YES. UNKNOWN and NO are both false. */ +extern bool spock_quorum_have_quorum(void); +extern bool spock_quorum_is_leader(void); + +/* NIL when there is no provider or the answer is unavailable. */ +extern List *spock_quorum_members(void); + +/* + * Is this member live in the cluster's judgement? UNKNOWN when there is no + * provider, which is what keeps a default build behaving exactly as it does + * today. + */ +extern SpockQuorumAnswer spock_quorum_member_live(const char *node_name); + +/* Backing spock.quorum_status(). */ +extern const char *spock_quorum_provider_name(void); +extern const char *spock_quorum_last_error(void); +extern TimestampTz spock_quorum_last_consulted(void); + +/* Provider tables, each defined by its own file. */ +extern const SpockQuorumProvider spock_quorum_provider_none; +extern const SpockQuorumProvider spock_quorum_provider_etcd; +extern const SpockQuorumProvider spock_quorum_provider_pgraft; +extern const SpockQuorumProvider spock_quorum_provider_pgbully; + +#endif /* SPOCK_QUORUM_H */ diff --git a/sql/spock--6.0.0--6.1.0.sql b/sql/spock--6.0.0--6.1.0.sql new file mode 100644 index 000000000..3a4a91d7f --- /dev/null +++ b/sql/spock--6.0.0--6.1.0.sql @@ -0,0 +1,26 @@ +/* spock--6.0.0--6.1.0.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION spock UPDATE TO '6.1.0'" to load this file. \quit + +-- Quorum layer. +-- +-- Spock does not implement consensus; spock.quorum_provider selects the +-- external system consulted for quorum decisions. This view exists because +-- anything able to influence WAL retention has to be inspectable before it is +-- trusted to. has_quorum and is_leader are NULL, not false, when no answer +-- could be obtained: "we are not in a quorum" and "we could not ask" call for +-- different responses during an incident. +CREATE FUNCTION spock.quorum_status( + OUT provider text, + OUT has_quorum boolean, + OUT is_leader boolean, + OUT leader text, + OUT last_consulted timestamptz, + OUT last_error text) +-- VOLATILE, not STABLE: the function deliberately invalidates the cached +-- reading and consults the provider afresh, so two calls in one statement +-- can legitimately differ. STABLE would let the planner fold them together +-- and report a stale answer. +RETURNS record VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_quorum_status_sql'; +REVOKE ALL ON FUNCTION spock.quorum_status() FROM PUBLIC; diff --git a/src/spock.c b/src/spock.c index e64357eab..6b39b6339 100644 --- a/src/spock.c +++ b/src/spock.c @@ -62,6 +62,7 @@ #include "spock_conflict.h" #include "spock_rmgr.h" #include "spock_worker.h" +#include "spock_quorum.h" #include "spock_output_config.h" #include "spock_output_plugin.h" #include "spock_exception_handler.h" @@ -162,6 +163,19 @@ static const struct config_enum_entry apply_change_logging_options[] = { {NULL, 0, false} }; +static const struct config_enum_entry quorum_provider_options[] = { + {"none", SPOCK_QUORUM_PROVIDER_NONE, false}, + {"etcd", SPOCK_QUORUM_PROVIDER_ETCD, false}, + {"pgraft", SPOCK_QUORUM_PROVIDER_PGRAFT, false}, + {"pgbully", SPOCK_QUORUM_PROVIDER_PGBULLY, false}, + {NULL, 0, false} +}; + +int spock_quorum_provider = SPOCK_QUORUM_PROVIDER_NONE; +int spock_quorum_timeout = 2000; +char *spock_quorum_etcd_endpoints = ""; +char *spock_quorum_cluster_id = ""; + bool spock_synchronous_commit = false; char *spock_temp_directory = ""; static char *spock_temp_directory_config; @@ -1159,6 +1173,56 @@ _PG_init(void) PGC_SIGHUP, 0, NULL, NULL, NULL); + DefineCustomEnumVariable("spock.quorum_provider", + gettext_noop("External system consulted for quorum decisions."), + gettext_noop("Spock does not implement consensus; it asks the " + "selected system whether this node is in a quorum " + "and which members are live. With 'none' no system " + "is consulted and Spock behaves conservatively, " + "exactly as it does without this feature."), + &spock_quorum_provider, + SPOCK_QUORUM_PROVIDER_NONE, + quorum_provider_options, + PGC_SIGHUP, 0, + NULL, NULL, NULL); + + DefineCustomIntVariable("spock.quorum_timeout", + gettext_noop("Deadline for a single call to the quorum provider."), + gettext_noop("Overrunning this is treated as an unknown answer, " + "which is handled exactly like a lost quorum: " + "conservatively. It is never a reason to keep " + "waiting, because these calls run on the path that " + "decides whether WAL may be released."), + &spock_quorum_timeout, + 2000, + 100, + 60000, + PGC_SIGHUP, + GUC_UNIT_MS, + NULL, NULL, NULL); + + DefineCustomStringVariable("spock.quorum_etcd_endpoints", + gettext_noop("Comma-separated etcd base URLs."), + gettext_noop("For example http://127.0.0.1:2379. Endpoints are " + "tried in rotation, so one unreachable member " + "costs a single tick rather than every tick."), + &spock_quorum_etcd_endpoints, + "", + PGC_SIGHUP, 0, + NULL, NULL, NULL); + + DefineCustomStringVariable("spock.quorum_cluster_id", + gettext_noop("Key-space prefix identifying this Spock cluster."), + gettext_noop("Required whenever a provider other than 'none' is " + "selected. It has no default on purpose: two clusters " + "sharing one quorum system and one prefix would each " + "count the other's nodes as its own members, and a " + "default is exactly how that happens unnoticed."), + &spock_quorum_cluster_id, + "", + PGC_SIGHUP, 0, + NULL, NULL, NULL); + DefineCustomIntVariable("spock.stats_max_entries", "Maximum entries for statistics", "Maximum number of entries that can be " diff --git a/src/spock_quorum.c b/src/spock_quorum.c new file mode 100644 index 000000000..bfd457876 --- /dev/null +++ b/src/spock_quorum.c @@ -0,0 +1,558 @@ +/*------------------------------------------------------------------------- + * + * spock_quorum.c + * Provider dispatch and the fail-safe rules around it. + * + * Nothing outside this file calls a provider directly. Every question goes + * through a wrapper here, and every wrapper turns an unusable answer into + * the conservative one. That is the whole point of the indirection: a + * caller cannot forget to handle UNKNOWN, because it never sees it. + * + * Copyright (c) 2022-2026, pgEdge, Inc. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "funcapi.h" +#include "miscadmin.h" + +#include "utils/builtins.h" +#include "utils/memutils.h" +#include "utils/timestamp.h" + +#include "spock.h" +#include "spock_quorum.h" + +PG_FUNCTION_INFO_V1(spock_quorum_status_sql); + +/* GUC storage; the variables themselves are defined in spock.c. */ + +/* + * The provider in force for this worker. Resolved at startup rather than + * read from the GUC per call: swapping providers underneath a half-finished + * decision is not a state worth supporting, and a SIGHUP that changes the + * provider restarts the worker anyway. + */ +static const SpockQuorumProvider *active = NULL; +static bool active_started = false; + +/* Diagnostics for spock.quorum_status(); never load-bearing. */ +static char *last_error = NULL; +static TimestampTz last_consulted = 0; + +/* + * One consult per tick, cached. + * + * Efficiency is the lesser reason. The real one is consistency: without a + * snapshot, deciding about five members asked the provider ten times, and + * nothing stopped quorum from being YES for the first question and NO for + * the fourth. Decisions within one tick would then rest on a cluster state + * that never existed at any single instant. Taking one reading and deciding + * against it is both cheaper and honest. + */ +static bool snap_valid = false; +static SpockQuorumAnswer snap_quorum = SPOCK_QUORUM_UNKNOWN; +static SpockQuorumAnswer snap_leader = SPOCK_QUORUM_UNKNOWN; +static List *snap_members = NIL; +static char *snap_leader_name = NULL; +static MemoryContext snap_ctx = NULL; + +/* The GUC value the active provider was resolved from. */ +static int active_provider_id = -1; + +static bool snapshot_take(void); +static void spock_quorum_invalidate(void); + +/* + * Record why the most recent consult failed. Kept in TopMemoryContext + * because the worker's per-tick context is reset underneath us, and this + * string has to outlive the tick that produced it in order to be worth + * anything to an operator. + */ +static void +note_error(const char *detail) +{ + MemoryContext old; + + if (last_error != NULL) + { + pfree(last_error); + last_error = NULL; + } + + if (detail == NULL) + return; + + old = MemoryContextSwitchTo(TopMemoryContext); + last_error = pstrdup(detail); + MemoryContextSwitchTo(old); +} + +/* Resolve the GUC to a provider table. */ +static const SpockQuorumProvider * +provider_for(int id) +{ + switch ((SpockQuorumProviderId) id) + { + case SPOCK_QUORUM_PROVIDER_NONE: + return &spock_quorum_provider_none; + case SPOCK_QUORUM_PROVIDER_ETCD: + return &spock_quorum_provider_etcd; + case SPOCK_QUORUM_PROVIDER_PGRAFT: + return &spock_quorum_provider_pgraft; + case SPOCK_QUORUM_PROVIDER_PGBULLY: + return &spock_quorum_provider_pgbully; + } + return &spock_quorum_provider_none; +} + +/* + * Drop the cached reading. Called at the top of each tick, and by the status + * view so an operator always sees a fresh answer rather than whatever the + * last tick happened to observe. + */ +static void +spock_quorum_invalidate(void) +{ + snap_valid = false; + snap_quorum = SPOCK_QUORUM_UNKNOWN; + snap_leader = SPOCK_QUORUM_UNKNOWN; + snap_members = NIL; + snap_leader_name = NULL; + if (snap_ctx != NULL) + MemoryContextReset(snap_ctx); +} + +void +spock_quorum_startup(void) +{ + char *detail = NULL; + + if (snap_ctx == NULL) + snap_ctx = AllocSetContextCreate(TopMemoryContext, + "spock quorum snapshot", + ALLOCSET_SMALL_SIZES); + + spock_quorum_invalidate(); + active = provider_for(spock_quorum_provider); + active_provider_id = spock_quorum_provider; + active_started = false; + + /* + * Every provider but 'none' addresses a shared key space, and the prefix + * is the only thing separating one cluster's members from another's. + * Refuse rather than fall back to a default: a wrong answer here mixes + * two clusters' membership, which is far worse than having no answer. + */ + if (spock_quorum_provider != SPOCK_QUORUM_PROVIDER_NONE && + (spock_quorum_cluster_id == NULL || spock_quorum_cluster_id[0] == '\0')) + { + note_error("spock.quorum_cluster_id is not set"); + ereport(WARNING, + (errmsg("spock quorum: provider \"%s\" needs spock.quorum_cluster_id", + active->name), + errhint("Set it to a value unique to this cluster. There is no " + "default, because two clusters sharing one would count " + "each other's nodes as their own."))); + return; + } + + if (active->startup == NULL) + { + active_started = true; + return; + } + + if (active->startup(&detail)) + { + active_started = true; + note_error(NULL); + return; + } + + /* + * Startup failed. Stay on the provider so the status view still reports + * what was configured and why it is not working, but leave it unstarted + * so every question below short-circuits to the conservative answer. + */ + note_error(detail ? detail : "provider startup failed"); + ereport(WARNING, + (errmsg("spock quorum: provider \"%s\" failed to start: %s", + active->name, last_error), + errhint("Spock continues with no quorum information, which is " + "the conservative behaviour."))); +} + +void +spock_quorum_shutdown(void) +{ + if (active != NULL && active_started && active->shutdown != NULL) + active->shutdown(); + active = NULL; + active_started = false; +} + +/* + * Renew whatever registration the provider needs. Called at the top of each + * tick, before any question is asked, so a lease that has lapsed is refused + * rather than answered from stale state. + */ +void +spock_quorum_refresh(void) +{ + char *detail = NULL; + + if (active == NULL || !active_started || active->refresh == NULL) + return; + + spock_quorum_invalidate(); + + if (!active->refresh(&detail)) + { + /* + * Registration could not be renewed, so this node may already have + * been dropped from the provider's view. Asking about quorum now + * could return YES from a position the cluster no longer counts, and + * would overwrite the reason the refresh failed. Leave the snapshot + * invalid: every question this tick then answers conservatively. + */ + note_error(detail ? detail : "refresh failed"); + return; + } + + note_error(NULL); + last_consulted = GetCurrentTimestamp(); + + /* + * Take the tick's single reading now, so everything decided below this + * point sees one consistent picture of the cluster. + */ + (void) snapshot_take(); +} + +/* + * Resolve the provider on first use. A consuming worker calls + * spock_quorum_startup() explicitly, but spock.quorum_status() can be called + * from any backend, and a status view reporting "none" merely because + * nothing had initialised the layer would be actively misleading. + */ +static bool +spock_quorum_ensure_started(void) +{ + /* + * Re-resolve when the GUC has moved. The provider is PGC_SIGHUP, and a + * worker restarts on one, but a long-lived backend does not -- it would + * otherwise keep answering from the provider that was configured when it + * first connected, indefinitely. + */ + if (active != NULL && active_provider_id != spock_quorum_provider) + { + spock_quorum_shutdown(); + spock_quorum_startup(); + } + else if (active == NULL) + spock_quorum_startup(); + + return active != NULL && active_started; +} + +/* + * Take one reading, if this tick has not already. + * + * Members are copied into snap_ctx: the provider allocates them in whatever + * context is current, which for a worker is reset between ticks, and the + * snapshot has to outlive that. + */ +static bool +snapshot_take(void) +{ + char *detail = NULL; + List *members; + + if (!spock_quorum_ensure_started()) + return false; + if (snap_valid) + return true; + + snap_quorum = active->have_quorum(&detail); + if (snap_quorum == SPOCK_QUORUM_UNKNOWN && detail != NULL) + note_error(detail); + else if (snap_quorum != SPOCK_QUORUM_UNKNOWN) + { + note_error(NULL); + last_consulted = GetCurrentTimestamp(); + } + + /* + * Leadership and membership are only asked for once quorum is held. A + * partitioned minority can still believe it leads and can still see some + * peers; acting on either is the failure this layer exists to prevent, + * so there is nothing to learn from asking. + */ + if (snap_quorum == SPOCK_QUORUM_YES) + { + detail = NULL; + snap_leader = active->is_leader(&detail); + if (snap_leader == SPOCK_QUORUM_UNKNOWN && detail != NULL) + note_error(detail); + + /* + * The leader's name belongs to this reading too. Fetching it later, + * when the status view is rendered, would be a separate consult and + * could name a leader from a different moment than the quorum answer + * shown beside it. + */ + if (active->leader != NULL) + { + char *who; + + detail = NULL; + who = active->leader(&detail); + if (who != NULL) + { + MemoryContext old = MemoryContextSwitchTo(snap_ctx); + + snap_leader_name = pstrdup(who); + MemoryContextSwitchTo(old); + } + } + + detail = NULL; + members = active->members(&detail); + if (members == NIL && detail != NULL) + note_error(detail); + else + { + MemoryContext old = MemoryContextSwitchTo(snap_ctx); + ListCell *lc; + + foreach(lc, members) + { + SpockQuorumMember *src = (SpockQuorumMember *) lfirst(lc); + SpockQuorumMember *cp = palloc0(sizeof(SpockQuorumMember)); + + cp->name = pstrdup(src->name); + cp->live = src->live; + cp->voting = src->voting; + cp->last_seen = src->last_seen; + snap_members = lappend(snap_members, cp); + } + MemoryContextSwitchTo(old); + } + } + + snap_valid = true; + return true; +} + +bool +spock_quorum_have_quorum(void) +{ + if (!snapshot_take()) + return false; + return snap_quorum == SPOCK_QUORUM_YES; +} + +bool +spock_quorum_is_leader(void) +{ + if (!snapshot_take()) + return false; + + /* snapshot_take only asks about leadership while quorum is held. */ + return snap_quorum == SPOCK_QUORUM_YES && snap_leader == SPOCK_QUORUM_YES; +} + +List * +spock_quorum_members(void) +{ + if (!snapshot_take()) + return NIL; + return snap_members; +} + +SpockQuorumAnswer +spock_quorum_member_live(const char *node_name) +{ + ListCell *lc; + + if (node_name == NULL || !snapshot_take()) + return SPOCK_QUORUM_UNKNOWN; + + /* + * Liveness is only trustworthy from inside a quorum. Without one this + * node may be the isolated party, and its opinion about who else is + * reachable says more about its own connectivity than about the cluster. + */ + if (snap_quorum != SPOCK_QUORUM_YES) + return SPOCK_QUORUM_UNKNOWN; + + foreach(lc, snap_members) + { + SpockQuorumMember *m = (SpockQuorumMember *) lfirst(lc); + + if (strcmp(m->name, node_name) == 0) + return m->live ? SPOCK_QUORUM_YES : SPOCK_QUORUM_NO; + } + + /* + * The quorum system has never heard of this node. That is not evidence + * that it is down -- it may simply not be registered -- so it is not + * grounds for releasing anything. + */ + return SPOCK_QUORUM_UNKNOWN; +} + +const char * +spock_quorum_provider_name(void) +{ + return active != NULL ? active->name : "none"; +} + +const char * +spock_quorum_last_error(void) +{ + return last_error; +} + +TimestampTz +spock_quorum_last_consulted(void) +{ + return last_consulted; +} + +/* + * spock.quorum_status() + * + * Anything able to move the WAL horizon has to be inspectable before it is + * allowed to. + */ +Datum +spock_quorum_status_sql(PG_FUNCTION_ARGS) +{ + TupleDesc tupdesc; + Datum values[6]; + bool nulls[6]; + HeapTuple tuple; + + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + memset(nulls, 0, sizeof(nulls)); + + /* + * Force a fresh reading. An operator running this is asking about now, + * not about whatever the last tick happened to see. + */ + spock_quorum_invalidate(); + (void) snapshot_take(); + + values[0] = CStringGetTextDatum(spock_quorum_provider_name()); + + if (active == NULL || !active_started) + { + nulls[1] = true; /* has_quorum */ + nulls[2] = true; /* is_leader */ + nulls[3] = true; /* leader */ + } + else + { + if (snap_quorum == SPOCK_QUORUM_UNKNOWN) + nulls[1] = true; + else + values[1] = BoolGetDatum(snap_quorum == SPOCK_QUORUM_YES); + + /* + * Reported through the same rule the rest of Spock acts on: without + * quorum, leadership is not something this node may act on, so + * showing the provider's raw opinion here would describe a decision + * Spock would never make. + */ + if (snap_quorum != SPOCK_QUORUM_YES || + snap_leader == SPOCK_QUORUM_UNKNOWN) + nulls[2] = true; + else + values[2] = BoolGetDatum(snap_leader == SPOCK_QUORUM_YES); + + if (snap_leader_name != NULL) + values[3] = CStringGetTextDatum(snap_leader_name); + else + nulls[3] = true; + } + + if (last_consulted == 0) + nulls[4] = true; + else + values[4] = TimestampTzGetDatum(last_consulted); + + if (last_error == NULL) + nulls[5] = true; + else + values[5] = CStringGetTextDatum(last_error); + + tuple = heap_form_tuple(tupdesc, values, nulls); + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} + +/* ---------------------------------------------------------------------- * + * The 'none' provider. + * + * Not a stub: it is the default, and it is what every other provider + * degrades to. It answers UNKNOWN rather than NO so that callers which + * distinguish the two (the status view, the logs) report "no information" + * instead of asserting a negative it has no basis for. + * ---------------------------------------------------------------------- */ + +static bool +none_startup(char **errdetail) +{ + return true; +} + +static void +none_shutdown(void) +{ +} + +static bool +none_refresh(char **errdetail) +{ + return true; +} + +static SpockQuorumAnswer +none_have_quorum(char **errdetail) +{ + return SPOCK_QUORUM_UNKNOWN; +} + +static List * +none_members(char **errdetail) +{ + return NIL; +} + +static SpockQuorumAnswer +none_is_leader(char **errdetail) +{ + return SPOCK_QUORUM_UNKNOWN; +} + +static char * +none_leader(char **errdetail) +{ + return NULL; +} + +const SpockQuorumProvider spock_quorum_provider_none = { + .name = "none", + .startup = none_startup, + .shutdown = none_shutdown, + .refresh = none_refresh, + .have_quorum = none_have_quorum, + .members = none_members, + .is_leader = none_is_leader, + .leader = none_leader +}; diff --git a/src/spock_quorum_cluster.c b/src/spock_quorum_cluster.c new file mode 100644 index 000000000..8c92c1e97 --- /dev/null +++ b/src/spock_quorum_cluster.c @@ -0,0 +1,512 @@ +/*------------------------------------------------------------------------- + * + * spock_quorum_cluster.c + * Quorum provider for the shared cluster-manager API. + * + * pgraft and pgBully now expose the same in-database interface, differing + * only in the schema it lives under: + * + * .get_cluster_status() node_id, term, leader_id, state, ... + * .get_nodes() node_id, address, port, is_leader + * .is_leader() + * .kv_put() / kv_get() replicated key/value + * + * So they are one provider here, parameterised by schema name, rather than + * two near-identical files drifting apart. Everything a quorum decision + * needs comes from that common surface. + * + * The one real difference is liveness, and it is not a matter of naming. + * pgBully additionally publishes pgbully.peers(), which carries `reachable` + * per peer -- the per-member liveness that Raft tracks internally to drive + * heartbeats but that pgraft does not expose. Where that is available it is + * used; where it is not, every configured member is reported live. + * + * Reporting all-live is the safe reading rather than a pretence that nothing + * has failed: under the fail-safe rules a live member is one that is never + * evicted, so a caller cannot release anything on the strength of a provider + * that has no opinion. It does mean the quorum layer buys pgraft users + * leadership and quorum but not eviction, until a last-contact column exists + * upstream. + * + * Copyright (c) 2022-2026, pgEdge, Inc. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/xact.h" +#include "executor/spi.h" +#include "utils/guc.h" +#include "utils/resowner.h" +#include "utils/builtins.h" +#include "utils/elog.h" +#include "utils/memutils.h" +#include "utils/timestamp.h" + +#include "spock.h" +#include "spock_node.h" +#include "spock_quorum.h" + +/* + * What differs between the two backends. Everything else below is shared. + */ +typedef struct ClusterApiConfig +{ + const char *schema; /* "pgraft" or "pgbully" */ + + /* + * SQL fragment yielding this member's liveness, evaluated against the + * get_nodes() row aliased `n`. A backend with no reachability + * information supplies a literal true. + */ + const char *live_expr; + + /* Extra FROM-list text the liveness expression needs, or "". */ + const char *live_from; + + /* + * SQL fragment yielding when this member was last heard from, or the + * literal NULL for a backend that does not track it. + */ + const char *seen_expr; + + /* Set once startup() has confirmed the extension is present. */ + bool available; + + /* This node's Spock name, published so peers can map ids to names. */ + char *self_name; +} ClusterApiConfig; + +/* + * pgBully joins its own peer table for real reachability. A LEFT JOIN, and + * coalesced to true, because a peer pgBully has not yet formed an opinion + * about must not read as dead: "no opinion" is not evidence of failure and + * must never license an eviction. + */ +static ClusterApiConfig cfg_pgbully = { + .schema = "pgbully", + .live_expr = "coalesce(p.reachable, true)", + .live_from = " LEFT JOIN pgbully.peers() p ON p.node_id = n.node_id ", + .seen_expr = "p.last_seen", + .available = false, + .self_name = NULL +}; + +static ClusterApiConfig cfg_pgraft = { + .schema = "pgraft", + .live_expr = "true", + .live_from = "", + .seen_expr = "NULL::timestamptz", + .available = false, + .self_name = NULL +}; + +/* + * Every consult below runs inside an internal subtransaction. + * + * Catching an error with PG_TRY and FlushErrorState() alone is not enough: + * the surrounding transaction stays aborted, and the caller's next command + * fails with "current transaction is aborted". These providers are consulted + * from spock.quorum_status(), inside whatever transaction the operator is + * already in, so that would turn a merely unreachable backend into a broken + * session. A subtransaction is the only way to catch an error and carry on. + * + * spock.quorum_timeout is applied as statement_timeout inside that + * subtransaction -- the in-database equivalent of etcd's HTTP deadline. + * SET LOCAL alone is not enough, because releasing a subtransaction + * propagates the setting up to the parent rather than restoring it, so the + * previous value is saved and put back on every exit path. Rollback-only + * cleanup would not do either: cluster_refresh() performs a kv_put() whose + * effect has to survive. + */ +typedef struct ClusterSpiScope +{ + MemoryContext oldcxt; + ResourceOwner oldowner; + char *prev_timeout; +} ClusterSpiScope; + +static void +cluster_spi_begin(ClusterSpiScope *scope) +{ + const char *prev = GetConfigOption("statement_timeout", false, false); + + scope->oldcxt = CurrentMemoryContext; + scope->oldowner = CurrentResourceOwner; + scope->prev_timeout = pstrdup(prev ? prev : "0"); + + BeginInternalSubTransaction(NULL); +} + +/* Put back what the caller had, whichever way the consult ended. */ +static void +cluster_spi_end(ClusterSpiScope *scope) +{ + MemoryContextSwitchTo(scope->oldcxt); + CurrentResourceOwner = scope->oldowner; + SetConfigOption("statement_timeout", scope->prev_timeout, + PGC_SUSET, PGC_S_SESSION); +} + +/* Apply the quorum deadline. Caller must already be connected to SPI. */ +static bool +cluster_spi_apply_timeout(char **errdetail) +{ + char sql[64]; + + snprintf(sql, sizeof(sql), + "SET LOCAL statement_timeout = %d", spock_quorum_timeout); + if (SPI_execute(sql, false, 0) != SPI_OK_UTILITY) + { + *errdetail = pstrdup("could not apply quorum timeout"); + return false; + } + + return true; +} + +/* Copy the error text somewhere that outlives the subtransaction. */ +static void +cluster_capture_error(ClusterSpiScope *scope, char **errdetail) +{ + ErrorData *edata; + + MemoryContextSwitchTo(scope->oldcxt); + edata = CopyErrorData(); + FlushErrorState(); + *errdetail = pstrdup(edata->message); + FreeErrorData(edata); +} + +/* + * Run a query yielding one text value, returning NULL rather than throwing. + * + * The backend is a separate extension that may be absent, mid-upgrade, or + * erroring, and a provider is contractually forbidden from raising. + */ +static char * +cluster_one_text(const char *sql, char **errdetail) +{ + volatile bool ok = true; + char *volatile result = NULL; + ClusterSpiScope scope; + + *errdetail = NULL; + cluster_spi_begin(&scope); + + PG_TRY(); + { + if (SPI_connect() != SPI_OK_CONNECT) + ok = false; + else + { + if (!cluster_spi_apply_timeout(errdetail)) + ok = false; + else if (SPI_execute(sql, true, 1) == SPI_OK_SELECT && SPI_processed >= 1) + { + char *raw = SPI_getvalue(SPI_tuptable->vals[0], + SPI_tuptable->tupdesc, 1); + + /* Copied out before SPI_finish frees the context it lives in. */ + if (raw != NULL) + { + MemoryContext old = MemoryContextSwitchTo(scope.oldcxt); + + result = pstrdup(raw); + MemoryContextSwitchTo(old); + } + } + SPI_finish(); + } + ReleaseCurrentSubTransaction(); + } + PG_CATCH(); + { + cluster_capture_error(&scope, errdetail); + RollbackAndReleaseCurrentSubTransaction(); + ok = false; + } + PG_END_TRY(); + + cluster_spi_end(&scope); + + if (!ok) + { + if (*errdetail == NULL) + *errdetail = pstrdup("query against the cluster manager failed"); + return NULL; + } + return result; +} + +/* Prefix under which node ids are mapped to Spock node names. */ +static char * +name_key_prefix(void) +{ + return psprintf("%s/nodes/", spock_quorum_cluster_id); +} + +static bool +cluster_startup(ClusterApiConfig *cfg, char **errdetail) +{ + SpockLocalNode *local; + MemoryContext old; + char *present; + + local = get_local_node(false, true); + if (local == NULL) + { + *errdetail = pstrdup("no local spock node"); + return false; + } + + old = MemoryContextSwitchTo(TopMemoryContext); + cfg->self_name = pstrdup(local->node->name); + MemoryContextSwitchTo(old); + + present = cluster_one_text( + psprintf("SELECT to_regprocedure('%s.get_cluster_status()') IS NOT NULL" + " AND to_regprocedure('%s.is_leader()') IS NOT NULL", + cfg->schema, cfg->schema), errdetail); + + if (present == NULL) + return false; + if (strcmp(present, "t") != 0) + { + *errdetail = psprintf("the %s extension is not installed in this database", + cfg->schema); + return false; + } + + cfg->available = true; + return true; +} + +/* + * Publish this node's id-to-name mapping. + * + * Not a heartbeat and no evidence of liveness: the replicated KV has no + * expiry, so an entry outlives the node that wrote it. It exists only so + * members() and leader() can render the integer ids the cluster manager + * speaks in as the node names the rest of Spock speaks in. + */ +static bool +cluster_refresh(ClusterApiConfig *cfg, char **errdetail) +{ + char *id; + + if (!cfg->available) + return false; + + id = cluster_one_text(psprintf("SELECT node_id::text FROM %s.get_cluster_status()", + cfg->schema), errdetail); + if (id == NULL) + return false; + + return cluster_one_text( + psprintf("SELECT %s.kv_put(%s, %s)::text", + cfg->schema, + quote_literal_cstr(psprintf("%s%s", name_key_prefix(), id)), + quote_literal_cstr(cfg->self_name)), errdetail) != NULL; +} + +/* + * A leader is elected only from within a majority, so a leader id that is + * set is itself the proof of quorum. There is no separate question to ask. + * + * coalesced because the two backends differ on how they say "nobody": + * pgraft reports 0 and pgBully reports NULL. Left bare, the comparison + * would yield NULL for pgBully and be reported as "unknown" when what it + * actually said was a definite "no leader, so no quorum". Both are safe -- + * the caller treats them alike -- but only one is true. + */ +static SpockQuorumAnswer +cluster_have_quorum(ClusterApiConfig *cfg, char **errdetail) +{ + char *leader; + + if (!cfg->available) + return SPOCK_QUORUM_UNKNOWN; + + leader = cluster_one_text( + psprintf("SELECT (coalesce(leader_id, 0) <> 0)::text " + " FROM %s.get_cluster_status()", cfg->schema), errdetail); + + if (leader == NULL) + return SPOCK_QUORUM_UNKNOWN; + return strcmp(leader, "t") == 0 ? SPOCK_QUORUM_YES : SPOCK_QUORUM_NO; +} + +static SpockQuorumAnswer +cluster_is_leader(ClusterApiConfig *cfg, char **errdetail) +{ + char *v; + + if (!cfg->available) + return SPOCK_QUORUM_UNKNOWN; + + v = cluster_one_text(psprintf("SELECT %s.is_leader()::text", cfg->schema), + errdetail); + if (v == NULL) + return SPOCK_QUORUM_UNKNOWN; + return strcmp(v, "t") == 0 ? SPOCK_QUORUM_YES : SPOCK_QUORUM_NO; +} + +static char * +cluster_leader(ClusterApiConfig *cfg, char **errdetail) +{ + char *id; + + if (!cfg->available) + return NULL; + + id = cluster_one_text(psprintf("SELECT leader_id::text " + " FROM %s.get_cluster_status()", cfg->schema), + errdetail); + if (id == NULL || strcmp(id, "0") == 0) + return NULL; + + return cluster_one_text( + psprintf("SELECT %s.kv_get(%s)", cfg->schema, + quote_literal_cstr(psprintf("%s%s", name_key_prefix(), id))), + errdetail); +} + +/* + * Membership, rendered as Spock node names, with each member's liveness + * according to whatever the backend can attest. + * + * A node that has not published a name mapping yet is skipped rather than + * reported under its integer id: a name matching no spock.node row is worse + * than no row at all, because it looks like an answer. + */ +static List * +cluster_members(ClusterApiConfig *cfg, char **errdetail) +{ + volatile bool ok = true; + List *volatile result = NIL; + ClusterSpiScope scope; + char *sql; + + if (!cfg->available) + return NIL; + + *errdetail = NULL; + + sql = psprintf( + "SELECT k.v, (%s)::text, (%s)::text " + " FROM %s.get_nodes() n %s" + " , LATERAL (SELECT %s.kv_get(%s || n.node_id::text) AS v) k " + " WHERE k.v IS NOT NULL", + cfg->live_expr, cfg->seen_expr, cfg->schema, cfg->live_from, + cfg->schema, quote_literal_cstr(name_key_prefix())); + + cluster_spi_begin(&scope); + + PG_TRY(); + { + if (SPI_connect() != SPI_OK_CONNECT) + ok = false; + else + { + if (!cluster_spi_apply_timeout(errdetail)) + ok = false; + else if (SPI_execute(sql, true, 0) == SPI_OK_SELECT) + { + uint64 i; + + for (i = 0; i < SPI_processed; i++) + { + HeapTuple tup = SPI_tuptable->vals[i]; + TupleDesc desc = SPI_tuptable->tupdesc; + char *name = SPI_getvalue(tup, desc, 1); + char *live = SPI_getvalue(tup, desc, 2); + char *seen = SPI_getvalue(tup, desc, 3); + MemoryContext old; + SpockQuorumMember *m; + + if (name == NULL) + continue; + + old = MemoryContextSwitchTo(scope.oldcxt); + m = palloc0(sizeof(SpockQuorumMember)); + m->name = pstrdup(name); + m->live = (live == NULL || strcmp(live, "t") == 0); + m->voting = true; + + /* + * Carry the backend's own last-contact time when it has + * one. SQL NULL stays 0, which the struct documents as + * "not tracked" -- the two mean the same to a caller. + */ + m->last_seen = (seen == NULL) ? 0 : + DatumGetTimestampTz(DirectFunctionCall3(timestamptz_in, + CStringGetDatum(seen), + ObjectIdGetDatum(InvalidOid), + Int32GetDatum(-1))); + result = lappend(result, m); + MemoryContextSwitchTo(old); + } + } + else + ok = false; + SPI_finish(); + } + ReleaseCurrentSubTransaction(); + } + PG_CATCH(); + { + cluster_capture_error(&scope, errdetail); + RollbackAndReleaseCurrentSubTransaction(); + ok = false; + } + PG_END_TRY(); + + cluster_spi_end(&scope); + + if (!ok) + { + if (*errdetail == NULL) + *errdetail = psprintf("could not read %s membership", cfg->schema); + return NIL; + } + return result; +} + +/* --- per-backend callback tables -------------------------------------- */ + +#define CLUSTER_PROVIDER_SHIMS(tag, cfgvar) \ +static bool tag##_startup(char **e) { return cluster_startup(&cfgvar, e); } \ +static void tag##_shutdown(void) { cfgvar.available = false; } \ +static bool tag##_refresh(char **e) { return cluster_refresh(&cfgvar, e); } \ +static SpockQuorumAnswer tag##_have_quorum(char **e) \ + { return cluster_have_quorum(&cfgvar, e); } \ +static List *tag##_members(char **e) { return cluster_members(&cfgvar, e); } \ +static SpockQuorumAnswer tag##_is_leader(char **e) \ + { return cluster_is_leader(&cfgvar, e); } \ +static char *tag##_leader(char **e) { return cluster_leader(&cfgvar, e); } + +CLUSTER_PROVIDER_SHIMS(pgraft, cfg_pgraft) +CLUSTER_PROVIDER_SHIMS(pgbully, cfg_pgbully) + +const SpockQuorumProvider spock_quorum_provider_pgraft = { + .name = "pgraft", + .startup = pgraft_startup, + .shutdown = pgraft_shutdown, + .refresh = pgraft_refresh, + .have_quorum = pgraft_have_quorum, + .members = pgraft_members, + .is_leader = pgraft_is_leader, + .leader = pgraft_leader +}; + +const SpockQuorumProvider spock_quorum_provider_pgbully = { + .name = "pgbully", + .startup = pgbully_startup, + .shutdown = pgbully_shutdown, + .refresh = pgbully_refresh, + .have_quorum = pgbully_have_quorum, + .members = pgbully_members, + .is_leader = pgbully_is_leader, + .leader = pgbully_leader +}; diff --git a/src/spock_quorum_etcd.c b/src/spock_quorum_etcd.c new file mode 100644 index 000000000..a5ae37f25 --- /dev/null +++ b/src/spock_quorum_etcd.c @@ -0,0 +1,758 @@ +/*------------------------------------------------------------------------- + * + * spock_quorum_etcd.c + * Quorum provider backed by an external etcd daemon. + * + * etcd runs as its own daemon, so this provider is a client: it speaks the + * v3 HTTP/JSON gateway, which keeps the dependency to an HTTP library rather + * than a gRPC stack. The whole file is guarded by SPOCK_HAVE_LIBCURL; a build + * without it still compiles and still offers the provider, which then + * reports why it cannot be used. Selecting a provider you did not build is + * a configuration mistake, not a reason to fail to start. + * + * Liveness model. etcd's own member list describes etcd, not Spock, so it + * cannot answer "is node n3 up". Instead each Spock node registers itself + * under a prefix with a lease and renews that lease every tick: + * + * /nodes/ -> (lease TTL) + * + * A node that stops renewing has its key expired by etcd, so presence under + * the prefix *is* liveness, judged by the cluster rather than by whichever + * node happens to be asking. That is the property Spock cannot get locally + * and the reason for the whole layer. + * + * Copyright (c) 2022-2026, pgEdge, Inc. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/xact.h" +#include "common/base64.h" +#include "utils/resowner.h" +#include "lib/stringinfo.h" +#include "utils/builtins.h" +#include "utils/jsonb.h" +#include "utils/memutils.h" +#include "parser/scansup.h" + +#include "spock.h" +#include "spock_node.h" +#include "spock_quorum.h" + +#ifdef SPOCK_HAVE_LIBCURL +#include +#endif + +/* + * Lease TTL, in seconds. Must comfortably exceed the worker interval or a + * slow tick would expire our own registration and make this node look dead + * to its peers. Six times the default 5s tick leaves room for a stalled + * tick or two before anyone draws conclusions. + */ +#define ETCD_LEASE_TTL_SECONDS 30 + +#define ETCD_NODES_INFIX "/nodes/" +#define ETCD_LEADER_SUFFIX "/leader" + +#ifdef SPOCK_HAVE_LIBCURL + +static bool curl_initialized = false; +static int64 etcd_lease_id = 0; +static char *etcd_self_name = NULL; + +static char *etcd_leader_name(char **errdetail); + +/* + * Split a comma-separated endpoint list. + * + * Deliberately not SplitIdentifierString(): that downcases unquoted text and + * truncates each element to NAMEDATALEN, both of which silently corrupt a + * URL. Endpoints are opaque strings here, so only whitespace is trimmed. + */ +static List * +split_endpoints(const char *raw) +{ + List *result = NIL; + char *copy = pstrdup(raw); + char *cursor = copy; + char *comma; + + for (;;) + { + char *item = cursor; + char *tail; + + comma = strchr(cursor, ','); + if (comma != NULL) + { + *comma = '\0'; + cursor = comma + 1; + } + + while (*item != '\0' && scanner_isspace(*item)) + item++; + tail = item + strlen(item); + while (tail > item && scanner_isspace(*(tail - 1))) + *(--tail) = '\0'; + + if (*item != '\0') + result = lappend(result, item); + + if (comma == NULL) + break; + } + + return result; +} + +/* Accumulates a response body. */ +static size_t +write_cb(void *contents, size_t size, size_t nmemb, void *userp) +{ + StringInfo buf = (StringInfo) userp; + size_t total = size * nmemb; + + appendBinaryStringInfo(buf, (const char *) contents, (int) total); + return total; +} + +/* + * POST a JSON body to one etcd endpoint and return the response body, or + * NULL with *errdetail set. + * + * Only the first endpoint in spock.quorum_etcd_endpoints is tried per call, + * rotating on failure, so a dead etcd member costs one tick rather than + * making every tick pay for the retry. + */ +static char * +etcd_post(const char *path, const char *body, char **errdetail) +{ + static int endpoint_cursor = 0; + CURL *curl; + CURLcode res; + StringInfoData resp; + StringInfoData url; + struct curl_slist *headers = NULL; + long http_code = 0; + List *endpoints; + ListCell *lc; + int n = 0; + char *chosen = NULL; + + if (spock_quorum_etcd_endpoints == NULL || + spock_quorum_etcd_endpoints[0] == '\0') + { + *errdetail = pstrdup("spock.quorum_etcd_endpoints is not set"); + return NULL; + } + + endpoints = split_endpoints(spock_quorum_etcd_endpoints); + if (endpoints == NIL) + { + *errdetail = pstrdup("spock.quorum_etcd_endpoints is malformed"); + return NULL; + } + + foreach(lc, endpoints) + { + if (n == (endpoint_cursor % list_length(endpoints))) + chosen = (char *) lfirst(lc); + n++; + } + if (chosen == NULL) + chosen = (char *) linitial(endpoints); + + initStringInfo(&url); + appendStringInfo(&url, "%s%s", chosen, path); + + curl = curl_easy_init(); + if (curl == NULL) + { + *errdetail = pstrdup("could not initialise HTTP client"); + return NULL; + } + + initStringInfo(&resp); + headers = curl_slist_append(headers, "Content-Type: application/json"); + + curl_easy_setopt(curl, CURLOPT_URL, url.data); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &resp); + /* The deadline is the contract; without it a hung etcd hangs the tick. */ + curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, (long) spock_quorum_timeout); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, (long) spock_quorum_timeout); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + + res = curl_easy_perform(curl); + if (res == CURLE_OK) + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + + if (res != CURLE_OK) + { + endpoint_cursor++; + *errdetail = psprintf("etcd %s: %s", chosen, curl_easy_strerror(res)); + return NULL; + } + if (http_code != 200) + { + endpoint_cursor++; + *errdetail = psprintf("etcd %s returned HTTP %ld", chosen, http_code); + return NULL; + } + + return resp.data; +} + +/* Base64, as the v3 gateway requires for every key and value. */ +static char * +b64(const char *src) +{ + int srclen = (int) strlen(src); + int maxlen = pg_b64_enc_len(srclen) + 1; + char *dst = palloc(maxlen); + int len = pg_b64_encode((const uint8 *) src, srclen, dst, maxlen - 1); + + if (len < 0) + return pstrdup(""); + dst[len] = '\0'; + return dst; +} + +static char * +unb64(const char *src) +{ + int srclen = (int) strlen(src); + int maxlen = pg_b64_dec_len(srclen) + 1; + char *dst = palloc(maxlen); + int len = pg_b64_decode(src, srclen, (uint8 *) dst, maxlen - 1); + + if (len < 0) + return pstrdup(""); + dst[len] = '\0'; + return dst; +} + +/* + * Parse a response body, returning NULL rather than throwing. + * + * etcd's replies are small and infrequent, so the server's own parser is + * used rather than a hand-rolled scanner. It has to be wrapped, though: + * jsonb_in raises on malformed input, and a provider is contractually + * forbidden from throwing. `ok` is volatile because it is written in the + * handler and read after PG_END_TRY. + */ +static Jsonb * +parse_json(const char *json) +{ + volatile bool ok = true; + Jsonb *volatile result = NULL; + MemoryContext oldcxt = CurrentMemoryContext; + ResourceOwner oldowner = CurrentResourceOwner; + + if (json == NULL) + return NULL; + + /* + * An internal subtransaction, not a bare PG_TRY. Catching the error with + * FlushErrorState() alone would leave the surrounding transaction + * aborted, and this runs inside whatever transaction the operator called + * spock.quorum_status() from -- so a single malformed reply would break + * their session rather than just this reading. + */ + BeginInternalSubTransaction(NULL); + + PG_TRY(); + { + result = DatumGetJsonbP(DirectFunctionCall1(jsonb_in, + CStringGetDatum(json))); + + /* Copy out before the subtransaction that allocated it goes away. */ + MemoryContextSwitchTo(oldcxt); + result = (Jsonb *) PG_DETOAST_DATUM_COPY(PointerGetDatum(result)); + + ReleaseCurrentSubTransaction(); + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcxt); + FlushErrorState(); + RollbackAndReleaseCurrentSubTransaction(); + ok = false; + } + PG_END_TRY(); + + MemoryContextSwitchTo(oldcxt); + CurrentResourceOwner = oldowner; + + return ok ? result : NULL; +} + +/* + * One top-level field, as text, or NULL when absent. + * + * The container API is used in preference to jsonb_object_field_text + * because DirectFunctionCall raises "function returned NULL" whenever the + * field is missing -- which for an optional field is the normal case, not + * an error. + */ +static char * +jb_field(Jsonb *jb, const char *field) +{ + JsonbValue *v; + + if (jb == NULL) + return NULL; + + v = getKeyJsonValueFromContainer(&jb->root, field, (int) strlen(field), + NULL); + if (v == NULL) + return NULL; + + switch (v->type) + { + case jbvString: + return pnstrdup(v->val.string.val, v->val.string.len); + case jbvBool: + return pstrdup(v->val.boolean ? "true" : "false"); + case jbvNumeric: + return DatumGetCString(DirectFunctionCall1(numeric_out, + NumericGetDatum(v->val.numeric))); + default: + return NULL; + } +} + +/* Convenience for the common "parse, then take one field" shape. */ +static char * +json_field(const char *json, const char *field) +{ + return jb_field(parse_json(json), field); +} + +/* The prefix under which this cluster's nodes register. */ +static char * +nodes_prefix(void) +{ + return psprintf("%s%s", spock_quorum_cluster_id, ETCD_NODES_INFIX); +} + +/* + * range_end for a prefix scan is the prefix with its last byte incremented, + * which is how etcd expresses "everything under this prefix". + */ +static char * +prefix_end(const char *prefix) +{ + char *end = pstrdup(prefix); + int len = (int) strlen(end); + + if (len > 0) + end[len - 1]++; + return end; +} + +static bool +etcd_grant_lease(char **errdetail) +{ + char *body = psprintf("{\"TTL\":\"%d\"}", ETCD_LEASE_TTL_SECONDS); + char *resp = etcd_post("/v3/lease/grant", body, errdetail); + char *id; + + if (resp == NULL) + return false; + + id = json_field(resp, "ID"); + if (id == NULL) + { + *errdetail = pstrdup("etcd lease grant returned no ID"); + return false; + } + + etcd_lease_id = strtoll(id, NULL, 10); + return etcd_lease_id != 0; +} + +/* Register (or re-register) this node under the nodes prefix. */ +static bool +etcd_put_self(char **errdetail) +{ + char *key = psprintf("%s%s", nodes_prefix(), etcd_self_name); + char *body = psprintf("{\"key\":\"%s\",\"value\":\"%s\",\"lease\":\"%lld\"}", + b64(key), b64(etcd_self_name), + (long long) etcd_lease_id); + + return etcd_post("/v3/kv/put", body, errdetail) != NULL; +} + +/* --- provider entry points -------------------------------------------- */ + +/* + * Startup deliberately touches neither etcd nor a lease. + * + * Registration is owned by the single long-lived worker that calls + * refresh(); an ordinary backend asking spock.quorum_status() must be able + * to read the cluster's view without minting a lease of its own and + * registering this node a second time. So startup only resolves identity, + * and everything with a side effect lives in refresh(). + */ +static bool +etcd_startup(char **errdetail) +{ + SpockLocalNode *local; + MemoryContext old; + + if (!curl_initialized) + { + curl_global_init(CURL_GLOBAL_DEFAULT); + curl_initialized = true; + } + + local = get_local_node(false, true); + if (local == NULL) + { + *errdetail = pstrdup("no local spock node"); + return false; + } + + old = MemoryContextSwitchTo(TopMemoryContext); + etcd_self_name = pstrdup(local->node->name); + MemoryContextSwitchTo(old); + + return true; +} + +static void +etcd_shutdown(void) +{ + char *detail = NULL; + + /* + * Revoke rather than waiting for the TTL, so a clean shutdown is visible + * to peers immediately instead of looking like a node that died. + */ + if (etcd_lease_id != 0) + { + char *body = psprintf("{\"ID\":\"%lld\"}", (long long) etcd_lease_id); + + (void) etcd_post("/v3/lease/revoke", body, &detail); + etcd_lease_id = 0; + } +} + +/* + * Renew the lease. If it has already expired -- a long stall, or etcd was + * unreachable for longer than the TTL -- grant a fresh one and re-register, + * rather than silently continuing to look dead to every peer. + */ +static bool +etcd_refresh(char **errdetail) +{ + char *body; + char *resp; + char *ttl; + + if (etcd_lease_id == 0) + return etcd_grant_lease(errdetail) && etcd_put_self(errdetail); + + body = psprintf("{\"ID\":\"%lld\"}", (long long) etcd_lease_id); + resp = etcd_post("/v3/lease/keepalive", body, errdetail); + if (resp == NULL) + return false; + + ttl = json_field(resp, "TTL"); + if (ttl == NULL || strtoll(ttl, NULL, 10) <= 0) + { + etcd_lease_id = 0; + return etcd_grant_lease(errdetail) && etcd_put_self(errdetail); + } + return true; +} + +/* + * Quorum by linearizable read. + * + * /v3/maintenance/status is deliberately NOT used: the Status RPC is answered + * from the queried member's own state, so a member isolated in a minority + * partition happily reports the leader it last knew about. Treating that as + * proof of quorum is exactly the mistake this layer exists to avoid. + * + * A Range read, by contrast, is linearizable unless serializable is asked for, + * which means etcd only answers it from within a majority. A successful reply + * therefore is the proof. Reading our own nodes prefix with limit 1 keeps the + * response small and needs no extra key. + * + * A minority member cannot complete the read, so the call runs out its + * deadline and the answer is UNKNOWN rather than NO. Both are handled + * identically by the caller; UNKNOWN is simply the truthful one, because + * etcd never got far enough to say no. + */ +static SpockQuorumAnswer +etcd_have_quorum(char **errdetail) +{ + char *prefix = nodes_prefix(); + char *body = psprintf("{\"key\":\"%s\",\"range_end\":\"%s\"," + "\"limit\":\"1\",\"serializable\":false}", + b64(prefix), b64(prefix_end(prefix))); + + if (etcd_post("/v3/kv/range", body, errdetail) == NULL) + return SPOCK_QUORUM_UNKNOWN; + + return SPOCK_QUORUM_YES; +} + +/* + * Pull the "kvs" array out of a range response. Returns the container and + * its length, or NULL when the key is absent -- which etcd uses to mean "no + * matches", not an error. + */ +static JsonbContainer * +kvs_array(const char *resp, int *count, char **errdetail) +{ + Jsonb *jb = parse_json(resp); + JsonbValue *kvs; + + *count = 0; + if (jb == NULL) + { + *errdetail = pstrdup("etcd returned unparseable JSON"); + return NULL; + } + + kvs = getKeyJsonValueFromContainer(&jb->root, "kvs", 3, NULL); + if (kvs == NULL || kvs->type != jbvBinary) + return NULL; /* no key matched */ + + *count = (int) JsonContainerSize(kvs->val.binary.data); + return kvs->val.binary.data; +} + +/* One string field of the i'th array element, base64-decoded. */ +static char * +kv_field(JsonbContainer *arr, int i, const char *field) +{ + JsonbValue *elem = getIthJsonbValueFromContainer(arr, (uint32) i); + JsonbValue *v; + + if (elem == NULL || elem->type != jbvBinary) + return NULL; + + v = getKeyJsonValueFromContainer(elem->val.binary.data, field, + (int) strlen(field), NULL); + if (v == NULL || v->type != jbvString) + return NULL; + + return unb64(pnstrdup(v->val.string.val, v->val.string.len)); +} + +static List * +etcd_members(char **errdetail) +{ + char *prefix = nodes_prefix(); + char *body = psprintf("{\"key\":\"%s\",\"range_end\":\"%s\"}", + b64(prefix), b64(prefix_end(prefix))); + char *resp = etcd_post("/v3/kv/range", body, errdetail); + JsonbContainer *arr; + List *result = NIL; + int count; + int i; + + if (resp == NULL) + return NIL; + + arr = kvs_array(resp, &count, errdetail); + if (arr == NULL) + return NIL; /* nobody registered yet */ + + for (i = 0; i < count; i++) + { + char *key = kv_field(arr, i, "key"); + char *name; + SpockQuorumMember *m; + + if (key == NULL) + continue; + + /* The node name is the last path element of the key. */ + name = strrchr(key, '/'); + name = (name != NULL) ? name + 1 : key; + if (*name == '\0') + continue; + + m = palloc0(sizeof(SpockQuorumMember)); + m->name = pstrdup(name); + + /* + * Presence under the prefix is liveness: etcd drops the key when its + * owner's lease lapses, so anything still here renewed recently. + */ + m->live = true; + m->voting = true; + m->last_seen = GetCurrentTimestamp(); + result = lappend(result, m); + } + + return result; +} + +/* + * Leadership by create-if-absent on a leased key. The txn compares the + * key's create_revision against 0, which is etcd's idiom for "does not + * exist", so exactly one node can win. The lease means a leader that dies + * releases the key without anyone having to notice and intervene. + */ +static SpockQuorumAnswer +etcd_is_leader(char **errdetail) +{ + char *key = psprintf("%s%s", spock_quorum_cluster_id, ETCD_LEADER_SUFFIX); + char *kb = b64(key); + char *body; + char *resp; + char *succeeded; + char *holder; + + /* + * Only the lease-owning worker campaigns. A read-only backend answers by + * comparing the recorded holder, so asking the question can never change + * who leads. + */ + if (etcd_lease_id == 0) + { + char *who; + + /* + * startup() leaves this unset when there is no local node, and a + * comparison against it would dereference NULL. Without an identity + * there is no question to answer. + */ + if (etcd_self_name == NULL) + return SPOCK_QUORUM_UNKNOWN; + + who = etcd_leader_name(errdetail); + if (who == NULL) + return SPOCK_QUORUM_UNKNOWN; + return strcmp(who, etcd_self_name) == 0 + ? SPOCK_QUORUM_YES : SPOCK_QUORUM_NO; + } + + body = psprintf("{\"compare\":[{\"key\":\"%s\",\"target\":\"CREATE\"," + "\"result\":\"EQUAL\",\"create_revision\":\"0\"}]," + "\"success\":[{\"requestPut\":{\"key\":\"%s\"," + "\"value\":\"%s\",\"lease\":\"%lld\"}}]," + "\"failure\":[{\"requestRange\":{\"key\":\"%s\"}}]}", + kb, kb, b64(etcd_self_name), (long long) etcd_lease_id, kb); + + resp = etcd_post("/v3/kv/txn", body, errdetail); + if (resp == NULL) + return SPOCK_QUORUM_UNKNOWN; + + succeeded = json_field(resp, "succeeded"); + if (succeeded != NULL && strcmp(succeeded, "true") == 0) + return SPOCK_QUORUM_YES; /* we took it */ + + /* + * Someone holds it. It may still be us from an earlier tick, which is + * the common case, so compare rather than assuming we lost. + */ + holder = etcd_leader_name(errdetail); + if (holder == NULL) + return SPOCK_QUORUM_UNKNOWN; + return strcmp(holder, etcd_self_name) == 0 ? SPOCK_QUORUM_YES : SPOCK_QUORUM_NO; +} + +static char * +etcd_leader_name(char **errdetail) +{ + char *key = psprintf("%s%s", spock_quorum_cluster_id, ETCD_LEADER_SUFFIX); + char *body = psprintf("{\"key\":\"%s\"}", b64(key)); + char *resp = etcd_post("/v3/kv/range", body, errdetail); + JsonbContainer *arr; + int count; + + if (resp == NULL) + return NULL; + + arr = kvs_array(resp, &count, errdetail); + if (arr == NULL || count < 1) + return NULL; /* nobody holds it */ + + return kv_field(arr, 0, "value"); +} + +#else /* !SPOCK_HAVE_LIBCURL */ + +/* + * Built without an HTTP client. The provider still exists so that selecting + * it produces a clear explanation instead of a mysterious silence, and so + * that it degrades to exactly the conservative behaviour of 'none'. + */ +static const char * +etcd_unavailable(void) +{ + return "this build of Spock has no HTTP client, so the etcd provider is unavailable"; +} + +static bool +etcd_startup(char **errdetail) +{ + *errdetail = pstrdup(etcd_unavailable()); + return false; +} + +static void +etcd_shutdown(void) +{ +} + +static bool +etcd_refresh(char **errdetail) +{ + *errdetail = pstrdup(etcd_unavailable()); + return false; +} + +static SpockQuorumAnswer +etcd_have_quorum(char **errdetail) +{ + *errdetail = pstrdup(etcd_unavailable()); + return SPOCK_QUORUM_UNKNOWN; +} + +static List * +etcd_members(char **errdetail) +{ + *errdetail = pstrdup(etcd_unavailable()); + return NIL; +} + +static SpockQuorumAnswer +etcd_is_leader(char **errdetail) +{ + *errdetail = pstrdup(etcd_unavailable()); + return SPOCK_QUORUM_UNKNOWN; +} + +static char * +etcd_leader_name(char **errdetail) +{ + *errdetail = pstrdup(etcd_unavailable()); + return NULL; +} + +#endif /* SPOCK_HAVE_LIBCURL */ + +const SpockQuorumProvider spock_quorum_provider_etcd = { + .name = "etcd", + .startup = etcd_startup, + .shutdown = etcd_shutdown, + .refresh = etcd_refresh, + .have_quorum = etcd_have_quorum, + .members = etcd_members, + .is_leader = etcd_is_leader, + .leader = etcd_leader_name +}; diff --git a/tests/tap/schedule b/tests/tap/schedule index 7e8c457ea..f78d0cbf2 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -58,6 +58,7 @@ test: 037_wire_format_datestyle test: 038_reserved_schema_ddl_guard test: 044_apply_change_logging test: 045_lsn_from_commit_ts +test: 106_quorum_layer # Upgrade schema match test (builds from source, slow): #test: 018_upgrade_schema_match # diff --git a/tests/tap/t/106_quorum_layer.pl b/tests/tap/t/106_quorum_layer.pl new file mode 100644 index 000000000..b534eee88 --- /dev/null +++ b/tests/tap/t/106_quorum_layer.pl @@ -0,0 +1,270 @@ +use strict; +use warnings; +use Test::More; +use lib '.'; +use lib 't'; +use SpockTest qw( + create_cluster destroy_cluster + get_test_config scalar_query psql_or_bail +); + +# ============================================================================= +# Test: 106_quorum_layer.pl +# +# Covers the quorum layer's fail-safe surface. +# +# Deliberately runs no external quorum system. The property that matters most +# is what happens when the layer CANNOT get an answer -- provider absent, +# endpoint unset, endpoint unreachable, provider switched at runtime -- and all +# of that is reachable without an etcd daemon or a Raft cluster. Keeping the +# suite free of external services also keeps it runnable in CI. +# +# The one invariant every case below shares: an answer that could not be +# obtained is reported as NULL, never as false, and never as an error to the +# caller. +# ============================================================================= + +create_cluster(1, 'Create 1-node cluster for quorum-layer tests'); + +my $cfg = get_test_config(); +my $bin = $cfg->{pg_bin}; +my $host = $cfg->{host}; +my $port = $cfg->{node_ports}[0]; +my $db = $cfg->{db_name}; +my $user = $cfg->{db_user}; +my $datadir; + +# Run SQL, returning (combined output, exit code). +sub psql_try { + my ($sql) = @_; + local $ENV{PGOPTIONS} = '-c client_min_messages=error'; + my $out = `$bin/psql -X -h $host -p $port -d $db -U $user -v ON_ERROR_STOP=1 -tAc "$sql" 2>&1`; + my $rc = ($? >> 8); + chomp $out; + return ($out, $rc); +} + +# One field of the single quorum_status() row, with NULL rendered as 'NULL'. +sub status_field { + my ($field) = @_; + my ($out, $rc) = psql_try( + "SELECT coalesce($field\::text, 'NULL') FROM spock.quorum_status()"); + return $rc == 0 ? $out : "ERROR:$out"; +} + +# Change a GUC in postgresql.conf and reload. ALTER SYSTEM is avoided so the +# provider can be switched even when a value is rejected as out of range. +sub set_guc_reload { + my ($name, $value) = @_; + psql_or_bail(1, "ALTER SYSTEM SET $name = '$value'"); + psql_or_bail(1, "SELECT pg_reload_conf()"); + sleep(1); +} + +sub reset_guc_reload { + my ($name) = @_; + psql_or_bail(1, "ALTER SYSTEM RESET $name"); + psql_or_bail(1, "SELECT pg_reload_conf()"); + sleep(1); +} + +# -------------------------------------------------------------------------- +# Version and surface +# -------------------------------------------------------------------------- +is(scalar_query(1, "SELECT extversion FROM pg_extension WHERE extname = 'spock'"), + '6.1.0', 'extension reports version 6.1.0'); + +is(scalar_query(1, + "SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace " . + " WHERE n.nspname = 'spock' AND p.proname = 'quorum_status'"), + '1', 'spock.quorum_status() exists'); + +is(scalar_query(1, "SELECT count(*) FROM spock.quorum_status()"), + '1', 'quorum_status() returns exactly one row'); + +# VOLATILE, not STABLE: the function re-reads the provider, so the planner +# must not fold two calls in one statement into a single evaluation. +is(scalar_query(1, + "SELECT provolatile FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace " . + " WHERE n.nspname = 'spock' AND p.proname = 'quorum_status'"), + 'v', 'quorum_status() is declared VOLATILE'); + +# The function can influence nothing by itself, but it names the cluster and +# its provider, so it is not world-readable. +is(scalar_query(1, + "SELECT has_function_privilege('public', 'spock.quorum_status()', 'EXECUTE')"), + 'f', 'quorum_status() is not executable by PUBLIC'); + +# Every GUC the layer defines must be present, or a deployment cannot be +# configured at all. +for my $guc (qw(spock.quorum_provider spock.quorum_timeout + spock.quorum_cluster_id spock.quorum_etcd_endpoints)) { + is(scalar_query(1, "SELECT count(*) FROM pg_settings WHERE name = '$guc'"), + '1', "$guc is defined"); +} + +# -------------------------------------------------------------------------- +# The default: nothing is consulted +# +# This is the regression that matters most. With no provider configured the +# layer must be inert, so enabling the feature is always a deliberate act. +# -------------------------------------------------------------------------- +is(scalar_query(1, "SELECT setting FROM pg_settings WHERE name = 'spock.quorum_provider'"), + 'none', 'the default provider is none'); + +is(status_field('provider'), 'none', 'status reports the none provider'); +is(status_field('has_quorum'), 'NULL', 'has_quorum is NULL, not false, with no provider'); +is(status_field('is_leader'), 'NULL', 'is_leader is NULL with no provider'); +is(status_field('leader'), 'NULL', 'leader is NULL with no provider'); +is(status_field('last_error'), 'NULL', 'no error is reported when nothing was attempted'); + +# -------------------------------------------------------------------------- +# A provider that is selected but not installed +# +# The layer must say so plainly rather than failing to start or pretending to +# have an answer. +# -------------------------------------------------------------------------- +# A cluster id is now mandatory for every provider but 'none': the prefix is +# the only thing keeping two clusters' members apart, so there is no default. +set_guc_reload('spock.quorum_cluster_id', 'tap_cluster'); + +for my $prov (qw(pgraft pgbully)) { + set_guc_reload('spock.quorum_provider', $prov); + + is(status_field('provider'), $prov, "status reports the $prov provider"); + is(status_field('has_quorum'), 'NULL', + "has_quorum is NULL when $prov is not installed"); + like(status_field('last_error'), qr/\Q$prov\E/, + "last_error names $prov as the missing extension"); +} + +# -------------------------------------------------------------------------- +# A missing cluster id is refused, not defaulted +# +# Sharing one prefix between two clusters would make each count the other's +# nodes as its own, so the layer declines to start rather than guess. +# -------------------------------------------------------------------------- +reset_guc_reload('spock.quorum_cluster_id'); +is(status_field('has_quorum'), 'NULL', 'no answer while the cluster id is unset'); +like(status_field('last_error'), qr/quorum_cluster_id/, + 'last_error names the missing cluster id'); +set_guc_reload('spock.quorum_cluster_id', 'tap_cluster'); + +# -------------------------------------------------------------------------- +# etcd with nothing to talk to +# -------------------------------------------------------------------------- +set_guc_reload('spock.quorum_provider', 'etcd'); + +# No endpoints configured at all. +is(status_field('has_quorum'), 'NULL', 'has_quorum is NULL with no etcd endpoints'); +like(status_field('last_error'), qr/endpoints/, + 'last_error points at the unset endpoint list'); + +# An endpoint that is syntactically fine but has nothing listening. Port 1 is +# used because it is reserved and will never have a real service on it. +set_guc_reload('spock.quorum_etcd_endpoints', 'http://127.0.0.1:1'); +is(status_field('has_quorum'), 'NULL', 'has_quorum is NULL when etcd is unreachable'); +like(status_field('last_error'), qr/127\.0\.0\.1:1/, + 'last_error names the endpoint that could not be reached'); + +# Asking again must not raise: a provider is forbidden from throwing, and the +# status view has to stay usable while the cluster is unhealthy. +my (undef, $again_rc) = psql_try("SELECT * FROM spock.quorum_status()"); +is($again_rc, 0, 'the status view keeps working while the provider is unreachable'); + +# Malformed endpoint lists are a configuration error, not a crash. +set_guc_reload('spock.quorum_etcd_endpoints', ',,,'); +is(status_field('has_quorum'), 'NULL', 'a malformed endpoint list yields no answer'); +my (undef, $mal_rc) = psql_try("SELECT * FROM spock.quorum_status()"); +is($mal_rc, 0, 'a malformed endpoint list does not raise'); + +reset_guc_reload('spock.quorum_etcd_endpoints'); + +# -------------------------------------------------------------------------- +# A failing provider must not poison the caller's transaction +# +# Catching the error without an internal subtransaction leaves the surrounding +# transaction aborted, so the operator's next statement fails with "current +# transaction is aborted". The status view is consulted from whatever +# transaction they happen to be in, so this has to hold. +# -------------------------------------------------------------------------- +my ($txn, $txn_rc) = psql_try( + "BEGIN; SELECT 1 AS before; SELECT has_quorum FROM spock.quorum_status(); " . + "SELECT 2 AS after; COMMIT"); +is($txn_rc, 0, 'a transaction survives consulting an unreachable provider'); +like($txn, qr/\b2\b/, 'statements after the failed consult still run'); + +# -------------------------------------------------------------------------- +# Runtime reconfiguration +# +# The provider is PGC_SIGHUP. A long-lived backend must notice a change rather +# than answering from whatever was configured when it first connected. +# -------------------------------------------------------------------------- +set_guc_reload('spock.quorum_provider', 'none'); +is(status_field('provider'), 'none', 'switching back to none is picked up'); +is(status_field('last_error'), 'NULL', 'switching provider clears the stale error'); + +set_guc_reload('spock.quorum_provider', 'pgraft'); +is(status_field('provider'), 'pgraft', 'switching away from none is picked up'); + +# Within a single session, too: the checks above each used a fresh backend, +# which would hide a provider cached for the life of a connection. Statements +# go in on stdin rather than through -c, because ALTER SYSTEM cannot run inside +# a transaction block and -c wraps its whole string in one. +my $session = `$bin/psql -X -h $host -p $port -d $db -U $user -tA 2>&1 <<'EOSQL' +SELECT 'first=' || provider FROM spock.quorum_status(); +ALTER SYSTEM SET spock.quorum_provider = 'none'; +SELECT pg_reload_conf(); +SELECT pg_sleep(1); +SELECT 'second=' || provider FROM spock.quorum_status(); +EOSQL`; +like($session, qr/first=pgraft/, 'the session starts on the configured provider'); +like($session, qr/second=none/, + 'the same backend reports the new provider after a reload'); + +reset_guc_reload('spock.quorum_provider'); + +# -------------------------------------------------------------------------- +# GUC bounds +# +# The timeout is the deadline that keeps a wedged provider from stalling the +# caller, so its bounds are load-bearing rather than cosmetic. +# -------------------------------------------------------------------------- +is(scalar_query(1, + "SELECT min_val || '..' || max_val FROM pg_settings WHERE name = 'spock.quorum_timeout'"), + '100..60000', 'the timeout is bounded to a sane millisecond range'); + +# PGC_SIGHUP: a session cannot change it at all, whatever the value. That is +# deliberate -- the deadline protects a shared worker, not one backend. +my (undef, $set_rc) = psql_try("SET spock.quorum_timeout = '5s'"); +isnt($set_rc, 0, 'the timeout cannot be changed by a single session'); +my (undef, $set_prov_rc) = psql_try("SET spock.quorum_provider = 'etcd'"); +isnt($set_prov_rc, 0, 'the provider cannot be changed by a single session'); + +# Values are still validated when set the supported way. +my (undef, $lo_rc) = psql_try("ALTER SYSTEM SET spock.quorum_timeout = '1ms'"); +isnt($lo_rc, 0, 'a timeout below the minimum is rejected'); +my (undef, $hi_rc) = psql_try("ALTER SYSTEM SET spock.quorum_timeout = '10min'"); +isnt($hi_rc, 0, 'a timeout above the maximum is rejected'); +my (undef, $ok_rc) = psql_try("ALTER SYSTEM SET spock.quorum_timeout = '5s'"); +is($ok_rc, 0, 'a timeout inside the range is accepted'); +reset_guc_reload('spock.quorum_timeout'); + +my (undef, $bad_prov_rc) = psql_try("ALTER SYSTEM SET spock.quorum_provider = 'wobble'"); +isnt($bad_prov_rc, 0, 'an unknown provider name is rejected'); + +# -------------------------------------------------------------------------- +# Nothing the layer does perturbs replication +# +# The layer is inert by design; this is the check that it stays that way. +# -------------------------------------------------------------------------- +is(scalar_query(1, + "SELECT count(*) FROM pg_replication_slots WHERE slot_name LIKE '%quorum%'"), + '0', 'the quorum layer creates no replication slots'); + +is(scalar_query(1, + "SELECT count(*) FROM pg_stat_activity WHERE application_name LIKE '%quorum%'"), + '0', 'the quorum layer starts no background worker of its own'); + +destroy_cluster('Destroy quorum-layer test cluster'); +done_testing();