diff --git a/Makefile b/Makefile index 1105a4fb4..ad271c126 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,24 @@ ifdef SPOCK_RANDOM_DELAYS PG_CPPFLAGS += -DSPOCK_RANDOM_DELAYS endif SHLIB_LINK += $(libpq) $(filter -lintl, $(LIBS)) + +# ----------------------------------------------------------------------------- +# Optional HTTP client, for the etcd quorum provider +# ----------------------------------------------------------------------------- +# etcd runs as its own daemon and is reached over its v3 HTTP/JSON gateway, so +# that provider needs an HTTP client. libcurl is used when curl-config is on +# PATH and skipped otherwise: a dependency that only one optional provider +# needs must not be one everybody has to satisfy. Without it the provider is +# still built and still selectable; it reports why it cannot be used. +# +# Set NO_LIBCURL=1 to force it off even where libcurl is available. +ifndef NO_LIBCURL +CURL_CONFIG := $(shell command -v curl-config 2>/dev/null) +ifneq ($(CURL_CONFIG),) +PG_CPPFLAGS += -DSPOCK_HAVE_LIBCURL $(shell $(CURL_CONFIG) --cflags) +SHLIB_LINK += $(shell $(CURL_CONFIG) --libs) +endif +endif ifdef NO_LOG_OLD_VALUE PG_CPPFLAGS += -DNO_LOG_OLD_VALUE endif diff --git a/docs/internals-doc/specs/spock-quorum-layer-design.md b/docs/internals-doc/specs/spock-quorum-layer-design.md new file mode 100644 index 000000000..d5ae0cb27 --- /dev/null +++ b/docs/internals-doc/specs/spock-quorum-layer-design.md @@ -0,0 +1,262 @@ +# A pluggable quorum layer for Spock + +**Date:** 2026-09-01 (revised after implementing all three providers) +**Status:** Implemented — layer and providers; no consumer yet +**Branch:** `QURAM` (based on `main`) + +> **Revised from the original proposal.** Three things the design got wrong, +> corrected below and worth stating up front because they were the load-bearing +> assumptions: +> +> 1. **Capability tiers were dropped.** The interface is uniform. Tiering was +> solving a problem that disappeared once the interface stopped asking for +> storage. +> 2. **pgBully turned out to be the *most* capable provider for liveness, and +> pgraft the least** — the opposite of what was predicted. No upstream change +> to pgBully was needed. +> 3. **A per-tick snapshot was added** after the first implementation showed the +> layer could make decisions against a cluster state that never existed. + +## Problem + +Spock decides everything about WAL retention from local catalogs. +`group_slot_evaluate()` refuses to advance the group slot when any required +member has not reported progress recently, and `spock.progress` only records what +*this* node received *from* a member, never what that member has confirmed. + +So a single unreachable node pins WAL on every survivor, indefinitely — during an +incident, exactly when disk headroom matters most. There is also no notion of a +majority, so a topology change needs every node reachable. + +Closing that needs agreement between nodes. It does not need Spock to implement +consensus, or to marry itself to one implementation. + +## Goals + +- One interface, several backends, none privileged. +- With no provider configured, behaviour is exactly as today, bit for bit. +- Uncertainty never releases WAL. A broken quorum layer degrades to today's + conservative behaviour, never past it. +- Attaching a system requires no new build dependency in Spock core. + +## Non-goals + +- Spock does not implement consensus and does not arbitrate elections. +- No Raft vocabulary in the interface. No terms, no log indexes — those are one + implementation's concepts and would leak into an interface meant to outlast it. +- Not a general cluster manager. The scope is replication decisions. + +## Why the interface is uniform + +The original design tiered the interface, because the backends are not equally +capable and pgBully in particular has no replicated storage. That was the wrong +cut. The interface asks only for **judgements**, never for **storage**, and once +that is true there is nothing left to negotiate: Spock already keeps its durable +state in its own crash-safe catalogs, and what it lacks is not a place to write +things down but a second opinion about who is alive. + +So every provider implements the same seven entry points, and a provider with +nothing to renew supplies a `refresh()` that returns true. That is the whole +contract, in `include/spock_quorum.h`. + +Every answer is three-valued: + +```c +typedef enum SpockQuorumAnswer +{ + SPOCK_QUORUM_NO = 0, + SPOCK_QUORUM_YES, + SPOCK_QUORUM_UNKNOWN +} SpockQuorumAnswer; +``` + +`UNKNOWN` is deliberately not folded into `NO`. Callers treat them identically — +that is the fail-safe rule — but keeping them apart is what lets the status view +distinguish *a cluster that lost quorum* from *a provider that stopped +answering*. During an incident those demand different responses, and a boolean +cannot tell them apart. This distinction earned its keep immediately: with pgraft +selected, "pgraft is not installed" and "pgraft reports no leader" are both +failures to proceed, and an operator needs to know which one they have. + +## The fail-safe contract + +A layer that can release WAL can destroy a node's ability to catch up. These +rules make that impossible by construction: + +1. **Uncertainty means no.** Any error, timeout, or `UNKNOWN` produces today's + conservative behaviour. No configuration inverts this. +2. **Off the hot path.** Never from an apply worker, a walsender, or anything a + client waits on. Today the only consult is `spock.quorum_status()`, which + invalidates first. A consuming worker (group-slot eviction) is not built yet. +3. **Deadlines, not hope.** Every call is bounded by `spock.quorum_timeout` + (default 2s). Overrunning is `UNKNOWN`. etcd applies it as an HTTP deadline; + pgraft and pgBully apply it as `SET LOCAL statement_timeout`. +4. **Providers may not throw.** A callback that `ereport(ERROR)`s would abort the + very tick deciding whether releasing WAL is safe. Callbacks return a status + and an `errdetail` string. +5. **One reading per tick.** Added after implementation, see below. +6. **Releasing a member is bounded and loud.** See *Group-slot integration*. + +### Rule 5, and why it was added + +The first implementation consulted the provider per question. Deciding about five +members cost ten round trips — and, far worse, nothing stopped quorum from +answering YES to the first question and NO to the fourth. Decisions inside one +tick could rest on a cluster state that never existed at any instant. + +The layer now takes **one reading per tick** and decides against it. Efficiency +is the lesser benefit; self-consistency is the point. Leadership and membership +are not even asked for unless quorum is held, since a partitioned minority can +still believe it leads and still see some peers, and acting on either is the +failure this layer exists to prevent. + +`spock.quorum_status()` explicitly invalidates first, because an operator running +it is asking about now, not about whatever the last tick saw. + +## Where providers live + +Spock core ships all three providers and depends on nothing new. etcd's HTTP +client is the only external dependency, and it is detected at build time via +`curl-config` rather than required: without it the provider still compiles and is +still selectable, and reports why it cannot be used. `NO_LIBCURL=1` forces it +off. Selecting a provider you did not build is a configuration mistake, not a +reason to fail to start. + +Selection is one GUC: `spock.quorum_provider = none | etcd | pgraft | pgbully`. + +## What the three providers actually turned out to be + +The original table was a prediction. This one is measured, and it changed again +once pgraft reached 2.0 and pgBully grew a cluster-manager API. + +| | etcd | pgraft 2.0 | pgBully | +|---|---|---|---| +| Transport | HTTP/JSON (daemon) | SPI (in-database) | SPI (in-database) | +| API shape | etcd v3 | `pgraft.*` cluster manager | `pgbully.*`, **the same one** | +| Quorum | native | `leader_id` set | `leader_id` set | +| Leadership | leased key | native | native | +| **Per-member liveness** | **yes** (lease TTL) | **no** | **yes** (`peers().reachable`) | +| Name mapping | own registration | replicated KV | replicated KV | + +**pgraft and pgBully share one implementation.** They remain two providers, selected by distinct values of `spock.quorum_provider`, but they expose the identical +in-database interface — `get_cluster_status()`, `get_nodes()`, `is_leader()`, +`kv_put`/`kv_get` — differing only in the schema it lives under. Keeping two +near-identical files would have guaranteed drift, so `spock_quorum_cluster.c` +implements both provider tables, parameterised by schema name. + +That collapse also retired a piece of scaffolding: pgBully previously had no +replicated storage, so node names had to be recovered by joining connection +strings between `pgbully.peers()` and `spock.node_interface`. With a KV +available it registers its id-to-name mapping the same way pgraft does, and the +conninfo join is gone. + +**One real difference remains, and it is not cosmetic.** pgBully publishes +`peers().reachable`; pgraft still exposes no reachability at all. So the shared +implementation takes a per-backend liveness expression: pgBully joins its peer +table, pgraft reports every configured member live. + +Reporting all-live is the safe reading rather than a pretence that nothing has +failed: under rule 1 a live member is never evicted, so nothing can be released +on the strength of a backend with no opinion. The practical consequence is that +pgraft users get quorum and leadership from this layer but not eviction, until a +last-contact column exists upstream. Raft already tracks it internally to drive +heartbeats; it is simply not published. + +The lesson worth keeping: **consensus strength and observability are independent +axes.** The most sophisticated backend is the least useful here, because the +interface needs a fact it happens not to expose. + +## A pgraft bug found on the way + +`pgraft_shmem_startup_hook()` never called `prev_shmem_startup_hook`, though +`_PG_init` saves it and the matching *request* hook chains correctly. With +`shared_preload_libraries = 'spock,pgraft'`, pgraft's hook became head of the +chain and silently dropped spock's, so spock's shared memory was never +initialised and its supervisor segfaulted in a restart loop. This breaks any +extension loaded before pgraft. Fixed in `pgraft/src/pgraft.c`. + +## Known limitations + +**A leader id is weaker evidence than it looks.** Both in-database backends +report quorum from a non-zero `leader_id`. A follower keeps the last leader it +knew about until its election timeout expires, and pgraft leaves `CheckQuorum` +disabled, so an isolated leader can go on publishing its own id. Either can +therefore answer YES from a minority for a bounded window. + +Spock cannot close this from its side: neither backend publishes a majority +signal to ask for instead. The etcd provider avoids the problem entirely by +using a linearizable read, which only a member inside a majority can complete. +Until an equivalent exists in-database, the in-database backends should not be +trusted for anything that releases WAL — which is also why pgraft cannot +support eviction. + +**pgraft must carry the shared-memory hook fix.** `pgraft_shmem_startup_hook()` +historically failed to chain `prev_shmem_startup_hook`, which silently dropped +the shared-memory initialisation of every extension loaded before it and +segfaulted Spock's supervisor in a restart loop. Deployments must use a pgraft +build containing that fix; the extension carries no version constraint that +would enforce it, so it has to be checked by hand. + +## Group-slot integration + +Not yet built. This is where the correctness risk lives, so it is last and starts +disabled. + +`group_slot_evaluate()`'s `stale_progress` branch becomes conditional: + +| Provider | Quorum | Member | Behaviour | +|---|---|---|---| +| none | — | — | Exactly today: block. | +| any | no / unknown | — | Block. | +| any | yes | live | Block — it is up, just behind. A real backlog. | +| any | yes | not live | Eligible for release, subject to the rule below. | + +The rule matters more than the table. **Advancing past what a down node still +needs means it can never resume by replication and will require a full resync.** +PGD accepts that trade under majority; so should we, but only explicitly: + +- A member is released only after being continuously non-live for + `spock.quorum_member_eviction_timeout` (generous by default — minutes, not + seconds). A node rebooting must not lose its place. +- Release is logged at `LOG` naming the member and the horizon moved past, and + recorded in `group_slot_state`. An operator must be able to answer "why does n3 + need a resync?" from the log. +- All of it sits behind `spock.quorum_advance`, **default off**. + +Note that with pgraft this branch is unreachable, since it never reports a member +as non-live. That is correct, not a gap. + +## Testing + +Backend-specific tests only need to prove the mapping is right. Spock's own logic +should be proven against a driveable mock provider, which does not exist yet and +is the main gap in the current work — it is what would make the awkward states +(quorum lost mid-tick, leadership changing under a decision, a provider that +times out) deterministic in CI without an etcd daemon or a Raft cluster. + +The `none` provider carries a standing regression obligation: with no provider +configured, behaviour must be byte-identical to today. + +## Status and remaining work + +Done: the layer, all three providers, `spock.quorum_status()`, GUCs, build +wiring. All three verified against live daemons. The existing pg_regress suite +still passes; there are no quorum-specific regress or TAP tests yet. + +Remaining, in order: + +1. **Mock provider and TAP coverage.** The correctness of everything above rests + on states that are currently only reachable by hand. +2. **Group-slot integration** behind `spock.quorum_advance = off`. Not a + QURAM-layer remaining item — it is the consumer, on a separate branch. +3. Optional: the additive pgraft last-contact column, which would make its + provider a peer of the other two. + +## Open questions + +- Should `members()` reconcile against `spock.node`, or report the provider's + view raw and let the caller intersect? Currently the latter, since a quorum + system may legitimately govern more nodes than Spock does. +- Does anything besides the group slot want this layer? Read-only mode and + failover-slot promotion are plausible second consumers, and if either is + likely, nothing here should be named after group slots. diff --git a/include/spock.h b/include/spock.h index 61e150556..a26a7f969 100644 --- a/include/spock.h +++ b/include/spock.h @@ -24,8 +24,8 @@ #include "spock_fe.h" #include "spock_node.h" -#define SPOCK_VERSION "6.0.0" -#define SPOCK_VERSION_NUM 60000 +#define SPOCK_VERSION "6.1.0" +#define SPOCK_VERSION_NUM 60100 #define EXTENSION_NAME "spock" diff --git a/include/spock_quorum.h b/include/spock_quorum.h new file mode 100644 index 000000000..6d0040d11 --- /dev/null +++ b/include/spock_quorum.h @@ -0,0 +1,162 @@ +/*------------------------------------------------------------------------- + * + * spock_quorum.h + * Pluggable quorum provider interface. + * + * Spock does not implement consensus. It asks an external system a small + * number of questions and stays conservative when it cannot get an answer. + * This header is the whole contract. + * + * Every provider implements the same interface -- there are no optional + * entry points and no capability negotiation. That is affordable because + * the interface asks only for judgements, never for storage: Spock already + * keeps its durable state in its own crash-safe catalogs. Keeping shared + * storage out is what lets a leader-election-only system such as pgBully sit + * behind the identical interface as etcd, which has a replicated key space. + * + * Copyright (c) 2022-2026, pgEdge, Inc. + * + *------------------------------------------------------------------------- + */ +#ifndef SPOCK_QUORUM_H +#define SPOCK_QUORUM_H + +#include "postgres.h" +#include "nodes/pg_list.h" +#include "storage/latch.h" +#include "utils/timestamp.h" + +/* + * Which provider is active. Selected by spock.quorum_provider; the order + * here is the order of the GUC's enum table. + */ +typedef enum SpockQuorumProviderId +{ + SPOCK_QUORUM_PROVIDER_NONE = 0, + SPOCK_QUORUM_PROVIDER_ETCD, + SPOCK_QUORUM_PROVIDER_PGRAFT, + SPOCK_QUORUM_PROVIDER_PGBULLY +} SpockQuorumProviderId; + +/* + * Every answer is three-valued. UNKNOWN is not an error code: it is the + * honest reply when the provider is unreachable, slow, or partitioned. + * Callers treat it exactly as they treat NO -- that is the fail-safe rule -- + * but the two are kept apart so the status view and the log can distinguish + * a cluster that lost quorum from a provider that stopped answering. During + * an incident those call for different responses, and a boolean cannot tell + * them apart. + */ +typedef enum SpockQuorumAnswer +{ + SPOCK_QUORUM_NO = 0, + SPOCK_QUORUM_YES, + SPOCK_QUORUM_UNKNOWN +} SpockQuorumAnswer; + +/* One member, as the quorum system sees it -- not as spock.node sees it. */ +typedef struct SpockQuorumMember +{ + char *name; /* matches spock.node.node_name */ + bool live; /* reachable, in the provider's judgement */ + bool voting; /* counts toward a majority */ + TimestampTz last_seen; /* 0 when the provider does not track it */ +} SpockQuorumMember; + +/* + * Provider callbacks. All of them are mandatory: a provider with nothing to + * do in refresh() supplies a function that returns true. + * + * Contract for every entry point: + * + * - Never from an apply worker, a walsender, or any path a client waits + * on. A wedged provider must not be able to stall replication. Today + * the only consult is spock.quorum_status(), which invalidates first. + * A consuming worker (group-slot eviction) is not built yet. + * - Must respect spock.quorum_timeout. Overrunning it is UNKNOWN, not a + * reason to keep waiting. etcd applies it as an HTTP deadline; the + * in-database providers apply it as SET LOCAL statement_timeout. + * - Must not ereport(ERROR). Return UNKNOWN/false and put a human-readable + * reason in *errdetail (palloc'd in the caller's context) instead. An + * error thrown here would abort the very tick that is deciding whether it + * is safe to release WAL. + * - Must be free of side effects, with the deliberate exception of + * refresh(), which is where a provider renews whatever registration it + * needs to stay visible to its peers. + */ +typedef struct SpockQuorumProvider +{ + const char *name; /* shown in spock.quorum_status() */ + + /* Called once when the worker starts, and once when it stops. */ + bool (*startup) (char **errdetail); + void (*shutdown) (void); + + /* + * Called at the top of every worker tick. This is where a provider + * renews a lease or heartbeat. Providers whose peers track liveness for + * them return true without doing anything. + */ + bool (*refresh) (char **errdetail); + + /* Does this node currently belong to a quorum? */ + SpockQuorumAnswer (*have_quorum) (char **errdetail); + + /* + * The cluster's view of its members: a List of SpockQuorumMember *, or + * NIL with *errdetail set. Names with no matching spock.node row are + * ignored by the caller, since the quorum system may govern more than + * Spock does. + */ + List *(*members) (char **errdetail); + + /* Is this node the one that should act for the cluster? */ + SpockQuorumAnswer (*is_leader) (char **errdetail); + + /* Name of the current leader, or NULL when unknown. */ + char *(*leader) (char **errdetail); +} SpockQuorumProvider; + +/* --- GUCs (defined in spock.c) ----------------------------------------- */ + +extern int spock_quorum_provider; /* SpockQuorumProviderId */ +extern int spock_quorum_timeout; /* milliseconds */ +extern char *spock_quorum_etcd_endpoints; +extern char *spock_quorum_cluster_id; + +/* --- Consumed by the rest of Spock ------------------------------------- */ + +/* + * These wrap the active provider and apply the fail-safe rules, so callers + * never touch a provider directly and cannot forget to handle UNKNOWN. + */ +extern void spock_quorum_startup(void); +extern void spock_quorum_shutdown(void); +extern void spock_quorum_refresh(void); + +/* True only for an unambiguous YES. UNKNOWN and NO are both false. */ +extern bool spock_quorum_have_quorum(void); +extern bool spock_quorum_is_leader(void); + +/* NIL when there is no provider or the answer is unavailable. */ +extern List *spock_quorum_members(void); + +/* + * Is this member live in the cluster's judgement? UNKNOWN when there is no + * provider, which is what keeps a default build behaving exactly as it does + * today. + */ +extern SpockQuorumAnswer spock_quorum_member_live(const char *node_name); + +/* Backing spock.quorum_status(). */ +extern const char *spock_quorum_provider_name(void); +extern const char *spock_quorum_last_error(void); +extern TimestampTz spock_quorum_last_consulted(void); + +/* Provider tables, each defined by its own file. */ +extern const SpockQuorumProvider spock_quorum_provider_none; +extern const SpockQuorumProvider spock_quorum_provider_etcd; +extern const SpockQuorumProvider spock_quorum_provider_pgraft; +extern const SpockQuorumProvider spock_quorum_provider_pgbully; + +#endif /* SPOCK_QUORUM_H */ diff --git a/sql/spock--6.0.0--6.1.0.sql b/sql/spock--6.0.0--6.1.0.sql new file mode 100644 index 000000000..3a4a91d7f --- /dev/null +++ b/sql/spock--6.0.0--6.1.0.sql @@ -0,0 +1,26 @@ +/* spock--6.0.0--6.1.0.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION spock UPDATE TO '6.1.0'" to load this file. \quit + +-- Quorum layer. +-- +-- Spock does not implement consensus; spock.quorum_provider selects the +-- external system consulted for quorum decisions. This view exists because +-- anything able to influence WAL retention has to be inspectable before it is +-- trusted to. has_quorum and is_leader are NULL, not false, when no answer +-- could be obtained: "we are not in a quorum" and "we could not ask" call for +-- different responses during an incident. +CREATE FUNCTION spock.quorum_status( + OUT provider text, + OUT has_quorum boolean, + OUT is_leader boolean, + OUT leader text, + OUT last_consulted timestamptz, + OUT last_error text) +-- VOLATILE, not STABLE: the function deliberately invalidates the cached +-- reading and consults the provider afresh, so two calls in one statement +-- can legitimately differ. STABLE would let the planner fold them together +-- and report a stale answer. +RETURNS record VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_quorum_status_sql'; +REVOKE ALL ON FUNCTION spock.quorum_status() FROM PUBLIC; diff --git a/src/spock.c b/src/spock.c index e64357eab..6b39b6339 100644 --- a/src/spock.c +++ b/src/spock.c @@ -62,6 +62,7 @@ #include "spock_conflict.h" #include "spock_rmgr.h" #include "spock_worker.h" +#include "spock_quorum.h" #include "spock_output_config.h" #include "spock_output_plugin.h" #include "spock_exception_handler.h" @@ -162,6 +163,19 @@ static const struct config_enum_entry apply_change_logging_options[] = { {NULL, 0, false} }; +static const struct config_enum_entry quorum_provider_options[] = { + {"none", SPOCK_QUORUM_PROVIDER_NONE, false}, + {"etcd", SPOCK_QUORUM_PROVIDER_ETCD, false}, + {"pgraft", SPOCK_QUORUM_PROVIDER_PGRAFT, false}, + {"pgbully", SPOCK_QUORUM_PROVIDER_PGBULLY, false}, + {NULL, 0, false} +}; + +int spock_quorum_provider = SPOCK_QUORUM_PROVIDER_NONE; +int spock_quorum_timeout = 2000; +char *spock_quorum_etcd_endpoints = ""; +char *spock_quorum_cluster_id = ""; + bool spock_synchronous_commit = false; char *spock_temp_directory = ""; static char *spock_temp_directory_config; @@ -1159,6 +1173,56 @@ _PG_init(void) PGC_SIGHUP, 0, NULL, NULL, NULL); + DefineCustomEnumVariable("spock.quorum_provider", + gettext_noop("External system consulted for quorum decisions."), + gettext_noop("Spock does not implement consensus; it asks the " + "selected system whether this node is in a quorum " + "and which members are live. With 'none' no system " + "is consulted and Spock behaves conservatively, " + "exactly as it does without this feature."), + &spock_quorum_provider, + SPOCK_QUORUM_PROVIDER_NONE, + quorum_provider_options, + PGC_SIGHUP, 0, + NULL, NULL, NULL); + + DefineCustomIntVariable("spock.quorum_timeout", + gettext_noop("Deadline for a single call to the quorum provider."), + gettext_noop("Overrunning this is treated as an unknown answer, " + "which is handled exactly like a lost quorum: " + "conservatively. It is never a reason to keep " + "waiting, because these calls run on the path that " + "decides whether WAL may be released."), + &spock_quorum_timeout, + 2000, + 100, + 60000, + PGC_SIGHUP, + GUC_UNIT_MS, + NULL, NULL, NULL); + + DefineCustomStringVariable("spock.quorum_etcd_endpoints", + gettext_noop("Comma-separated etcd base URLs."), + gettext_noop("For example http://127.0.0.1:2379. Endpoints are " + "tried in rotation, so one unreachable member " + "costs a single tick rather than every tick."), + &spock_quorum_etcd_endpoints, + "", + PGC_SIGHUP, 0, + NULL, NULL, NULL); + + DefineCustomStringVariable("spock.quorum_cluster_id", + gettext_noop("Key-space prefix identifying this Spock cluster."), + gettext_noop("Required whenever a provider other than 'none' is " + "selected. It has no default on purpose: two clusters " + "sharing one quorum system and one prefix would each " + "count the other's nodes as its own members, and a " + "default is exactly how that happens unnoticed."), + &spock_quorum_cluster_id, + "", + PGC_SIGHUP, 0, + NULL, NULL, NULL); + DefineCustomIntVariable("spock.stats_max_entries", "Maximum entries for statistics", "Maximum number of entries that can be " diff --git a/src/spock_quorum.c b/src/spock_quorum.c new file mode 100644 index 000000000..bfd457876 --- /dev/null +++ b/src/spock_quorum.c @@ -0,0 +1,558 @@ +/*------------------------------------------------------------------------- + * + * spock_quorum.c + * Provider dispatch and the fail-safe rules around it. + * + * Nothing outside this file calls a provider directly. Every question goes + * through a wrapper here, and every wrapper turns an unusable answer into + * the conservative one. That is the whole point of the indirection: a + * caller cannot forget to handle UNKNOWN, because it never sees it. + * + * Copyright (c) 2022-2026, pgEdge, Inc. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "funcapi.h" +#include "miscadmin.h" + +#include "utils/builtins.h" +#include "utils/memutils.h" +#include "utils/timestamp.h" + +#include "spock.h" +#include "spock_quorum.h" + +PG_FUNCTION_INFO_V1(spock_quorum_status_sql); + +/* GUC storage; the variables themselves are defined in spock.c. */ + +/* + * The provider in force for this worker. Resolved at startup rather than + * read from the GUC per call: swapping providers underneath a half-finished + * decision is not a state worth supporting, and a SIGHUP that changes the + * provider restarts the worker anyway. + */ +static const SpockQuorumProvider *active = NULL; +static bool active_started = false; + +/* Diagnostics for spock.quorum_status(); never load-bearing. */ +static char *last_error = NULL; +static TimestampTz last_consulted = 0; + +/* + * One consult per tick, cached. + * + * Efficiency is the lesser reason. The real one is consistency: without a + * snapshot, deciding about five members asked the provider ten times, and + * nothing stopped quorum from being YES for the first question and NO for + * the fourth. Decisions within one tick would then rest on a cluster state + * that never existed at any single instant. Taking one reading and deciding + * against it is both cheaper and honest. + */ +static bool snap_valid = false; +static SpockQuorumAnswer snap_quorum = SPOCK_QUORUM_UNKNOWN; +static SpockQuorumAnswer snap_leader = SPOCK_QUORUM_UNKNOWN; +static List *snap_members = NIL; +static char *snap_leader_name = NULL; +static MemoryContext snap_ctx = NULL; + +/* The GUC value the active provider was resolved from. */ +static int active_provider_id = -1; + +static bool snapshot_take(void); +static void spock_quorum_invalidate(void); + +/* + * Record why the most recent consult failed. Kept in TopMemoryContext + * because the worker's per-tick context is reset underneath us, and this + * string has to outlive the tick that produced it in order to be worth + * anything to an operator. + */ +static void +note_error(const char *detail) +{ + MemoryContext old; + + if (last_error != NULL) + { + pfree(last_error); + last_error = NULL; + } + + if (detail == NULL) + return; + + old = MemoryContextSwitchTo(TopMemoryContext); + last_error = pstrdup(detail); + MemoryContextSwitchTo(old); +} + +/* Resolve the GUC to a provider table. */ +static const SpockQuorumProvider * +provider_for(int id) +{ + switch ((SpockQuorumProviderId) id) + { + case SPOCK_QUORUM_PROVIDER_NONE: + return &spock_quorum_provider_none; + case SPOCK_QUORUM_PROVIDER_ETCD: + return &spock_quorum_provider_etcd; + case SPOCK_QUORUM_PROVIDER_PGRAFT: + return &spock_quorum_provider_pgraft; + case SPOCK_QUORUM_PROVIDER_PGBULLY: + return &spock_quorum_provider_pgbully; + } + return &spock_quorum_provider_none; +} + +/* + * Drop the cached reading. Called at the top of each tick, and by the status + * view so an operator always sees a fresh answer rather than whatever the + * last tick happened to observe. + */ +static void +spock_quorum_invalidate(void) +{ + snap_valid = false; + snap_quorum = SPOCK_QUORUM_UNKNOWN; + snap_leader = SPOCK_QUORUM_UNKNOWN; + snap_members = NIL; + snap_leader_name = NULL; + if (snap_ctx != NULL) + MemoryContextReset(snap_ctx); +} + +void +spock_quorum_startup(void) +{ + char *detail = NULL; + + if (snap_ctx == NULL) + snap_ctx = AllocSetContextCreate(TopMemoryContext, + "spock quorum snapshot", + ALLOCSET_SMALL_SIZES); + + spock_quorum_invalidate(); + active = provider_for(spock_quorum_provider); + active_provider_id = spock_quorum_provider; + active_started = false; + + /* + * Every provider but 'none' addresses a shared key space, and the prefix + * is the only thing separating one cluster's members from another's. + * Refuse rather than fall back to a default: a wrong answer here mixes + * two clusters' membership, which is far worse than having no answer. + */ + if (spock_quorum_provider != SPOCK_QUORUM_PROVIDER_NONE && + (spock_quorum_cluster_id == NULL || spock_quorum_cluster_id[0] == '\0')) + { + note_error("spock.quorum_cluster_id is not set"); + ereport(WARNING, + (errmsg("spock quorum: provider \"%s\" needs spock.quorum_cluster_id", + active->name), + errhint("Set it to a value unique to this cluster. There is no " + "default, because two clusters sharing one would count " + "each other's nodes as their own."))); + return; + } + + if (active->startup == NULL) + { + active_started = true; + return; + } + + if (active->startup(&detail)) + { + active_started = true; + note_error(NULL); + return; + } + + /* + * Startup failed. Stay on the provider so the status view still reports + * what was configured and why it is not working, but leave it unstarted + * so every question below short-circuits to the conservative answer. + */ + note_error(detail ? detail : "provider startup failed"); + ereport(WARNING, + (errmsg("spock quorum: provider \"%s\" failed to start: %s", + active->name, last_error), + errhint("Spock continues with no quorum information, which is " + "the conservative behaviour."))); +} + +void +spock_quorum_shutdown(void) +{ + if (active != NULL && active_started && active->shutdown != NULL) + active->shutdown(); + active = NULL; + active_started = false; +} + +/* + * Renew whatever registration the provider needs. Called at the top of each + * tick, before any question is asked, so a lease that has lapsed is refused + * rather than answered from stale state. + */ +void +spock_quorum_refresh(void) +{ + char *detail = NULL; + + if (active == NULL || !active_started || active->refresh == NULL) + return; + + spock_quorum_invalidate(); + + if (!active->refresh(&detail)) + { + /* + * Registration could not be renewed, so this node may already have + * been dropped from the provider's view. Asking about quorum now + * could return YES from a position the cluster no longer counts, and + * would overwrite the reason the refresh failed. Leave the snapshot + * invalid: every question this tick then answers conservatively. + */ + note_error(detail ? detail : "refresh failed"); + return; + } + + note_error(NULL); + last_consulted = GetCurrentTimestamp(); + + /* + * Take the tick's single reading now, so everything decided below this + * point sees one consistent picture of the cluster. + */ + (void) snapshot_take(); +} + +/* + * Resolve the provider on first use. A consuming worker calls + * spock_quorum_startup() explicitly, but spock.quorum_status() can be called + * from any backend, and a status view reporting "none" merely because + * nothing had initialised the layer would be actively misleading. + */ +static bool +spock_quorum_ensure_started(void) +{ + /* + * Re-resolve when the GUC has moved. The provider is PGC_SIGHUP, and a + * worker restarts on one, but a long-lived backend does not -- it would + * otherwise keep answering from the provider that was configured when it + * first connected, indefinitely. + */ + if (active != NULL && active_provider_id != spock_quorum_provider) + { + spock_quorum_shutdown(); + spock_quorum_startup(); + } + else if (active == NULL) + spock_quorum_startup(); + + return active != NULL && active_started; +} + +/* + * Take one reading, if this tick has not already. + * + * Members are copied into snap_ctx: the provider allocates them in whatever + * context is current, which for a worker is reset between ticks, and the + * snapshot has to outlive that. + */ +static bool +snapshot_take(void) +{ + char *detail = NULL; + List *members; + + if (!spock_quorum_ensure_started()) + return false; + if (snap_valid) + return true; + + snap_quorum = active->have_quorum(&detail); + if (snap_quorum == SPOCK_QUORUM_UNKNOWN && detail != NULL) + note_error(detail); + else if (snap_quorum != SPOCK_QUORUM_UNKNOWN) + { + note_error(NULL); + last_consulted = GetCurrentTimestamp(); + } + + /* + * Leadership and membership are only asked for once quorum is held. A + * partitioned minority can still believe it leads and can still see some + * peers; acting on either is the failure this layer exists to prevent, + * so there is nothing to learn from asking. + */ + if (snap_quorum == SPOCK_QUORUM_YES) + { + detail = NULL; + snap_leader = active->is_leader(&detail); + if (snap_leader == SPOCK_QUORUM_UNKNOWN && detail != NULL) + note_error(detail); + + /* + * The leader's name belongs to this reading too. Fetching it later, + * when the status view is rendered, would be a separate consult and + * could name a leader from a different moment than the quorum answer + * shown beside it. + */ + if (active->leader != NULL) + { + char *who; + + detail = NULL; + who = active->leader(&detail); + if (who != NULL) + { + MemoryContext old = MemoryContextSwitchTo(snap_ctx); + + snap_leader_name = pstrdup(who); + MemoryContextSwitchTo(old); + } + } + + detail = NULL; + members = active->members(&detail); + if (members == NIL && detail != NULL) + note_error(detail); + else + { + MemoryContext old = MemoryContextSwitchTo(snap_ctx); + ListCell *lc; + + foreach(lc, members) + { + SpockQuorumMember *src = (SpockQuorumMember *) lfirst(lc); + SpockQuorumMember *cp = palloc0(sizeof(SpockQuorumMember)); + + cp->name = pstrdup(src->name); + cp->live = src->live; + cp->voting = src->voting; + cp->last_seen = src->last_seen; + snap_members = lappend(snap_members, cp); + } + MemoryContextSwitchTo(old); + } + } + + snap_valid = true; + return true; +} + +bool +spock_quorum_have_quorum(void) +{ + if (!snapshot_take()) + return false; + return snap_quorum == SPOCK_QUORUM_YES; +} + +bool +spock_quorum_is_leader(void) +{ + if (!snapshot_take()) + return false; + + /* snapshot_take only asks about leadership while quorum is held. */ + return snap_quorum == SPOCK_QUORUM_YES && snap_leader == SPOCK_QUORUM_YES; +} + +List * +spock_quorum_members(void) +{ + if (!snapshot_take()) + return NIL; + return snap_members; +} + +SpockQuorumAnswer +spock_quorum_member_live(const char *node_name) +{ + ListCell *lc; + + if (node_name == NULL || !snapshot_take()) + return SPOCK_QUORUM_UNKNOWN; + + /* + * Liveness is only trustworthy from inside a quorum. Without one this + * node may be the isolated party, and its opinion about who else is + * reachable says more about its own connectivity than about the cluster. + */ + if (snap_quorum != SPOCK_QUORUM_YES) + return SPOCK_QUORUM_UNKNOWN; + + foreach(lc, snap_members) + { + SpockQuorumMember *m = (SpockQuorumMember *) lfirst(lc); + + if (strcmp(m->name, node_name) == 0) + return m->live ? SPOCK_QUORUM_YES : SPOCK_QUORUM_NO; + } + + /* + * The quorum system has never heard of this node. That is not evidence + * that it is down -- it may simply not be registered -- so it is not + * grounds for releasing anything. + */ + return SPOCK_QUORUM_UNKNOWN; +} + +const char * +spock_quorum_provider_name(void) +{ + return active != NULL ? active->name : "none"; +} + +const char * +spock_quorum_last_error(void) +{ + return last_error; +} + +TimestampTz +spock_quorum_last_consulted(void) +{ + return last_consulted; +} + +/* + * spock.quorum_status() + * + * Anything able to move the WAL horizon has to be inspectable before it is + * allowed to. + */ +Datum +spock_quorum_status_sql(PG_FUNCTION_ARGS) +{ + TupleDesc tupdesc; + Datum values[6]; + bool nulls[6]; + HeapTuple tuple; + + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + memset(nulls, 0, sizeof(nulls)); + + /* + * Force a fresh reading. An operator running this is asking about now, + * not about whatever the last tick happened to see. + */ + spock_quorum_invalidate(); + (void) snapshot_take(); + + values[0] = CStringGetTextDatum(spock_quorum_provider_name()); + + if (active == NULL || !active_started) + { + nulls[1] = true; /* has_quorum */ + nulls[2] = true; /* is_leader */ + nulls[3] = true; /* leader */ + } + else + { + if (snap_quorum == SPOCK_QUORUM_UNKNOWN) + nulls[1] = true; + else + values[1] = BoolGetDatum(snap_quorum == SPOCK_QUORUM_YES); + + /* + * Reported through the same rule the rest of Spock acts on: without + * quorum, leadership is not something this node may act on, so + * showing the provider's raw opinion here would describe a decision + * Spock would never make. + */ + if (snap_quorum != SPOCK_QUORUM_YES || + snap_leader == SPOCK_QUORUM_UNKNOWN) + nulls[2] = true; + else + values[2] = BoolGetDatum(snap_leader == SPOCK_QUORUM_YES); + + if (snap_leader_name != NULL) + values[3] = CStringGetTextDatum(snap_leader_name); + else + nulls[3] = true; + } + + if (last_consulted == 0) + nulls[4] = true; + else + values[4] = TimestampTzGetDatum(last_consulted); + + if (last_error == NULL) + nulls[5] = true; + else + values[5] = CStringGetTextDatum(last_error); + + tuple = heap_form_tuple(tupdesc, values, nulls); + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} + +/* ---------------------------------------------------------------------- * + * The 'none' provider. + * + * Not a stub: it is the default, and it is what every other provider + * degrades to. It answers UNKNOWN rather than NO so that callers which + * distinguish the two (the status view, the logs) report "no information" + * instead of asserting a negative it has no basis for. + * ---------------------------------------------------------------------- */ + +static bool +none_startup(char **errdetail) +{ + return true; +} + +static void +none_shutdown(void) +{ +} + +static bool +none_refresh(char **errdetail) +{ + return true; +} + +static SpockQuorumAnswer +none_have_quorum(char **errdetail) +{ + return SPOCK_QUORUM_UNKNOWN; +} + +static List * +none_members(char **errdetail) +{ + return NIL; +} + +static SpockQuorumAnswer +none_is_leader(char **errdetail) +{ + return SPOCK_QUORUM_UNKNOWN; +} + +static char * +none_leader(char **errdetail) +{ + return NULL; +} + +const SpockQuorumProvider spock_quorum_provider_none = { + .name = "none", + .startup = none_startup, + .shutdown = none_shutdown, + .refresh = none_refresh, + .have_quorum = none_have_quorum, + .members = none_members, + .is_leader = none_is_leader, + .leader = none_leader +}; diff --git a/src/spock_quorum_cluster.c b/src/spock_quorum_cluster.c new file mode 100644 index 000000000..8c92c1e97 --- /dev/null +++ b/src/spock_quorum_cluster.c @@ -0,0 +1,512 @@ +/*------------------------------------------------------------------------- + * + * spock_quorum_cluster.c + * Quorum provider for the shared cluster-manager API. + * + * pgraft and pgBully now expose the same in-database interface, differing + * only in the schema it lives under: + * + * .get_cluster_status() node_id, term, leader_id, state, ... + * .get_nodes() node_id, address, port, is_leader + * .is_leader() + * .kv_put() / kv_get() replicated key/value + * + * So they are one provider here, parameterised by schema name, rather than + * two near-identical files drifting apart. Everything a quorum decision + * needs comes from that common surface. + * + * The one real difference is liveness, and it is not a matter of naming. + * pgBully additionally publishes pgbully.peers(), which carries `reachable` + * per peer -- the per-member liveness that Raft tracks internally to drive + * heartbeats but that pgraft does not expose. Where that is available it is + * used; where it is not, every configured member is reported live. + * + * Reporting all-live is the safe reading rather than a pretence that nothing + * has failed: under the fail-safe rules a live member is one that is never + * evicted, so a caller cannot release anything on the strength of a provider + * that has no opinion. It does mean the quorum layer buys pgraft users + * leadership and quorum but not eviction, until a last-contact column exists + * upstream. + * + * Copyright (c) 2022-2026, pgEdge, Inc. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/xact.h" +#include "executor/spi.h" +#include "utils/guc.h" +#include "utils/resowner.h" +#include "utils/builtins.h" +#include "utils/elog.h" +#include "utils/memutils.h" +#include "utils/timestamp.h" + +#include "spock.h" +#include "spock_node.h" +#include "spock_quorum.h" + +/* + * What differs between the two backends. Everything else below is shared. + */ +typedef struct ClusterApiConfig +{ + const char *schema; /* "pgraft" or "pgbully" */ + + /* + * SQL fragment yielding this member's liveness, evaluated against the + * get_nodes() row aliased `n`. A backend with no reachability + * information supplies a literal true. + */ + const char *live_expr; + + /* Extra FROM-list text the liveness expression needs, or "". */ + const char *live_from; + + /* + * SQL fragment yielding when this member was last heard from, or the + * literal NULL for a backend that does not track it. + */ + const char *seen_expr; + + /* Set once startup() has confirmed the extension is present. */ + bool available; + + /* This node's Spock name, published so peers can map ids to names. */ + char *self_name; +} ClusterApiConfig; + +/* + * pgBully joins its own peer table for real reachability. A LEFT JOIN, and + * coalesced to true, because a peer pgBully has not yet formed an opinion + * about must not read as dead: "no opinion" is not evidence of failure and + * must never license an eviction. + */ +static ClusterApiConfig cfg_pgbully = { + .schema = "pgbully", + .live_expr = "coalesce(p.reachable, true)", + .live_from = " LEFT JOIN pgbully.peers() p ON p.node_id = n.node_id ", + .seen_expr = "p.last_seen", + .available = false, + .self_name = NULL +}; + +static ClusterApiConfig cfg_pgraft = { + .schema = "pgraft", + .live_expr = "true", + .live_from = "", + .seen_expr = "NULL::timestamptz", + .available = false, + .self_name = NULL +}; + +/* + * Every consult below runs inside an internal subtransaction. + * + * Catching an error with PG_TRY and FlushErrorState() alone is not enough: + * the surrounding transaction stays aborted, and the caller's next command + * fails with "current transaction is aborted". These providers are consulted + * from spock.quorum_status(), inside whatever transaction the operator is + * already in, so that would turn a merely unreachable backend into a broken + * session. A subtransaction is the only way to catch an error and carry on. + * + * spock.quorum_timeout is applied as statement_timeout inside that + * subtransaction -- the in-database equivalent of etcd's HTTP deadline. + * SET LOCAL alone is not enough, because releasing a subtransaction + * propagates the setting up to the parent rather than restoring it, so the + * previous value is saved and put back on every exit path. Rollback-only + * cleanup would not do either: cluster_refresh() performs a kv_put() whose + * effect has to survive. + */ +typedef struct ClusterSpiScope +{ + MemoryContext oldcxt; + ResourceOwner oldowner; + char *prev_timeout; +} ClusterSpiScope; + +static void +cluster_spi_begin(ClusterSpiScope *scope) +{ + const char *prev = GetConfigOption("statement_timeout", false, false); + + scope->oldcxt = CurrentMemoryContext; + scope->oldowner = CurrentResourceOwner; + scope->prev_timeout = pstrdup(prev ? prev : "0"); + + BeginInternalSubTransaction(NULL); +} + +/* Put back what the caller had, whichever way the consult ended. */ +static void +cluster_spi_end(ClusterSpiScope *scope) +{ + MemoryContextSwitchTo(scope->oldcxt); + CurrentResourceOwner = scope->oldowner; + SetConfigOption("statement_timeout", scope->prev_timeout, + PGC_SUSET, PGC_S_SESSION); +} + +/* Apply the quorum deadline. Caller must already be connected to SPI. */ +static bool +cluster_spi_apply_timeout(char **errdetail) +{ + char sql[64]; + + snprintf(sql, sizeof(sql), + "SET LOCAL statement_timeout = %d", spock_quorum_timeout); + if (SPI_execute(sql, false, 0) != SPI_OK_UTILITY) + { + *errdetail = pstrdup("could not apply quorum timeout"); + return false; + } + + return true; +} + +/* Copy the error text somewhere that outlives the subtransaction. */ +static void +cluster_capture_error(ClusterSpiScope *scope, char **errdetail) +{ + ErrorData *edata; + + MemoryContextSwitchTo(scope->oldcxt); + edata = CopyErrorData(); + FlushErrorState(); + *errdetail = pstrdup(edata->message); + FreeErrorData(edata); +} + +/* + * Run a query yielding one text value, returning NULL rather than throwing. + * + * The backend is a separate extension that may be absent, mid-upgrade, or + * erroring, and a provider is contractually forbidden from raising. + */ +static char * +cluster_one_text(const char *sql, char **errdetail) +{ + volatile bool ok = true; + char *volatile result = NULL; + ClusterSpiScope scope; + + *errdetail = NULL; + cluster_spi_begin(&scope); + + PG_TRY(); + { + if (SPI_connect() != SPI_OK_CONNECT) + ok = false; + else + { + if (!cluster_spi_apply_timeout(errdetail)) + ok = false; + else if (SPI_execute(sql, true, 1) == SPI_OK_SELECT && SPI_processed >= 1) + { + char *raw = SPI_getvalue(SPI_tuptable->vals[0], + SPI_tuptable->tupdesc, 1); + + /* Copied out before SPI_finish frees the context it lives in. */ + if (raw != NULL) + { + MemoryContext old = MemoryContextSwitchTo(scope.oldcxt); + + result = pstrdup(raw); + MemoryContextSwitchTo(old); + } + } + SPI_finish(); + } + ReleaseCurrentSubTransaction(); + } + PG_CATCH(); + { + cluster_capture_error(&scope, errdetail); + RollbackAndReleaseCurrentSubTransaction(); + ok = false; + } + PG_END_TRY(); + + cluster_spi_end(&scope); + + if (!ok) + { + if (*errdetail == NULL) + *errdetail = pstrdup("query against the cluster manager failed"); + return NULL; + } + return result; +} + +/* Prefix under which node ids are mapped to Spock node names. */ +static char * +name_key_prefix(void) +{ + return psprintf("%s/nodes/", spock_quorum_cluster_id); +} + +static bool +cluster_startup(ClusterApiConfig *cfg, char **errdetail) +{ + SpockLocalNode *local; + MemoryContext old; + char *present; + + local = get_local_node(false, true); + if (local == NULL) + { + *errdetail = pstrdup("no local spock node"); + return false; + } + + old = MemoryContextSwitchTo(TopMemoryContext); + cfg->self_name = pstrdup(local->node->name); + MemoryContextSwitchTo(old); + + present = cluster_one_text( + psprintf("SELECT to_regprocedure('%s.get_cluster_status()') IS NOT NULL" + " AND to_regprocedure('%s.is_leader()') IS NOT NULL", + cfg->schema, cfg->schema), errdetail); + + if (present == NULL) + return false; + if (strcmp(present, "t") != 0) + { + *errdetail = psprintf("the %s extension is not installed in this database", + cfg->schema); + return false; + } + + cfg->available = true; + return true; +} + +/* + * Publish this node's id-to-name mapping. + * + * Not a heartbeat and no evidence of liveness: the replicated KV has no + * expiry, so an entry outlives the node that wrote it. It exists only so + * members() and leader() can render the integer ids the cluster manager + * speaks in as the node names the rest of Spock speaks in. + */ +static bool +cluster_refresh(ClusterApiConfig *cfg, char **errdetail) +{ + char *id; + + if (!cfg->available) + return false; + + id = cluster_one_text(psprintf("SELECT node_id::text FROM %s.get_cluster_status()", + cfg->schema), errdetail); + if (id == NULL) + return false; + + return cluster_one_text( + psprintf("SELECT %s.kv_put(%s, %s)::text", + cfg->schema, + quote_literal_cstr(psprintf("%s%s", name_key_prefix(), id)), + quote_literal_cstr(cfg->self_name)), errdetail) != NULL; +} + +/* + * A leader is elected only from within a majority, so a leader id that is + * set is itself the proof of quorum. There is no separate question to ask. + * + * coalesced because the two backends differ on how they say "nobody": + * pgraft reports 0 and pgBully reports NULL. Left bare, the comparison + * would yield NULL for pgBully and be reported as "unknown" when what it + * actually said was a definite "no leader, so no quorum". Both are safe -- + * the caller treats them alike -- but only one is true. + */ +static SpockQuorumAnswer +cluster_have_quorum(ClusterApiConfig *cfg, char **errdetail) +{ + char *leader; + + if (!cfg->available) + return SPOCK_QUORUM_UNKNOWN; + + leader = cluster_one_text( + psprintf("SELECT (coalesce(leader_id, 0) <> 0)::text " + " FROM %s.get_cluster_status()", cfg->schema), errdetail); + + if (leader == NULL) + return SPOCK_QUORUM_UNKNOWN; + return strcmp(leader, "t") == 0 ? SPOCK_QUORUM_YES : SPOCK_QUORUM_NO; +} + +static SpockQuorumAnswer +cluster_is_leader(ClusterApiConfig *cfg, char **errdetail) +{ + char *v; + + if (!cfg->available) + return SPOCK_QUORUM_UNKNOWN; + + v = cluster_one_text(psprintf("SELECT %s.is_leader()::text", cfg->schema), + errdetail); + if (v == NULL) + return SPOCK_QUORUM_UNKNOWN; + return strcmp(v, "t") == 0 ? SPOCK_QUORUM_YES : SPOCK_QUORUM_NO; +} + +static char * +cluster_leader(ClusterApiConfig *cfg, char **errdetail) +{ + char *id; + + if (!cfg->available) + return NULL; + + id = cluster_one_text(psprintf("SELECT leader_id::text " + " FROM %s.get_cluster_status()", cfg->schema), + errdetail); + if (id == NULL || strcmp(id, "0") == 0) + return NULL; + + return cluster_one_text( + psprintf("SELECT %s.kv_get(%s)", cfg->schema, + quote_literal_cstr(psprintf("%s%s", name_key_prefix(), id))), + errdetail); +} + +/* + * Membership, rendered as Spock node names, with each member's liveness + * according to whatever the backend can attest. + * + * A node that has not published a name mapping yet is skipped rather than + * reported under its integer id: a name matching no spock.node row is worse + * than no row at all, because it looks like an answer. + */ +static List * +cluster_members(ClusterApiConfig *cfg, char **errdetail) +{ + volatile bool ok = true; + List *volatile result = NIL; + ClusterSpiScope scope; + char *sql; + + if (!cfg->available) + return NIL; + + *errdetail = NULL; + + sql = psprintf( + "SELECT k.v, (%s)::text, (%s)::text " + " FROM %s.get_nodes() n %s" + " , LATERAL (SELECT %s.kv_get(%s || n.node_id::text) AS v) k " + " WHERE k.v IS NOT NULL", + cfg->live_expr, cfg->seen_expr, cfg->schema, cfg->live_from, + cfg->schema, quote_literal_cstr(name_key_prefix())); + + cluster_spi_begin(&scope); + + PG_TRY(); + { + if (SPI_connect() != SPI_OK_CONNECT) + ok = false; + else + { + if (!cluster_spi_apply_timeout(errdetail)) + ok = false; + else if (SPI_execute(sql, true, 0) == SPI_OK_SELECT) + { + uint64 i; + + for (i = 0; i < SPI_processed; i++) + { + HeapTuple tup = SPI_tuptable->vals[i]; + TupleDesc desc = SPI_tuptable->tupdesc; + char *name = SPI_getvalue(tup, desc, 1); + char *live = SPI_getvalue(tup, desc, 2); + char *seen = SPI_getvalue(tup, desc, 3); + MemoryContext old; + SpockQuorumMember *m; + + if (name == NULL) + continue; + + old = MemoryContextSwitchTo(scope.oldcxt); + m = palloc0(sizeof(SpockQuorumMember)); + m->name = pstrdup(name); + m->live = (live == NULL || strcmp(live, "t") == 0); + m->voting = true; + + /* + * Carry the backend's own last-contact time when it has + * one. SQL NULL stays 0, which the struct documents as + * "not tracked" -- the two mean the same to a caller. + */ + m->last_seen = (seen == NULL) ? 0 : + DatumGetTimestampTz(DirectFunctionCall3(timestamptz_in, + CStringGetDatum(seen), + ObjectIdGetDatum(InvalidOid), + Int32GetDatum(-1))); + result = lappend(result, m); + MemoryContextSwitchTo(old); + } + } + else + ok = false; + SPI_finish(); + } + ReleaseCurrentSubTransaction(); + } + PG_CATCH(); + { + cluster_capture_error(&scope, errdetail); + RollbackAndReleaseCurrentSubTransaction(); + ok = false; + } + PG_END_TRY(); + + cluster_spi_end(&scope); + + if (!ok) + { + if (*errdetail == NULL) + *errdetail = psprintf("could not read %s membership", cfg->schema); + return NIL; + } + return result; +} + +/* --- per-backend callback tables -------------------------------------- */ + +#define CLUSTER_PROVIDER_SHIMS(tag, cfgvar) \ +static bool tag##_startup(char **e) { return cluster_startup(&cfgvar, e); } \ +static void tag##_shutdown(void) { cfgvar.available = false; } \ +static bool tag##_refresh(char **e) { return cluster_refresh(&cfgvar, e); } \ +static SpockQuorumAnswer tag##_have_quorum(char **e) \ + { return cluster_have_quorum(&cfgvar, e); } \ +static List *tag##_members(char **e) { return cluster_members(&cfgvar, e); } \ +static SpockQuorumAnswer tag##_is_leader(char **e) \ + { return cluster_is_leader(&cfgvar, e); } \ +static char *tag##_leader(char **e) { return cluster_leader(&cfgvar, e); } + +CLUSTER_PROVIDER_SHIMS(pgraft, cfg_pgraft) +CLUSTER_PROVIDER_SHIMS(pgbully, cfg_pgbully) + +const SpockQuorumProvider spock_quorum_provider_pgraft = { + .name = "pgraft", + .startup = pgraft_startup, + .shutdown = pgraft_shutdown, + .refresh = pgraft_refresh, + .have_quorum = pgraft_have_quorum, + .members = pgraft_members, + .is_leader = pgraft_is_leader, + .leader = pgraft_leader +}; + +const SpockQuorumProvider spock_quorum_provider_pgbully = { + .name = "pgbully", + .startup = pgbully_startup, + .shutdown = pgbully_shutdown, + .refresh = pgbully_refresh, + .have_quorum = pgbully_have_quorum, + .members = pgbully_members, + .is_leader = pgbully_is_leader, + .leader = pgbully_leader +}; diff --git a/src/spock_quorum_etcd.c b/src/spock_quorum_etcd.c new file mode 100644 index 000000000..a5ae37f25 --- /dev/null +++ b/src/spock_quorum_etcd.c @@ -0,0 +1,758 @@ +/*------------------------------------------------------------------------- + * + * spock_quorum_etcd.c + * Quorum provider backed by an external etcd daemon. + * + * etcd runs as its own daemon, so this provider is a client: it speaks the + * v3 HTTP/JSON gateway, which keeps the dependency to an HTTP library rather + * than a gRPC stack. The whole file is guarded by SPOCK_HAVE_LIBCURL; a build + * without it still compiles and still offers the provider, which then + * reports why it cannot be used. Selecting a provider you did not build is + * a configuration mistake, not a reason to fail to start. + * + * Liveness model. etcd's own member list describes etcd, not Spock, so it + * cannot answer "is node n3 up". Instead each Spock node registers itself + * under a prefix with a lease and renews that lease every tick: + * + * /nodes/ -> (lease TTL) + * + * A node that stops renewing has its key expired by etcd, so presence under + * the prefix *is* liveness, judged by the cluster rather than by whichever + * node happens to be asking. That is the property Spock cannot get locally + * and the reason for the whole layer. + * + * Copyright (c) 2022-2026, pgEdge, Inc. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/xact.h" +#include "common/base64.h" +#include "utils/resowner.h" +#include "lib/stringinfo.h" +#include "utils/builtins.h" +#include "utils/jsonb.h" +#include "utils/memutils.h" +#include "parser/scansup.h" + +#include "spock.h" +#include "spock_node.h" +#include "spock_quorum.h" + +#ifdef SPOCK_HAVE_LIBCURL +#include +#endif + +/* + * Lease TTL, in seconds. Must comfortably exceed the worker interval or a + * slow tick would expire our own registration and make this node look dead + * to its peers. Six times the default 5s tick leaves room for a stalled + * tick or two before anyone draws conclusions. + */ +#define ETCD_LEASE_TTL_SECONDS 30 + +#define ETCD_NODES_INFIX "/nodes/" +#define ETCD_LEADER_SUFFIX "/leader" + +#ifdef SPOCK_HAVE_LIBCURL + +static bool curl_initialized = false; +static int64 etcd_lease_id = 0; +static char *etcd_self_name = NULL; + +static char *etcd_leader_name(char **errdetail); + +/* + * Split a comma-separated endpoint list. + * + * Deliberately not SplitIdentifierString(): that downcases unquoted text and + * truncates each element to NAMEDATALEN, both of which silently corrupt a + * URL. Endpoints are opaque strings here, so only whitespace is trimmed. + */ +static List * +split_endpoints(const char *raw) +{ + List *result = NIL; + char *copy = pstrdup(raw); + char *cursor = copy; + char *comma; + + for (;;) + { + char *item = cursor; + char *tail; + + comma = strchr(cursor, ','); + if (comma != NULL) + { + *comma = '\0'; + cursor = comma + 1; + } + + while (*item != '\0' && scanner_isspace(*item)) + item++; + tail = item + strlen(item); + while (tail > item && scanner_isspace(*(tail - 1))) + *(--tail) = '\0'; + + if (*item != '\0') + result = lappend(result, item); + + if (comma == NULL) + break; + } + + return result; +} + +/* Accumulates a response body. */ +static size_t +write_cb(void *contents, size_t size, size_t nmemb, void *userp) +{ + StringInfo buf = (StringInfo) userp; + size_t total = size * nmemb; + + appendBinaryStringInfo(buf, (const char *) contents, (int) total); + return total; +} + +/* + * POST a JSON body to one etcd endpoint and return the response body, or + * NULL with *errdetail set. + * + * Only the first endpoint in spock.quorum_etcd_endpoints is tried per call, + * rotating on failure, so a dead etcd member costs one tick rather than + * making every tick pay for the retry. + */ +static char * +etcd_post(const char *path, const char *body, char **errdetail) +{ + static int endpoint_cursor = 0; + CURL *curl; + CURLcode res; + StringInfoData resp; + StringInfoData url; + struct curl_slist *headers = NULL; + long http_code = 0; + List *endpoints; + ListCell *lc; + int n = 0; + char *chosen = NULL; + + if (spock_quorum_etcd_endpoints == NULL || + spock_quorum_etcd_endpoints[0] == '\0') + { + *errdetail = pstrdup("spock.quorum_etcd_endpoints is not set"); + return NULL; + } + + endpoints = split_endpoints(spock_quorum_etcd_endpoints); + if (endpoints == NIL) + { + *errdetail = pstrdup("spock.quorum_etcd_endpoints is malformed"); + return NULL; + } + + foreach(lc, endpoints) + { + if (n == (endpoint_cursor % list_length(endpoints))) + chosen = (char *) lfirst(lc); + n++; + } + if (chosen == NULL) + chosen = (char *) linitial(endpoints); + + initStringInfo(&url); + appendStringInfo(&url, "%s%s", chosen, path); + + curl = curl_easy_init(); + if (curl == NULL) + { + *errdetail = pstrdup("could not initialise HTTP client"); + return NULL; + } + + initStringInfo(&resp); + headers = curl_slist_append(headers, "Content-Type: application/json"); + + curl_easy_setopt(curl, CURLOPT_URL, url.data); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &resp); + /* The deadline is the contract; without it a hung etcd hangs the tick. */ + curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, (long) spock_quorum_timeout); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, (long) spock_quorum_timeout); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + + res = curl_easy_perform(curl); + if (res == CURLE_OK) + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + + if (res != CURLE_OK) + { + endpoint_cursor++; + *errdetail = psprintf("etcd %s: %s", chosen, curl_easy_strerror(res)); + return NULL; + } + if (http_code != 200) + { + endpoint_cursor++; + *errdetail = psprintf("etcd %s returned HTTP %ld", chosen, http_code); + return NULL; + } + + return resp.data; +} + +/* Base64, as the v3 gateway requires for every key and value. */ +static char * +b64(const char *src) +{ + int srclen = (int) strlen(src); + int maxlen = pg_b64_enc_len(srclen) + 1; + char *dst = palloc(maxlen); + int len = pg_b64_encode((const uint8 *) src, srclen, dst, maxlen - 1); + + if (len < 0) + return pstrdup(""); + dst[len] = '\0'; + return dst; +} + +static char * +unb64(const char *src) +{ + int srclen = (int) strlen(src); + int maxlen = pg_b64_dec_len(srclen) + 1; + char *dst = palloc(maxlen); + int len = pg_b64_decode(src, srclen, (uint8 *) dst, maxlen - 1); + + if (len < 0) + return pstrdup(""); + dst[len] = '\0'; + return dst; +} + +/* + * Parse a response body, returning NULL rather than throwing. + * + * etcd's replies are small and infrequent, so the server's own parser is + * used rather than a hand-rolled scanner. It has to be wrapped, though: + * jsonb_in raises on malformed input, and a provider is contractually + * forbidden from throwing. `ok` is volatile because it is written in the + * handler and read after PG_END_TRY. + */ +static Jsonb * +parse_json(const char *json) +{ + volatile bool ok = true; + Jsonb *volatile result = NULL; + MemoryContext oldcxt = CurrentMemoryContext; + ResourceOwner oldowner = CurrentResourceOwner; + + if (json == NULL) + return NULL; + + /* + * An internal subtransaction, not a bare PG_TRY. Catching the error with + * FlushErrorState() alone would leave the surrounding transaction + * aborted, and this runs inside whatever transaction the operator called + * spock.quorum_status() from -- so a single malformed reply would break + * their session rather than just this reading. + */ + BeginInternalSubTransaction(NULL); + + PG_TRY(); + { + result = DatumGetJsonbP(DirectFunctionCall1(jsonb_in, + CStringGetDatum(json))); + + /* Copy out before the subtransaction that allocated it goes away. */ + MemoryContextSwitchTo(oldcxt); + result = (Jsonb *) PG_DETOAST_DATUM_COPY(PointerGetDatum(result)); + + ReleaseCurrentSubTransaction(); + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcxt); + FlushErrorState(); + RollbackAndReleaseCurrentSubTransaction(); + ok = false; + } + PG_END_TRY(); + + MemoryContextSwitchTo(oldcxt); + CurrentResourceOwner = oldowner; + + return ok ? result : NULL; +} + +/* + * One top-level field, as text, or NULL when absent. + * + * The container API is used in preference to jsonb_object_field_text + * because DirectFunctionCall raises "function returned NULL" whenever the + * field is missing -- which for an optional field is the normal case, not + * an error. + */ +static char * +jb_field(Jsonb *jb, const char *field) +{ + JsonbValue *v; + + if (jb == NULL) + return NULL; + + v = getKeyJsonValueFromContainer(&jb->root, field, (int) strlen(field), + NULL); + if (v == NULL) + return NULL; + + switch (v->type) + { + case jbvString: + return pnstrdup(v->val.string.val, v->val.string.len); + case jbvBool: + return pstrdup(v->val.boolean ? "true" : "false"); + case jbvNumeric: + return DatumGetCString(DirectFunctionCall1(numeric_out, + NumericGetDatum(v->val.numeric))); + default: + return NULL; + } +} + +/* Convenience for the common "parse, then take one field" shape. */ +static char * +json_field(const char *json, const char *field) +{ + return jb_field(parse_json(json), field); +} + +/* The prefix under which this cluster's nodes register. */ +static char * +nodes_prefix(void) +{ + return psprintf("%s%s", spock_quorum_cluster_id, ETCD_NODES_INFIX); +} + +/* + * range_end for a prefix scan is the prefix with its last byte incremented, + * which is how etcd expresses "everything under this prefix". + */ +static char * +prefix_end(const char *prefix) +{ + char *end = pstrdup(prefix); + int len = (int) strlen(end); + + if (len > 0) + end[len - 1]++; + return end; +} + +static bool +etcd_grant_lease(char **errdetail) +{ + char *body = psprintf("{\"TTL\":\"%d\"}", ETCD_LEASE_TTL_SECONDS); + char *resp = etcd_post("/v3/lease/grant", body, errdetail); + char *id; + + if (resp == NULL) + return false; + + id = json_field(resp, "ID"); + if (id == NULL) + { + *errdetail = pstrdup("etcd lease grant returned no ID"); + return false; + } + + etcd_lease_id = strtoll(id, NULL, 10); + return etcd_lease_id != 0; +} + +/* Register (or re-register) this node under the nodes prefix. */ +static bool +etcd_put_self(char **errdetail) +{ + char *key = psprintf("%s%s", nodes_prefix(), etcd_self_name); + char *body = psprintf("{\"key\":\"%s\",\"value\":\"%s\",\"lease\":\"%lld\"}", + b64(key), b64(etcd_self_name), + (long long) etcd_lease_id); + + return etcd_post("/v3/kv/put", body, errdetail) != NULL; +} + +/* --- provider entry points -------------------------------------------- */ + +/* + * Startup deliberately touches neither etcd nor a lease. + * + * Registration is owned by the single long-lived worker that calls + * refresh(); an ordinary backend asking spock.quorum_status() must be able + * to read the cluster's view without minting a lease of its own and + * registering this node a second time. So startup only resolves identity, + * and everything with a side effect lives in refresh(). + */ +static bool +etcd_startup(char **errdetail) +{ + SpockLocalNode *local; + MemoryContext old; + + if (!curl_initialized) + { + curl_global_init(CURL_GLOBAL_DEFAULT); + curl_initialized = true; + } + + local = get_local_node(false, true); + if (local == NULL) + { + *errdetail = pstrdup("no local spock node"); + return false; + } + + old = MemoryContextSwitchTo(TopMemoryContext); + etcd_self_name = pstrdup(local->node->name); + MemoryContextSwitchTo(old); + + return true; +} + +static void +etcd_shutdown(void) +{ + char *detail = NULL; + + /* + * Revoke rather than waiting for the TTL, so a clean shutdown is visible + * to peers immediately instead of looking like a node that died. + */ + if (etcd_lease_id != 0) + { + char *body = psprintf("{\"ID\":\"%lld\"}", (long long) etcd_lease_id); + + (void) etcd_post("/v3/lease/revoke", body, &detail); + etcd_lease_id = 0; + } +} + +/* + * Renew the lease. If it has already expired -- a long stall, or etcd was + * unreachable for longer than the TTL -- grant a fresh one and re-register, + * rather than silently continuing to look dead to every peer. + */ +static bool +etcd_refresh(char **errdetail) +{ + char *body; + char *resp; + char *ttl; + + if (etcd_lease_id == 0) + return etcd_grant_lease(errdetail) && etcd_put_self(errdetail); + + body = psprintf("{\"ID\":\"%lld\"}", (long long) etcd_lease_id); + resp = etcd_post("/v3/lease/keepalive", body, errdetail); + if (resp == NULL) + return false; + + ttl = json_field(resp, "TTL"); + if (ttl == NULL || strtoll(ttl, NULL, 10) <= 0) + { + etcd_lease_id = 0; + return etcd_grant_lease(errdetail) && etcd_put_self(errdetail); + } + return true; +} + +/* + * Quorum by linearizable read. + * + * /v3/maintenance/status is deliberately NOT used: the Status RPC is answered + * from the queried member's own state, so a member isolated in a minority + * partition happily reports the leader it last knew about. Treating that as + * proof of quorum is exactly the mistake this layer exists to avoid. + * + * A Range read, by contrast, is linearizable unless serializable is asked for, + * which means etcd only answers it from within a majority. A successful reply + * therefore is the proof. Reading our own nodes prefix with limit 1 keeps the + * response small and needs no extra key. + * + * A minority member cannot complete the read, so the call runs out its + * deadline and the answer is UNKNOWN rather than NO. Both are handled + * identically by the caller; UNKNOWN is simply the truthful one, because + * etcd never got far enough to say no. + */ +static SpockQuorumAnswer +etcd_have_quorum(char **errdetail) +{ + char *prefix = nodes_prefix(); + char *body = psprintf("{\"key\":\"%s\",\"range_end\":\"%s\"," + "\"limit\":\"1\",\"serializable\":false}", + b64(prefix), b64(prefix_end(prefix))); + + if (etcd_post("/v3/kv/range", body, errdetail) == NULL) + return SPOCK_QUORUM_UNKNOWN; + + return SPOCK_QUORUM_YES; +} + +/* + * Pull the "kvs" array out of a range response. Returns the container and + * its length, or NULL when the key is absent -- which etcd uses to mean "no + * matches", not an error. + */ +static JsonbContainer * +kvs_array(const char *resp, int *count, char **errdetail) +{ + Jsonb *jb = parse_json(resp); + JsonbValue *kvs; + + *count = 0; + if (jb == NULL) + { + *errdetail = pstrdup("etcd returned unparseable JSON"); + return NULL; + } + + kvs = getKeyJsonValueFromContainer(&jb->root, "kvs", 3, NULL); + if (kvs == NULL || kvs->type != jbvBinary) + return NULL; /* no key matched */ + + *count = (int) JsonContainerSize(kvs->val.binary.data); + return kvs->val.binary.data; +} + +/* One string field of the i'th array element, base64-decoded. */ +static char * +kv_field(JsonbContainer *arr, int i, const char *field) +{ + JsonbValue *elem = getIthJsonbValueFromContainer(arr, (uint32) i); + JsonbValue *v; + + if (elem == NULL || elem->type != jbvBinary) + return NULL; + + v = getKeyJsonValueFromContainer(elem->val.binary.data, field, + (int) strlen(field), NULL); + if (v == NULL || v->type != jbvString) + return NULL; + + return unb64(pnstrdup(v->val.string.val, v->val.string.len)); +} + +static List * +etcd_members(char **errdetail) +{ + char *prefix = nodes_prefix(); + char *body = psprintf("{\"key\":\"%s\",\"range_end\":\"%s\"}", + b64(prefix), b64(prefix_end(prefix))); + char *resp = etcd_post("/v3/kv/range", body, errdetail); + JsonbContainer *arr; + List *result = NIL; + int count; + int i; + + if (resp == NULL) + return NIL; + + arr = kvs_array(resp, &count, errdetail); + if (arr == NULL) + return NIL; /* nobody registered yet */ + + for (i = 0; i < count; i++) + { + char *key = kv_field(arr, i, "key"); + char *name; + SpockQuorumMember *m; + + if (key == NULL) + continue; + + /* The node name is the last path element of the key. */ + name = strrchr(key, '/'); + name = (name != NULL) ? name + 1 : key; + if (*name == '\0') + continue; + + m = palloc0(sizeof(SpockQuorumMember)); + m->name = pstrdup(name); + + /* + * Presence under the prefix is liveness: etcd drops the key when its + * owner's lease lapses, so anything still here renewed recently. + */ + m->live = true; + m->voting = true; + m->last_seen = GetCurrentTimestamp(); + result = lappend(result, m); + } + + return result; +} + +/* + * Leadership by create-if-absent on a leased key. The txn compares the + * key's create_revision against 0, which is etcd's idiom for "does not + * exist", so exactly one node can win. The lease means a leader that dies + * releases the key without anyone having to notice and intervene. + */ +static SpockQuorumAnswer +etcd_is_leader(char **errdetail) +{ + char *key = psprintf("%s%s", spock_quorum_cluster_id, ETCD_LEADER_SUFFIX); + char *kb = b64(key); + char *body; + char *resp; + char *succeeded; + char *holder; + + /* + * Only the lease-owning worker campaigns. A read-only backend answers by + * comparing the recorded holder, so asking the question can never change + * who leads. + */ + if (etcd_lease_id == 0) + { + char *who; + + /* + * startup() leaves this unset when there is no local node, and a + * comparison against it would dereference NULL. Without an identity + * there is no question to answer. + */ + if (etcd_self_name == NULL) + return SPOCK_QUORUM_UNKNOWN; + + who = etcd_leader_name(errdetail); + if (who == NULL) + return SPOCK_QUORUM_UNKNOWN; + return strcmp(who, etcd_self_name) == 0 + ? SPOCK_QUORUM_YES : SPOCK_QUORUM_NO; + } + + body = psprintf("{\"compare\":[{\"key\":\"%s\",\"target\":\"CREATE\"," + "\"result\":\"EQUAL\",\"create_revision\":\"0\"}]," + "\"success\":[{\"requestPut\":{\"key\":\"%s\"," + "\"value\":\"%s\",\"lease\":\"%lld\"}}]," + "\"failure\":[{\"requestRange\":{\"key\":\"%s\"}}]}", + kb, kb, b64(etcd_self_name), (long long) etcd_lease_id, kb); + + resp = etcd_post("/v3/kv/txn", body, errdetail); + if (resp == NULL) + return SPOCK_QUORUM_UNKNOWN; + + succeeded = json_field(resp, "succeeded"); + if (succeeded != NULL && strcmp(succeeded, "true") == 0) + return SPOCK_QUORUM_YES; /* we took it */ + + /* + * Someone holds it. It may still be us from an earlier tick, which is + * the common case, so compare rather than assuming we lost. + */ + holder = etcd_leader_name(errdetail); + if (holder == NULL) + return SPOCK_QUORUM_UNKNOWN; + return strcmp(holder, etcd_self_name) == 0 ? SPOCK_QUORUM_YES : SPOCK_QUORUM_NO; +} + +static char * +etcd_leader_name(char **errdetail) +{ + char *key = psprintf("%s%s", spock_quorum_cluster_id, ETCD_LEADER_SUFFIX); + char *body = psprintf("{\"key\":\"%s\"}", b64(key)); + char *resp = etcd_post("/v3/kv/range", body, errdetail); + JsonbContainer *arr; + int count; + + if (resp == NULL) + return NULL; + + arr = kvs_array(resp, &count, errdetail); + if (arr == NULL || count < 1) + return NULL; /* nobody holds it */ + + return kv_field(arr, 0, "value"); +} + +#else /* !SPOCK_HAVE_LIBCURL */ + +/* + * Built without an HTTP client. The provider still exists so that selecting + * it produces a clear explanation instead of a mysterious silence, and so + * that it degrades to exactly the conservative behaviour of 'none'. + */ +static const char * +etcd_unavailable(void) +{ + return "this build of Spock has no HTTP client, so the etcd provider is unavailable"; +} + +static bool +etcd_startup(char **errdetail) +{ + *errdetail = pstrdup(etcd_unavailable()); + return false; +} + +static void +etcd_shutdown(void) +{ +} + +static bool +etcd_refresh(char **errdetail) +{ + *errdetail = pstrdup(etcd_unavailable()); + return false; +} + +static SpockQuorumAnswer +etcd_have_quorum(char **errdetail) +{ + *errdetail = pstrdup(etcd_unavailable()); + return SPOCK_QUORUM_UNKNOWN; +} + +static List * +etcd_members(char **errdetail) +{ + *errdetail = pstrdup(etcd_unavailable()); + return NIL; +} + +static SpockQuorumAnswer +etcd_is_leader(char **errdetail) +{ + *errdetail = pstrdup(etcd_unavailable()); + return SPOCK_QUORUM_UNKNOWN; +} + +static char * +etcd_leader_name(char **errdetail) +{ + *errdetail = pstrdup(etcd_unavailable()); + return NULL; +} + +#endif /* SPOCK_HAVE_LIBCURL */ + +const SpockQuorumProvider spock_quorum_provider_etcd = { + .name = "etcd", + .startup = etcd_startup, + .shutdown = etcd_shutdown, + .refresh = etcd_refresh, + .have_quorum = etcd_have_quorum, + .members = etcd_members, + .is_leader = etcd_is_leader, + .leader = etcd_leader_name +}; diff --git a/tests/tap/schedule b/tests/tap/schedule index 7e8c457ea..f78d0cbf2 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -58,6 +58,7 @@ test: 037_wire_format_datestyle test: 038_reserved_schema_ddl_guard test: 044_apply_change_logging test: 045_lsn_from_commit_ts +test: 106_quorum_layer # Upgrade schema match test (builds from source, slow): #test: 018_upgrade_schema_match # diff --git a/tests/tap/t/106_quorum_layer.pl b/tests/tap/t/106_quorum_layer.pl new file mode 100644 index 000000000..b534eee88 --- /dev/null +++ b/tests/tap/t/106_quorum_layer.pl @@ -0,0 +1,270 @@ +use strict; +use warnings; +use Test::More; +use lib '.'; +use lib 't'; +use SpockTest qw( + create_cluster destroy_cluster + get_test_config scalar_query psql_or_bail +); + +# ============================================================================= +# Test: 106_quorum_layer.pl +# +# Covers the quorum layer's fail-safe surface. +# +# Deliberately runs no external quorum system. The property that matters most +# is what happens when the layer CANNOT get an answer -- provider absent, +# endpoint unset, endpoint unreachable, provider switched at runtime -- and all +# of that is reachable without an etcd daemon or a Raft cluster. Keeping the +# suite free of external services also keeps it runnable in CI. +# +# The one invariant every case below shares: an answer that could not be +# obtained is reported as NULL, never as false, and never as an error to the +# caller. +# ============================================================================= + +create_cluster(1, 'Create 1-node cluster for quorum-layer tests'); + +my $cfg = get_test_config(); +my $bin = $cfg->{pg_bin}; +my $host = $cfg->{host}; +my $port = $cfg->{node_ports}[0]; +my $db = $cfg->{db_name}; +my $user = $cfg->{db_user}; +my $datadir; + +# Run SQL, returning (combined output, exit code). +sub psql_try { + my ($sql) = @_; + local $ENV{PGOPTIONS} = '-c client_min_messages=error'; + my $out = `$bin/psql -X -h $host -p $port -d $db -U $user -v ON_ERROR_STOP=1 -tAc "$sql" 2>&1`; + my $rc = ($? >> 8); + chomp $out; + return ($out, $rc); +} + +# One field of the single quorum_status() row, with NULL rendered as 'NULL'. +sub status_field { + my ($field) = @_; + my ($out, $rc) = psql_try( + "SELECT coalesce($field\::text, 'NULL') FROM spock.quorum_status()"); + return $rc == 0 ? $out : "ERROR:$out"; +} + +# Change a GUC in postgresql.conf and reload. ALTER SYSTEM is avoided so the +# provider can be switched even when a value is rejected as out of range. +sub set_guc_reload { + my ($name, $value) = @_; + psql_or_bail(1, "ALTER SYSTEM SET $name = '$value'"); + psql_or_bail(1, "SELECT pg_reload_conf()"); + sleep(1); +} + +sub reset_guc_reload { + my ($name) = @_; + psql_or_bail(1, "ALTER SYSTEM RESET $name"); + psql_or_bail(1, "SELECT pg_reload_conf()"); + sleep(1); +} + +# -------------------------------------------------------------------------- +# Version and surface +# -------------------------------------------------------------------------- +is(scalar_query(1, "SELECT extversion FROM pg_extension WHERE extname = 'spock'"), + '6.1.0', 'extension reports version 6.1.0'); + +is(scalar_query(1, + "SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace " . + " WHERE n.nspname = 'spock' AND p.proname = 'quorum_status'"), + '1', 'spock.quorum_status() exists'); + +is(scalar_query(1, "SELECT count(*) FROM spock.quorum_status()"), + '1', 'quorum_status() returns exactly one row'); + +# VOLATILE, not STABLE: the function re-reads the provider, so the planner +# must not fold two calls in one statement into a single evaluation. +is(scalar_query(1, + "SELECT provolatile FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace " . + " WHERE n.nspname = 'spock' AND p.proname = 'quorum_status'"), + 'v', 'quorum_status() is declared VOLATILE'); + +# The function can influence nothing by itself, but it names the cluster and +# its provider, so it is not world-readable. +is(scalar_query(1, + "SELECT has_function_privilege('public', 'spock.quorum_status()', 'EXECUTE')"), + 'f', 'quorum_status() is not executable by PUBLIC'); + +# Every GUC the layer defines must be present, or a deployment cannot be +# configured at all. +for my $guc (qw(spock.quorum_provider spock.quorum_timeout + spock.quorum_cluster_id spock.quorum_etcd_endpoints)) { + is(scalar_query(1, "SELECT count(*) FROM pg_settings WHERE name = '$guc'"), + '1', "$guc is defined"); +} + +# -------------------------------------------------------------------------- +# The default: nothing is consulted +# +# This is the regression that matters most. With no provider configured the +# layer must be inert, so enabling the feature is always a deliberate act. +# -------------------------------------------------------------------------- +is(scalar_query(1, "SELECT setting FROM pg_settings WHERE name = 'spock.quorum_provider'"), + 'none', 'the default provider is none'); + +is(status_field('provider'), 'none', 'status reports the none provider'); +is(status_field('has_quorum'), 'NULL', 'has_quorum is NULL, not false, with no provider'); +is(status_field('is_leader'), 'NULL', 'is_leader is NULL with no provider'); +is(status_field('leader'), 'NULL', 'leader is NULL with no provider'); +is(status_field('last_error'), 'NULL', 'no error is reported when nothing was attempted'); + +# -------------------------------------------------------------------------- +# A provider that is selected but not installed +# +# The layer must say so plainly rather than failing to start or pretending to +# have an answer. +# -------------------------------------------------------------------------- +# A cluster id is now mandatory for every provider but 'none': the prefix is +# the only thing keeping two clusters' members apart, so there is no default. +set_guc_reload('spock.quorum_cluster_id', 'tap_cluster'); + +for my $prov (qw(pgraft pgbully)) { + set_guc_reload('spock.quorum_provider', $prov); + + is(status_field('provider'), $prov, "status reports the $prov provider"); + is(status_field('has_quorum'), 'NULL', + "has_quorum is NULL when $prov is not installed"); + like(status_field('last_error'), qr/\Q$prov\E/, + "last_error names $prov as the missing extension"); +} + +# -------------------------------------------------------------------------- +# A missing cluster id is refused, not defaulted +# +# Sharing one prefix between two clusters would make each count the other's +# nodes as its own, so the layer declines to start rather than guess. +# -------------------------------------------------------------------------- +reset_guc_reload('spock.quorum_cluster_id'); +is(status_field('has_quorum'), 'NULL', 'no answer while the cluster id is unset'); +like(status_field('last_error'), qr/quorum_cluster_id/, + 'last_error names the missing cluster id'); +set_guc_reload('spock.quorum_cluster_id', 'tap_cluster'); + +# -------------------------------------------------------------------------- +# etcd with nothing to talk to +# -------------------------------------------------------------------------- +set_guc_reload('spock.quorum_provider', 'etcd'); + +# No endpoints configured at all. +is(status_field('has_quorum'), 'NULL', 'has_quorum is NULL with no etcd endpoints'); +like(status_field('last_error'), qr/endpoints/, + 'last_error points at the unset endpoint list'); + +# An endpoint that is syntactically fine but has nothing listening. Port 1 is +# used because it is reserved and will never have a real service on it. +set_guc_reload('spock.quorum_etcd_endpoints', 'http://127.0.0.1:1'); +is(status_field('has_quorum'), 'NULL', 'has_quorum is NULL when etcd is unreachable'); +like(status_field('last_error'), qr/127\.0\.0\.1:1/, + 'last_error names the endpoint that could not be reached'); + +# Asking again must not raise: a provider is forbidden from throwing, and the +# status view has to stay usable while the cluster is unhealthy. +my (undef, $again_rc) = psql_try("SELECT * FROM spock.quorum_status()"); +is($again_rc, 0, 'the status view keeps working while the provider is unreachable'); + +# Malformed endpoint lists are a configuration error, not a crash. +set_guc_reload('spock.quorum_etcd_endpoints', ',,,'); +is(status_field('has_quorum'), 'NULL', 'a malformed endpoint list yields no answer'); +my (undef, $mal_rc) = psql_try("SELECT * FROM spock.quorum_status()"); +is($mal_rc, 0, 'a malformed endpoint list does not raise'); + +reset_guc_reload('spock.quorum_etcd_endpoints'); + +# -------------------------------------------------------------------------- +# A failing provider must not poison the caller's transaction +# +# Catching the error without an internal subtransaction leaves the surrounding +# transaction aborted, so the operator's next statement fails with "current +# transaction is aborted". The status view is consulted from whatever +# transaction they happen to be in, so this has to hold. +# -------------------------------------------------------------------------- +my ($txn, $txn_rc) = psql_try( + "BEGIN; SELECT 1 AS before; SELECT has_quorum FROM spock.quorum_status(); " . + "SELECT 2 AS after; COMMIT"); +is($txn_rc, 0, 'a transaction survives consulting an unreachable provider'); +like($txn, qr/\b2\b/, 'statements after the failed consult still run'); + +# -------------------------------------------------------------------------- +# Runtime reconfiguration +# +# The provider is PGC_SIGHUP. A long-lived backend must notice a change rather +# than answering from whatever was configured when it first connected. +# -------------------------------------------------------------------------- +set_guc_reload('spock.quorum_provider', 'none'); +is(status_field('provider'), 'none', 'switching back to none is picked up'); +is(status_field('last_error'), 'NULL', 'switching provider clears the stale error'); + +set_guc_reload('spock.quorum_provider', 'pgraft'); +is(status_field('provider'), 'pgraft', 'switching away from none is picked up'); + +# Within a single session, too: the checks above each used a fresh backend, +# which would hide a provider cached for the life of a connection. Statements +# go in on stdin rather than through -c, because ALTER SYSTEM cannot run inside +# a transaction block and -c wraps its whole string in one. +my $session = `$bin/psql -X -h $host -p $port -d $db -U $user -tA 2>&1 <<'EOSQL' +SELECT 'first=' || provider FROM spock.quorum_status(); +ALTER SYSTEM SET spock.quorum_provider = 'none'; +SELECT pg_reload_conf(); +SELECT pg_sleep(1); +SELECT 'second=' || provider FROM spock.quorum_status(); +EOSQL`; +like($session, qr/first=pgraft/, 'the session starts on the configured provider'); +like($session, qr/second=none/, + 'the same backend reports the new provider after a reload'); + +reset_guc_reload('spock.quorum_provider'); + +# -------------------------------------------------------------------------- +# GUC bounds +# +# The timeout is the deadline that keeps a wedged provider from stalling the +# caller, so its bounds are load-bearing rather than cosmetic. +# -------------------------------------------------------------------------- +is(scalar_query(1, + "SELECT min_val || '..' || max_val FROM pg_settings WHERE name = 'spock.quorum_timeout'"), + '100..60000', 'the timeout is bounded to a sane millisecond range'); + +# PGC_SIGHUP: a session cannot change it at all, whatever the value. That is +# deliberate -- the deadline protects a shared worker, not one backend. +my (undef, $set_rc) = psql_try("SET spock.quorum_timeout = '5s'"); +isnt($set_rc, 0, 'the timeout cannot be changed by a single session'); +my (undef, $set_prov_rc) = psql_try("SET spock.quorum_provider = 'etcd'"); +isnt($set_prov_rc, 0, 'the provider cannot be changed by a single session'); + +# Values are still validated when set the supported way. +my (undef, $lo_rc) = psql_try("ALTER SYSTEM SET spock.quorum_timeout = '1ms'"); +isnt($lo_rc, 0, 'a timeout below the minimum is rejected'); +my (undef, $hi_rc) = psql_try("ALTER SYSTEM SET spock.quorum_timeout = '10min'"); +isnt($hi_rc, 0, 'a timeout above the maximum is rejected'); +my (undef, $ok_rc) = psql_try("ALTER SYSTEM SET spock.quorum_timeout = '5s'"); +is($ok_rc, 0, 'a timeout inside the range is accepted'); +reset_guc_reload('spock.quorum_timeout'); + +my (undef, $bad_prov_rc) = psql_try("ALTER SYSTEM SET spock.quorum_provider = 'wobble'"); +isnt($bad_prov_rc, 0, 'an unknown provider name is rejected'); + +# -------------------------------------------------------------------------- +# Nothing the layer does perturbs replication +# +# The layer is inert by design; this is the check that it stays that way. +# -------------------------------------------------------------------------- +is(scalar_query(1, + "SELECT count(*) FROM pg_replication_slots WHERE slot_name LIKE '%quorum%'"), + '0', 'the quorum layer creates no replication slots'); + +is(scalar_query(1, + "SELECT count(*) FROM pg_stat_activity WHERE application_name LIKE '%quorum%'"), + '0', 'the quorum layer starts no background worker of its own'); + +destroy_cluster('Destroy quorum-layer test cluster'); +done_testing();