diff --git a/deps/libscram/src/scram.c b/deps/libscram/src/scram.c index c7685b95cb..6fa176772e 100644 --- a/deps/libscram/src/scram.c +++ b/deps/libscram/src/scram.c @@ -502,21 +502,34 @@ char *build_client_first_message(ScramState *scram_state) goto failed; scram_state->client_nonce[encoded_len] = '\0'; - len = 8 + strlen(scram_state->client_nonce) + 1; - result = malloc(len); - if (result == NULL) - goto failed; - if (scram_state->client_cbind_input != NULL) { - /* Channel-bound client: gs2 cbind flag 'p' with tls-server-end-point - * type. The PostgreSQL convention is an empty SCRAM username (the - * real username travels in the StartupMessage), so the header is - * "p=tls-server-end-point,,". */ - snprintf(result, len, "p=tls-server-end-point,,n=,r=%s", scram_state->client_nonce); - } else { - snprintf(result, len, "n,,n=,r=%s", scram_state->client_nonce); - } + /* gs2 header: "n,," for plain SCRAM, "p=tls-server-end-point,," when the + * caller installed a channel-binding input (SCRAM-SHA-256-PLUS). Its length + * drives BOTH the allocation and the offset used to derive + * client_first_message_bare, so the two cannot disagree. + * + * Previously the buffer was sized for the 8-char plain prefix "n,,n=,r=", + * which silently truncated the 29-char channel-bound message, and the bare + * message was taken as "result + 3", which skipped only 3 of the 24 header + * bytes and corrupted the AuthMessage the client proof is computed over. + * + * The PostgreSQL convention is an empty SCRAM username (the real username + * travels in the StartupMessage), hence "n=". + */ + { + const char *gs2 = (scram_state->client_cbind_input != NULL) + ? "p=tls-server-end-point,," : "n,,"; + const size_t gs2_len = strlen(gs2); + + /* gs2 + "n=,r=" (5) + nonce + NUL. For the plain header this is + * 3 + 5 + nonce + 1, identical to the previous 8 + nonce + 1. */ + len = gs2_len + 5 + strlen(scram_state->client_nonce) + 1; + result = malloc(len); + if (result == NULL) + goto failed; + snprintf(result, len, "%sn=,r=%s", gs2, scram_state->client_nonce); - scram_state->client_first_message_bare = strdup(result + 3); + scram_state->client_first_message_bare = strdup(result + gs2_len); + } if (scram_state->client_first_message_bare == NULL) goto failed; @@ -545,9 +558,11 @@ char *build_client_final_message(ScramState *scram_state, if (scram_state->client_cbind_input != NULL) { /* Channel-bound client: c=base64(gs2-header || cbind-data). - * 86 bytes buffer = 22 (header) + 64 (max digest we accept) = 86; - * base64-encoded = 116 chars max. The full prefix - * "c=,r=" easily fits in 512. */ + * The gs2 header "p=tls-server-end-point,," is 24 bytes, so the + * cbind input is at most 24 + 64 (max digest we accept) = 88 bytes; + * base64-encoded = 4*ceil(88/3) = 120 chars, 121 with the NUL that + * is written below -- so b64[128] has 7 bytes of headroom. The full + * prefix "c=,r=" easily fits in 512. */ char b64[128]; int blen = pg_b64_encode(scram_state->client_cbind_input, scram_state->client_cbind_input_len, diff --git a/docs/superpowers/specs/2026-08-03-pgsql-native-protocol-adversarial-tests-design.md b/docs/superpowers/specs/2026-08-03-pgsql-native-protocol-adversarial-tests-design.md new file mode 100644 index 0000000000..5bb51162d8 --- /dev/null +++ b/docs/superpowers/specs/2026-08-03-pgsql-native-protocol-adversarial-tests-design.md @@ -0,0 +1,483 @@ +# PostgreSQL Native Backend Protocol — Adversarial Test Coverage + +**Date:** 2026-08-03 +**Status:** Approved design, pending implementation plan +**Update (2026-08-19):** D3 was subsequently fixed in this same PR (#6112) and +`pgsql-native_framer_retention-t` now passes; the statements below that D3 is +unfixed and out of scope (§4.1, §5, §6) record the original design intent. The +separate recv+feed read-loop accumulation is unrelated to D3 and remains open. +**Scope:** Close the test gaps in the native PostgreSQL backend protocol +(`feature/pgsql-native-backend-protocol`) and prove three defects found by code +audit and direct measurement. +**Related:** `2026-06-11-pgsql-native-protocol-design.md` (the implementation this +tests), `2026-06-14-pgsql-native-scram-plus-design.md`, +`2026-07-07-pgsql-native-extq-stmt-pipeline-design.md` + +--- + +## 1. Motivation + +The native backend protocol replaces libpq on the ProxySQL → PostgreSQL data +path. It ships with eleven TAP tests and five unit tests, all of which compare +the native path against libpq as an oracle **using a healthy, well-behaved +PostgreSQL backend over plaintext**. + +That shape leaves three classes of behaviour untested: + +1. **Everything a cooperative backend never does.** Malformed framing, truncated + messages, forged authentication, unexpected message types, and mid-stream + disconnects are all handled by code in `PgSQL_Connection.cpp` and + `PgSQL_Backend_Protocol.cpp` that no test can reach, because a real + PostgreSQL will not emit those bytes. +2. **Backend TLS.** `use_ssl` appears in none of the native tests, yet the + infra's PostgreSQL runs with `ssl = on`. The `SSL_read`/BIO branch of + `native_recv_into_framer()` and the whole SCRAM-SHA-256-PLUS channel-binding + path have never executed end to end. +3. **Pool lifecycle.** The differential tests exercise queries on a connection. + They do not exercise what happens to a native connection when it is *returned + to the pool* and *reset for a different client*. + +A code audit of those three areas found three defects, each confirmed with +evidence. This document specifies the tests that prove them, plus the harness +work needed to reach the untestable surface. + +A fourth candidate — a no-op `async_ping()` in native mode — was investigated +and **rejected**; it keeps its identifier D2, marked withdrawn, so the analysis +is not repeated and mis-filed later. D1/D3/D4 keep the identifiers they were +first reported under. + +--- + +## 2. Confirmed Defects + +These are stated as findings, not hypotheses. Each names the code and the +evidence. + +### D1 — `async_reset_session()` does nothing in native mode + +`lib/PgSQL_Connection.cpp:3322` + +```c +if (native_mode) { + async_state_machine = ASYNC_RESET_SESSION_SUCCESSFUL; + return 0; +} +``` + +The libpq path (`reset_session_start()`, `lib/PgSQL_Connection.cpp:4206`) sends +`DISCARD ALL`, or `ROLLBACK` when the connection is inside a transaction. The +native branch sends nothing and reports success. `PgSQL_Session.cpp:1169` then +calls `myconn->reset()`, clearing ProxySQL's own record of the connection's +session state. + +The result is that ProxySQL marks the connection clean while the backend still +holds the previous client's `SET`s, temp tables, `LISTEN` registrations, open +cursors, and session-level prepared statements. + +Reachability is not theoretical: `handler_again___verify_backend_user_db()` +(`lib/PgSQL_Session.cpp:1343`) routes to `RESETTING_CONNECTION_V2` whenever +`requires_RESETTING_CONNECTION()` finds the pooled backend connection carrying +tracked variables the incoming client did not ask for. + +**Severity:** session state crosses between clients. Under a shared application +user this is a correctness bug; where distinct end users share a ProxySQL user +it is a confidentiality bug. + +### D2 — WITHDRAWN — "`async_ping()` does nothing in native mode" + +Originally reported as a native-mode divergence on the strength of the early +return at `lib/PgSQL_Connection.cpp:3410`: + +```c +if (native_mode) { + async_state_machine = ASYNC_PING_SUCCESSFUL; + return 0; +} +``` + +**This is not a defect, and not a divergence.** The libpq path does not ping +either: + +```c +case ASYNC_IDLE: + async_state_machine = ASYNC_PING_START; +default: + //handler(event); // lib/PgSQL_Connection.cpp:3430 + async_state_machine = ASYNC_PING_SUCCESSFUL; +``` + +`handler(event)` is commented out, and the `default:` branch unconditionally +reports success. There is no `ping_start()` or `ping_cont()` anywhere in the +PgSQL sources — unlike `reset_session_start()`, which D1 relies on. Whatever +`native_mode` is, `async_ping()` returns 0 without touching the socket. + +The native early return is therefore **redundant, not defective**. It was +reported because it is textually near-identical to the `async_reset_session()` +early return in D1, where the libpq path *is* live. The two functions look +alike and behave differently; only reading both to the end distinguishes them. + +**Why this matters for the test plan.** A `pgsql-native_pool_ping-t` +differential would have compared a no-op against a no-op, passed, and been +recorded as evidence that idle-connection health checking works. A green +differential is only meaningful when the oracle actually does the thing. + +**The real observation underneath**, recorded here and deliberately left out of +scope: ProxySQL never health-checks idle pooled PostgreSQL connections at all, +where the MySQL path does. A backend connection killed server-side stays in the +pool advertised as healthy in **both** modes. That is a pre-existing gap in the +PostgreSQL implementation, not a regression from this branch, and by +construction it cannot be demonstrated by a libpq-vs-native differential. It +belongs in its own issue against the PgSQL connection pool, with a test that +asserts absolute behaviour rather than parity. + +### D3 — The message framer never reclaims consumed bytes + +`lib/PgSQL_Backend_Protocol.cpp:40` + +```c +pos += total; +if (pos == len) { pos = 0; len = 0; } // fully drained -> cheap reset +``` + +`feed()` appends at `buf + len`. The consumed prefix `[0, pos)` is reclaimed +only when a drain happens to land exactly on a message boundary. Retention +therefore grows until `k × chunk ≡ 0 (mod msglen)`, bounding peak retention at +`chunk × msglen / gcd(msglen, chunk)` bytes. + +Measured against the real framer compiled standalone, feeding through the same +16384-byte reads `native_recv_into_framer()` uses: + +| message size | bytes fed | peak retained | +|---|---|---| +| 105 B | 64 MiB | 1.9 MiB | +| **8197 B** | 256 MiB | **130.0 MiB** | +| 65536 B | 512 MiB | 0 MiB | +| 1 MiB | 512 MiB | 0.9 MiB | + +Powers of two share a large GCD with the read size and reset often; **any odd +message length is coprime with 16384 and never resets until the stream ends**. + +Stated precisely, peak retention is + +``` +min( result_set_size, chunk × msglen / gcd(msglen, chunk) ) +``` + +so the pathological bound is only reached by a result set large enough to get +there: ~130 MiB of retention needs ~130 MiB of 8197-byte rows, and the ~1.6 GiB +implied by a 100 KB odd-length row needs a ~1.6 GiB result set. The +amplification is roughly *one extra full copy of the result set* for coprime +message sizes, on top of the `PgSQL_Query_Result` copy the design's whole point +was to make the *only* copy. + +`chunk` is 16384 on the plaintext path (`tmp[16384]`, +`lib/PgSQL_Connection.cpp:2191`) but **8192** on the TLS path (`MY_SSL_BUFFER`, +`include/PgSQL_Data_Stream.h:17`), which halves the bound there without changing +the shape of the problem. + +Note that the framer *does* reset between queries: `ReadyForQuery` is normally +the last message in the buffer, so `next()` sees `pos == len` and rewinds. The +growth is confined to a single result set — which is exactly what the +measurement above models. + +**Severity:** a normal `SELECT` over a wide text column is a memory-amplification +vector. This directly contradicts design §5's "a backend row's bytes are copied +exactly once". + +### D4 — Plaintext reads discard a complete result on EOF + +`lib/PgSQL_Connection.cpp:2201` + +```c +if (n == 0) { + return -1; // peer closed +} +``` + +`got` is ignored. The TLS branch of the same function gets this right +(`lib/PgSQL_Connection.cpp:2181`: `return got ? 1 : -1`). + +The plaintext loop only exits early when `recv()` returns a short read. If a +`recv()` fills the 16384-byte buffer exactly and the peer has already sent FIN, +the next iteration returns 0 and the function returns `-1`, discarding a +complete, already-framed result whose terminating `ReadyForQuery` is sitting in +the buffer. The caller reports "backend closed during result fetch". + +Triggered when a backend response is an exact multiple of 16384 bytes and the +backend closes immediately after — a self-terminating backend, an admin +shutdown, an `idle_session_timeout` race. + +**Triggering it from a black-box test is not deterministic, and the test design +must not pretend otherwise.** The condition is that the *final* `recv()` returns +exactly `sizeof(tmp)`, which requires at least 16384 bytes pending at that +instant. A test cannot arrange that over TCP: + +- ProxySQL wakes on the first readable segment, which over a Docker bridge + (MTU 1500) is ~1448 bytes, not 16384. +- The read loop exits on the first short read, so any short read mid-stream + destroys the alignment and the remaining tail is delivered normally — the + query then completes and the FIN is never observed. +- Whether the writer outruns the reader by exactly a 16384 multiple at the end + is a scheduling artefact, not something the test controls. + +The defect is nonetheless established by inspection, and the TLS branch is the +proof that the plaintext branch is wrong rather than deliberately different: +the two paths of the same function disagree about whether buffered data +survives an EOF. + +**Severity:** a successfully-completed query is reported to the client as a +connection failure. + +--- + +## 3. Coverage Gaps + +Distinct from the defects above: surface with no test at all. + +- **No hostile backend exists.** Every `FRAME_ERROR` branch, every "unexpected + message during auth", "short Authentication message", "short MD5 salt", + "malformed backend message during result fetch", and every capability-gap + fallback is unreachable from a cooperative PostgreSQL. +- **Server authentication is unverified.** `pg_scram_verify_server_final()` + (`include/PgSQL_Backend_Protocol.h:121`) is the sole defence against a + malicious or spoofed backend impersonating the real server. Its rejection + branch has no test. +- **Backend TLS has never run.** `native_recv_into_framer()`'s BIO/`SSL_read` + loop, certificate verification, and the SCRAM-SHA-256-PLUS + `tls-server-end-point` channel binding are covered only by crypto-level unit + tests of the helper functions. +- **The framer unit test is seven assertions** (`unit/pgsql_backend_framing-t.cpp`). + It omits `msglen` 0–3, `msglen == 4`, the exact 1 GiB cap boundary, sticky + failure semantics, `reset()` recovery, byte-at-a-time feeding, and retention. + +--- + +## 4. Design + +Four test groups. Groups 1 and 2 add new capability; groups 3 and 4 extend the +existing libpq-oracle differential pattern to untested axes. + +### 4.1 Group 1 — `test/tap/tests/unit/pgsql_backend_framing-t.cpp` (extended) + +Pure unit test, no infrastructure. Grows from 7 assertions to roughly 25. + +| Case | Assertion | +|---|---| +| `msglen` = 0, 1, 2, 3 | `FRAME_ERROR` for each (length field includes itself, so < 4 is malformed) | +| `msglen` = 4 | `FRAME_OK`, `payload_len == 0` | +| `msglen` = `PGSQL_MAX_BACKEND_MSG_LEN` | `FRAME_NEED_MORE` — at the cap is legal, only the header has been fed | +| `msglen` = cap + 1 | `FRAME_ERROR` | +| after `FRAME_ERROR` | `feed()` is ignored; `next()` stays `FRAME_ERROR` | +| after `reset()` | failure cleared; framing resumes correctly | +| 3 messages, 1 byte per `feed()` | all framed in order, payloads intact | +| N messages in one `feed()` | all framed in order, types and payloads intact | +| **retention (D3)** | 256 MiB of 8197-byte messages in 16384-byte chunks; `VmRSS` delta must stay under 8 MiB | + +The retention case reads `VmRSS` from `/proc/self/status` before the loop and +tracks the peak during it. A coarse instrument is adequate for a 130 MiB signal +against an 8 MiB bar. The test skips (rather than fails) if `/proc` is +unavailable, so it stays portable. + +**Expected result: the retention case fails.** Everything else passes. + +### 4.2 Group 2 — mock backend harness + `pgsql-native_hostile_backend-t` + +New reusable helper `test/tap/tests/pgsql_mock_backend.{h,cpp}`, plus the test +that drives it. + +**Harness.** A listener thread inside the test process speaks enough of the +PostgreSQL wire protocol to complete a scripted handshake, then emits a +per-case script of attacker-chosen bytes. It supports: reading the client +startup packet; answering with any `Authentication*` subtype including a real +SCRAM server-first with correct nonce extension; emitting arbitrary framed or +deliberately malformed messages; controlling delivery granularity down to one +byte per `write()`; and closing at any chosen point. + +**Wiring.** The test-runner container shares the Docker network with ProxySQL +(`run-tests-isolated.bash:276` and `start-proxysql-isolated.bash:228` both use +`${NETWORK_NAME}`). The test discovers its own container IP at runtime and +registers `ip:port` in `pgsql_servers` under a hostgroup dedicated to this test, +with a `pgsql_users` entry routed there. Runtime IP discovery is used rather +than Docker DNS so the test does not depend on how the runner container's name +or hostname is registered. + +**Preconditions — without these, most cases fail for reasons unrelated to what +they test.** A backend that deliberately breaks handshakes trips two independent +mechanisms that remove it from rotation: + +1. **Monitor shunning.** `PgSQL_Monitor::shunn_non_resp_srv()` + (`lib/PgSQL_Monitor.cpp:1729`) calls `PgHGM->shun_and_killall()` for a server + that misses heartbeats. The monitor probes with libpq, which every hostile + handshake defeats. The test sets `pgsql-monitor_enabled = false` for its + duration. +2. **Error-counter shunning.** `pgsql-shun_on_failures` defaults to **5** with + `pgsql-shun_recovery_time_sec` at **10** (`lib/PgSQL_Thread.cpp:1042`). The + case list below is roughly twenty failures; the mock backend would be shunned + part-way through and the remaining cases would never reach it. The test + raises `shun_on_failures` well above the case count for its duration. + +Both are restored in memory at the end, including on the `BAIL_OUT` path, and +neither is written to disk. The test asserts the mock backend is `ONLINE` in +`runtime_pgsql_servers` before each case, so that a case failing because the +server was shunned is reported as such rather than as a protocol finding. + +**Auth cases.** + +| Case | Expectation | +|---|---| +| `ErrorResponse` in place of `R` | error surfaced from the backend's fields; connection torn down | +| unknown message type during auth | "unexpected message during auth"; torn down | +| `R` with `payload_len < 4` | "short Authentication message" | +| `R` type 5 with fewer than 4 salt bytes | "short MD5 salt" | +| `R` type 7 / 9 (GSSAPI / SSPI) | capability gap: single warning, libpq fallback attempted | +| `R` with an unrecognised type code | no hang; deterministic teardown | +| SASL with an empty mechanism list | capability gap | +| **SASL final with a forged server signature** | **rejected — auth must fail** | +| SASL server-first whose nonce does not extend the client nonce | rejected | +| FIN immediately after the startup packet | clean connect failure | +| FIN after `AuthenticationOk`, before `ReadyForQuery` | clean connect failure | +| entire handshake delivered one byte per write | succeeds; partial-message handling holds | + +**Result-phase cases** (after a successful cleartext handshake). + +| Case | Expectation | +|---|---| +| complete result of exactly *N*×16384 bytes, then FIN, repeated in a bounded loop | **result delivered (D4 — probabilistic prober, see below)** | +| same at *N*×16384 + 1 bytes, then FIN | result delivered (control; must pass) | +| truncated `DataRow`, then FIN | clean error; connection destroyed | +| declared length 900 MB, 100 bytes sent, then silence | no OOM; times out and tears down | +| unrecognised message type mid-result | no crash; deterministic client-visible outcome | +| a second `Z` after `Z`, then a query on the same pooled connection | the next query is not corrupted by the stray message | +| `E` whose final field value has no NUL terminator | parsed within bounds; no over-read | +| `S` whose value has no NUL terminator | parsed within bounds | +| whole result delivered one byte per write | framed correctly | +| `NotificationResponse` delivered while the connection idles in the pool | defined behaviour for the next client on that connection | + +**Invariants asserted after every case**: ProxySQL is alive, the admin interface +answers, and the process fd count has not grown. A crash, a hang, or an fd leak +fails the case regardless of the protocol-level outcome. + +**The D4 case is a prober, not a proof.** Per §2 D4, the trigger condition is +not controllable over TCP. The case writes a large exact-multiple response and +closes, repeated a bounded number of times (target: 50 iterations, capped by +wall-clock so it cannot dominate the group's runtime). A "backend closed during +result fetch" on a response the mock fully delivered is a **failure and a +confirmed hit**. Completing all iterations without a hit is reported as +`# D4 not observed in N iterations` — explicitly **not** as evidence the defect +is absent, since the same run against known-defective code can legitimately +miss it. The case must never be read as a regression guard. + +**Expected result for the rest of the group: exploratory.** Predictions exist +for the framing and bounds cases; the response to a forged SCRAM signature, a +stray second `ReadyForQuery`, and an idle-pool `NotificationResponse` is not +known in advance. Any failure there is a new finding, reported as such and never +normalised away. + +### 4.3 Group 3 — pool lifecycle differential + +One test — `pgsql-native_pool_reset-t` — following the established libpq-oracle +pattern: run the identical scenario with `pgsql-use_native_backend_protocol` +false then true, and require the client-visible outcomes to match. + +A second test for idle-connection health checking was specified in an earlier +draft and **removed**: per §2 D2, both modes share the same no-op `async_ping()`, +so the differential would have passed while proving nothing. + +**`pgsql-native_pool_reset-t` (D1).** Client A sets a tracked dynamic variable — +`bytea_output`, in the `PGSQL_NAME_LAST_LOW_WM`..`PGSQL_NAME_LAST_HIGH_WM` block +that `requires_RESETTING_CONNECTION()` compares — and records +`pg_backend_pid()`. A disconnects. Client B connects as the same user, confirms +via `pg_backend_pid()` that it inherited the same backend connection, and reads +the variable back with `current_setting()`. The libpq run is the oracle; the +native run must match it. A test-local hostgroup with `max_connections = 1` +forces reuse, and the case retries a bounded number of times if the PID does not +match rather than asserting on a coincidence. + +The test also covers the transaction variant: a connection returned while inside +a transaction, where libpq issues `ROLLBACK`. + +Backend variable tracking is protocol-agnostic — `PgSQL_Variables` writes +`session->mybe->server_myds->myconn->var_hash[idx]` with no `native_mode` +branch — so `requires_RESETTING_CONNECTION()` fires identically in both modes +and the reset path is genuinely reached in the native run. This was verified +before specifying the test; had the native path not recorded backend variables, +the reset would never have been requested and the scenario would have needed a +different trigger. + +The test restores every runtime variable it changes, in memory only — no +`SAVE ... TO DISK`. + +### 4.4 Group 4 — `pgsql-native_tls-t` + +Sets `use_ssl = 1` on the backend row and runs the standard two-phase +differential. + +The infra supports this without new fixtures: `postgresql.conf:15` sets +`ssl = 'on'`, and `pg_hba.conf:35` (`host all all all scram-sha-256`) matches +before the `hostssl ... cert` rule at line 46, so TLS connections authenticate +with SCRAM and no client certificate is required. + +Corpus: the broad query set from `pgsql-native_query_differential-t` plus a +10,000-row result. The large result is the point — it is what forces TLS record +boundaries to fall across protocol message boundaries, exercising the +`SSL_read`/BIO branch of `native_recv_into_framer()` that plaintext testing +cannot reach. + +**Channel binding is asserted implicitly but strictly.** PostgreSQL 16 offers +both `SCRAM-SHA-256` and `SCRAM-SHA-256-PLUS`; over TLS the mechanism selection +at `lib/PgSQL_Connection.cpp:2345` takes `-PLUS`. A wrong `tls-server-end-point` +digest produces a SCRAM proof the server rejects, so a successful TLS connect +proves `pg_tls_server_end_point()` and +`pg_scram_build_cbind_input_tls_server_end_point()` are correct end to end. +There is no log line naming the chosen mechanism, so the test additionally +asserts TLS is genuinely in use for the backend connection by joining the +observed backend PID against `pg_stat_ssl` on a direct connection — otherwise a +silent plaintext connect would masquerade as a passing TLS test. + +A log tripwire asserts no capability-gap or libpq-fallback warning appeared. + +**Expected result: passes.** If it does not, that is a new finding in the TLS or +channel-binding path. + +--- + +## 5. Placement and Reporting + +The three TAP tests register in the **same five groups** the existing +`pgsql-native_*` tests use — `legacy-g1`, +`mysql-auto_increment_delay_multiplex=0-g1`, `mysql-multiplexing=false-g1`, +`mysql-query_digests=0-g1`, `mysql-query_digests_keep_comment=1-g1` — matching +the pattern at `test/tap/groups/groups.json:166-175`. Entries must be inserted +in sorted order or `CI-lint-groups-json` fails. The unit test is registered the +way `unit/` tests already are. No new TAP group and no new CI workflow wiring. + +Per the decision recorded for this work, **defect-proving assertions land red**. +Each test header states explicitly which assertions are expected to fail, cites +the file and line of the implementation defect, and quotes the measured +evidence. The word "flaky" appears nowhere: a failing assertion here is a +reproducible defect with a known cause, and the header says so, so that a +reader encountering a red run reaches for the fix rather than the mute button. + +Two assertions carry a weaker guarantee and their headers must say so plainly: +the D4 prober (§4.2), which cannot prove absence, and the Group 2 exploratory +cases, whose expected outcome is not known in advance. + +D1, D3 and D4 are not fixed as part of this work. + +--- + +## 6. Out of Scope + +- Fixing D1, D3, D4. +- Idle-connection health checking for PostgreSQL (the real observation under the + withdrawn D2, §2.2). Pre-existing, not native-specific, needs its own issue and + an absolute-behaviour test rather than a differential. +- Extended-query pipelining (multiple Parse/Bind/Execute cycles before a single + Sync). Worth a separate investigation; not part of this suite. +- Statement and portal name truncation at PostgreSQL's 63-byte `NAMEDATALEN` + limit. ProxySQL keys its registries on the untruncated client name; the same + collision occurs against a direct PostgreSQL, so it is a shared-semantics + question rather than a proxy divergence. +- Monitor and genai paths, which the native design deliberately leaves on libpq. +- TLS support in the mock backend harness. The harness is plaintext; Group 4 + covers TLS against the real backend. diff --git a/include/PgSQL_Backend_Protocol.h b/include/PgSQL_Backend_Protocol.h index b46772e6a8..2ac14cb5c9 100644 --- a/include/PgSQL_Backend_Protocol.h +++ b/include/PgSQL_Backend_Protocol.h @@ -43,20 +43,36 @@ class PgSQL_Backend_Msg_Framer { // Writes the fixed 8-byte SSLRequest packet: length=8, code=80877103 (0x04d2162f). void pg_build_ssl_request(unsigned char out[8]); -// Encodes a protocol-3.0 StartupMessage into out[0..*out_len). -// Layout: int32 length (incl. itself), int32 protocol (196608 = 0x00030000), -// then "user\0\0database\0\0" and a terminating empty key (\0). -// Bounds: writes nothing past out_cap. If the encoded message would exceed -// out_cap, sets *out_len = 0 and returns false (no partial/oversized write). -// Returns true on success with *out_len set to the number of bytes written. +// Builds the protocol-3.0 StartupMessage that opens a backend connection, carrying the +// user and database to connect as and, when given, the options, application_name and +// client_encoding to start the session with. Pass NULL or an empty string to leave any +// of those three out; options is a "-c name=value ..." string of settings for the +// backend to apply, and application_name is what the session reports in +// pg_stat_activity. +// Writes nothing past out_cap: if the message would not fit it sets *out_len = 0 and +// returns false, leaving no partial write. On success returns true with *out_len set to +// the number of bytes written. bool pg_build_startup(unsigned char* out, size_t* out_len, size_t out_cap, - const char* user, const char* database); + const char* user, const char* database, + const char* client_encoding, const char* options, + const char* application_name); // Builds the PostgreSQL AuthenticationMD5Password response into out[36]: // "md5" + hex(md5( hex(md5(password+user)) + salt[4] )) // Result is the 35-char "md5..." string plus a terminating NUL (36 bytes total). void pg_build_md5(char out[36], const char* user, const char* password, const unsigned char salt[4]); +// Same response, built from a STORED md5 secret instead of a plaintext password. +// `md5_secret` is the pg_authid.rolpassword form that pgsql_users.password holds for an +// md5-stored user: "md5" + 32 lowercase hex chars, where the hex IS the inner +// md5(password+user). Only the outer hash over (inner_hex || salt) is left to compute -- +// running pg_build_md5() over such a secret would hash it a second time and the backend +// would reject the login. +// Returns false, leaving `out` untouched, unless the secret is exactly that form: 35 +// chars, "md5" prefix, 32 lowercase hex digits. A partially written response must never +// reach the wire, so validation happens before the first byte is stored. +bool pg_build_md5_from_secret(char out[36], const char* md5_secret, const unsigned char salt[4]); + // Computes the tls-server-end-point channel-binding data for a finished TLS // session: the digest of the peer cert's DER encoding, using the cert's own // signature hash algorithm, upgraded to SHA-256 if it would otherwise be @@ -76,8 +92,8 @@ int pg_scram_build_cbind_input_tls_server_end_point( // --- SCRAM-SHA-256 client exchange (thin wrappers over vendored libscram) --- // -// Plain SCRAM-SHA-256 only: gs2 channel-binding flag is 'n' (no channel binding). -// Channel binding (gs2 flag 'p'/'y') is a separate task. The wrapper owns a libscram +// SCRAM-SHA-256, and SCRAM-SHA-256-PLUS when a cbind input has been installed via +// pg_scram_set_cbind() before building client-first. The wrapper owns a libscram // ScramState plus a cached PgCredentials; it is defined in lib/PgSQL_Backend_Auth.cpp // so this header stays free of scram.h. Usage mirrors a SCRAM client driving the // PostgreSQL SASL handshake: @@ -103,7 +119,10 @@ void pg_scram_free(PgSQL_Scram_State* s); // gs2 header is "n,," (no channel binding) and the username field is empty ("n="), // matching the PostgreSQL convention where the real username travels in the startup // packet. Returns the owned message string, or nullptr on error (see scram_error()). -// channel_binding=true is not supported by this task and returns nullptr. +// With channel_binding=true the gs2 header is "p=tls-server-end-point,," and the +// caller MUST have installed a matching cbind input via pg_scram_set_cbind() first; +// returns nullptr if it has not, rather than emit a header that contradicts the +// advertised -PLUS mechanism. const char* pg_scram_client_first(PgSQL_Scram_State* s, bool channel_binding); // Consumes the server-first message (AuthenticationSASLContinue body) and the plaintext @@ -111,9 +130,27 @@ const char* pg_scram_client_first(PgSQL_Scram_State* s, bool channel_binding); // NUL-terminated; server_first_len bytes are used. The password is treated as a SCRAM // plaintext secret (keys derived ad-hoc by libscram). Returns the owned message string, // or nullptr on error (nonce mismatch, malformed input, etc; see scram_error()). +// +// password may be nullptr ONLY after pg_scram_set_keys() has injected a ClientKey/ServerKey +// pair; the exchange then runs off those keys and the salt and iteration count in +// server_first are correctly unused. Without injected keys a nullptr password is an error. const char* pg_scram_client_final(PgSQL_Scram_State* s, const char* password, const char* server_first, size_t server_first_len); +// Installs a harvested ClientKey and the stored verifier's ServerKey (32 bytes each) so +// the exchange authenticates FROM THE KEYS, skipping SASLprep + PBKDF2 over a password we +// do not have. This is the backend leg of verifier pass-through: the ClientKey recovered +// from the client's frontend SCRAM proof, plus the ServerKey read out of the stored +// verifier, are exactly the two secrets the rest of the handshake needs. +// +// BOTH keys are required. With only a ClientKey the proof would still be accepted but +// pg_scram_verify_server_final() would have nothing genuine to check against, silently +// dropping the server half of mutual authentication -- so a half-injection is refused and +// the state is left untouched. Returns false on a NULL state or a NULL key. +// +// Call before pg_scram_client_final(), which may then be passed password == nullptr. +bool pg_scram_set_keys(PgSQL_Scram_State* s, const uint8_t* client_key, const uint8_t* server_key); + // Verifies the server-final message (AuthenticationSASLFinal body). server_final need // not be NUL-terminated; len bytes are used. Returns true iff the server signature // matches the one expected from this exchange. Must be called after a successful diff --git a/include/PgSQL_Connection.h b/include/PgSQL_Connection.h index 8a30236ac7..a59be59141 100644 --- a/include/PgSQL_Connection.h +++ b/include/PgSQL_Connection.h @@ -264,6 +264,21 @@ class PgSQL_Connection { PG_ASYNC_ST handler(short event); void connect_start(); + // Builds the session settings a backend StartupMessage must carry: the + // client_encoding value and the "-c name=value ..." options string (tracked + // variables followed by the client's own untracked options). Also records what is + // being sent via server_set_hash_and_value(), so requires_RESETTING_CONNECTION() + // does not afterwards see a false mismatch and resync settings already applied. + // Returns false when there is no client session to read settings from, leaving + // both outputs untouched. + // Escaping differs by transport. Conninfo goes through libpq's parser, which strips + // one level of backslashes before the value reaches the wire; the native path writes + // the wire bytes directly, so it must emit exactly what the backend's pg_split_opts() + // expects (one level less). + enum class StartupParamEscape { Conninfo, Wire }; + bool build_and_record_startup_session_params(std::string& client_encoding_out, + std::string& options_out, + StartupParamEscape escape_mode); void connect_cont(short event); // Consults PgSQL_Monitor::dns_lookup; returns the cached IP on a hit, // empty std::string on a miss. Used by connect_start() to set @@ -498,38 +513,58 @@ class PgSQL_Connection { inline const PGconn* get_pg_connection() const { return pgsql_conn; } inline int get_pg_server_version() { if (native_mode) { - // native_params["server_version"] is e.g. "16.2" or "9.6.1"; libpq encodes - // PQserverVersion as major*10000 + minor*100 + rev. Parse best-effort. auto it = native_params.find("server_version"); if (it == native_params.end()) return 0; - int maj = 0, min = 0, rev = 0; - sscanf(it->second.c_str(), "%d.%d.%d", &maj, &min, &rev); - return maj * 10000 + min * 100 + rev; + // PostgreSQL changed the numeric version encoding at 10: major*10000 + minor + // from 10 onwards, major*10000 + minor*100 + revision before it. + int vmaj = 0, vmin = 0, vrev = 0; + const int cnt = sscanf(it->second.c_str(), "%d.%d.%d", &vmaj, &vmin, &vrev); + // The backend controls this string; the multiplies below overflow int for + // absurd values, so anything implausible is reported as unknown. + if (vmaj < 0 || vmaj > 9999 || vmin < 0 || vmin > 9999 || vrev < 0 || vrev > 9999) return 0; + if (cnt == 3) return (100 * vmaj + vmin) * 100 + vrev; + if (cnt == 2) return (vmaj >= 10) ? (100 * 100 * vmaj + vmin) : ((100 * vmaj + vmin) * 100); + if (cnt == 1) return 100 * 100 * vmaj; + return 0; } return PQserverVersion(pgsql_conn); } inline int get_pg_protocol_version() { return native_mode ? 3 : PQprotocolVersion(pgsql_conn); } inline const char* get_pg_host() { return native_mode ? native_host.c_str() : PQhost(pgsql_conn); } - inline const char* get_pg_hostaddr() { return PQhostaddr(pgsql_conn); } - inline const char* get_pg_port() { return PQport(pgsql_conn); } + inline const char* get_pg_hostaddr() { return native_mode ? native_hostaddr.c_str() : PQhostaddr(pgsql_conn); } + inline const char* get_pg_port() { return native_mode ? native_port.c_str() : PQport(pgsql_conn); } inline const char* get_pg_dbname() { return native_mode ? (userinfo ? userinfo->dbname : "") : PQdb(pgsql_conn); } inline const char* get_pg_user() { return native_mode ? (userinfo ? userinfo->username : "") : PQuser(pgsql_conn); } - inline const char* get_pg_password() { return PQpass(pgsql_conn); } - inline const char* get_pg_options() { return PQoptions(pgsql_conn); } + inline const char* get_pg_password() { return native_mode ? (userinfo && userinfo->password ? userinfo->password : "") : PQpass(pgsql_conn); } + inline const char* get_pg_options() { return native_mode ? native_options.c_str() : PQoptions(pgsql_conn); } inline int get_pg_socket_fd() { return native_mode ? fd : PQsocket(pgsql_conn); } inline int get_pg_backend_pid() { return native_mode ? native_backend_pid : PQbackendPID(pgsql_conn); } inline int get_pg_connection_needs_password() { return PQconnectionNeedsPassword(pgsql_conn); } inline int get_pg_connection_used_password() { return PQconnectionUsedPassword(pgsql_conn); } inline int get_pg_connection_used_gssapi() { return PQconnectionUsedGSSAPI(pgsql_conn); } - inline int get_pg_client_encoding() { return PQclientEncoding(pgsql_conn); } + inline int get_pg_client_encoding() { + if (native_mode) { + constexpr int SQL_ASCII = 0; // PG_SQL_ASCII; mb/pg_wchar.h is not included here + auto it = native_params.find("client_encoding"); + if (it == native_params.end()) return SQL_ASCII; + const int enc = char_to_encoding(it->second.c_str()); + return (enc < 0) ? SQL_ASCII : enc; + } + return PQclientEncoding(pgsql_conn); + } // Native TLS (1.6b): SSL is in use once the handshake handed the SSL* to myds. // Out-of-line in PgSQL_Connection.cpp because PgSQL_Data_Stream is incomplete here. int get_pg_ssl_in_use(); - inline ConnStatusType get_pg_connection_status() { - if (native_mode) return native_connected ? CONNECTION_OK : CONNECTION_BAD; + inline ConnStatusType get_pg_connection_status() const { + // Only the native side needs the check. libpq already reports a connection + // that is still being set up, and those must not be reported as bad. + if (native_mode) return backend_is_live() ? CONNECTION_OK : CONNECTION_BAD; return PQstatus(pgsql_conn); } - inline PGTransactionStatusType get_pg_transaction_status() { + inline PGTransactionStatusType get_pg_transaction_status() const { + // Says whether the connection can be reused. Use + // last_ready_for_query_status() if you want what the backend actually said. + if (!backend_is_live()) return PQTRANS_UNKNOWN; if (native_mode) { switch (native_txn_status) { case 'I': return PQTRANS_IDLE; @@ -540,6 +575,11 @@ class PgSQL_Connection { } return PQtransactionStatus(pgsql_conn); } + // The transaction letter the backend sent with its last reply. It says nothing + // about whether the connection is still usable -- callers that need that ask + // get_pg_transaction_status() instead. + inline char last_ready_for_query_status() const { return native_txn_status; } + inline void set_ready_for_query_status(char st) { native_txn_status = st; } inline int get_pg_is_nonblocking() { return native_mode ? 1 : PQisnonblocking(pgsql_conn); } inline int get_pg_is_threadsafe() { return PQisthreadsafe(); } inline const char* get_pg_error_message() { @@ -691,16 +731,21 @@ class PgSQL_Connection { // SEND_STARTUP to flush the remainder; this records the state to resume in // once the buffer drains (AUTH after a password/SASL message, etc.). PG_Native_Conn_St native_st_after_send = PG_Native_Conn_St::AUTH; + // True once login has finished and the connection can carry queries. Set when + // the backend sends its first ready-for-query, cleared when we start a new + // connect or tear the connection down. + bool native_connected = false; PgSQL_Backend_Msg_Framer native_framer; // frames inbound backend bytes PgSQL_Scram_State* native_scram = nullptr; // owned; freed in destructor / teardown std::string native_outbuf; // pending outbound bytes (partial send buffer) - bool native_connected = false; // true once ReadyForQuery received bool handler_first_call = true; // one-shot first-call detector for handler() (both libpq and native paths) std::map native_params; // ParameterStatus name->value std::string native_host; // backend host (parent->address, captured at connect) + std::string native_options; // the `options` value sent in the StartupMessage + std::string native_hostaddr; // resolved numeric IP, or "" — mirrors when the libpq path passes hostaddr= + std::string native_port; // backend port as a decimal string, matching PQport()'s shape int native_backend_pid = 0; // BackendKeyData PID int native_backend_secret = 0; // BackendKeyData secret key - char native_txn_status = 'I'; // ReadyForQuery status byte ('I'/'T'/'E') // --- Native simple-query / simple-command execution (Task 1.6c / Phase 2 core) --- // Set true once a ReadyForQuery ('Z') has been consumed for the in-flight query, @@ -796,7 +841,14 @@ class PgSQL_Connection { // EAGAIN/incomplete frame → async_exit_status = PG_EVENT_READ and return; a fatal // recv/frame error sets error_info and marks the fetch done. Sets // native_result_complete when ReadyForQuery is reached. - void native_fetch_result_cont(short event); + // Adds up the bytes it hands to query_result in *processed_bytes, so the + // caller can apply the same fetch-pause rule the libpq loop uses. + void native_fetch_result_cont(short event, uint64_t* processed_bytes = nullptr); + // Finish sending a reset command and consume its reply up to ReadyForQuery. + // The reply is discarded; a connection being reset has no client to send it to. + void native_reset_session_cont(); + // Record a ParameterStatus message into native_params. + void native_track_parameter_status(const unsigned char* payload, uint32_t len); // Flush the just-built extended-query step in native_outbuf and set // async_exit_status the way the stmt_*_start callers expect: PG_EVENT_WRITE while // bytes remain buffered (caller waits for POLLOUT), PG_EVENT_NONE once fully sent @@ -848,6 +900,10 @@ class PgSQL_Connection { int native_recv_into_framer(); void native_teardown(); // close fd, free scram (capability gap / failure) void native_capability_gap(const char* mechanism); // tear down native, restart via libpq + // Fatal error during the RESULT phase: records the error AND tears the socket + // down, so the connection is classified non-reusable instead of being pooled. + // See the definition in PgSQL_Connection.cpp for why the teardown is required. + void native_result_fatal(const char* code, const char* message); // Parse an ErrorResponse ('E') payload into error_info. void native_fill_error_from_E(const unsigned char* payload, uint32_t len); @@ -902,6 +958,21 @@ class PgSQL_Connection { PgSQL_SrvC *parent; PgSQL_Connection_userinfo* userinfo; PgSQL_Data_Stream* myds; + + // Native backend TLS. Owned by the connection so it shares the lifetime of + // `fd`, which the native path also owns; the data stream is per-session and + // would take the TLS session with it when the connection is pooled. + // + // SSL_set_bio() transfers both BIOs to the SSL, so SSL_free() releases all + // three -- done only in native_teardown() and ~PgSQL_Connection(), never on a + // pool return. myds->ssl stays NULL in native mode. + // Set when a result fetch stopped early because it had already moved enough + // bytes for this event. The next entry drains what is still framed instead + // of asking the socket for more, which may never come. + bool native_fetch_paused = false; + SSL* native_ssl = nullptr; + BIO* native_rbio = nullptr; + BIO* native_wbio = nullptr; //unsigned int warning_count; int fd; /** @@ -914,6 +985,21 @@ class PgSQL_Connection { bool unknown_transaction_status; private: + // The one place that decides whether this connection can still be used. Every + // answer about the connection's health is built on it. + bool backend_is_live() const; + + // True once this connection has been added to the global count of connected + // backends. The two native subtractions (teardown and destructor) check it, so + // a connection that was never added is never subtracted and the count cannot + // wrap. The libpq branch of the destructor still keys off is_connected(). + bool counted_in_connections_connected = false; + + // Kept private on purpose. It is stale whenever the connection is broken, so + // read it through one of the two accessors, which say which question you are + // asking. + char native_txn_status = 'I'; // ReadyForQuery status byte ('I'/'T'/'E') + // Set end state for the fetch result to indicate that it originates from a simple query or statement execution. ASYNC_ST fetch_result_end_st = ASYNC_QUERY_END; inline void set_fetch_result_end_state(ASYNC_ST st) { @@ -925,6 +1011,10 @@ class PgSQL_Connection { // Handles the COPY OUT response from the server. // Returns true if it consumes all buffer data, or false if the threshold for result size is reached bool handle_copy_out(const PGresult* result, uint64_t* processed_bytes); + // True when this event has moved enough bytes that the result fetch should + // pause and let the client catch up. Shared by the libpq and native result + // loops so both honour pgsql-threshold_resultset_size the same way. + bool suspend_resultset_fetch(uint64_t processed_bytes) const; static void notice_handler_cb(void* arg, const PGresult* result); static void unhandled_notice_cb(void* arg, const PGresult* result); void init_query_result(); diff --git a/include/PgSQL_Data_Stream.h b/include/PgSQL_Data_Stream.h index fb9a33c776..aff24705c8 100644 --- a/include/PgSQL_Data_Stream.h +++ b/include/PgSQL_Data_Stream.h @@ -228,6 +228,10 @@ class PgSQL_Data_Stream // // we have a similar code in MySQL_Connection // in case of ASYNC_CONNECT_SUCCESSFUL + // + // For futher details: + // - without ssl: we use the file descriptor from pgsql connection + // - with ssl: we use the SSL structure from pgsql connection if (sess != NULL && sess->session_fast_forward) { // Relaying without the backend's TLS would put plaintext on an encrypted // socket. Close the session instead; the connection is already flagged. diff --git a/include/PgSQL_PreparedStatement.h b/include/PgSQL_PreparedStatement.h index 6d5007b795..e59d6b7887 100644 --- a/include/PgSQL_PreparedStatement.h +++ b/include/PgSQL_PreparedStatement.h @@ -150,6 +150,16 @@ class PgSQL_STMT_Local { */ void client_close_all(); + /** + * Close all backend-side prepared statement mappings (is_client_ == false). + * + * Decrements the server refcount for each associated global statement and + * clears the backend maps. Mirrors the backend branch of ~PgSQL_STMT_Local(), + * for use when the backend's prepared statements are known to be gone (e.g. + * after forwarding DEALLOCATE ALL) but the connection object lives on. + */ + void backend_close_all(); + /** * Generate a new backend statement ID. * diff --git a/include/gen_utils.h b/include/gen_utils.h index 5559551797..ad1abca866 100644 --- a/include/gen_utils.h +++ b/include/gen_utils.h @@ -515,7 +515,10 @@ char *trim_spaces_and_quotes_in_place(char *str); bool mywildcmp(const char *p, const char *str); std::string trim(const std::string& s); char* escape_string_single_quotes_and_backslashes(char* input, bool free_it); -const char* escape_string_backslash_spaces(const char* input); +// Appends `input` to `out`, escaped for a PostgreSQL StartupMessage 'options' value: every +// space and every backslash is prefixed with a backslash, all other characters are copied +// unchanged. `out` is appended to, not replaced. +void pg_append_escaped_option_value(std::string& out, const char* input); time_t monotonic_time_to_realtime(time_t mt); time_t realtime_to_monotonic_time(time_t rt); diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index bccdc6251a..a3e0e60959 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -328,9 +328,16 @@ enum session_status { SETTING_SESSION_TRACK_VARIABLES, SETTING_SESSION_TRACK_STATE, SETTING_USER_VARIABLES, - // Append-only: changing existing values can leave incrementally built - // translation units disagreeing on this enum's numeric values. + // NOTE: append-only. PROCESSING_STMT_BIND (Task P1) is placed at the END of the + // enum on purpose: inserting it mid-list would renumber every following value, and + // the build does not track header→object dependencies, so any translation unit not + // recompiled would silently disagree on the numeric values (observed: a stale + // pgsql_tracked_variables[] holding the old SETTING_VARIABLE value crashed + // verify_server_variable with "Wrong status"). Keep new statuses here. PROCESSING_STMT_BIND, + // Named-portal Close (Task P2): a real backend Close('P', name) round-trip + // (CloseComplete '3' forwarded, registry entry evicted). Append-only, same + // rationale as PROCESSING_STMT_BIND above. PROCESSING_STMT_CLOSE, session_status___NONE // special marker }; diff --git a/lib/PgSQL_Backend_Auth.cpp b/lib/PgSQL_Backend_Auth.cpp index bff9e60839..f4bdba25b9 100644 --- a/lib/PgSQL_Backend_Auth.cpp +++ b/lib/PgSQL_Backend_Auth.cpp @@ -23,11 +23,24 @@ void pg_build_ssl_request(unsigned char out[8]) { } bool pg_build_startup(unsigned char* out, size_t* out_len, size_t out_cap, - const char* user, const char* database) { + const char* user, const char* database, + const char* client_encoding, const char* options, + const char* application_name) { + const bool has_enc = (client_encoding != nullptr && client_encoding[0] != '\0'); + const bool has_opts = (options != nullptr && options[0] != '\0'); + const bool has_app = (application_name != nullptr && application_name[0] != '\0'); + // Compute the required size first so a bound check can reject before any write, - // guaranteeing no partial/oversized output is left in the caller buffer. - // length(4) + protocol(4) + "user\0" + user\0 + "database\0" + database\0 + \0 - size_t need = 8 + 5 + (strlen(user) + 1) + 9 + (strlen(database) + 1) + 1; + // guaranteeing no partial/oversized output is left in the caller buffer. Every + // parameter costs "key\0" + "value\0"; sizeof() on the key literal already counts + // its NUL, so renaming a key can never leave a stale hand-counted length behind. + size_t need = 4 + 4 // length + protocol + + sizeof("user") + (strlen(user) + 1) + + sizeof("database") + (strlen(database) + 1) + + 1; // terminating empty key + if (has_opts) need += sizeof("options") + strlen(options) + 1; + if (has_app) need += sizeof("application_name") + strlen(application_name) + 1; + if (has_enc) need += sizeof("client_encoding") + strlen(client_encoding) + 1; if (need > out_cap) { *out_len = 0; return false; @@ -35,8 +48,13 @@ bool pg_build_startup(unsigned char* out, size_t* out_len, size_t out_cap, size_t off = 8; // reserve length(4) + protocol(4) auto add = [&](const char* s) { size_t l = strlen(s) + 1; memcpy(out + off, s, l); off += l; }; + // Emitted in the same order libpq uses, so a packet capture lines up parameter for + // parameter with one from the libpq path. add("user"); add(user); add("database"); add(database); + if (has_opts) { add("options"); add(options); } + if (has_app) { add("application_name"); add(application_name); } + if (has_enc) { add("client_encoding"); add(client_encoding); } out[off++] = 0; // terminating empty key put_be32(out, (uint32_t)off); // total length (includes the length field itself) @@ -58,35 +76,57 @@ static void md5_hex(const unsigned char* in, size_t inlen, char out_hex[33]) { out_hex[MD5_DIGEST_LENGTH * 2] = '\0'; } +// Shared outer step of the AuthenticationMD5Password response: +// out = "md5" + hex(md5( inner_hex[32] || salt[4] )) +// inner_hex is NOT NUL-terminated; exactly 32 bytes are read. +static void md5_response_from_inner(char out[36], const char* inner_hex, + const unsigned char salt[4]) { + unsigned char outer_in[MD5_DIGEST_LENGTH * 2 + 4]; + memcpy(outer_in, inner_hex, MD5_DIGEST_LENGTH * 2); + memcpy(outer_in + MD5_DIGEST_LENGTH * 2, salt, 4); + + char outer_hex[33]; + md5_hex(outer_in, sizeof(outer_in), outer_hex); + + memcpy(out, "md5", 3); + memcpy(out + 3, outer_hex, 33); // 32 hex chars + NUL -> out[3..35] +} + void pg_build_md5(char out[36], const char* user, const char* password, const unsigned char salt[4]) { // inner = hex(md5(password + user)). Hash the concatenation without an // intermediate NUL-terminated copy by passing each part length explicitly. - size_t plen = strlen(password); - size_t ulen = strlen(user); - { - unsigned char digest[MD5_DIGEST_LENGTH]; - MD5_CTX ctx; - MD5_Init(&ctx); - MD5_Update(&ctx, password, plen); - MD5_Update(&ctx, user, ulen); - MD5_Final(digest, &ctx); - static const char hexd[] = "0123456789abcdef"; - char inner_hex[33]; - for (int i = 0; i < MD5_DIGEST_LENGTH; i++) { - inner_hex[i * 2] = hexd[(digest[i] >> 4) & 0xf]; - inner_hex[i * 2 + 1] = hexd[digest[i] & 0xf]; - } - // outer input = 32 inner hex chars + 4 raw salt bytes (NOT NUL-terminated). - unsigned char outer_in[MD5_DIGEST_LENGTH * 2 + 4]; - memcpy(outer_in, inner_hex, MD5_DIGEST_LENGTH * 2); - memcpy(outer_in + MD5_DIGEST_LENGTH * 2, salt, 4); - - char outer_hex[33]; - md5_hex(outer_in, sizeof(outer_in), outer_hex); - - memcpy(out, "md5", 3); - memcpy(out + 3, outer_hex, 33); // 32 hex chars + NUL -> out[3..35] + unsigned char digest[MD5_DIGEST_LENGTH]; + MD5_CTX ctx; + MD5_Init(&ctx); + MD5_Update(&ctx, password, strlen(password)); + MD5_Update(&ctx, user, strlen(user)); + MD5_Final(digest, &ctx); + + static const char hexd[] = "0123456789abcdef"; + char inner_hex[MD5_DIGEST_LENGTH * 2]; + for (int i = 0; i < MD5_DIGEST_LENGTH; i++) { + inner_hex[i * 2] = hexd[(digest[i] >> 4) & 0xf]; + inner_hex[i * 2 + 1] = hexd[digest[i] & 0xf]; } + md5_response_from_inner(out, inner_hex, salt); +} + +bool pg_build_md5_from_secret(char out[36], const char* md5_secret, const unsigned char salt[4]) { + if (out == nullptr || md5_secret == nullptr || salt == nullptr) return false; + // "md5" + exactly 32 LOWERCASE hex digits and nothing else -- the only form PostgreSQL + // stores. Validate in full before writing the first byte: a half-built response on a + // rejected secret is indistinguishable from a good one at the call site. + if (strlen(md5_secret) != 3 + (size_t)MD5_DIGEST_LENGTH * 2) return false; + if (memcmp(md5_secret, "md5", 3) != 0) return false; + const char* inner_hex = md5_secret + 3; + for (int i = 0; i < MD5_DIGEST_LENGTH * 2; i++) { + const char c = inner_hex[i]; + if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'))) return false; + } + // The stored secret IS the inner hash, so only the outer step is left. Hashing it a + // second time through pg_build_md5() is what made md5-stored users fail on this path. + md5_response_from_inner(out, inner_hex, salt); + return true; } // --- SCRAM-SHA-256 client exchange (thin wrappers over libscram) --- @@ -115,17 +155,25 @@ void pg_scram_free(PgSQL_Scram_State* s) { if (s->st) free_scram_state(s->st); // frees ScramState's owned buffers + the struct free(s->client_first); free(s->client_final); + // creds holds password-equivalent material (a plaintext password, or an injected + // ClientKey/ServerKey pair). Scrub it non-elidably before the memory goes back. + OPENSSL_cleanse(&s->creds, sizeof(s->creds)); delete s; } const char* pg_scram_client_first(PgSQL_Scram_State* s, bool channel_binding) { if (s == nullptr || s->st == nullptr) return nullptr; - // Channel binding ('p'/'y' gs2 flag) is a separate task; this wrapper only does - // plain SCRAM-SHA-256 with gs2 flag 'n' ("n,," header). - if (channel_binding) return nullptr; + // channel_binding=true selects SCRAM-SHA-256-PLUS. The gs2 header libscram + // writes is driven by the cbind input the caller installed with + // pg_scram_set_cbind(), NOT by this flag, so the two must agree: advertising + // -PLUS while no cbind input is set would emit a plain "n,," header against a + // -PLUS mechanism name and the server would reject the proof. Refuse that + // combination rather than produce a mismatched handshake. + if (channel_binding && s->st->client_cbind_input == nullptr) return nullptr; scram_reset_error(); - // libscram emits "n,,n=,r=" and stashes client_nonce / client_first_message_bare - // ("n=,r=") into the ScramState for the later proof computation. + // libscram emits "n,,n=,r=", or "p=tls-server-end-point,,n=,r=" + // when a cbind input is set, and stashes client_nonce / client_first_message_bare + // into the ScramState for the later proof computation. char* msg = build_client_first_message(s->st); if (msg == nullptr) return nullptr; free(s->client_first); @@ -135,8 +183,10 @@ const char* pg_scram_client_first(PgSQL_Scram_State* s, bool channel_binding) { const char* pg_scram_client_final(PgSQL_Scram_State* s, const char* password, const char* server_first, size_t server_first_len) { - if (s == nullptr || s->st == nullptr || password == nullptr || server_first == nullptr) - return nullptr; + if (s == nullptr || s->st == nullptr || server_first == nullptr) return nullptr; + // A password is required UNLESS pg_scram_set_keys() injected a ClientKey/ServerKey pair, + // in which case libscram derives nothing from a password and never reads creds.passwd. + if (password == nullptr && !s->creds.has_scram_keys) return nullptr; scram_reset_error(); // read_server_first_message() mutates its input (read_attr_value writes NULs and @@ -152,9 +202,13 @@ const char* pg_scram_client_final(PgSQL_Scram_State* s, const char* password, return nullptr; } - // The password is the SCRAM plaintext secret; libscram derives keys ad-hoc. - // has_scram_keys stays false (value-initialized) so the plaintext path is used. - snprintf(s->creds.passwd, sizeof(s->creds.passwd), "%s", password); + if (!s->creds.has_scram_keys) { + // The password is the SCRAM plaintext secret; libscram derives keys ad-hoc. + snprintf(s->creds.passwd, sizeof(s->creds.passwd), "%s", password); + } + // With keys injected, creds already carries ClientKey/ServerKey and passwd stays empty: + // build_client_final_message() uses the ClientKey directly and skips SASLprep + PBKDF2, + // so the salt and iteration count the backend just sent are (correctly) unused. char* msg = build_client_final_message(s->st, &s->creds, server_nonce, salt, saltlen, iterations); // server_nonce / salt point into the parsed buffers: server_nonce into the local @@ -220,6 +274,23 @@ int pg_scram_build_cbind_input_tls_server_end_point( return (int)total; } +bool pg_scram_set_keys(PgSQL_Scram_State* s, const uint8_t* client_key, const uint8_t* server_key) { + if (s == nullptr || s->st == nullptr) return false; + // Both or neither. A ClientKey on its own still produces a proof the backend accepts, + // but leaves verify_server_signature() nothing genuine to check against -- the server + // half of mutual authentication would be dropped without any visible failure. libscram + // gates both keys on one flag, so a half-filled PgCredentials is not representable + // there either; refuse the call rather than approximate it. + if (client_key == nullptr || server_key == nullptr) return false; + memcpy(s->creds.scram_ClientKey, client_key, sizeof(s->creds.scram_ClientKey)); + memcpy(s->creds.scram_ServerKey, server_key, sizeof(s->creds.scram_ServerKey)); + s->creds.has_scram_keys = true; + // A verifier is not a password: an empty passwd means nothing downstream can fall back + // to SASLprep + PBKDF2 over verifier text. Cleansed so an earlier call's cannot linger. + OPENSSL_cleanse(s->creds.passwd, sizeof(s->creds.passwd)); + return true; +} + void pg_scram_set_cbind(PgSQL_Scram_State* s, const char* cbind_input, int cbind_input_len) { if (s == nullptr) return; scram_state_set_cbind_input(s->st, cbind_input, cbind_input_len); diff --git a/lib/PgSQL_Backend_Protocol.cpp b/lib/PgSQL_Backend_Protocol.cpp index c84a4e39f0..3e257c0077 100644 --- a/lib/PgSQL_Backend_Protocol.cpp +++ b/lib/PgSQL_Backend_Protocol.cpp @@ -2,6 +2,9 @@ #include #include #include +#ifdef DEBUG +#include +#endif static inline uint32_t be32(const unsigned char* p) { return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16) | ((uint32_t)p[2] << 8) | p[3]; @@ -9,6 +12,60 @@ static inline uint32_t be32(const unsigned char* p) { void PgSQL_Backend_Msg_Framer::feed(const unsigned char* data, size_t n) { if (failed) return; // already in error state; ignore further bytes + + // Reclaim the consumed prefix before appending. + // + // next() rewinds the buffer only when a message ends exactly at len (its + // `if (pos == len) { pos = 0; len = 0; }` reset). When a read stopped + // mid-message that never fired, so the bytes below pos — already framed, + // handed to the caller and copied into the result buffer — stayed resident + // while feed() appended above them. The buffer grew for the whole result + // set, and cap never shrinks, so the peak stayed allocated for the life of + // the connection. + // + // Whether the reset fires is arithmetic, not a property of the data: with + // reads of `chunk` bytes and messages of `msglen` it lands every + // msglen/gcd(msglen, chunk) reads, so retention reached + // chunk * msglen / gcd(msglen, chunk). Reads are 16384 bytes + // (native_recv_into_framer) and 8192 over TLS (MY_SSL_BUFFER), both powers + // of two, so any ODD message size never rewound until the stream ended. + // Measured: 48 MiB streamed as 2049-byte messages retained 38.6 MiB, against + // 28 KiB for the same run with 2048-byte messages. + // + // The fix slides the unread tail down to offset 0 and drops the consumed + // prefix, keeping the buffer at roughly one read in size. It compacts only + // when that prefix is at least as large as the tail, so the move never costs + // more than it reclaims and a large message is left in place while it is + // still being assembled across reads. + if (pos > 0) { + const size_t live = len - pos; + if (pos >= live) { + memmove(buf, buf + pos, live); // live may be 0; memmove(_,_,0) is a no-op + len = live; + pos = 0; + } + } + +#ifdef DEBUG + // Everything below pos has already been framed and copied out to the client + // -- it is finished with. If the compaction above stops reclaiming it, those + // dead bytes stay in the buffer while feed() keeps appending above them, and + // it grows for the whole result set. next() has a cheap rewind for this, but + // it only fires when a drain lands exactly on len, which for a 2049-byte row + // read in 16384-byte chunks first happens after 33 MB. Measured: 58 MB of + // delivered rows retained on one connection, and held for the connection's + // whole life because cap never shrinks. + // + // After compaction one of these always holds: pos == 0 (the dead prefix was + // reclaimed) or pos < live (not yet worth the move). That is the compaction + // condition restated, so correct code cannot trip it -- and it fires on the + // first feed after a drain, not once tens of MB have piled up. + // + // MUST stay #ifdef DEBUG: NDEBUG is not set anywhere in this build, so a + // bare assert() would stay live in release and abort a production proxy. + assert(pos == 0 || pos < len - pos); +#endif + if (n > SIZE_MAX - len) { failed = true; return; } // would overflow len+n if (len + n > cap) { size_t need = len + n; diff --git a/lib/PgSQL_Connection.cpp b/lib/PgSQL_Connection.cpp index 5807e6d822..be366627a6 100644 --- a/lib/PgSQL_Connection.cpp +++ b/lib/PgSQL_Connection.cpp @@ -17,7 +17,7 @@ #include "openssl/x509v3.h" // X509_VERIFY_PARAM_set1_host / set_hostflags (native backend TLS) #include "openssl/evp.h" // EVP_MAX_MD_SIZE for cbind digest buffer (SCRAM-PLUS) #include "PgSQL_Backend_Protocol.h" // pg_tls_server_end_point / pg_scram_build_cbind_input_* / pg_scram_set_cbind (SCRAM-PLUS) -#include // OPENSSL_cleanse — non-elidable wipe of harvested SCRAM key material +#include "openssl/crypto.h" // OPENSSL_cleanse — non-elidable wipe of harvested SCRAM key material #include "../deps/json/json.hpp" using json = nlohmann::json; @@ -253,7 +253,7 @@ PgSQL_Connection::~PgSQL_Connection() { // so the block above is skipped: mirror its connected-counter decrement and // free the native socket + SCRAM state here. if (native_mode) { - if (native_connected) { + if (counted_in_connections_connected) { __sync_fetch_and_sub(&PgHGM->status.server_connections_connected, 1); } if (native_scram) { @@ -264,9 +264,21 @@ PgSQL_Connection::~PgSQL_Connection() { ::close(fd); fd = -1; } + // The TLS session is owned by this connection (see PgSQL_Connection.h), so it + // must be released here as well as in native_teardown(): a pooled connection + // evicted by destroy_MyConn_from_pool() is `delete`d WITHOUT going through + // teardown, and would otherwise leak the SSL and both its BIOs. SSL_free() + // releases the BIOs too (SSL_set_bio transferred them). + if (native_ssl) { + SSL_free(native_ssl); + native_ssl = nullptr; + native_rbio = nullptr; + native_wbio = nullptr; + } // native_ssl_ctx is normally freed at SSL_new() time (the SSL holds a ref) or // in native_teardown(); free here as a safety net if a connection is destroyed - // before either ran. The SSL* itself lives on myds and is freed by ~PgSQL_Data_Stream(). + // before either ran. The SSL and its BIOs belong to this connection, not to + // myds: a fast_forward data stream only borrows them (adopt_backend_tls()). if (native_ssl_ctx) { SSL_CTX_free(native_ssl_ctx); native_ssl_ctx = nullptr; @@ -419,6 +431,26 @@ PG_ASYNC_ST PgSQL_Connection::handler(short event) { assert(0); // shouldn't ever reach here, we have messed up the state machine if (get_pg_ssl_in_use()) { + if (native_mode && myds && myds->sess && myds->sess->session_fast_forward) { + // fast_forward relays raw bytes and wants the backend SSL on the data + // stream. Handing it ours is not fatal -- detach_connection() nulls + // myds->ssl for fast_forward without freeing it (a deliberate + // borrowed-pointer design), so there is no double free -- but the + // block below calls SSL_set_bio(), which REPLACES and frees the BIOs + // this connection still holds pointers to, leaving native_rbio / + // native_wbio dangling. + // + // MEASURED: with this guard disabled, 10 native fast_forward TLS + // sessions produced no crash, no assert and no double free; the + // queries failed either way. fast_forward + backend TLS is broken for + // BOTH the native and libpq paths (verified: libpq fails identically), + // so this guard changes no user-visible outcome. It is kept only so + // the connection is never left holding freed BIO pointers; the + // fallback also routes the session to libpq, which is the path that + // owns this combination. + native_capability_gap("fast_forward with native TLS"); + return async_state_machine; + } if (myds && myds->sess && myds->sess->session_fast_forward) { assert(myds->ssl == NULL); if (myds->adopt_backend_tls() == false) { @@ -431,6 +463,7 @@ PG_ASYNC_ST PgSQL_Connection::handler(short event) { } } __sync_fetch_and_add(&PgHGM->status.server_connections_connected, 1); + counted_in_connections_connected = true; __sync_fetch_and_add(&parent->connect_OK, 1); // Seed the PgSQL DNS cache from the just-established connection so // the next connect for this hostname can skip getaddrinfo even if @@ -477,6 +510,34 @@ PG_ASYNC_ST PgSQL_Connection::handler(short event) { if (is_error_present()) { NEXT_IMMEDIATE(ASYNC_QUERY_END); } + // Record where this query should go once its reply has been read. + // + // Two functions reach this case. async_query() runs ordinary client queries, + // and async_send_simple_command() is what ProxySQL uses internally to configure + // a backend connection, for example the "SET client_encoding" it sends when a + // pooled connection is given to a client that asked for a different encoding. + // Both send a single 'Q' message and both finish in ASYNC_QUERY_END, so that is + // the value stored here. + // + // ASYNC_QUERY_CONT below stores the same value, but it cannot be relied on to + // do it. query_start() will often write the whole 'Q' in one syscall, which is + // the normal outcome in native mode for something as short as a SET. When that + // happens there is nothing left to wait for, so we go straight to the result + // drain and never pass through ASYNC_QUERY_CONT at all. + // + // Nothing else ever clears this field. Without the line below it would still + // hold whatever an earlier extended-query step left on this connection, such as + // ASYNC_STMT_EXECUTE_END, and the result dispatch would jump there when the + // reply arrived. async_query() copes with that, because it accepts any *_END + // state as success. async_send_simple_command() does not: it accepts only + // ASYNC_QUERY_END, so anything else makes it answer "not finished yet" every + // time it is called, and the session then waits in SETTING_VARIABLE forever + // because nothing times it out. + // + // Only the native path can get into that state. libpq's flush never reports + // that it sent everything in one go, so a libpq connection always goes through + // ASYNC_QUERY_CONT and picks up the assignment there. + set_fetch_result_end_state(ASYNC_QUERY_END); NEXT_IMMEDIATE(ASYNC_USE_RESULT_START); } break; @@ -525,7 +586,7 @@ PG_ASYNC_ST PgSQL_Connection::handler(short event) { // handles the native path and must NOT fall through to any libpq // PGresult dispatch below. if (native_mode) { - native_fetch_result_cont(event); + native_fetch_result_cont(event, &processed_bytes); if (async_exit_status) { // Need more bytes from the socket → wait for READ. next_event(ASYNC_USE_RESULT_CONT); @@ -537,6 +598,13 @@ PG_ASYNC_ST PgSQL_Connection::handler(short event) { // or the configured fetch_result_end_st). NEXT_IMMEDIATE(fetch_result_end_st); } + // Enough bytes moved in this event: pause and let the client drain, + // exactly as the libpq loop below does, so pgsql-threshold_resultset_size + // behaves the same on both paths. + if (suspend_resultset_fetch(processed_bytes)) { + next_event(ASYNC_USE_RESULT_CONT); // we temporarily pause + break; + } // Neither complete nor error nor waiting: loop to drain/recv more. NEXT_IMMEDIATE(ASYNC_USE_RESULT_CONT); } @@ -729,17 +797,7 @@ PG_ASYNC_ST PgSQL_Connection::handler(short event) { update_bytes_recv(bytes_recv); processed_bytes += bytes_recv; // issue #527 : this variable will store the amount of bytes processed during this event - bool suspend_resultset_fetch = (processed_bytes > overflow_safe_multiply<8,unsigned int>(pgsql_thread___threshold_resultset_size)); - - if (suspend_resultset_fetch == true && myds->sess && myds->sess->qpo && myds->sess->qpo->cache_ttl > 0) { - suspend_resultset_fetch = (processed_bytes > ((uint64_t)pgsql_thread___query_cache_size_MB) * 1024ULL * 1024ULL); - } - - if ( - suspend_resultset_fetch - || - (pgsql_thread___throttle_ratio_server_to_client && pgsql_thread___throttle_max_bytes_per_second_to_client && (processed_bytes > (unsigned long long)pgsql_thread___throttle_max_bytes_per_second_to_client / 10 * (unsigned long long)pgsql_thread___throttle_ratio_server_to_client)) - ) { + if (suspend_resultset_fetch(processed_bytes)) { next_event(ASYNC_USE_RESULT_CONT); // we temporarily pause break; } else { @@ -757,17 +815,7 @@ PG_ASYNC_ST PgSQL_Connection::handler(short event) { update_bytes_recv(bytes_recv); processed_bytes += bytes_recv; // issue #527 : this variable will store the amount of bytes processed during this event - bool suspend_resultset_fetch = (processed_bytes > overflow_safe_multiply<8,unsigned int>(pgsql_thread___threshold_resultset_size)); - - if (suspend_resultset_fetch == true && myds->sess && myds->sess->qpo && myds->sess->qpo->cache_ttl > 0) { - suspend_resultset_fetch = (processed_bytes > ((uint64_t)pgsql_thread___query_cache_size_MB) * 1024ULL * 1024ULL); - } - - if ( - suspend_resultset_fetch - || - (pgsql_thread___throttle_ratio_server_to_client && pgsql_thread___throttle_max_bytes_per_second_to_client && (processed_bytes > (unsigned long long)pgsql_thread___throttle_max_bytes_per_second_to_client / 10 * (unsigned long long)pgsql_thread___throttle_ratio_server_to_client)) - ) { + if (suspend_resultset_fetch(processed_bytes)) { next_event(ASYNC_USE_RESULT_CONT); // we temporarily pause break; } else { @@ -1118,8 +1166,14 @@ PG_ASYNC_ST PgSQL_Connection::handler(short event) { break; default: - // not implemented yet - assert(0); + // The connection is in a state nothing here knows how to handle. Log which + // state it was, so the abort below is not a bare assert with no clue, then stop. + proxy_error("Unhandled state %d in PgSQL_Connection::handler() for backend %s:%d (native_mode=%d, fd=%d). Aborting.\n", + (int)async_state_machine, + (parent && parent->address) ? parent->address : "(unknown)", + parent ? parent->port : -1, + native_mode ? 1 : 0, fd); + assert(0); } return async_state_machine; } @@ -1209,23 +1263,6 @@ bool pgsql_append_conninfo_credentials(std::ostringstream& conninfo, const char* return false; } -// escape_string_backslash_spaces() already emits DOUBLE backslashes for a space (and doubles a -// literal backslash), i.e. it produces a value that survives libpq stripping one escape level out -// of a single-quoted conninfo value. What it does NOT handle is the apostrophe: an unescaped ' ends -// the quoted value and everything after it is parsed by libpq as further conninfo KEYWORDS -// (host=, sslmode=, ...). This adds exactly that missing level and nothing else -- escaping -// backslashes here as well would double what the helper already doubled and corrupt every value -// containing a space (observed: DateStyle "ISO, MDY" arriving at the backend as "ISO,\\"). -static std::string pg_conninfo_escape_quotes(const char* v) { - std::string out; - if (v == nullptr) return out; - for (const char* c = v; *c; c++) { - if (*c == '\'') out += '\\'; - out += *c; - } - return out; -} - std::string PgSQL_Connection::connect_start_DNS_lookup() { // PgSQL_Monitor::dns_lookup() returns an IP on cache hit, or empty // on miss / when 'parent->address' is itself an IP / when the cache is @@ -1236,6 +1273,80 @@ std::string PgSQL_Connection::connect_start_DNS_lookup() { return ip; } +// Raises a wire-form value to the level a libpq conninfo needs. libpq parses the conninfo +// and strips one level of backslash escaping before the value reaches the wire, so doubling +// every backslash of the wire form is what makes the backend see that exact wire form. +// The spaces separating the "-c key=value" tokens need nothing: both values are single-quoted +// in the conninfo, so they pass through untouched. The apostrophe does need it, for a different +// reason: an unescaped ' ends the quoted value, and everything after it is parsed by libpq as +// further conninfo KEYWORDS (host=, sslmode=, ...). Escaping it here keeps a client-supplied +// option value a literal instead of a way to redirect the backend connection. +static std::string pg_conninfo_escape_level(const std::string& wire) { + std::string out; + // Worst case is every character needing an escape, so reserve once rather than + // regrowing part-way through. + out.reserve(wire.size() * 2); + for (char c : wire) { + if (c == '\\' || c == '\'') out += '\\'; + out += c; + } + return out; +} + +bool PgSQL_Connection::build_and_record_startup_session_params(std::string& client_encoding_out, + std::string& options_out, + StartupParamEscape escape_mode) { + if (!(myds && myds->sess && myds->sess->client_myds)) return false; + + // Client encoding is always set; it travels as its own startup key, not inside options. + const char* client_charset = pgsql_variables.client_get_value(myds->sess, PGSQL_CLIENT_ENCODING); + assert(client_charset); + const uint32_t client_charset_hash = pgsql_variables.client_get_hash(myds->sess, PGSQL_CLIENT_ENCODING); + assert(client_charset_hash); + // A startup key's value is a plain NUL-terminated string, so the wire form is the raw + // value; the conninfo form is derived from it at the end of this function. + client_encoding_out.assign(client_charset); + // charset validation is already done + pgsql_variables.server_set_hash_and_value(myds->sess, PGSQL_CLIENT_ENCODING, client_charset, client_charset_hash); + + // The tracked variables, as "-c name=value" tokens, escaped for the wire. + std::string opts; + const char* separator = ""; + for (int idx = 1; idx < PGSQL_NAME_LAST_LOW_WM; idx++) { + const char* value = pgsql_variables.client_get_value(myds->sess, idx); + opts += separator; + opts += "-c "; + opts += pgsql_tracked_variables[idx].set_variable_name; + opts += "="; + pg_append_escaped_option_value(opts, value); + separator = " "; + const uint32_t hash = pgsql_variables.client_get_hash(myds->sess, idx); + pgsql_variables.server_set_hash_and_value(myds->sess, idx, value, hash); + } + // The client's own connection options, which it supplied as options='-c ...'. + if (myds->sess->untracked_option_parameters.empty() == false) { + opts += separator; + opts += myds->sess->untracked_option_parameters; + } + options_out = std::move(opts); + + // Snapshot variables[] into startup_parameters[] so requires_RESETTING_CONNECTION() + // knows these are already applied. server_set_hash_and_value() above wrote into + // sess->mybe->server_myds->myconn, and this copy is intra-object (variables[] -> + // startup_parameters[] on whichever connection it is called on), so it has to run on + // that same connection -- hence the same expression rather than `this`. + myds->sess->mybe->server_myds->myconn->copy_pgsql_variables_to_startup_parameters(true); + + // Everything above is the wire form, which is what untracked_option_parameters is + // stored in too. The libpq path needs one level more, since libpq strips one while + // parsing the conninfo. + if (escape_mode == StartupParamEscape::Conninfo) { + client_encoding_out = pg_conninfo_escape_level(client_encoding_out); + options_out = pg_conninfo_escape_level(options_out); + } + return true; +} + void PgSQL_Connection::connect_start() { PROXY_TRACE(); assert(pgsql_conn == NULL); // already there is a connection @@ -1313,62 +1424,16 @@ void PgSQL_Connection::connect_start() { conninfo << "sslmode='disable' "; // not supporting SSL } - if (myds && myds->sess && myds->sess->client_myds) { - // Client Encoding should be always set - const char* client_charset = pgsql_variables.client_get_value(myds->sess, PGSQL_CLIENT_ENCODING); - assert(client_charset); - uint32_t client_charset_hash = pgsql_variables.client_get_hash(myds->sess, PGSQL_CLIENT_ENCODING); - assert(client_charset_hash); - { - // escape_string_backslash_spaces() covers spaces and backslashes for BOTH the - // conninfo quoting layer and the backend's options tokeniser (it emits two - // backslashes per space). pg_conninfo_escape_quotes() adds the one case it misses: - // the apostrophe, which would otherwise close the quoted conninfo value early. - const char* wire = escape_string_backslash_spaces(client_charset); - conninfo << "client_encoding='" << pg_conninfo_escape_quotes(wire) << "' "; - if (wire != client_charset) free((char*)wire); - } - - // charset validation is already done - pgsql_variables.server_set_hash_and_value(myds->sess, PGSQL_CLIENT_ENCODING, client_charset, client_charset_hash); - - // optimized way to set client parameters on backend connection when creating a new connection - // Join the "-c key=value" tokens with a leading separator so the options value has - // no trailing space before the closing quote. PgBouncer rejects a startup packet - // whose options value ends in whitespace (#5801). - // Build the whole options value in its WIRE form first, then apply the conninfo - // quoting layer once over the finished string (see the comment above). Assembling it - // straight into `conninfo` cannot work: the second layer has to see the complete - // value, including untracked_option_parameters, which is also stored wire-escaped. - std::string opts; - const char* separator = ""; - // excluding client_encoding, which is already set above - for (int idx = 1; idx < PGSQL_NAME_LAST_LOW_WM; idx++) { - const char* value = pgsql_variables.client_get_value(myds->sess, idx); - const char* escaped_str = escape_string_backslash_spaces(value); - opts += separator; - opts += "-c "; - opts += pgsql_tracked_variables[idx].set_variable_name; - opts += "="; - opts += escaped_str; - separator = " "; - if (escaped_str != value) - free((char*)escaped_str); - - const uint32_t hash = pgsql_variables.client_get_hash(myds->sess, idx); - pgsql_variables.server_set_hash_and_value(myds->sess, idx, value, hash); - } - - myds->sess->mybe->server_myds->myconn->copy_pgsql_variables_to_startup_parameters(true); - - // if there are untracked parameters, the session should lock on the host group - if (myds->sess->untracked_option_parameters.empty() == false) { - opts += separator; - opts += myds->sess->untracked_option_parameters; - } - - conninfo << "options='" << pg_conninfo_escape_quotes(opts.c_str()) << "'"; - + { + std::string startup_encoding, startup_options; + if (build_and_record_startup_session_params(startup_encoding, startup_options, + StartupParamEscape::Conninfo)) { + conninfo << "client_encoding='" << startup_encoding << "' "; + // Join the "-c key=value" tokens with a leading separator so the options value + // has no trailing space before the closing quote. PgBouncer rejects a startup + // packet whose options value ends in whitespace (#5801). + conninfo << "options='" << startup_options << "'"; + } } /*conninfo << "postgres://"; @@ -1529,13 +1594,13 @@ bool PgSQL_Connection::native_ssl_pump_wbio_to_fd(bool& would_block) { // First, pull any freshly produced ciphertext out of wbio into native_ssl_outbuf. char buf[MY_SSL_BUFFER]; for (;;) { - int n = BIO_read(myds->wbio_ssl, buf, sizeof(buf)); + int n = BIO_read(native_wbio, buf, sizeof(buf)); if (n > 0) { native_ssl_outbuf.append(buf, (size_t)n); continue; } // No more bytes pending; BIO_should_retry distinguishes empty from error. - if (!BIO_should_retry(myds->wbio_ssl)) { + if (!BIO_should_retry(native_wbio)) { // For a mem BIO an "empty" read also returns !should_retry; that is normal. } break; @@ -1562,7 +1627,7 @@ bool PgSQL_Connection::native_ssl_pump_wbio_to_fd(bool& would_block) { bool PgSQL_Connection::native_flush_outbuf() { // Encrypted path: native_outbuf holds *plaintext* protocol bytes. Feed them to // SSL_write, which produces ciphertext into wbio_ssl, then drain wbio to the fd. - if (myds && myds->encrypted && myds->ssl) { + if (native_ssl != nullptr) { // If there is leftover ciphertext from a previous partial socket write, flush // it first before producing more (preserves ordering). if (!native_ssl_outbuf.empty()) { @@ -1572,7 +1637,7 @@ bool PgSQL_Connection::native_flush_outbuf() { } while (!native_outbuf.empty()) { ERR_clear_error(); - int w = SSL_write(myds->ssl, native_outbuf.data(), (int)native_outbuf.size()); + int w = SSL_write(native_ssl, native_outbuf.data(), (int)native_outbuf.size()); if (w > 0) { native_outbuf.erase(0, (size_t)w); bool wb = false; @@ -1580,7 +1645,7 @@ bool PgSQL_Connection::native_flush_outbuf() { if (wb) return true; // socket full; remaining plaintext stays buffered continue; } - int err = SSL_get_error(myds->ssl, w); + int err = SSL_get_error(native_ssl, w); if (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ) { // SSL needs to do I/O before it can accept more plaintext. Drain // whatever ciphertext it produced and wait for the socket. @@ -1617,6 +1682,26 @@ bool PgSQL_Connection::native_flush_outbuf() { return true; } +// A fatal error during the RESULT phase kills the CONNECTION, not just the query, +// so the socket must be torn down and not merely flagged. +// +// Two kinds of exit reach here and both are unrecoverable for the connection: +// * CONNECTION_FAILURE -- the peer closed, or a send()/recv() failed outright. +// * PROTOCOL_VIOLATION -- the byte stream is desynchronised. We no longer know +// where the next message begins, so nothing can ever be read from it safely +// again, even though the socket is still technically open. +// +// Closing the socket here is what marks the connection as unusable, so it gets +// thrown away instead of going back into the pool. +// +// The auth and startup phases already do this -- their "backend closed during +// auth" / "during startup" exits call native_teardown() -- the result phase simply +// never did, on any of its exits. +void PgSQL_Connection::native_result_fatal(const char* code, const char* message) { + set_error(code, message, false); + native_teardown(); +} + void PgSQL_Connection::native_teardown() { if (native_scram) { pg_scram_free(native_scram); @@ -1626,12 +1711,32 @@ void PgSQL_Connection::native_teardown() { ::close(fd); fd = -1; } + // Drop this connection from the count of connected backends. The check makes + // sure we only subtract if we added in the first place, so a teardown followed + // by the destructor still subtracts exactly once. + if (counted_in_connections_connected) { + __sync_fetch_and_sub(&PgHGM->status.server_connections_connected, 1); + counted_in_connections_connected = false; + } + native_connected = false; native_framer.reset(); native_outbuf.clear(); native_ssl_outbuf.clear(); - // The SSL object (if any) lives on myds and is freed by ~PgSQL_Data_Stream(); - // it uses mem BIOs so SSL_free()'s shutdown writes harmlessly into a mem buffer - // even though the fd is now closed. We only own the per-connection SSL_CTX here. + // The TLS session belongs to this connection (see PgSQL_Connection.h), so we + // free it here. SSL_set_bio() transferred both BIOs to the SSL, so SSL_free() + // releases all three; freeing the BIOs separately would be a double free. It + // uses mem BIOs, so SSL_free()'s shutdown writes harmlessly into a mem buffer + // even though the fd is already closed. + // + // This runs only on REAL teardown. A pool return must never reach here -- that + // was precisely finding A7, where the TLS context was destroyed while the + // socket stayed open and pooled. + if (native_ssl) { + SSL_free(native_ssl); + native_ssl = nullptr; + native_rbio = nullptr; + native_wbio = nullptr; + } if (native_ssl_ctx) { SSL_CTX_free(native_ssl_ctx); native_ssl_ctx = nullptr; @@ -1642,12 +1747,12 @@ void PgSQL_Connection::native_teardown() { // type at the header's accessor declarations. Native TLS reports SSL-in-use once the // handshake handed the SSL* to myds; the libpq path defers to PQsslInUse(). int PgSQL_Connection::get_pg_ssl_in_use() { - if (native_mode) return (myds && myds->encrypted && myds->ssl) ? 1 : 0; + if (native_mode) return (native_ssl != nullptr) ? 1 : 0; return PQsslInUse(pgsql_conn); } SSL* PgSQL_Connection::get_pg_ssl_object() { - if (native_mode) return (myds && myds->encrypted) ? myds->ssl : nullptr; + if (native_mode) return native_ssl; return (SSL*)PQsslStruct(pgsql_conn, "OpenSSL"); } @@ -1738,6 +1843,11 @@ void PgSQL_Connection::native_connect_start() { this->fd = sock; native_host = parent->address ? parent->address : ""; + // Mirror the libpq path's rule for `hostaddr` (connect_start(): passed only when + // the DNS cache resolved something DIFFERENT from parent->address) so that both + // paths report the same value for the same server configuration. + native_hostaddr = (!ip.empty() && parent->address && ip != std::string(parent->address)) ? ip : ""; + native_port = portstr; native_st = PG_Native_Conn_St::TCP_CONNECTING; native_framer.reset(); native_outbuf.clear(); @@ -1829,8 +1939,8 @@ void PgSQL_Connection::native_connect_cont(short event) { native_teardown(); return; } - myds->ssl = SSL_new(native_ssl_ctx); - if (myds->ssl == nullptr) { + native_ssl = SSL_new(native_ssl_ctx); + if (native_ssl == nullptr) { set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_CONNECTION_FAILURE), "SSL_new() failed", false); native_teardown(); return; @@ -1840,11 +1950,11 @@ void PgSQL_Connection::native_connect_cont(short event) { SSL_CTX_free(native_ssl_ctx); native_ssl_ctx = nullptr; - SSL_set_connect_state(myds->ssl); // client role + SSL_set_connect_state(native_ssl); // client role // verify-full: enforce hostname verification at the TLS layer. if (native_ssl_mode == PG_Native_SSL_Mode::VERIFY_FULL) { const char* host = (parent->address && parent->address[0]) ? parent->address : native_host.c_str(); - X509_VERIFY_PARAM* vp = SSL_get0_param(myds->ssl); + X509_VERIFY_PARAM* vp = SSL_get0_param(native_ssl); X509_VERIFY_PARAM_set_hostflags(vp, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); if (X509_VERIFY_PARAM_set1_host(vp, host, 0) != 1) { set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_CONNECTION_FAILURE), "failed to set TLS verify host", false); @@ -1854,17 +1964,23 @@ void PgSQL_Connection::native_connect_cont(short event) { } // SNI: present the backend hostname (best-effort; ignored for IP literals). if (parent->address && parent->address[0]) { - SSL_set_tlsext_host_name(myds->ssl, parent->address); + SSL_set_tlsext_host_name(native_ssl, parent->address); } - myds->encrypted = true; - myds->rbio_ssl = BIO_new(BIO_s_mem()); - myds->wbio_ssl = BIO_new(BIO_s_mem()); - if (myds->rbio_ssl == nullptr || myds->wbio_ssl == nullptr) { + native_rbio = BIO_new(BIO_s_mem()); + native_wbio = BIO_new(BIO_s_mem()); + if (native_rbio == nullptr || native_wbio == nullptr) { set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_OUT_OF_MEMORY), "BIO_new() failed", false); + // Free them HERE, not via native_teardown(). Ownership passes to the + // SSL only at SSL_set_bio() below, which has not run yet -- so + // teardown's SSL_free(native_ssl) would not release them and the one + // that DID allocate would leak. Teardown nulls the pointers, so it + // cannot clean up after us either. + if (native_rbio) { BIO_free(native_rbio); native_rbio = nullptr; } + if (native_wbio) { BIO_free(native_wbio); native_wbio = nullptr; } native_teardown(); return; } - SSL_set_bio(myds->ssl, myds->rbio_ssl, myds->wbio_ssl); + SSL_set_bio(native_ssl, native_rbio, native_wbio); native_st = PG_Native_Conn_St::SSL_HANDSHAKE; // Kick the handshake immediately (it will emit ClientHello into wbio). native_connect_cont(event); @@ -1949,16 +2065,33 @@ void PgSQL_Connection::native_connect_cont(short event) { } bool PgSQL_Connection::native_send_startup() { - unsigned char startup[2048]; size_t slen2 = 0; const char* user = userinfo->username ? userinfo->username : ""; const char* db = (userinfo->dbname && userinfo->dbname[0]) ? userinfo->dbname : user; - if (!pg_build_startup(startup, &slen2, sizeof(startup), user, db)) { + + // Carry the session settings the libpq path sends in its conninfo. Without these a + // client's connection options are silently dropped, and every new backend connection + // pays a SET round-trip because requires_RESETTING_CONNECTION() sees a mismatch. + std::string startup_encoding, startup_options; + const bool have_params = build_and_record_startup_session_params(startup_encoding, startup_options, + StartupParamEscape::Wire); + + // The untracked half of the options string is client-controlled, so size the buffer + // from the content rather than assuming a fixed ceiling. + std::vector startup(512 + strlen(user) + strlen(db) + + startup_encoding.size() + startup_options.size()); + if (!pg_build_startup(startup.data(), &slen2, startup.size(), user, db, + have_params ? startup_encoding.c_str() : nullptr, + have_params ? startup_options.c_str() : nullptr, + "proxysql")) { set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_CONNECTION_FAILURE), "startup message too large", false); return false; } - native_outbuf.assign((const char*)startup, slen2); + // Keep the options value for reporting (PROXYSQL INTERNAL SESSION / stats), matching + // what PQoptions() returns on the libpq path. + native_options = have_params ? startup_options : std::string(); + native_outbuf.assign((const char*)startup.data(), slen2); // After the StartupMessage flushes, wait for the AuthenticationRequest. On the // TLS path native_send_or_buffer routes the plaintext through SSL_write. if (!native_send_or_buffer(PG_Native_Conn_St::AUTH)) { @@ -2111,14 +2244,14 @@ int PgSQL_Connection::native_drive_ssl_handshake() { for (;;) { ERR_clear_error(); - int ret = SSL_do_handshake(myds->ssl); + int ret = SSL_do_handshake(native_ssl); if (ret == 1) { // Handshake complete. For VERIFY_CA / VERIFY_FULL, confirm the result. // (For VERIFY_FULL the hostname check is folded into SSL_get_verify_result // because we set the verify host on the SSL object before the handshake.) if (native_ssl_mode == PG_Native_SSL_Mode::VERIFY_CA || native_ssl_mode == PG_Native_SSL_Mode::VERIFY_FULL) { - X509* peer = SSL_get_peer_certificate(myds->ssl); + X509* peer = SSL_get_peer_certificate(native_ssl); if (peer == nullptr) { set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_CONNECTION_FAILURE), "TLS verification required but server presented no certificate", false); @@ -2126,7 +2259,7 @@ int PgSQL_Connection::native_drive_ssl_handshake() { return -1; } X509_free(peer); - long vr = SSL_get_verify_result(myds->ssl); + long vr = SSL_get_verify_result(native_ssl); if (vr != X509_V_OK) { char msg[256]; snprintf(msg, sizeof(msg), "TLS certificate verification failed: %s", @@ -2148,7 +2281,7 @@ int PgSQL_Connection::native_drive_ssl_handshake() { return 1; } - int err = SSL_get_error(myds->ssl, ret); + int err = SSL_get_error(native_ssl, ret); if (err == SSL_ERROR_WANT_WRITE) { bool wb = false; if (!native_ssl_pump_wbio_to_fd(wb)) { @@ -2185,9 +2318,9 @@ int PgSQL_Connection::native_drive_ssl_handshake() { unsigned char* src = cipher; int len = (int)n; while (len > 0) { - int w = BIO_write(myds->rbio_ssl, src, len); + int w = BIO_write(native_rbio, src, len); if (w <= 0) { - if (!BIO_should_retry(myds->rbio_ssl)) { + if (!BIO_should_retry(native_rbio)) { set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_CONNECTION_FAILURE), "BIO_write during TLS handshake failed", false); native_teardown(); return -1; @@ -2240,7 +2373,7 @@ int PgSQL_Connection::native_recv_into_framer() { // Encrypted path: read ciphertext from fd into rbio, then SSL_read plaintext // protocol bytes out and feed them to the framer. Mirrors the BIO-mem decrypt // loop of PgSQL_Data_Stream::read_from_net(), but drives the raw fd directly. - if (myds && myds->encrypted && myds->ssl) { + if (native_ssl != nullptr) { bool got = false; unsigned char cipher[MY_SSL_BUFFER]; // Pull whatever ciphertext is available from the socket into rbio. A single @@ -2265,9 +2398,9 @@ int PgSQL_Connection::native_recv_into_framer() { unsigned char* src = cipher; int len = (int)n; while (len > 0) { - int w = BIO_write(myds->rbio_ssl, src, len); + int w = BIO_write(native_rbio, src, len); if (w <= 0) { - if (!BIO_should_retry(myds->rbio_ssl)) return -1; + if (!BIO_should_retry(native_rbio)) return -1; continue; } src += w; @@ -2278,13 +2411,13 @@ int PgSQL_Connection::native_recv_into_framer() { for (;;) { unsigned char plain[MY_SSL_BUFFER]; ERR_clear_error(); - int r = SSL_read(myds->ssl, plain, sizeof(plain)); + int r = SSL_read(native_ssl, plain, sizeof(plain)); if (r > 0) { native_framer.feed(plain, (size_t)r); got = true; continue; } - int err = SSL_get_error(myds->ssl, r); + int err = SSL_get_error(native_ssl, r); if (err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_WRITE) { break; // need more ciphertext from the socket; wait for next event } @@ -2405,6 +2538,16 @@ void PgSQL_Connection::native_drive_auth(short /*event*/) { case 3: { // AuthenticationCleartextPassword const char* pw = userinfo->password ? userinfo->password : ""; + // Only a plaintext secret can answer this challenge: the backend wants the password + // itself, and a stored md5 hash or SCRAM verifier is a one-way derivation we cannot + // invert. libpq fails the same combination on its shared "no password supplied" + // guard in pg_fe_sendauth(), so refusing here keeps the two paths identical. + if (get_password_type(pw) != PASSWORD_TYPE_PLAINTEXT) { + set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_INVALID_PASSWORD), + "backend requested a cleartext password but the stored credential is not a plaintext password", false); + native_teardown(); + return; + } size_t pwlen = strlen(pw); native_outbuf.clear(); pg_append_typed_msg(native_outbuf, 'p', (const unsigned char*)pw, pwlen + 1); // include NUL @@ -2426,7 +2569,34 @@ void PgSQL_Connection::native_drive_auth(short /*event*/) { char md5buf[36]; const char* user = userinfo->username ? userinfo->username : ""; const char* pw = userinfo->password ? userinfo->password : ""; - pg_build_md5(md5buf, user, pw, salt); // "md5"+32hex+NUL (35 chars + NUL) + // An md5-stored secret IS hex(md5(password+user)) -- the inner hash this response is + // built from. Running pg_build_md5() over it hashes it a SECOND time and the backend + // rejects the login -- the md5 divergence from libpq, which reuses the stored hash + // via the patched md5_secret conninfo parameter. + switch (get_password_type(pw)) { + case PASSWORD_TYPE_MD5: + // get_password_type() applies the same test (length 35, "md5", 32 lowercase hex), + // so this branch is unreachable from here; it is the postcondition that keeps a + // half-built response off the wire if the two ever diverge. Covered directly by + // pgsql_backend_auth-t rather than end to end. + if (!pg_build_md5_from_secret(md5buf, pw, salt)) { + set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_INVALID_PASSWORD), + "stored md5 credential is malformed; expected \"md5\" followed by 32 lowercase hex digits", false); + native_teardown(); + return; + } + break; + case PASSWORD_TYPE_PLAINTEXT: + pg_build_md5(md5buf, user, pw, salt); // "md5"+32hex+NUL (35 chars + NUL) + break; + default: + // A SCRAM verifier cannot answer an md5 challenge at all: the two derivations + // share nothing. libpq reaches its no-password guard here and fails likewise. + set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_INVALID_PASSWORD), + "backend requested md5 authentication but the stored credential is a SCRAM verifier", false); + native_teardown(); + return; + } native_outbuf.clear(); pg_append_typed_msg(native_outbuf, 'p', (const unsigned char*)md5buf, strlen(md5buf) + 1); if (!native_send_or_buffer(PG_Native_Conn_St::AUTH)) { @@ -2454,7 +2624,7 @@ void PgSQL_Connection::native_drive_auth(short /*event*/) { // both, TLS -> PLUS (set cbind below) <-- the upgrade // both, !TLS -> plain // neither -> capability gap - const bool tls_in_use = (myds && myds->encrypted && myds->ssl); + const bool tls_in_use = (native_ssl != nullptr); bool use_scram_plus = false; if (has_scram_plus && tls_in_use) { use_scram_plus = true; @@ -2475,12 +2645,51 @@ void PgSQL_Connection::native_drive_auth(short /*event*/) { return; } + // Verifier pass-through. A verifier-stored user has no plaintext to + // derive from, so the exchange runs off the ClientKey harvested during that user's + // FRONTEND SCRAM login plus the verifier's ServerKey -- PgSQL_Protocol.cpp records + // both on the userinfo. Installed before client-first so client-final has them. + { + const char* stored = userinfo->password ? userinfo->password : ""; + if (userinfo->has_scram_keys) { + if (!pg_scram_set_keys(native_scram, userinfo->scram_client_key, + userinfo->scram_server_key)) { + set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_INVALID_PASSWORD), + "could not install the harvested SCRAM keys for the backend handshake", false); + native_teardown(); + return; + } + } else switch (get_password_type(stored)) { + case PASSWORD_TYPE_PLAINTEXT: + break; // libscram derives the keys ad-hoc from the plaintext + case PASSWORD_TYPE_SCRAM_SHA_256: + // A verifier with no harvested ClientKey: a proof derived from the verifier + // TEXT is always rejected. libpq refuses to build the conninfo at all here + // (pgsql_append_conninfo_credentials); fail for the same reason. + set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_INVALID_PASSWORD), + "SCRAM verifier stored but no harvested ClientKey; cannot authenticate to the backend without a frontend SCRAM login", false); + native_teardown(); + return; + default: + // An md5 secret shares no derivation with SCRAM, so there is nothing to reuse. + // A role's FRONTEND auth-method floor and its backend pg_hba method are chosen + // independently, so an md5-stored user meeting a scram-sha-256 backend is + // reachable -- and without this the md5 hash TEXT would go through PBKDF2 and + // fail as an opaque "password authentication failed". libpq stops on its + // no-password guard here (only md5_secret was set, never password). + set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_INVALID_PASSWORD), + "backend requested SCRAM authentication but the stored credential is an md5 hash", false); + native_teardown(); + return; + } + } + // If using -PLUS, set the cbind input BEFORE building client-first // so the gs2 header in client-first is "p=tls-server-end-point,,". if (use_scram_plus) { unsigned char digest[EVP_MAX_MD_SIZE]; size_t digest_len = 0; - if (pg_tls_server_end_point(myds->ssl, digest, &digest_len) < 0) { + if (pg_tls_server_end_point(native_ssl, digest, &digest_len) < 0) { // Digest failed: degrade to plain if also offered, else // capability gap. Log once via the capability-gap path. if (has_scram) { @@ -2538,7 +2747,10 @@ void PgSQL_Connection::native_drive_auth(short /*event*/) { // Copy server-first BEFORE building (client_final reads it; no further feed here, // but copying keeps us robust against the dangling-pointer rule). std::string server_first((const char*)rest, rest_len); - const char* pw = userinfo->password ? userinfo->password : ""; + // With keys injected there is no password to send. + const char* pw = userinfo->has_scram_keys + ? nullptr + : (userinfo->password ? userinfo->password : ""); const char* client_final = pg_scram_client_final(native_scram, pw, server_first.data(), server_first.size()); if (client_final == nullptr) { set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_CONNECTION_FAILURE), "SCRAM client-final failed", false); @@ -2587,6 +2799,23 @@ void PgSQL_Connection::native_drive_auth(short /*event*/) { } } +// Record a ParameterStatus ('S'): two NUL-separated strings, name then value. The +// backend sends one whenever a reported setting changes, and DISCARD ALL changes +// every one of them back to its default. Dropping these would leave native_params +// describing settings the connection no longer has. +void PgSQL_Connection::native_track_parameter_status(const unsigned char* payload, uint32_t len) { + if (payload == nullptr || len == 0) return; + uint32_t i = 0; + const char* name = (const char*)payload; + while (i < len && payload[i] != 0) i++; + if (i >= len) return; // malformed; ignore + std::string nm(name, (const char*)(payload + i)); + i++; // skip the NUL between the two strings + const char* val = (const char*)(payload + i); + while (i < len && payload[i] != 0) i++; + native_params[nm] = std::string(val, (const char*)(payload + i)); +} + void PgSQL_Connection::native_drive_startup_tail(short /*event*/) { // Consume ParameterStatus(S)/BackendKeyData(K)/NoticeResponse(N) until // ReadyForQuery(Z). This may be called immediately after AuthenticationOk @@ -2611,23 +2840,9 @@ void PgSQL_Connection::native_drive_startup_tail(short /*event*/) { } // FRAME_OK. Copy any payload we retain before a subsequent recv()/feed(). switch (msg.type) { - case 'S': { // ParameterStatus: two C-strings name, value - const unsigned char* p = msg.payload; - uint32_t len = msg.payload_len; - uint32_t i = 0; - const char* name = (const char*)p; - while (i < len && p[i] != 0) i++; - if (i >= len) break; // malformed; ignore - std::string nm(name, (const char*)(p + i)); - i++; // skip NUL - const char* val = (const char*)(p + i); - uint32_t vstart = i; - while (i < len && p[i] != 0) i++; - std::string vl(val, (const char*)(p + i)); - (void)vstart; - native_params[nm] = vl; + case 'S': // ParameterStatus + native_track_parameter_status(msg.payload, msg.payload_len); break; - } case 'K': { // BackendKeyData: int32 pid, int32 secret if (msg.payload_len >= 8) { native_backend_pid = (int)pg_read_be32(msg.payload); @@ -2643,7 +2858,7 @@ void PgSQL_Connection::native_drive_startup_tail(short /*event*/) { native_teardown(); return; case 'Z': { // ReadyForQuery: 1 status byte - if (msg.payload_len >= 1) native_txn_status = (char)msg.payload[0]; + if (msg.payload_len >= 1) set_ready_for_query_status((char)msg.payload[0]); native_connected = true; native_st = PG_Native_Conn_St::DONE; async_exit_status = PG_EVENT_NONE; // connect/auth phase COMPLETE @@ -2902,7 +3117,11 @@ void PgSQL_Connection::native_publish_describe_cache() { eqi->stmt_info->publish_describe_cache(cand); } -void PgSQL_Connection::native_fetch_result_cont(short /*event*/) { +void PgSQL_Connection::native_fetch_result_cont(short /*event*/, uint64_t* processed_bytes) { + // Every byte handed to query_result counts towards this event's total, which + // the caller compares against pgsql-threshold_resultset_size to decide when + // to pause the fetch. + auto count_bytes = [&](unsigned int n) { if (processed_bytes) *processed_bytes += n; }; // Native result fetch (Task 1.6c / Phase 2). Pull backend bytes into the // framer, then drain every complete message into query_result as raw // client-wire bytes. Non-blocking throughout. @@ -2911,7 +3130,7 @@ void PgSQL_Connection::native_fetch_result_cont(short /*event*/) { // query_result must have been allocated in ASYNC_USE_RESULT_START via // init_query_result(). Guard defensively so we never deref a null result. if (query_result == nullptr) { - set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_INTERNAL_ERROR), "native result fetch with no query_result", false); + native_result_fatal(PGSQL_GET_ERROR_CODE_STR(ERRCODE_INTERNAL_ERROR), "native result fetch with no query_result"); return; } @@ -2923,7 +3142,7 @@ void PgSQL_Connection::native_fetch_result_cont(short /*event*/) { // ReadyForQuery that complete the cycle. Mirrors query_cont()'s native branch. if (!native_outbuf.empty() || !native_ssl_outbuf.empty()) { if (!native_flush_outbuf()) { - set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_CONNECTION_FAILURE), "send failed during result fetch", false); + native_result_fatal(PGSQL_GET_ERROR_CODE_STR(ERRCODE_CONNECTION_FAILURE), "send failed during result fetch"); return; } if (!native_outbuf.empty() || !native_ssl_outbuf.empty()) { @@ -2933,15 +3152,21 @@ void PgSQL_Connection::native_fetch_result_cont(short /*event*/) { } } - int r = native_recv_into_framer(); - if (r < 0) { - set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_CONNECTION_FAILURE), "backend closed during result fetch", false); - return; - } - if (r == 0) { - // EAGAIN: no bytes available yet → wait for the socket to become readable. - async_exit_status = PG_EVENT_READ; - return; + if (native_fetch_paused) { + // Resuming a fetch that stopped on the byte threshold: the messages are + // already framed, and the backend may have nothing left to send. + native_fetch_paused = false; + } else { + int r = native_recv_into_framer(); + if (r < 0) { + native_result_fatal(PGSQL_GET_ERROR_CODE_STR(ERRCODE_CONNECTION_FAILURE), "backend closed during result fetch"); + return; + } + if (r == 0) { + // EAGAIN: no bytes available yet → wait for the socket to become readable. + async_exit_status = PG_EVENT_READ; + return; + } } // Drain all complete messages. msg.payload points INTO the framer buffer and @@ -2949,6 +3174,12 @@ void PgSQL_Connection::native_fetch_result_cont(short /*event*/) { // buffer) before looping, and we never feed() again inside this loop, so the // dangling-pointer rule is respected. for (;;) { + // Enough bytes for this event: stop before taking another message and let + // the client drain, the same rule the libpq fetch loop applies per row. + if (processed_bytes && suspend_resultset_fetch(*processed_bytes)) { + native_fetch_paused = true; + return; + } PgSQL_Backend_Msg msg; PgSQL_Frame_Result fr = native_framer.next(msg); if (fr == FRAME_OK) { @@ -2969,7 +3200,7 @@ void PgSQL_Connection::native_fetch_result_cont(short /*event*/) { // during the connect handshake; it is dead here (post-connect, // mid-fetch) — only the flush result and async_exit_status count. if (!native_send_or_buffer(PG_Native_Conn_St::DONE)) { - set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_CONNECTION_FAILURE), "send(CopyFail) failed", false); + native_result_fatal(PGSQL_GET_ERROR_CODE_STR(ERRCODE_CONNECTION_FAILURE), "send(CopyFail) failed"); return; } if (async_exit_status == PG_EVENT_WRITE || !native_outbuf.empty() || !native_ssl_outbuf.empty()) { @@ -2999,7 +3230,7 @@ void PgSQL_Connection::native_fetch_result_cont(short /*event*/) { // waits for its 'Z' below. if (t == '2') { if (native_stmt_step == PG_Native_Stmt_Step::BIND) { - query_result->add_native_backend_message(t, msg.payload, msg.payload_len); + count_bytes(query_result->add_native_backend_message(t, msg.payload, msg.payload_len)); if (!native_stmt_sync_terminated) { native_result_complete = true; return; @@ -3017,7 +3248,7 @@ void PgSQL_Connection::native_fetch_result_cont(short /*event*/) { // unexpected in native extq (unnamed Close is synthesized) - forward it // defensively rather than drop it. if (t == '3') { - query_result->add_native_backend_message(t, msg.payload, msg.payload_len); + count_bytes(query_result->add_native_backend_message(t, msg.payload, msg.payload_len)); if (native_stmt_step == PG_Native_Stmt_Step::CLOSE_P && !native_stmt_sync_terminated) { native_result_complete = true; @@ -3032,7 +3263,7 @@ void PgSQL_Connection::native_fetch_result_cont(short /*event*/) { // for its 'Z'. if (t == '1') { if (!native_suppress_parse_complete) { - query_result->add_native_backend_message(t, msg.payload, msg.payload_len); + count_bytes(query_result->add_native_backend_message(t, msg.payload, msg.payload_len)); } if (native_stmt_step == PG_Native_Stmt_Step::PARSE && !native_stmt_sync_terminated) { native_result_complete = true; @@ -3044,7 +3275,7 @@ void PgSQL_Connection::native_fetch_result_cont(short /*event*/) { // ErrorResponse: forward it (its side effect fills error_info, so the // session sees rc -1), then get the backend back to ReadyForQuery. if (t == 'E') { - query_result->add_native_backend_message(t, msg.payload, msg.payload_len); + count_bytes(query_result->add_native_backend_message(t, msg.payload, msg.payload_len)); if (native_stmt_sync_terminated) { // A Sync already reached the backend, so it WILL emit 'Z' after // the error; keep draining until we consume it. @@ -3073,7 +3304,7 @@ void PgSQL_Connection::native_fetch_result_cont(short /*event*/) { } pg_build_sync(native_outbuf); if (!native_send_or_buffer(PG_Native_Conn_St::DONE)) { - set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_CONNECTION_FAILURE), "send(Sync) failed", false); + native_result_fatal(PGSQL_GET_ERROR_CODE_STR(ERRCODE_CONNECTION_FAILURE), "send(Sync) failed"); return; } if (async_exit_status == PG_EVENT_WRITE || !native_outbuf.empty() || !native_ssl_outbuf.empty()) { @@ -3091,7 +3322,7 @@ void PgSQL_Connection::native_fetch_result_cont(short /*event*/) { // ReadyForQuery: completes any Sync-terminated step (and the injected- // Sync error recovery above). if (t == 'Z') { - query_result->add_native_backend_message(t, msg.payload, msg.payload_len); + count_bytes(query_result->add_native_backend_message(t, msg.payload, msg.payload_len)); // A Sync-terminated statement-level Describe streamed its 't'+'T'|'n' // through the generic case below and completes here — publish the // captured metadata now (no-op if nothing valid was captured). @@ -3105,7 +3336,7 @@ void PgSQL_Connection::native_fetch_result_cont(short /*event*/) { // Everything else (ParameterDescription 't', RowDescription 'T', NoData // 'n', DataRow 'D', CommandComplete 'C', EmptyQueryResponse 'I', // ParameterStatus 'S', NoticeResponse 'N', etc.) streams through. - query_result->add_native_backend_message(t, msg.payload, msg.payload_len); + count_bytes(query_result->add_native_backend_message(t, msg.payload, msg.payload_len)); // Statement-level Describe metadata capture (set-once cache): copy the // backend's raw 't' body and 'T'/'n' state as they stream past, for @@ -3163,7 +3394,7 @@ void PgSQL_Connection::native_fetch_result_cont(short /*event*/) { continue; } - query_result->add_native_backend_message(msg.type, msg.payload, msg.payload_len); + count_bytes(query_result->add_native_backend_message(msg.type, msg.payload, msg.payload_len)); if (msg.type == 'Z') { // ReadyForQuery: the result stream for this query is complete. native_result_complete = true; @@ -3177,7 +3408,7 @@ void PgSQL_Connection::native_fetch_result_cont(short /*event*/) { return; } // FRAME_ERROR: malformed backend message length. - set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_PROTOCOL_VIOLATION), "malformed backend message during result fetch", false); + native_result_fatal(PGSQL_GET_ERROR_CODE_STR(ERRCODE_PROTOCOL_VIOLATION), "malformed backend message during result fetch"); return; } } @@ -3239,15 +3470,21 @@ int PgSQL_Connection::async_connect(short event) { return 1; } -bool PgSQL_Connection::is_connected() const { +bool PgSQL_Connection::backend_is_live() const { if (native_mode) { - // Native handshake completed (ReadyForQuery received) => usable in the pool. - return native_connected; - } - if (pgsql_conn == nullptr || PQstatus(pgsql_conn) != CONNECTION_OK) { - return false; - } - return true; + // Usable means the socket is still open and login finished. Do not use + // native_st here: it moves back to a sending state whenever a large query + // cannot be written in one go, which would make a healthy connection look + // dead. + return fd >= 0 && native_connected; + } + // The same check libpq makes internally, so this never rejects a connection + // libpq would have accepted. + return pgsql_conn != nullptr && PQstatus(pgsql_conn) == CONNECTION_OK; +} + +bool PgSQL_Connection::is_connected() const { + return backend_is_live(); } void PgSQL_Connection::compute_unknown_transaction_status() { @@ -3440,15 +3677,11 @@ int PgSQL_Connection::async_query(short event, const char* stmt, unsigned long l int PgSQL_Connection::async_reset_session(short event) { PROXY_TRACE(); PROXY_TRACE2(); - // In native_mode pgsql_conn is permanently NULL (the native state machine - // owns the socket and is reset on a different code path). The libpq-only - // invariant asserted below does not hold for native connections; bail out - // early with a successful reset rather than crashing the process. - if (native_mode) { - async_state_machine = ASYNC_RESET_SESSION_SUCCESSFUL; - return 0; - } - assert(pgsql_conn); + // A native connection has no pgsql_conn and never will; only the libpq branches + // below dereference it. Everything else in this function -- the timeout, the error + // mapping, returning the connection to ASYNC_IDLE once the backend has acknowledged + // the reset -- serves both kinds of connection. + assert(native_mode || pgsql_conn); server_status = parent->status; // we copy it here to avoid race condition. The caller will see this if (IsServerOffline()) @@ -3578,10 +3811,12 @@ int PgSQL_Connection::async_ping(short event) { } bool PgSQL_Connection::IsKnownActiveTransaction() { + // Callers use this to decide whether a failed statement can safely be run + // again on a different connection. A connection that died in the middle of a + // transaction must still say it has one, otherwise the statement would be + // re-run on its own, outside that transaction. Do not add a liveness check + // here -- the answer has to survive the connection dying. if (native_mode) { - // Native state machine tracks txn status in `native_txn_status` ('I'/'T'/'E'), - // the same byte the backend emits in ReadyForQuery. pgsql_conn is null for - // native connections, so the libpq path below does not apply. return native_txn_status == 'T' || native_txn_status == 'E'; } if (!pgsql_conn) return false; @@ -3636,23 +3871,15 @@ void PgSQL_Connection::set_is_client() { } bool PgSQL_Connection::is_connection_in_reusable_state() const { - // In native mode pgsql_conn is NULL, so PQtransactionStatus() would return - // PQTRANS_UNKNOWN and wrongly classify a normal query error (backend sent - // ErrorResponse then ReadyForQuery — connection still idle and reusable) as a - // broken connection. Derive the transaction status from the last ReadyForQuery - // byte tracked natively. - PGTransactionStatusType txn_status; - if (native_mode) { - switch (native_txn_status) { - case 'I': txn_status = PQTRANS_IDLE; break; - case 'T': txn_status = PQTRANS_INTRANS; break; - case 'E': txn_status = PQTRANS_INERROR; break; - default: txn_status = PQTRANS_UNKNOWN; break; - } - } else { - txn_status = PQtransactionStatus(pgsql_conn); + // Native only, and it has to answer before the check below: a connection that + // never finished connecting is unusable but has no error recorded against it. + // libpq falls through on purpose, so a dead libpq connection with no error + // still trips that check the way it always did. + if (native_mode && !backend_is_live()) { + return false; } - bool conn_usable = !(txn_status == PQTRANS_UNKNOWN || txn_status == PQTRANS_ACTIVE); + const PGTransactionStatusType txn_status = get_pg_transaction_status(); + const bool conn_usable = !(txn_status == PQTRANS_UNKNOWN || txn_status == PQTRANS_ACTIVE); assert(!(conn_usable == false && is_error_present() == false)); return conn_usable; } @@ -4330,6 +4557,26 @@ void PgSQL_Connection::stmt_execute_cont(short event) { void PgSQL_Connection::reset_session_start() { PROXY_TRACE(); + if (native_mode) { + // Two commands, and the order is forced: the backend refuses DISCARD ALL while + // a transaction is open, so an open one is rolled back first and DISCARD ALL + // goes out on the next pass. + reset_session_in_pipeline = false; // nothing here ever runs in pipeline mode + reset_session_in_txn = IsKnownActiveTransaction(); + const char* cmd = (reset_session_in_txn == false ? "DISCARD ALL" : "ROLLBACK"); + set_query(cmd, strlen(cmd)); + query_start(); + if (async_exit_status == PG_EVENT_NONE && is_error_present() == false) { + // Reached only when the whole command actually went out: query_start() asks + // for writability instead if any of it is still buffered, and leaves an error + // set if the send failed outright. Nothing is left to send, so what we wait + // for is the reply. Say so, or the cycle finishes here without ever entering + // ASYNC_RESET_SESSION_CONT -- taking the reset timeout, which lives in that + // state, with it. + async_exit_status = PG_EVENT_READ; + } + return; + } assert(pgsql_conn); reset_error(); async_exit_status = PG_EVENT_NONE; @@ -4353,8 +4600,69 @@ void PgSQL_Connection::reset_session_start() { flush(); } +// Finish sending a reset command and read its reply, which is thrown away: a +// connection being reset has no client waiting for it. Reading stops at +// ReadyForQuery. Two things in the reply are kept -- the transaction status, which +// decides whether a second command is still owed, and an error, because a reset that +// failed must not be reported as done or a dirty connection goes back in the pool. +void PgSQL_Connection::native_reset_session_cont() { + async_exit_status = PG_EVENT_NONE; + + // A command that did not fit in one write has to be finished before its reply + // can arrive. + if (!native_outbuf.empty() || !native_ssl_outbuf.empty()) { + if (!native_flush_outbuf()) { + native_result_fatal(PGSQL_GET_ERROR_CODE_STR(ERRCODE_CONNECTION_FAILURE), "send failed during reset"); + return; + } + if (!native_outbuf.empty() || !native_ssl_outbuf.empty()) { + async_exit_status = PG_EVENT_WRITE; + return; + } + } + + for (;;) { + PgSQL_Backend_Msg msg; + PgSQL_Frame_Result fr = native_framer.next(msg); + if (fr == FRAME_NEED_MORE) { + int r = native_recv_into_framer(); + if (r == 0) { // EAGAIN + async_exit_status = PG_EVENT_READ; + return; + } + if (r < 0) { + native_result_fatal(PGSQL_GET_ERROR_CODE_STR(ERRCODE_CONNECTION_FAILURE), "backend closed during reset"); + return; + } + continue; + } + if (fr == FRAME_ERROR) { + native_result_fatal(PGSQL_GET_ERROR_CODE_STR(ERRCODE_PROTOCOL_VIOLATION), "malformed backend message during reset"); + return; + } + switch (msg.type) { + case 'E': // the backend refused the command; ReadyForQuery still follows it + native_fill_error_from_E(msg.payload, msg.payload_len); + break; + case 'S': // ParameterStatus: DISCARD ALL reverts reported settings and says so + native_track_parameter_status(msg.payload, msg.payload_len); + break; + case 'Z': // ReadyForQuery: the reply is complete + if (msg.payload_len >= 1) set_ready_for_query_status((char)msg.payload[0]); + return; + default: + // CommandComplete and NoticeResponse: nothing to keep. + break; + } + } +} + void PgSQL_Connection::reset_session_cont(short event) { PROXY_TRACE(); + if (native_mode) { + native_reset_session_cont(); + return; + } proxy_debug(PROXY_DEBUG_MYSQL_PROTOCOL, 6, "event=%d\n", event); async_exit_status = PG_EVENT_NONE; if (event & POLLOUT) { @@ -4454,6 +4762,18 @@ char PgSQL_Connection::get_transaction_status_char() { return txn_status; } +bool PgSQL_Connection::suspend_resultset_fetch(uint64_t processed_bytes) const { + bool suspend = (processed_bytes > overflow_safe_multiply<8,unsigned int>(pgsql_thread___threshold_resultset_size)); + // A cacheable query is allowed to buffer up to the whole query cache instead, + // otherwise it would be paused before it could ever be stored. + if (suspend == true && myds->sess && myds->sess->qpo && myds->sess->qpo->cache_ttl > 0) { + suspend = (processed_bytes > ((uint64_t)pgsql_thread___query_cache_size_MB) * 1024ULL * 1024ULL); + } + if (suspend == true) return true; + return (pgsql_thread___throttle_ratio_server_to_client && pgsql_thread___throttle_max_bytes_per_second_to_client + && (processed_bytes > (unsigned long long)pgsql_thread___throttle_max_bytes_per_second_to_client / 10 * (unsigned long long)pgsql_thread___throttle_ratio_server_to_client)); +} + void PgSQL_Connection::update_bytes_recv(uint64_t bytes_recv) { __sync_fetch_and_add(&parent->bytes_recv, bytes_recv); myds->sess->thread->status_variables.stvar[st_var_queries_backends_bytes_recv] += bytes_recv; diff --git a/lib/PgSQL_Data_Stream.cpp b/lib/PgSQL_Data_Stream.cpp index f7f516fc20..e89b783fa6 100644 --- a/lib/PgSQL_Data_Stream.cpp +++ b/lib/PgSQL_Data_Stream.cpp @@ -1204,6 +1204,17 @@ bool PgSQL_Data_Stream::adopt_backend_tls() { } encrypted = true; ssl = ssl_obj; + if (myconn->native_mode) { + // The native path built this SSL with two memory BIOs and keeps reading and + // writing through those same pointers. Share them rather than installing a + // pair here: SSL_set_bio() would free the ones still in use, and the next + // query on the connection would touch freed memory. Nothing is displaced, + // so there is nothing for release_backend_tls() to put back. + assert(myconn->native_rbio != NULL && myconn->native_wbio != NULL); + rbio_ssl = myconn->native_rbio; + wbio_ssl = myconn->native_wbio; + return true; + } backend_tls_adopted = true; // libpq's BIO carries the PGconn as app data and cannot be rebuilt from out // here, so hold a reference: SSL_set_bio() frees whatever it replaces. @@ -1244,7 +1255,18 @@ bool PgSQL_Data_Stream::adopt_backend_tls() { // libpq keeps writing into our buffers and the next query never reaches the // backend, in this session or in whichever one gets the connection next. void PgSQL_Data_Stream::release_backend_tls() { - if (backend_tls_adopted == false) return; // nothing was borrowed here + if (backend_tls_adopted == false) { + if (myconn != NULL && myconn->native_mode && ssl != NULL) { + // A native borrow shares the connection's own BIOs, so there is nothing to + // hand back. The stream still has to stop claiming the TLS: leaving these + // set makes the next query on this session take the encrypted path for a + // transport the connection is driving itself. The BIO pointers stay as the + // connection owns them. + encrypted = false; + ssl = NULL; + } + return; // nothing was borrowed here + } if (myconn == NULL || ssl == NULL || myconn->saved_backend_rbio == NULL) { // Cannot hand it back. Clear our side anyway: leaving 'encrypted' set would // make ~PgSQL_Data_Stream() SSL_free() the connection's own SSL, which libpq diff --git a/lib/PgSQL_HostGroups_Manager.cpp b/lib/PgSQL_HostGroups_Manager.cpp index 453fb4eebd..acf016aaf5 100644 --- a/lib/PgSQL_HostGroups_Manager.cpp +++ b/lib/PgSQL_HostGroups_Manager.cpp @@ -3110,7 +3110,9 @@ SQLite3_result * PgSQL_HostGroups_Manager::SQL3_Free_Connections() { j["port"] = conn->parent ? conn->parent->port : 0; j["user"] = (conn->userinfo && conn->userinfo->username) ? conn->userinfo->username : ""; j["database"] = (conn->userinfo && conn->userinfo->dbname) ? conn->userinfo->dbname : ""; - j["transaction_status"] = string(1, conn->native_txn_status); + j["backend_pid"] = conn->get_pg_backend_pid(); + j["using_ssl"] = conn->get_pg_ssl_in_use() ? "YES" : "NO"; + j["transaction_status"] = string(1, conn->last_ready_for_query_status()); } else { j["native_mode"] = false; j["host"] = conn->get_pg_host(); diff --git a/lib/PgSQL_PreparedStatement.cpp b/lib/PgSQL_PreparedStatement.cpp index 3d37d92b0d..5e416812fe 100644 --- a/lib/PgSQL_PreparedStatement.cpp +++ b/lib/PgSQL_PreparedStatement.cpp @@ -200,6 +200,17 @@ void PgSQL_STMT_Local::client_close_all() { stmt_name_to_global_info.clear(); } +void PgSQL_STMT_Local::backend_close_all() { + // Same server-refcount release as ~PgSQL_STMT_Local()'s backend branch: one + // ref_count_server(-1) per backend statement. Also clears global_stmt_to_backend_ids + // (the destructor skips it only because the object is about to be freed). + for (auto& [_, global_stmt_info] : backend_stmt_to_global_info) { + GloPgStmt->ref_count_server(global_stmt_info.get(), -1); + } + backend_stmt_to_global_info.clear(); + global_stmt_to_backend_ids.clear(); +} + uint32_t PgSQL_STMT_Local::generate_new_backend_stmt_id() { assert(is_client_ == false); if (free_backend_ids.empty() == false) { diff --git a/lib/PgSQL_Protocol.cpp b/lib/PgSQL_Protocol.cpp index e88fde6e2c..08ddf234c3 100644 --- a/lib/PgSQL_Protocol.cpp +++ b/lib/PgSQL_Protocol.cpp @@ -1112,7 +1112,11 @@ EXECUTION_STATE PgSQL_Protocol::process_handshake_response_packet(unsigned char* PgCredentials stored_user_info{ '\0' }; snprintf(stored_user_info.name, sizeof(stored_user_info.name), "%.*s", (int)(sizeof(stored_user_info.name) - 1), user); - if (password) snprintf(stored_user_info.passwd, sizeof(stored_user_info.passwd), "%.*s", (int)(sizeof(stored_user_info.passwd) - 1), password); + // `password` is NULL on the anti-enumeration path (unknown frontend user -> mock=true): + // the `if (password || mock)` guard above lets NULL reach here, and "%.*s" with a NULL + // argument is UB. stored_user_info was value-initialised, so passwd stays "" for the mock. + if (password) + snprintf(stored_user_info.passwd, sizeof(stored_user_info.passwd), "%.*s", (int)(sizeof(stored_user_info.passwd) - 1), password); stored_user_info.mock_auth = mock; // unknown/too-weak -> mock SCRAM (deterministic fake salt), fails like a wrong password if (!(*myds)->scram_state->server_nonce) { @@ -1414,7 +1418,6 @@ EXECUTION_STATE PgSQL_Protocol::process_handshake_response_packet(unsigned char* // parameter provided is not part of the tracked variables. Will lock on hostgroup on next query. const char* val_cstr = param_val.c_str(); proxy_warning("Unrecognized connection parameter. Please report this as a bug for future enhancements:%s:%s\n", param_key.c_str(), val_cstr); - const char* escaped_str = escape_string_backslash_spaces(val_cstr); std::string& untracked = sess->untracked_option_parameters; // Append the "[ ]-c =" token in place, avoiding the // temporary strings a "-c " + key + "=" + value concatenation creates. @@ -1423,9 +1426,11 @@ EXECUTION_STATE PgSQL_Protocol::process_handshake_response_packet(unsigned char* untracked += "-c "; untracked += param_key; untracked += '='; - untracked += escaped_str; - if (escaped_str != val_cstr) - free((char*)escaped_str); + // Escaped for the StartupMessage wire form. The libpq path raises this to + // its own level when it builds the conninfo; storing the conninfo form here + // instead would reach the native path over-escaped, and the backend would + // reject it with `invalid value for parameter "": "\"`. + pg_append_escaped_option_value(untracked, val_cstr); } } @@ -2923,6 +2928,9 @@ unsigned int PgSQL_Query_Result::add_native_backend_message(char type, const uns // Find tag length up to the NUL terminator (defensive: bound by payload_len). uint32_t taglen = 0; while (taglen < payload_len && payload[taglen] != '\0') taglen++; + // Unterminated tag: nothing would stop strtoull below reading past the + // end of the message. -1 leaves the row count unrecorded. + if (taglen == payload_len) break; if (taglen > 0) { // Scan back over the trailing run of digits. uint32_t end = taglen; @@ -2977,7 +2985,7 @@ unsigned int PgSQL_Query_Result::add_native_backend_message(char type, const uns break; case 'Z': // ReadyForQuery: final message; records txn status and finalizes buffer. if (conn && payload_len >= 1) { - conn->native_txn_status = (char)payload[0]; + conn->set_ready_for_query_status((char)payload[0]); } result_packet_type |= PGSQL_QUERY_RESULT_READY; // Mirror add_ready_status(): flush the in-line buffer into PSarrayOUT so the diff --git a/lib/PgSQL_Session.cpp b/lib/PgSQL_Session.cpp index 9ff6dcfb6f..3955762618 100644 --- a/lib/PgSQL_Session.cpp +++ b/lib/PgSQL_Session.cpp @@ -1501,7 +1501,7 @@ bool PgSQL_Session::handler_again___status_SETTING_GENERIC_VARIABLE(int* _rc, co NEXT_IMMEDIATE_NEW(st); } else { - if (rc == -1) { + if (rc == -1 || rc == -2) { // the command failed bool error_present = myconn->is_error_present(); PgHGM->p_update_pgsql_error_counter( @@ -1515,9 +1515,19 @@ bool PgSQL_Session::handler_again___status_SETTING_GENERIC_VARIABLE(int* _rc, co bool retry_conn = false; // client error, serious detected_broken_connection(__FILE__, __LINE__, __func__, "while setting ", myconn); - if ((myds->myconn->reusable == true) && myds->myconn->IsActiveTransaction() == false && myds->myconn->MultiplexDisabled() == false && - myds->myconn->is_pipeline_active() == false) { - retry_conn = true; + // rc == -2: the backend answered a simple command with a RESULTSET. + // Nothing is wrong with the connection, so retrying only repeats + // the same reply. Worse, async_send_simple_command() returns + // -2 WITHOUT clearing query_result, so a caller that neither fails + // nor retries re-enters, re-detects the same resultset and re-logs: + // one client query produced 785k log lines before this was handled. + // Terminate the session instead, exactly as + // handler_again___status_SETTING_INIT_CONNECT() already does. + if (rc != -2) { + if ((myds->myconn->reusable == true) && myds->myconn->IsActiveTransaction() == false && myds->myconn->MultiplexDisabled() == false && + myds->myconn->is_pipeline_active() == false) { + retry_conn = true; + } } myds->destroy_MySQL_Connection_From_Pool(false); myds->fd = 0; @@ -3781,7 +3791,7 @@ int PgSQL_Session::handler() { // registry (clear_portals_at_boundary), so sticky_backend_connection below // is computed exactly as before (as if the clear had already happened); // only the destructive free is moved past the logging read. - bool clear_portals_at_boundary = (!has_pending_messages && myconn->native_txn_status == 'I'); + bool clear_portals_at_boundary = (!has_pending_messages && myconn->last_ready_for_query_status() == 'I'); // Pin the backend while named portals are open (same intent as the // active-transaction sticky pin) so a later Execute/Describe/Close of a // named portal routes to the connection that holds it. Kept SEPARATE from @@ -3926,9 +3936,9 @@ int PgSQL_Session::handler() { // backend still being the reusable connection; if it was torn down the // portals are gone with it and the session either ends (destructor clears // via reset()) or reconnects fresh. - if (processing_extended_query && rc == -1 && myconn && - myconn->is_connection_in_reusable_state() && - myconn->native_txn_status == 'I') { + if (processing_extended_query && rc == -1 && myds->myconn && + myds->myconn->last_ready_for_query_status() == 'I' && + myds->myconn->is_connection_in_reusable_state()) { clear_named_portals(); } } @@ -5122,7 +5132,7 @@ bool PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___handle_ } bool PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___handle_DEALLOCATE_command(const char* dig) { - + std::string nq = string((char*)CurrentQuery.QueryPointer, CurrentQuery.QueryLength); RE2::GlobalReplace(&nq, "(?U)/\\*.*\\*/", ""); @@ -5133,19 +5143,57 @@ bool PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___handle_ proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Parsing DEALLOCATE command = %s\n", nq.c_str()); const char* dealloc_value = nq.c_str(); - if (strncasecmp(dealloc_value, "ALL", 3) == 0) { - client_myds->myconn->local_stmts->client_close_all(); + if (strcasecmp(dealloc_value, "ALL") == 0) { + // Forward DEALLOCATE ALL to the backend so SQL-level PREPARE statements are + // actually freed there -- but only when the connection is pinned to a backend + // (locked or multiplex-disabled), so the forward reaches the connection that + // holds them. A mirror replay never forwards. + PgSQL_Connection* be = (mybe && mybe->server_myds) ? mybe->server_myds->myconn : nullptr; + const bool forward = (!mirror && be && (locked_on_hostgroup >= 0 || be->MultiplexDisabled())); + // In an aborted transaction the backend rejects DEALLOCATE ALL and every + // statement survives, so forward for the real error but keep our tracking + // intact -- clearing it here would desync us (client stmts wrongly reported + // gone, backend proxysql_ps_* orphaned) from a statement that still exists. + const bool aborted = forward && be->get_pg_transaction_status() == PQTRANS_INERROR; + if (!aborted) { + // Drop client-side tracking (SQL-level PREPARE names are not in this map; + // only binary/extended-query prepares are). + client_myds->myconn->local_stmts->client_close_all(); + } + if (forward) { + // DEALLOCATE ALL also drops the backend's renamed proxysql_ps_* statements, + // so release our backend-side tracking (backend_close_all) before forwarding + // -- the same release the connection does on teardown -- keeping the server + // refcounts and maps consistent. + if (!aborted && be->local_stmts) be->local_stmts->backend_close_all(); + return false; + } } else { if (client_myds->myconn->local_stmts->client_close(dealloc_value) == false) { - client_myds->DSS = STATE_QUERY_SENT_NET; - const std::string& errmsg = "prepared statement \"" + std::string(dealloc_value) + "\" does not exist"; - client_myds->myprot.generate_error_packet(true, true, errmsg.c_str(), PGSQL_ERROR_CODES::ERRCODE_INVALID_SQL_STATEMENT_NAME, false, true); - if (mirror == false) { - RequestEnd(NULL, true); - } else { + if (mirror) { + // A mirror replay never forwards DEALLOCATE: same as the ALL + // branch above and the tracked-statement path below. client_myds->DSS = STATE_SLEEP; status = WAITING_CLIENT_DATA; + return true; + } + // Untracked name: a SQL-level PREPARE (local_stmts holds only binary + // prepares) or a typo. A SQL PREPARE disables multiplexing, so its + // backend connection is still attached to this session -- forward the + // DEALLOCATE there. But if the connection is neither locked nor + // multiplex-disabled, no SQL PREPARE happened here and the statement + // cannot exist: answer locally rather than acquiring a backend + // connection only to fail (or hitting an unrelated statement left on a + // pooled connection). + PgSQL_Connection* be = (mybe && mybe->server_myds) ? mybe->server_myds->myconn : nullptr; + if (locked_on_hostgroup >= 0 || (be && be->MultiplexDisabled())) { + return false; } + client_myds->DSS = STATE_QUERY_SENT_NET; + const std::string& errmsg = "prepared statement \"" + std::string(dealloc_value) + "\" does not exist"; + client_myds->myprot.generate_error_packet(true, true, errmsg.c_str(), + PGSQL_ERROR_CODES::ERRCODE_INVALID_SQL_STATEMENT_NAME, false, true); + RequestEnd(NULL, true); return true; } } @@ -5766,6 +5814,7 @@ void PgSQL_Session::PgSQL_Result_to_PgSQL_wire(PgSQL_Connection* _conn, PgSQL_Da } CurrentQuery.rows_sent = num_rows; bool resultset_completed = query_result->get_resultset(client_myds->PSarrayOUT); + // Not known to be reachable. If this fires it is a bug -- please report it. if (status == PROCESSING_QUERY && _conn->processing_multi_statement == false) assert(resultset_completed); // the resultset should always be completed if PgSQL_Result_to_PgSQL_wire is called if (status == PROCESSING_QUERY && transfer_started == false && @@ -7084,36 +7133,6 @@ int PgSQL_Session::handle_post_sync_describe_message(PgSQL_Describe_Message* des extended_query_info.stmt_type = stmt_type; CurrentQuery.start_time = thread->curtime; - // ---------------------------------------------------------------------- - // Statement-level Describe metadata cache (set-once) — serve on hit. - // If the global statement already carries its ParameterDescription + - // RowDescription/NoData (populated by the first statement-level Describe in - // EITHER backend mode: native raw bytes or libpq rebuild), synthesize the - // response to the client directly, byte-identical to a backend round-trip, - // and complete the cycle WITHOUT any backend dispatch — mirroring the - // cache-hit ParseComplete synthesis. Portal Describes ('P') always round-trip - // (they depend on the bound result formats), so they never consult the cache. - if (stmt_type == 'S') { - const PgSQL_Describe_Cache* dc = stmt_info->get_describe_cache(); - if (dc) { - // Evidence mechanism for the cache hit (the PgSQL status-variable enum is - // currently a stub, so a visible counter is not yet wireable — see report). - // Debug-level ONLY: this is the common path by design (every repeat - // Describe of a cached statement lands here), so an always-on line would - // be per-query log flood. Tests scrape it by raising admin-debug_output - // to include stderr (3) with debug_mysql_com verbosity >= 5. - proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, - "Session=%p client_myds=%p. PgSQL statement-level Describe served from metadata cache (stmt_id=%llu)\n", - this, client_myds, (unsigned long long)stmt_info->statement_id); - client_myds->setDSS_STATE_QUERY_SENT_NET(); - char txn_state = NumActiveTransactions() > 0 ? 'T' : 'I'; - bool send_ready_packet = is_extended_query_ready_for_query(); - client_myds->myprot.generate_describe_from_cache(true, send_ready_packet, txn_state, dc); - RequestEnd(NULL, false); - return 0; - } - } - timespec begint; timespec endt; if (thread->variables.stats_time_query_processor) { @@ -7166,6 +7185,31 @@ int PgSQL_Session::handle_post_sync_describe_message(PgSQL_Describe_Message* des } } + // A repeat statement-level Describe is answered from the cached metadata with no + // backend round-trip. This sits after the query rules on purpose: the Describe + // must still count a rule hit, pin the pipeline's hostgroup and obey hostgroup + // locks exactly as an uncached one does. Portal Describes never use the cache. + if (stmt_type == 'S') { + const PgSQL_Describe_Cache* dc = stmt_info->get_describe_cache(); + if (dc) { + // Evidence mechanism for the cache hit (the PgSQL status-variable enum is + // currently a stub, so a visible counter is not yet wireable — see report). + // Debug-level ONLY: this is the common path by design (every repeat + // Describe of a cached statement lands here), so an always-on line would + // be per-query log flood. Tests scrape it by raising admin-debug_output + // to include stderr (3) with debug_mysql_com verbosity >= 5. + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, + "Session=%p client_myds=%p. PgSQL statement-level Describe served from metadata cache (stmt_id=%llu)\n", + this, client_myds, (unsigned long long)stmt_info->statement_id); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + char txn_state = NumActiveTransactions() > 0 ? 'T' : 'I'; + bool send_ready_packet = is_extended_query_ready_for_query(); + client_myds->myprot.generate_describe_from_cache(true, send_ready_packet, txn_state, dc); + RequestEnd(NULL, false); + return 0; + } + } + if (extended_query_frame.empty() == true) { extended_query_info.flags |= PGSQL_EXTENDED_QUERY_FLAG_SYNC; } diff --git a/lib/gen_utils.cpp b/lib/gen_utils.cpp index 57f84a9cb8..4f7e6e34fe 100644 --- a/lib/gen_utils.cpp +++ b/lib/gen_utils.cpp @@ -321,46 +321,30 @@ char* escape_string_single_quotes_and_backslashes(char* input, bool free_it) { return output; } -/** - * Escapes spaces in the input string by prepending "\\". - * If no spaces are present, the original input is returned. - * If spaces are escaped, a new string is returned, and the caller - * is responsible for freeing it. - * - * @param input The input string to process. - * @return A new string with spaces escaped, or the original input string if no escaping is needed. - */ -const char* escape_string_backslash_spaces(const char* input) { - const char* c; - int input_len = 0; - int escape_count = 0; - - for (c = input; *c != '\0'; c++) { - if ((*c == ' ')) { - escape_count += 3; - } else if ((*c == '\\')) { - escape_count += 2; - } - input_len++; - } - if (escape_count == 0) - return input; - - char* output = (char*)malloc(input_len + escape_count + 1); - char* p = output; - - for (c = input; *c != '\0'; c++) { - if ((*c == ' ')) { - memcpy(p, "\\\\", 2); - p += 2; - } else if (*c == '\\') { - *(p++) = '\\'; +void pg_append_escaped_option_value(std::string& out, const char* input) { + // Scan for the first character that needs escaping. Values like "on", "GMT" or + // "postgres" have none, and are appended with a single copy: one pass over the input, + // no strlen(), no temporary and no allocation of its own. + const char* p = input; + while (*p != '\0' && *p != ' ' && *p != '\\') p++; + if (*p == '\0') { + out.append(input, static_cast(p - input)); + return; + } + // Something does need escaping. Size the destination once for the worst case, then + // copy the runs between escapes in bulk instead of a character at a time. + const size_t len = static_cast(p - input) + strlen(p); + out.reserve(out.size() + len * 2); + const char* run = input; + for (const char* c = p; *c != '\0'; c++) { + if (*c == ' ' || *c == '\\') { + out.append(run, static_cast(c - run)); + out += '\\'; + run = c; } - *(p++) = *c; } - *(p++) = '\0'; - return output; + out.append(run, static_cast(input + len - run)); } /** diff --git a/test/infra/docker-pgsql16-single/bin/docker-pgsql-post.bash b/test/infra/docker-pgsql16-single/bin/docker-pgsql-post.bash index e3990938b3..8bb0b68e94 100755 --- a/test/infra/docker-pgsql16-single/bin/docker-pgsql-post.bash +++ b/test/infra/docker-pgsql16-single/bin/docker-pgsql-post.bash @@ -4,7 +4,22 @@ set -o pipefail . constants CONTAINER="${COMPOSE_PROJECT}-pgdb1-1" -PGUSERS="root testuser monitor" +# authtrust / authpw / authreject back the native backend-protocol auth matrix +# (pgsql-native_auth_matrix-t): pg_hba.conf grants each of them exactly one +# authentication METHOD (trust / password / reject) via a username-scoped rule +# placed above the scram-sha-256 catch-alls, so the native path's +# AuthenticationOk, AuthenticationCleartextPassword and ErrorResponse-during-auth +# branches can each be reached against a REAL PostgreSQL. md5 is covered by the +# pre-existing 'md5user' role, which needs no rule here. 'authscram' needs no +# rule either -- it falls through to the scram-sha-256 catch-all -- but it does +# need to be a role of its OWN rather than reusing 'testuser', because the +# matrix rewrites each role's pgsql_users row and 'testuser' is seeded by +# conf/proxysql/config.sql for the whole suite. +# +# All four are created exactly like every other role (password == username); +# for 'authtrust' the password is irrelevant by construction, which is what lets +# the test prove trust semantics by presenting a deliberately wrong one. +PGUSERS="root testuser monitor authtrust authpw authreject authscram" printf "[$(date)] PgSQL Provisioning (Container: ${CONTAINER}) ..." diff --git a/test/infra/docker-pgsql16-single/conf/pgsql/pgsql1/pg_hba.conf b/test/infra/docker-pgsql16-single/conf/pgsql/pgsql1/pg_hba.conf index fe7d433523..516a6b3132 100644 --- a/test/infra/docker-pgsql16-single/conf/pgsql/pgsql1/pg_hba.conf +++ b/test/infra/docker-pgsql16-single/conf/pgsql/pgsql1/pg_hba.conf @@ -29,6 +29,18 @@ host all md5user all md5 host all cleartextuser 127.0.0.1/32 password host all cleartextuser ::1/128 password host all cleartextuser all password +# Per-method roles for the native backend-protocol auth matrix +# (pgsql-native_auth_matrix-t). Like the md5user rules above, these are +# USERNAME-SCOPED and must precede the scram-sha-256 catch-alls below, because +# pg_hba is first-match-wins. Only these three roles are affected: root, +# testuser, monitor and every other role still authenticate with scram-sha-256, +# so no existing legacy-g* test changes behaviour. +# authtrust -> AuthenticationOk with no credential exchange (native case 0) +# authpw -> AuthenticationCleartextPassword (native case 3) +# authreject -> ErrorResponse where an Authentication belongs (native 'E' path) +host all authtrust all trust +host all authpw all password +host all authreject all reject # IPv4 local connections: host all all 127.0.0.1/32 scram-sha-256 # IPv6 local connections: diff --git a/test/infra/docker-pgsql16-single/conf/proxysql/config.sql b/test/infra/docker-pgsql16-single/conf/proxysql/config.sql index be04fcf78a..1d775baae1 100644 --- a/test/infra/docker-pgsql16-single/conf/proxysql/config.sql +++ b/test/infra/docker-pgsql16-single/conf/proxysql/config.sql @@ -8,6 +8,20 @@ SAVE PGSQL SERVERS TO DISK; DELETE FROM pgsql_users; INSERT INTO pgsql_users (username,password,active) values ('postgres','postgres',1); INSERT INTO pgsql_users (username,password,active) values ('testuser','testuser',1); +-- Per-auth-method users, paired with the PostgreSQL roles created in +-- bin/docker-pgsql-post.bash and the username-scoped pg_hba.conf rules that bind +-- each one to a single method (trust / password / reject / scram-sha-256 / md5). +-- Seeded here so tests can simply USE them: a PostgreSQL role alone is not enough +-- to reach a backend, the client must also authenticate to ProxySQL, which needs +-- a pgsql_users row. Password == username throughout, matching every other role. +-- 'authtrust' is the exception: pg_hba grants it `trust`, so the backend never +-- checks the password and a deliberately wrong one still connects -- which is how +-- a test proves no credential was exchanged. +INSERT INTO pgsql_users (username,password,active) values ('authtrust','authtrust',1); +INSERT INTO pgsql_users (username,password,active) values ('authpw','authpw',1); +INSERT INTO pgsql_users (username,password,active) values ('authreject','authreject',1); +INSERT INTO pgsql_users (username,password,active) values ('authscram','authscram',1); +INSERT INTO pgsql_users (username,password,active) values ('md5user','md5user',1); LOAD PGSQL USERS TO RUNTIME; SAVE PGSQL USERS TO DISK; diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 09e65b8d63..b94ea96abf 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -203,12 +203,19 @@ "pgsql-monitor_ssl_connections_test-t" : [ "legacy-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4" ], "pgsql-multiplex_status_test-t" : [ "legacy-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4" ], "pgsql-native_auth_differential-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], + "pgsql-native_auth_matrix-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], "pgsql-native_cancel-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], + "pgsql-native_concurrency-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], "pgsql-native_copy-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], + "pgsql-native_copy_tls_differential-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], + "pgsql-native_framer_retention-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], + "pgsql-native_hostile_backend-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], "pgsql-native_notify-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], + "pgsql-native_pool_reset-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], "pgsql-native_portals-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], "pgsql-native_prepared-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], "pgsql-native_query_differential-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], + "pgsql-native_ssl_pool_reuse-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], "pgsql-native_streaming-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], "pgsql-native_stress-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], "pgsql-native_transactions-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], @@ -267,10 +274,12 @@ "pgsql_backend_extq-t" : [ "unit-tests-g1" ], "pgsql_backend_framing-t" : [ "unit-tests-g1" ], "pgsql_command_complete_unit-t" : [ "unit-tests-g1" ], + "pgsql_conn_liveness_unit-t" : [ "unit-tests-g1" ], "pgsql_conninfo_credentials_unit-t" : [ "unit-tests-g1" ], "pgsql_error_classifier_unit-t" : [ "unit-tests-g1" ], "pgsql_error_helper_unit-t" : [ "unit-tests-g1" ], "pgsql_monitor_unit-t" : [ "unit-tests-g1" ], + "pgsql_native_params_unit-t" : [ "unit-tests-g1" ], "pgsql_query_logging_autodump-t" : [ "legacy-g6","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], "pgsql_query_logging_memory-t" : [ "legacy-g6","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1" ], "pgsql_query_processor_unit-t" : [ "unit-tests-g1" ], diff --git a/test/tap/tests/Makefile b/test/tap/tests/Makefile index b21856d1e4..7e6ed73ca3 100644 --- a/test/tap/tests/Makefile +++ b/test/tap/tests/Makefile @@ -501,8 +501,9 @@ test_wexecvp_syscall_failures-t: test_wexecvp_syscall_failures-t.cpp $(TAP_LDIR) endif # Every test that links pg_lite_client.cpp shares one link line. pg_lite_client -# implements PostgreSQL MD5 and SCRAM-SHA-256 client authentication, so it needs -# -lscram/-lusual; $(ALLOW_MULTI_DEF) resolves the duplicate symbols between +# implements PostgreSQL MD5 and SCRAM-SHA-256 client authentication (including the +# stepwise saslBegin/saslFinish pair the mid-handshake reload test drives), so it +# needs -lscram/-lusual; $(ALLOW_MULTI_DEF) resolves the duplicate symbols between # libscram/libusual and the other vendored static libs (test binaries only, and # it is empty on Darwin where the linker has no such flag). PG_LITE_CLIENT_TESTS := \ @@ -512,12 +513,12 @@ PG_LITE_CLIENT_TESTS := \ pgsql-reg_test_5300_threshold_resultset_deadlock-t \ pgsql-reg_test_5866_result_format-t \ pgsql-auth_method_matrix-t \ + pgsql-datatype_matrix-t \ + pgsql-server_side_cursors-t \ pgsql-scram_reload_midhandshake-t \ pgsql-scram_user_removed_midhandshake-t \ pgsql-scram_rotate_midhandshake_backend-t \ - pgsql-datatype_matrix-t \ pgsql-native_portals-t \ - pgsql-server_side_cursors-t \ test_ffto_pgsql_pipeline-t \ test_ffto_pgsql_stmt_portal-t @@ -534,7 +535,8 @@ $(PG_LITE_CLIENT_TESTS): %: %.cpp pg_lite_client.cpp pg_lite_client.h $(TAP_LDIR # -lusual to create duplicate symbols, so $(ALLOW_MULTI_DEF) does not apply. PGSQL_MOCK_BACKEND_TESTS := \ pgsql-reg_test_6109_midresult_disconnect-t \ - pgsql-reg_test_6110_invalid_reply_sequence-t + pgsql-reg_test_6110_invalid_reply_sequence-t \ + pgsql-native_hostile_backend-t $(PGSQL_MOCK_BACKEND_TESTS): %: %.cpp pgsql_mock_backend.cpp pgsql_mock_backend.h $(TAP_LDIR)/libtap$(SHLIB_EXT) $(CXX) $< pgsql_mock_backend.cpp $(IDIRS) $(LDIRS) $(OPT) $(MYLIBS) $(STATIC_LIBS) -o $@ diff --git a/test/tap/tests/pg_lite_client.cpp b/test/tap/tests/pg_lite_client.cpp index 759c8ce993..c9fd379f0b 100644 --- a/test/tap/tests/pg_lite_client.cpp +++ b/test/tap/tests/pg_lite_client.cpp @@ -6,14 +6,6 @@ #include "pg_lite_client.h" #include #include -#ifdef PG_LITE_CLIENT_SCRAM -// SCRAM-SHA-256 client support for direct-to-backend connections (pg_hba -// scram-sha-256). Enabled only by test rules that pass -DPG_LITE_CLIENT_SCRAM -// and link -lscram -lusual; other tests that share pg_lite_client.cpp compile -// this file without the flag and never pull the pg_scram_* symbols from -// libproxysql.a, so their link lines need no scram libraries. -#include "PgSQL_Backend_Protocol.h" -#endif #include #include #include @@ -324,17 +316,6 @@ void PgConnection::handleAuthentication(const std::string& password) { char type; std::vector buffer; -#ifdef PG_LITE_CLIENT_SCRAM - // The SCRAM state persists across the multi-round SASL handshake (10 -> 11 -> - // 12 -> 0). RAII-freed on every exit path (throw or return) so a mid-handshake - // failure cannot leak the libscram state. - PgSQL_Scram_State* scram = nullptr; - struct ScramGuard { - PgSQL_Scram_State** s; - ~ScramGuard() { if (*s) pg_scram_free(*s); } - } scram_guard{&scram}; -#endif - while (true) { readMessage(type, buffer); @@ -445,10 +426,23 @@ static std::string extractErrorMessage(const std::vector& buffer) { // Completes a SCRAM-SHA-256 SASL exchange as the CLIENT, reusing deps/libscram. // mechListMsg is the AuthenticationSASL(10) payload after the 4-byte authType: -// a sequence of null-terminated mechanism names terminated by an extra null. -// (We do not parse it; ProxySQL offers SCRAM-SHA-256 and we answer with that.) +// a sequence of null-terminated mechanism names terminated by an extra null, +// e.g. "SCRAM-SHA-256\0[SCRAM-SHA-256-PLUS\0]\0". void PgConnection::doSASLAuth(const std::string& password, - const std::vector& /*mechListMsg*/) { + const std::vector& mechListMsg) { + // This client only does plain SCRAM-SHA-256 (no channel binding), matching the "n,," + // gs2 header libscram emits, so refuse up front if the server does not offer it rather + // than answering with a mechanism it never advertised. + bool has_scram = false; + for (size_t i = 4; i < mechListMsg.size() && mechListMsg[i] != 0; ) { + const char* mech = reinterpret_cast(mechListMsg.data() + i); + size_t mlen = strnlen(mech, mechListMsg.size() - i); + if (mlen == strlen("SCRAM-SHA-256") && memcmp(mech, "SCRAM-SHA-256", mlen) == 0) + has_scram = true; + i += mlen + 1; + } + if (!has_scram) throw PgException("Server did not offer plain SCRAM-SHA-256"); + ScramState* st = scram_state_init(); PgCredentials cred; memset(&cred, 0, sizeof(cred)); @@ -488,7 +482,8 @@ void PgConnection::doSASLAuth(const std::string& password, } std::string server_first(reinterpret_cast(buffer.data()) + 4, buffer.size() - 4); char* server_nonce = nullptr; char* salt = nullptr; int saltlen = 0; int iterations = 0; - if (!read_server_first_message(st, const_cast(server_first.c_str()), + // &s[0], not const_cast(c_str()): read_server_first_message writes NULs into its input. + if (!read_server_first_message(st, &server_first[0], &server_nonce, &salt, &saltlen, &iterations)) { free(client_first); free_scram_state(st); throw PgException(std::string("scram read server-first: ") + scram_error()); @@ -518,7 +513,8 @@ void PgConnection::doSASLAuth(const std::string& password, { std::string server_final(reinterpret_cast(buffer.data()) + 4, buffer.size() - 4); char server_sig[256] = {0}; - if (!read_server_final_message(const_cast(server_final.c_str()), server_sig) || + // &s[0], not const_cast(c_str()): read_server_final_message writes NULs into its input. + if (!read_server_final_message(&server_final[0], server_sig) || !verify_server_signature(st, &cred, server_sig)) { free(client_first); free(client_final); free_scram_state(st); throw PgException("scram server signature verification failed"); @@ -628,11 +624,12 @@ std::string PgConnection::saslBegin(const std::string& user, const std::string& throw PgException("expected AuthenticationSASLContinue(11)"); } std::string server_first(reinterpret_cast(buffer.data()) + 4, buffer.size() - 4); - // server_nonce comes back pointing INTO server_first, which dies when this function returns -- - // copy it into the member before that happens. salt is malloc'd by read_server_first_message() - // and genuinely handed over, so it keeps its raw pointer. + // read_server_first_message() mutates its input in place (read_attr_value writes NULs), and + // hands back server_nonce as a pointer INTO that buffer -- which is this local, about to die. + // Copy the nonce into the member before returning; sasl_salt_ IS malloc'd and stays owned. + // &s[0], not const_cast(c_str()): the callee writes NULs into the buffer it is handed. char* server_nonce = nullptr; - if (!read_server_first_message(sasl_st_, const_cast(server_first.c_str()), + if (!read_server_first_message(sasl_st_, &server_first[0], &server_nonce, &sasl_salt_, &sasl_saltlen_, &sasl_iterations_)) { std::string e = scram_error(); freeSaslState(); throw PgException(std::string("scram read server-first: ") + e); @@ -683,7 +680,7 @@ int PgConnection::saslFinish() { { std::string server_final(reinterpret_cast(buffer.data()) + 4, buffer.size() - 4); char server_sig[256] = {0}; - if (!read_server_final_message(const_cast(server_final.c_str()), server_sig) || + if (!read_server_final_message(&server_final[0], server_sig) || !verify_server_signature(sasl_st_, &cred, server_sig)) { free(client_final); freeSaslState(); throw PgException("scram server signature verification failed"); diff --git a/test/tap/tests/pgsql-extended_query_protocol_test-t.cpp b/test/tap/tests/pgsql-extended_query_protocol_test-t.cpp index a277226de1..debc9cf652 100644 --- a/test/tap/tests/pgsql-extended_query_protocol_test-t.cpp +++ b/test/tap/tests/pgsql-extended_query_protocol_test-t.cpp @@ -3849,6 +3849,61 @@ void test_deallocate_non_existent_stmt() { } } +// Regression for the DEALLOCATE-forwarding fix (libpq path). A statement created +// with a SQL-level PREPARE is NOT tracked in local_stmts (only the extended-query +// Parse path records names), so ProxySQL used to answer DEALLOCATE locally with a +// fabricated "prepared statement does not exist" and never forwarded it. It must +// instead be forwarded to the backend and succeed. The tracked (binary-prepare) +// side is covered by test_deallocate_having_stmt_name_via_simple_query above; the +// native-path variants live in pgsql-native_prepared-t. +void test_deallocate_sql_prepared_via_simple_query() { + diag("Test %d: Simple Query - DEALLOCATE a SQL-level PREPARE is forwarded", test_count++); + auto conn = create_connection(); if (!conn) return; + + try { + conn->execute("PREPARE sql_pstmt AS SELECT 42"); + conn->waitForReady(); + conn->execute("EXECUTE sql_pstmt"); + conn->waitForReady(); + + // DEALLOCATE of the SQL-prepared name must succeed (a CommandComplete), + // not a fabricated ErrorResponse. + conn->execute("DEALLOCATE sql_pstmt"); + char type; + std::vector buffer; + conn->readMessage(type, buffer); + ok(type == PgConnection::COMMAND_COMPLETE, + "CommandComplete after DEALLOCATE of a SQL-prepared statement (forwarded, not fabricated)"); + conn->readMessage(type, buffer); + ok(type == PgConnection::READY_FOR_QUERY, "ReadyForQuery after DEALLOCATE"); + + // It really was deallocated on the backend: a later EXECUTE now errors. + conn->execute("EXECUTE sql_pstmt"); + conn->readMessage(type, buffer); + ok(type == PgConnection::ERROR_RESPONSE, "EXECUTE after DEALLOCATE errors, statement is gone"); + conn->waitForReady(); + + // ALL-prefix guard: a statement named "all_users" must be a normal + // DEALLOCATE (forwarded + freed), not mistaken for DEALLOCATE ALL. The + // CommandComplete arrives either way, so the EXECUTE-errors check below is + // what proves it was actually deallocated. + conn->execute("PREPARE all_users AS SELECT 7"); + conn->waitForReady(); + conn->execute("DEALLOCATE all_users"); + conn->readMessage(type, buffer); + ok(type == PgConnection::COMMAND_COMPLETE, "CommandComplete after DEALLOCATE all_users"); + conn->waitForReady(); + conn->execute("EXECUTE all_users"); + conn->readMessage(type, buffer); + ok(type == PgConnection::ERROR_RESPONSE, + "EXECUTE all_users after DEALLOCATE errors, so it was really deallocated (not swallowed as DEALLOCATE ALL)"); + conn->waitForReady(); + } + catch (const PgException& e) { + ok(false, "DEALLOCATE of a SQL-level PREPARE failed with error:%s", e.what()); + } +} + void test_describe_portal_returns_no_data() { diag("Test %d: Extended Query - Describe Returns No Data", test_count++); auto conn = create_connection(); if (!conn) return; @@ -5044,6 +5099,137 @@ void test_empty_query_without_describe_portal() { } } +// =========================================================================== +// DEALLOCATE ALL matrix (libpq backend path). Mirrors the native-path matrix in +// pgsql-native_prepared-t: DEALLOCATE ALL forwards to the pinned backend and +// releases ProxySQL's backend-side bookkeeping (backend_close_all) so SQL-level +// PREPARE statements are actually freed while binary statements and the shared +// statement cache stay consistent. Runs in the default (libpq) backend mode. +// S1 SQL-only S2 binary-only S3 mixed S4 nothing S5 cross-conn S6 cycles +// S7 aborted-txn (DEALLOCATE ALL rejected -> statements survive, guard keeps tracking) +// =========================================================================== +static const int N_DALLALL_MATRIX_LIBPQ = 36; + +static bool eq_ok(PGconn* c, const char* q) { + PGresult* r = PQexec(c, q); + bool good = PQresultStatus(r) == PGRES_COMMAND_OK || PQresultStatus(r) == PGRES_TUPLES_OK; + PQclear(r); + return good; +} +static bool eq_val(PGresult* r, const char* v) { + return PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) == 1 + && std::string(PQgetvalue(r, 0, 0)) == v; +} + +void test_dealloc_all_matrix() { + diag("Test %d: DEALLOCATE ALL matrix (libpq path)", test_count++); + auto fail = [](int n, const char* why) { for (int i = 0; i < n; i++) ok(false, "dealloc-all matrix: %s", why); }; + + // S1: SQL-only (pinned -> forward, statements actually freed). + { + auto cp = createNewConnection(ConnType::BACKEND); PGconn* c = cp.get(); + if (!cp || PQstatus(c) != CONNECTION_OK) { fail(5, "S1 conn"); } + else { + PGresult* r; + r = PQexec(c, "PREPARE lq_s1 AS SELECT 1"); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "S1 PREPARE lq_s1"); PQclear(r); + r = PQexec(c, "PREPARE lq_s2 AS SELECT 2"); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "S1 PREPARE lq_s2"); PQclear(r); + (void)eq_ok(c, "DEALLOCATE ALL"); + r = PQexec(c, "EXECUTE lq_s1"); ok(PQresultStatus(r) == PGRES_FATAL_ERROR, "S1 EXECUTE lq_s1 freed -> %s", PQresStatus(PQresultStatus(r))); PQclear(r); + r = PQexec(c, "EXECUTE lq_s2"); ok(PQresultStatus(r) == PGRES_FATAL_ERROR, "S1 EXECUTE lq_s2 freed -> %s", PQresStatus(PQresultStatus(r))); PQclear(r); + r = PQexec(c, "PREPARE lq_s1 AS SELECT 1"); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "S1 re-PREPARE lq_s1 (backend cleared) -> %s", PQresStatus(PQresultStatus(r))); PQclear(r); + } + } + // S2: binary-only (not pinned -> local; cached statement reusable, no desync). + { + auto cp = createNewConnection(ConnType::BACKEND); PGconn* c = cp.get(); + if (!cp || PQstatus(c) != CONNECTION_OK) { fail(5, "S2 conn"); } + else { + PGresult* r; + r = PQprepare(c, "lq_b1", "SELECT 88", 0, nullptr); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "S2 binary prepare lq_b1"); PQclear(r); + r = PQexecPrepared(c, "lq_b1", 0, nullptr, nullptr, nullptr, 0); ok(eq_val(r, "88"), "S2 EXECUTE lq_b1 = 88"); PQclear(r); + (void)eq_ok(c, "DEALLOCATE ALL"); + r = PQexecPrepared(c, "lq_b1", 0, nullptr, nullptr, nullptr, 0); ok(PQresultStatus(r) == PGRES_FATAL_ERROR, "S2 EXECUTE lq_b1 after DEALLOCATE ALL fails -> %s", PQresStatus(PQresultStatus(r))); PQclear(r); + r = PQprepare(c, "lq_b2", "SELECT 88", 0, nullptr); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "S2 re-prepare same-hash lq_b2 (no desync)"); PQclear(r); + r = PQexecPrepared(c, "lq_b2", 0, nullptr, nullptr, nullptr, 0); ok(eq_val(r, "88"), "S2 EXECUTE lq_b2 = 88"); PQclear(r); + } + } + // S3: mixed (SQL pins; binary lands on it) -> forward + backend_close_all. + { + auto cp = createNewConnection(ConnType::BACKEND); PGconn* c = cp.get(); + if (!cp || PQstatus(c) != CONNECTION_OK) { fail(7, "S3 conn"); } + else { + PGresult* r; + r = PQexec(c, "PREPARE lq_sp AS SELECT 5"); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "S3 SQL PREPARE lq_sp"); PQclear(r); + r = PQprepare(c, "lq_bp", "SELECT 88", 0, nullptr); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "S3 binary prepare lq_bp"); PQclear(r); + r = PQexecPrepared(c, "lq_bp", 0, nullptr, nullptr, nullptr, 0); ok(eq_val(r, "88"), "S3 EXECUTE lq_bp = 88"); PQclear(r); + (void)eq_ok(c, "DEALLOCATE ALL"); + r = PQexec(c, "EXECUTE lq_sp"); ok(PQresultStatus(r) == PGRES_FATAL_ERROR, "S3 EXECUTE lq_sp freed -> %s", PQresStatus(PQresultStatus(r))); PQclear(r); + r = PQexec(c, "PREPARE lq_sp AS SELECT 5"); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "S3 re-PREPARE lq_sp (backend cleared) -> %s", PQresStatus(PQresultStatus(r))); PQclear(r); + r = PQprepare(c, "lq_bp2", "SELECT 88", 0, nullptr); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "S3 re-prepare same-hash lq_bp2 (no desync after backend_close_all)"); PQclear(r); + r = PQexecPrepared(c, "lq_bp2", 0, nullptr, nullptr, nullptr, 0); ok(eq_val(r, "88"), "S3 EXECUTE lq_bp2 = 88"); PQclear(r); + } + } + // S4: nothing prepared -> harmless, session usable. + { + auto cp = createNewConnection(ConnType::BACKEND); PGconn* c = cp.get(); + if (!cp || PQstatus(c) != CONNECTION_OK) { fail(2, "S4 conn"); } + else { + ok(eq_ok(c, "DEALLOCATE ALL"), "S4 DEALLOCATE ALL on fresh connection ok"); + PGresult* r = PQexec(c, "SELECT 1"); ok(eq_val(r, "1"), "S4 session usable after DEALLOCATE ALL"); PQclear(r); + } + } + // S5: cross-connection isolation -- connB DEALLOCATE ALL must not corrupt connA's X. + { + auto ap = createNewConnection(ConnType::BACKEND); PGconn* a = ap.get(); + auto bp = createNewConnection(ConnType::BACKEND); PGconn* b = bp.get(); + if (!ap || PQstatus(a) != CONNECTION_OK || !bp || PQstatus(b) != CONNECTION_OK) { fail(5, "S5 conn"); } + else { + PGresult* r; + r = PQprepare(a, "lq_X", "SELECT 42", 0, nullptr); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "S5 connA prepare lq_X"); PQclear(r); + r = PQexecPrepared(a, "lq_X", 0, nullptr, nullptr, nullptr, 0); ok(eq_val(r, "42"), "S5 connA EXECUTE lq_X = 42"); PQclear(r); + r = PQexec(b, "PREPARE lq_spB AS SELECT 1"); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "S5 connB SQL PREPARE lq_spB (pins)"); PQclear(r); + r = PQprepare(b, "lq_X", "SELECT 42", 0, nullptr); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "S5 connB prepare lq_X (same hash)"); PQclear(r); + (void)eq_ok(b, "DEALLOCATE ALL"); + r = PQexecPrepared(a, "lq_X", 0, nullptr, nullptr, nullptr, 0); ok(eq_val(r, "42"), "S5 connA EXECUTE lq_X still = 42 (no corruption) -> %s", PQresStatus(PQresultStatus(r))); PQclear(r); + } + } + // S6: repeated PREPARE + DEALLOCATE ALL cycles -- clean each time, refcounts stable. + { + auto cp = createNewConnection(ConnType::BACKEND); PGconn* c = cp.get(); + if (!cp || PQstatus(c) != CONNECTION_OK) { fail(4, "S6 conn"); } + else { + for (int i = 1; i <= 3; i++) { + PGresult* r = PQexec(c, "PREPARE lq_cyc AS SELECT 1"); + ok(PQresultStatus(r) == PGRES_COMMAND_OK, "S6 cycle %d PREPARE lq_cyc -> %s", i, PQresStatus(PQresultStatus(r))); + PQclear(r); + (void)eq_ok(c, "DEALLOCATE ALL"); + } + PGresult* r = PQexec(c, "EXECUTE lq_cyc"); ok(PQresultStatus(r) == PGRES_FATAL_ERROR, "S6 EXECUTE lq_cyc after last DEALLOCATE ALL fails -> %s", PQresStatus(PQresultStatus(r))); PQclear(r); + } + } + // S7: aborted txn. DEALLOCATE ALL is rejected mid-abort so every statement + // survives; the aborted-txn guard keeps our tracking intact, so both the + // SQL PREPARE and the binary prepare still work after ROLLBACK -- matching + // real PostgreSQL exactly. + { + auto cp = createNewConnection(ConnType::BACKEND); PGconn* c = cp.get(); + if (!cp || PQstatus(c) != CONNECTION_OK) { fail(8, "S7 conn"); } + else { + PGresult* r; + r = PQexec(c, "PREPARE lq_sp7 AS SELECT 5"); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "S7 SQL PREPARE lq_sp7 (pins)"); PQclear(r); + r = PQprepare(c, "lq_bp7", "SELECT 88", 0, nullptr); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "S7 binary prepare lq_bp7"); PQclear(r); + r = PQexecPrepared(c, "lq_bp7", 0, nullptr, nullptr, nullptr, 0); ok(eq_val(r, "88"), "S7 EXECUTE lq_bp7 = 88"); PQclear(r); + (void)eq_ok(c, "BEGIN"); + r = PQexec(c, "SELECT 1/0"); ok(PQresultStatus(r) == PGRES_FATAL_ERROR, "S7 SELECT 1/0 aborts txn -> %s", PQresStatus(PQresultStatus(r))); PQclear(r); + r = PQexec(c, "DEALLOCATE ALL"); ok(PQresultStatus(r) == PGRES_FATAL_ERROR, "S7 DEALLOCATE ALL rejected in aborted txn -> %s", PQresStatus(PQresultStatus(r))); PQclear(r); + (void)eq_ok(c, "ROLLBACK"); + r = PQexec(c, "EXECUTE lq_sp7"); ok(eq_val(r, "5"), "S7 SQL lq_sp7 survives (guard kept tracking) -> %s", PQresStatus(PQresultStatus(r))); PQclear(r); + r = PQexecPrepared(c, "lq_bp7", 0, nullptr, nullptr, nullptr, 0); ok(eq_val(r, "88"), "S7 binary lq_bp7 survives (guard kept tracking) -> %s", PQresStatus(PQresultStatus(r))); PQclear(r); + r = PQexec(c, "SELECT 99"); ok(eq_val(r, "99"), "S7 session usable after aborted-txn DEALLOCATE ALL"); PQclear(r); + } + } +} + int main(int argc, char** argv) { if (cl.getEnv()) return exit_status(); @@ -5053,9 +5239,9 @@ int main(int argc, char** argv) { spawn_internal_noise(cl, internal_noise_rest_prometheus_poller, {{"enable_rest_api", "true"}}); if (cl.use_noise) { - plan(1061 + 3); + plan(1066 + N_DALLALL_MATRIX_LIBPQ + 3); } else { - plan(1061); + plan(1066 + N_DALLALL_MATRIX_LIBPQ); } std::string f_path{get_env("REGULAR_INFRA_DATADIR") + "/proxysql.log"}; @@ -5155,6 +5341,8 @@ int main(int argc, char** argv) { test_deallocate_all_via_simple_query(); test_deallocate_all_via_prepared(); test_deallocate_non_existent_stmt(); + test_deallocate_sql_prepared_via_simple_query(); + test_dealloc_all_matrix(); test_deallocate_statement_with_simple_query_mix(); // Tests for sending multiple simple queries and extended queries without waiting for response diff --git a/test/tap/tests/pgsql-native_auth_differential-t.cpp b/test/tap/tests/pgsql-native_auth_differential-t.cpp index 0b8d4bc626..459ad3dbb4 100644 --- a/test/tap/tests/pgsql-native_auth_differential-t.cpp +++ b/test/tap/tests/pgsql-native_auth_differential-t.cpp @@ -3,11 +3,20 @@ * @brief Differential test: ProxySQL's native PostgreSQL backend protocol vs. the libpq path. * * ============================================================================ - * STATUS: WRITTEN BUT UNRUN (Task 1.8) - * This test was authored while Docker was unavailable, so it has been - * COMPILE-VERIFIED ONLY. It has NEVER been executed against a live backend. - * See "FIRST-RUN CHECKLIST" at the bottom of this header before trusting a - * green run. + * STATUS: RUN AND GREEN -- 8/8 on 2026-09-03 against docker-pgsql16-single, + * binary 4.0.12-351-gc909215_DEBUG, group legacy-g1. + * + * Read that number carefully: only 2 of the 8 assertions execute anything. + * The other 6 are SKIPs for scenarios this infra cannot reach (see INFRA / + * SCENARIO COVERAGE below). The live coverage is exactly one scenario, + * scram-sha-256 without TLS. + * + * pgsql-native_auth_matrix-t now covers every auth method (trust, cleartext, + * md5, SCRAM, SCRAM-PLUS) against a real backend, over plaintext AND TLS, and + * runs a libpq leg beside every cell. It therefore supersedes this file's + * differential intent almost entirely. Keep this test only for its distinct + * assertion -- the proxysql.log scrape proving no silent libpq fallback -- or + * retire it and move that scrape into the matrix. * ============================================================================ * * PURPOSE @@ -98,15 +107,18 @@ * CREATE USER so the stored verifier is md5, not scram). * 3. Register an md5 pgsql_user in ProxySQL and flip MD5_SCENARIO_ENABLED below. * - * FIRST-RUN CHECKLIST (do these the first time Docker is up): - * [ ] Confirm the scram-sha-256 differential passes (results identical). - * [ ] Confirm NO fallback warning appears in proxysql.log during the native - * run — i.e. the "used native path" assertion genuinely passes, not just - * because the log file path was wrong. Temporarily flipping the native - * query path off should make this assertion FAIL; if it never fails, the - * log-scrape is not wired correctly. - * [ ] Confirm REGULAR_INFRA_DATADIR/proxysql.log is the live server log for - * this infra (it is for the isolated runner; see env-isolated.bash). + * FIRST-RUN CHECKLIST — status after the 2026-09-03 run: + * [x] The scram-sha-256 differential passes; native and libpq results identical. + * [x] REGULAR_INFRA_DATADIR/proxysql.log resolves to the live server log. The + * test BAILs when the log cannot be opened and it did not bail, so the + * path is right for the isolated runner (see env-isolated.bash). + * [ ] STILL OUTSTANDING — the negative control was never run. Nobody has + * confirmed that the "used native path" assertion can actually FAIL. Until + * someone forces a libpq fallback (e.g. point the cell at a GSSAPI-only + * backend, or temporarily make native_capability_gap() fire) and watches + * this assertion go red, a green result only proves the scrape found no + * warning — not that it would have caught one. Treat assertion 2 as + * unproven until then. */ #include diff --git a/test/tap/tests/pgsql-native_auth_matrix-t.cpp b/test/tap/tests/pgsql-native_auth_matrix-t.cpp new file mode 100644 index 0000000000..f6562855a1 --- /dev/null +++ b/test/tap/tests/pgsql-native_auth_matrix-t.cpp @@ -0,0 +1,737 @@ +/** + * @file pgsql-native_auth_matrix-t.cpp + * @brief The native PostgreSQL backend protocol's authentication matrix, against a REAL backend. + * + * PURPOSE + * ------- + * `native_drive_auth()` (lib/PgSQL_Connection.cpp:2238) implements four ways of + * authenticating to a PostgreSQL backend without libpq: + * + * case 0 AuthenticationOk (pg_hba `trust`) + * case 3 AuthenticationCleartextPassword (pg_hba `password`, also ldap/pam/radius) + * case 5 AuthenticationMD5Password (pg_hba `md5`) + * case 10/11/12 SASL / SCRAM-SHA-256(-PLUS) (pg_hba `scram-sha-256`) + * + * Before this file, the shared pgsql16 infra could only ever produce ONE of them. + * Its pg_hba.conf offered `scram-sha-256` to every role except `md5user`, and the + * one md5 test (pgsql-md5_passthrough-t) runs the LIBPQ path. So cases 0, 3 and 5 + * had never executed against a real PostgreSQL, and neither had the backend's + * ErrorResponse-during-auth path. + * + * The infra now provisions a role per method -- `authtrust`, `authpw`, + * `authreject`, `authscram` (test/infra/docker-pgsql16-single/bin/docker-pgsql-post.bash) + * with username-scoped pg_hba rules above the scram-sha-256 catch-alls + * (conf/pgsql/pgsql1/pg_hba.conf) -- plus the pre-existing `md5user`. This test + * drives each of them through ProxySQL on the native path. + * + * THE AXIS THAT ACTUALLY MATTERS: HOW THE SECRET IS STORED + * -------------------------------------------------------- + * ProxySQL can hold a user's credential in `pgsql_users.password` three ways: the + * plaintext, a SCRAM verifier ("SCRAM-SHA-256$:$:"), + * or an md5 hash ("md5"+32hex). The LIBPQ path handles all three -- it passes the + * harvested keys down as the patched conninfo params `scram_client_key` / + * `scram_server_key` / `md5_secret` (lib/PgSQL_Connection.cpp:1101-1123, gated on + * `userinfo->has_scram_keys`). + * + * The NATIVE path used to have no equivalent: it read `userinfo->password` and + * treated it as a plaintext password unconditionally at all three call sites, so a + * verifier STRING went through PBKDF2 as if it were the password and an md5-stored + * hash was hashed a second time. Both produced a wrong credential and the backend + * rejected the login. Native now mirrors the libpq policy in + * native_drive_auth(): a stored md5 secret is reused as the inner hash + * (pg_build_md5_from_secret), and a verifier-stored user's backend SASL exchange + * runs off the ClientKey harvested during that user's FRONTEND login plus the + * verifier's ServerKey (pg_scram_set_keys), skipping SASLprep + PBKDF2 entirely. + * + * That is the point of this file. Cells `scram-verifier` and `md5-hash` are the + * pass-through regression: they assert it works, AND that the connection which + * served them reported native_mode=true, so a silent fallback to libpq cannot make + * them pass. `cleartext-vs-md5hash` is the complement -- a credential that genuinely + * cannot answer the method the backend demands must fail cleanly on BOTH paths + * rather than send a hash where a password belongs. + * + * The libpq leg is run alongside every cell and reported, so any native-vs-libpq + * divergence is visible in the output rather than merely asserted. + * + * PROVING THE NATIVE PATH WAS ACTUALLY USED + * ------------------------------------------ + * A test like this passes trivially if ProxySQL quietly served the request over + * libpq. Existing native tests detect that by grepping the proxy log for the + * capability-gap warning. That signal is NOT reliable: native_capability_gap() + * guards its warning with `static thread_local bool warned` + * (lib/PgSQL_Connection.cpp:1545), so only the FIRST gap per worker thread is + * ever logged, despite the comment above it claiming "once per backend". A suite + * of 21 cells would see at most one warning no matter how many times it fell back. + * + * So we assert positively instead: `stats_pgsql_free_connections.pgsql_info` + * carries a per-connection `"native_mode":true|false` + * (lib/PgSQL_HostGroups_Manager.cpp:3077, surfaced via + * lib/ProxySQL_Admin_Stats.cpp:1332). After each successful cell we read it back + * for that hostgroup and user and require true. + * + * Backend TLS is confirmed the same way pgsql-native_tls-t does it: capture + * pg_backend_pid() through the proxy, then read pg_stat_ssl for that pid from a + * DIRECT superuser connection. A native path that silently downgraded to + * plaintext would otherwise look identical. + * + * THE MATRIX + * ---------- + * 7 base cells (backend method x stored-secret type) x backend use_ssl {0,1}, + * plus a 7-cell overlay at use_ssl=1 with the CLIENT leg on TLS too. Frontend TLS + * is orthogonal to backend authentication -- it changes nothing about which + * Authentication message the backend sends -- so it gets an overlay rather than a + * full cross product; the overlay's job is to prove both legs can be encrypted at + * once. 21 cells total. + * + * `pgsql-authentication_method` (the frontend floor) is NOT a free axis: it is + * fixed per cell by the stored secret type, because pgsql_reconcile_auth_method() + * (lib/PgSQL_Protocol.cpp:386) makes a verifier satisfy any floor, an md5 hash + * REJECT at floor 3, and plaintext take the floor's own method. Setting it + * correctly per cell is therefore mandatory, and it means the frontend leg gets + * covered across cleartext/md5/scram challenges as a side effect. + * + * ISOLATION + * --------- + * Everything runs on a dedicated hostgroup so the default hostgroup's pool is + * never disturbed. `authscram` exists as its own role rather than reusing + * `testuser` precisely because this test rewrites each role's pgsql_users row and + * `testuser` is seeded for the whole suite by conf/proxysql/config.sql. + * + * The monitor is disabled and shun_on_failures raised for the duration: three of + * the seven base cells fail their backend connect BY DESIGN, and without this the + * hostgroup would be shunned part-way through and later cells would fail for the + * wrong reason. + * + * Per project rule: only LOAD ... TO RUNTIME, never SAVE ... TO DISK. All runtime + * state touched here is restored in memory at the end. + */ +#include +#include +#include +#include +#include +#include +#include + +#include "libpq-fe.h" +#include "command_line.h" +#include "tap.h" +#include "utils.h" + +using PGConnPtr = std::unique_ptr; +CommandLine cl; + +// A hostgroup of our own, so flushing the pool between cells cannot disturb the +// default hostgroup that the rest of the group's tests are using. +static const int AUTH_HG = 91; + +// ---------------------------------------------------------------- connections + +static PGConnPtr openConn(const char* host, int port, const char* user, + const char* pass, const char* db, const char* sslmode) { + std::stringstream ss; + ss << "host=" << host << " port=" << port << " user=" << user << " password=" << pass; + if (db && *db) ss << " dbname=" << db; + ss << " sslmode=" << sslmode << " connect_timeout=10"; + return PGConnPtr(PQconnectdb(ss.str().c_str()), &PQfinish); +} + +static PGConnPtr adminConn() { + return openConn(cl.pgsql_admin_host, cl.pgsql_admin_port, + cl.admin_username, cl.admin_password, nullptr, "disable"); +} + +static PGConnPtr directBackendConn() { + return openConn(cl.pgsql_server_host, cl.pgsql_server_port, + cl.pgsql_server_username, cl.pgsql_server_password, "postgres", "disable"); +} + +static bool execAdmin(PGconn* a, const std::string& q) { + PGresult* r = PQexec(a, q.c_str()); + const ExecStatusType st = PQresultStatus(r); + const bool good = (st == PGRES_COMMAND_OK || st == PGRES_TUPLES_OK); + if (!good) diag("admin failed: %s -- %s", q.c_str(), PQerrorMessage(a)); + PQclear(r); + return good; +} + +static std::string scalar(PGconn* c, const std::string& q) { + PGresult* r = PQexec(c, q.c_str()); + std::string v; + if (PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) > 0 && !PQgetisnull(r, 0, 0)) + v = PQgetvalue(r, 0, 0); + PQclear(r); + return v; +} + +// Collapse a libpq error blob to a single line so TAP output stays readable. +static std::string oneLine(std::string s) { + for (char& ch : s) if (ch == '\n' || ch == '\r' || ch == '\t') ch = ' '; + while (!s.empty() && s.back() == ' ') s.pop_back(); + if (s.size() > 160) s = s.substr(0, 157) + "..."; + return s; +} + +// ---------------------------------------------------------------- the matrix + +enum class Secret { PLAINTEXT, VERIFIER, MD5HASH }; + +struct Cell { + const char* id; + const char* role; // PG role == ProxySQL username == that role's database + const char* password; // the plaintext the client presents + Secret secret; // how pgsql_users stores it + const char* floor; // pgsql-authentication_method for this cell + bool expect_ok; // is the NATIVE path expected to serve a query? + const char* rationale; + // Substring the NATIVE failure must contain, for expect_ok=false cells whose green + // depends on failing for one specific reason. Without it a cell that fails because the + // infra role is missing, or the database does not exist, is indistinguishable from a + // cell that fails because the credential genuinely cannot answer the method -- the pass + // would prove nothing. nullptr = any failure is acceptable evidence. + const char* expect_err_substr = nullptr; + // Database to connect to. Defaults to the role, which is how the infra provisions + // every per-method role -- except the ones created solely to exercise a pg_hba method, + // which own no database. Those name an existing one here so a cell's outcome is decided + // by authentication and never by a missing database. + const char* db = nullptr; +}; + +static const Cell BASE[] = { + { "trust", "authtrust", "deliberately-wrong-password", Secret::PLAINTEXT, "3", true, + "pg_hba trust: the backend sends AuthenticationOk with no challenge, so even a " + "wrong password connects -- which is exactly what proves no credential was exchanged (native case 0)" }, + + { "cleartext", "authpw", "authpw", Secret::PLAINTEXT, "1", true, + "pg_hba password: AuthenticationCleartextPassword, answered from the stored plaintext (native case 3)" }, + + { "md5", "md5user", "md5user", Secret::PLAINTEXT, "2", true, + "pg_hba md5: AuthenticationMD5Password, answered by pg_build_md5() over the stored plaintext (native case 5)" }, + + { "scram", "authscram", "authscram", Secret::PLAINTEXT, "3", true, + "pg_hba scram-sha-256: full SASL exchange derived from the stored plaintext (native cases 10/11/12)" }, + + { "reject", "authreject", "authreject", Secret::PLAINTEXT, "3", false, + "pg_hba reject: the backend sends ErrorResponse where an Authentication message belongs; " + "native must surface it as a clean client error and not crash or hang" }, + + { "scram-verifier", "authscram", "authscram", Secret::VERIFIER, "3", true, + "verifier pass-through: with no plaintext to derive from, the backend SASL exchange runs " + "off the ClientKey harvested during the frontend login plus the verifier's ServerKey " + "(pg_scram_set_keys). Previously the verifier STRING went through PBKDF2 instead" }, + + { "md5-hash", "md5user", "md5user", Secret::MD5HASH, "2", true, + "md5 pass-through: the stored hash IS the inner md5(password+user), so only the outer " + "hash over (inner||salt) is computed (pg_build_md5_from_secret). Previously " + "pg_build_md5() hashed the stored hash a second time" }, + + { "cleartext-vs-md5hash", "cleartextuser", "cleartextuser", Secret::MD5HASH, "2", false, + "an md5 hash cannot answer AuthenticationCleartextPassword -- the backend wants the " + "password itself and a hash is not invertible. Must fail cleanly on BOTH paths: sending " + "the hash verbatim would authenticate as whatever that literal text hashes to, and on the " + "libpq path this exact combination once reached strlen(NULL) and killed the process", + "not a plaintext password", "postgres" }, + + { "scram-vs-md5hash", "authscram", "authscram", Secret::MD5HASH, "2", false, + "the other half of the same rule, on the third call site: an md5 hash shares no " + "derivation with SCRAM, so there is nothing to reuse. Reachable because a role's " + "FRONTEND auth-method floor and its backend pg_hba method are chosen independently -- " + "here the floor is md5 while pg_hba's catch-all demands scram-sha-256. Native must say " + "so rather than run PBKDF2 over the hash TEXT and report an opaque auth failure", + "stored credential is an md5 hash" }, +}; +static const size_t NBASE = sizeof(BASE) / sizeof(BASE[0]); + +struct Run { + const Cell* cell; + int use_ssl; // backend leg + bool frontend_tls; // client leg +}; + +// ---------------------------------------------------------------- probing + +struct Probe { + bool conn_ok = false; + bool query_ok = false; + std::string err; + std::string backend_pid; // from the FIRST query + std::string backend_pid2; // from the SECOND query (must match the first) +}; + +// DO NOT use `SELECT pg_backend_pid()` anywhere in this file. +// +// ProxySQL INTERCEPTS it (lib/PgSQL_Session.cpp:5247) on a prefix match against +// the query digest and answers it locally with `this->thread_session_id` -- its +// own session counter, not a backend pid. The query never reaches a backend and +// never causes one to be opened. +// +// An earlier revision of this test used it as the probe, and every one of the 21 +// cells reported SERVED -- including the pg_hba `reject` cell, which PostgreSQL +// demonstrably refuses. It also made the pool lookup find nothing (no backend +// connection had been opened at all) and the pg_stat_ssl lookup miss (the pid was +// fabricated). One wrong query invalidated all three assertion families at once. +// +// The two queries below are the safe forms: +// PROBE_QUERY round-trips to the backend and matches no interception prefix. +// PID_QUERY evaluates pg_backend_pid() SERVER-side inside a catalog scan, so +// the digest does not begin with "SELECT pg_backend_pid()" and the +// value returned is the real backend pid. +static const char* PROBE_QUERY = "SELECT 42"; +static const char* PID_QUERY = "SELECT pid FROM pg_stat_activity WHERE pid = pg_backend_pid()"; + +// "Served" means the whole path worked: the client authenticated to ProxySQL AND +// a query round-tripped to the backend. The distinction matters because ProxySQL +// authenticates the client first and only opens the backend connection lazily on +// the first query -- so every backend-auth failure in this file surfaces at query +// time, with PQconnectdb having already returned CONNECTION_OK. +static Probe probeThroughProxy(const Cell& c, bool frontend_tls) { + Probe p; + PGConnPtr conn = openConn(cl.pgsql_host, cl.pgsql_port, c.role, c.password, + c.db ? c.db : c.role, frontend_tls ? "require" : "disable"); + if (!conn || PQstatus(conn.get()) != CONNECTION_OK) { + p.err = oneLine(conn ? PQerrorMessage(conn.get()) : "null connection"); + return p; + } + p.conn_ok = true; + + // Run PID_QUERY TWICE on the same client session. Both round-trip to the + // backend, so the first proves the path works and the second is exactly the + // query that the pooled-TLS bug used to break -- the one served after the connection + // has been detached and re-attached. Comparing the two backend PIDs proves + // the second query really was served by the SAME pooled backend connection + // rather than a freshly opened one, which is what makes this a regression + // test by construction instead of by luck. + for (int i = 0; i < 2; i++) { + PGresult* r = PQexec(conn.get(), PID_QUERY); + const bool good = (PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) == 1); + if (good) { + if (i == 0) p.backend_pid = PQgetvalue(r, 0, 0); + else p.backend_pid2 = PQgetvalue(r, 0, 0); + } else if (p.err.empty()) { + p.err = oneLine(PQerrorMessage(conn.get())); + } + PQclear(r); + if (!good) return p; + } + p.query_ok = true; + return p; +} + +// Positive proof that the pooled backend connection for this cell ran native. +// Returns "" when no free connection is recorded (which is itself diagnostic). +// Count pooled connections for this role. Used after a cell whose backend auth +// FAILED: a connection that never authenticated must not be left in the pool. +// That was reported against the mock backend (via its leftover counter); +// this checks it against a REAL PostgreSQL, using the per-method roles +// the infra provisions, which is the more trustworthy signal. +static int poolCount(PGconn* admin, const char* role) { + std::stringstream q; + q << "SELECT count(*) FROM stats_pgsql_free_connections WHERE hostgroup=" << AUTH_HG + << " AND user='" << role << "'"; + usleep(500000); // allow any return-to-pool to land before we look + const std::string v = scalar(admin, q.str()); + return v.empty() ? -1 : atoi(v.c_str()); +} + +// Hardened for CI, where the pool is NOT guaranteed to be in any particular +// state and machines are slow and loaded: +// - POLLS for the connection to appear rather than sleeping a fixed 300ms; a +// loaded runner can easily exceed any constant we pick, and an empty result +// would otherwise be misread as "fell back to libpq". +// - Aggregates over EVERY row for this hostgroup+user instead of LIMIT 1. With +// more than one pooled connection, LIMIT 1 could return a native row while a +// libpq row sat behind it, hiding a real fallback. +// Returns "true" only if at least one row was found and EVERY row is native. +static std::string poolNativeMode(PGconn* admin, const char* role) { + std::stringstream q; + q << "SELECT pgsql_info FROM stats_pgsql_free_connections WHERE hostgroup=" << AUTH_HG + << " AND user='" << role << "'"; + for (int waited = 0; waited <= 5000; waited += 100) { + PGresult* r = PQexec(admin, q.str().c_str()); + const bool okres = (PQresultStatus(r) == PGRES_TUPLES_OK); + const int n = okres ? PQntuples(r) : 0; + if (n > 0) { + int native = 0, libpq = 0, unparsed = 0; + for (int i = 0; i < n; i++) { + const std::string info = PQgetvalue(r, i, 0); + if (info.find("\"native_mode\":true") != std::string::npos) native++; + else if (info.find("\"native_mode\":false") != std::string::npos) libpq++; + else unparsed++; + } + PQclear(r); + if (unparsed) { std::stringstream m; m << "unparsed rows=" << unparsed; return m.str(); } + if (libpq) { std::stringstream m; m << "false (native=" << native << " libpq=" << libpq << ")"; return m.str(); } + return "true"; + } + PQclear(r); + usleep(100000); + } + return ""; // nothing ever appeared +} + +// Wait for the hostgroup's free-connection list to drain. Called after the server +// row is re-created so a cell never inspects a connection left behind by the +// previous cell -- or, in CI, by whatever ran before this test. +static bool waitPoolDrained(PGconn* admin) { + std::stringstream q; + q << "SELECT count(*) FROM stats_pgsql_free_connections WHERE hostgroup=" << AUTH_HG; + for (int waited = 0; waited <= 5000; waited += 100) { + if (scalar(admin, q.str()) == "0") return true; + usleep(100000); + } + return false; +} + +int main(int, char**) { + // Build the run list first so the plan is derived, never hand-counted. + // + // Cells run in natural matrix order, expect-ok and expect-fail INTERLEAVED. + // + // This previously had to be ordered expect-ok-first: a backend auth failure + // poisoned the connection it left behind and the next connect aborted the + // whole process, so the suite lost every assertion after the + // first failing cell. That is fixed -- is_connection_in_reusable_state() now + // refuses to call a torn-down native connection reusable, so it is destroyed + // instead of re-pooled. Interleaving is the regression check: if that ever + // returns, cell 5 (reject) poisons cell 6 and the run dies here rather than + // at the very end. + std::vector runs; + for (int ssl = 0; ssl <= 1; ssl++) + for (size_t i = 0; i < NBASE; i++) runs.push_back({ &BASE[i], ssl, false }); + for (size_t i = 0; i < NBASE; i++) runs.push_back({ &BASE[i], 1, true }); + + int planned = 1; // the final proxy-still-alive assertion + for (const Run& r : runs) { + planned += 1; // outcome vs expectation + if (r.cell->expect_ok) planned += 2; // native_mode + same-connection reuse + else planned += 1; // no connection stranded in the pool + if (!r.cell->expect_ok && r.cell->expect_err_substr) + planned += 1; // failed for the RIGHT reason + if (r.cell->expect_ok && r.use_ssl) planned += 1; // backend really encrypted + } + plan(planned); + + if (cl.getEnv()) return exit_status(); + + PGConnPtr adminOwner = adminConn(); + if (!adminOwner || PQstatus(adminOwner.get()) != CONNECTION_OK) + BAIL_OUT("cannot proceed without an admin connection"); + PGconn* admin = adminOwner.get(); + + // ---- save every piece of runtime state we are about to change ---------- + auto getVar = [&](const char* n) { + return scalar(admin, std::string("SELECT variable_value FROM global_variables " + "WHERE variable_name='") + n + "'"); + }; + const std::string saved_native = getVar("pgsql-use_native_backend_protocol"); + const std::string saved_floor = getVar("pgsql-authentication_method"); + const std::string saved_monitor = getVar("pgsql-monitor_enabled"); + const std::string saved_shun = getVar("pgsql-shun_on_failures"); + const std::string saved_conn_to = getVar("pgsql-connect_timeout_server_max"); + + // Snapshot any pgsql_users rows for the roles we are about to overwrite, so a + // pre-existing row (md5user is injected by other tests) is put back verbatim. + // + // frontend/backend MUST be part of both the snapshot and the restore. The admin + // table keeps a SEPARATE row per role for each flag -- one (frontend=1,backend=0) + // and one (frontend=0,backend=1) -- under PRIMARY KEY (username, backend) and + // UNIQUE (username, frontend). Selecting without them yields two indistinguishable + // rows per role and re-inserting without them collapses both onto the column + // defaults, so the second INSERT per role hits the primary key and the restore + // silently loses the frontend/backend split. + struct SavedUser { std::string username, password, active, hg, frontend, backend; }; + std::vector saved_users; + { + std::stringstream q; + q << "SELECT username,password,active,default_hostgroup,frontend,backend" + " FROM pgsql_users WHERE username IN ("; + for (size_t i = 0; i < NBASE; i++) q << (i ? "," : "") << "'" << BASE[i].role << "'"; + q << ")"; + PGresult* r = PQexec(admin, q.str().c_str()); + if (PQresultStatus(r) == PGRES_TUPLES_OK) + for (int i = 0; i < PQntuples(r); i++) + saved_users.push_back({ PQgetvalue(r,i,0), PQgetvalue(r,i,1), + PQgetvalue(r,i,2), PQgetvalue(r,i,3), + PQgetvalue(r,i,4), PQgetvalue(r,i,5) }); + PQclear(r); + } + + auto setVar = [&](const char* n, const std::string& v) { + return execAdmin(admin, std::string("SET ") + n + "='" + v + "'") && + execAdmin(admin, "LOAD PGSQL VARIABLES TO RUNTIME"); + }; + + auto restore = [&]() { + // Derived from BASE, not hand-listed: a hand-listed set silently stops cleaning up + // as soon as a cell is added, leaving that cell's row active for the rest of the group. + { + std::stringstream d; + d << "DELETE FROM pgsql_users WHERE username IN ("; + for (size_t i = 0; i < NBASE; i++) d << (i ? "," : "") << "'" << BASE[i].role << "'"; + d << ")"; + execAdmin(admin, d.str()); + } + for (const SavedUser& u : saved_users) { + std::stringstream q; + q << "INSERT INTO pgsql_users" + " (username,password,active,default_hostgroup,frontend,backend) VALUES ('" + << u.username << "','" << u.password << "'," << u.active << "," << u.hg + << "," << u.frontend << "," << u.backend << ")"; + execAdmin(admin, q.str()); + } + execAdmin(admin, "LOAD PGSQL USERS TO RUNTIME"); + + std::stringstream d; + d << "DELETE FROM pgsql_servers WHERE hostgroup_id=" << AUTH_HG; + execAdmin(admin, d.str()); + execAdmin(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + + if (!saved_native.empty()) setVar("pgsql-use_native_backend_protocol", saved_native); + if (!saved_floor.empty()) setVar("pgsql-authentication_method", saved_floor); + if (!saved_monitor.empty()) setVar("pgsql-monitor_enabled", saved_monitor); + if (!saved_shun.empty()) setVar("pgsql-shun_on_failures", saved_shun); + if (!saved_conn_to.empty()) setVar("pgsql-connect_timeout_server_max", saved_conn_to); + }; + + // ---- preconditions ----------------------------------------------------- + // Three of the seven base cells fail their backend connect by design. Without + // these two settings the hostgroup gets shunned part-way through the matrix + // and every later cell fails for the wrong reason. + if (!setVar("pgsql-monitor_enabled", "false")) BAIL_OUT("cannot disable the monitor"); + if (!setVar("pgsql-shun_on_failures", "10000")) BAIL_OUT("cannot raise shun_on_failures"); + setVar("pgsql-connect_timeout_server_max", "5000"); // a hang becomes a failed cell, not a hung run + + // ---- the dedicated hostgroup, pointed at the real pgsql16 backend ------ + { + std::stringstream d, i; + d << "DELETE FROM pgsql_servers WHERE hostgroup_id=" << AUTH_HG; + i << "INSERT INTO pgsql_servers (hostgroup_id,hostname,port,use_ssl) VALUES (" + << AUTH_HG << ",'" << cl.pgsql_server_host << "'," << cl.pgsql_server_port << ",0)"; + if (!execAdmin(admin, d.str()) || !execAdmin(admin, i.str()) || + !execAdmin(admin, "LOAD PGSQL SERVERS TO RUNTIME")) { + restore(); + BAIL_OUT("could not create the dedicated auth hostgroup %d", AUTH_HG); + } + } + + // Re-create the hostgroup's server row from scratch, with the use_ssl this + // cell wants. This is BOTH the use_ssl switch and the pool flush, on purpose. + // + // Mandatory between cells: a connection picks native-vs-libpq AND its TLS mode + // at creation and never switches (lib/PgSQL_Connection.cpp:350, :1642), so a + // survivor from the previous cell silently serves the next one in the wrong + // mode. + // + // DELETE + re-INSERT, not UPDATE status OFFLINE_HARD -> ONLINE. The latter was + // tried first and does NOT reliably evict at this cadence: cells asserted + // pg_stat_ssl.ssl=false while a direct check on a freshly created pool showed + // native use_ssl=1 negotiating TLSv1.3 correctly. The whole "native ignores + // use_ssl" reading was an artefact of plaintext connections surviving from the + // preceding use_ssl=0 cell. Removing a server drops its free connections + // immediately (PgSQL_HostGroups_Manager purge -> drop_all_connections). + auto applyServer = [&](int use_ssl) { + std::stringstream d, i; + d << "DELETE FROM pgsql_servers WHERE hostgroup_id=" << AUTH_HG; + i << "INSERT INTO pgsql_servers (hostgroup_id,hostname,port,use_ssl) VALUES (" + << AUTH_HG << ",'" << cl.pgsql_server_host << "'," << cl.pgsql_server_port + << "," << use_ssl << ")"; + execAdmin(admin, d.str()); + execAdmin(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + usleep(200000); + execAdmin(admin, i.str()); + execAdmin(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + // Confirm the hostgroup's free list is actually empty before the cell + // probes it. A fixed sleep is not safe on a loaded CI runner, and a + // leftover connection would be read as this cell's result. + if (!waitPoolDrained(admin)) + diag("pool for hg %d did not drain; this cell's native_mode reading may be unreliable", AUTH_HG); + }; + + auto setNative = [&](bool on) { + return setVar("pgsql-use_native_backend_protocol", on ? "true" : "false"); + }; + + // Write this cell's user row: the same plaintext password, stored in whichever + // form the cell is exercising. Verifier and md5 hash are generated at runtime + // by libpq (never hardcoded), exactly as pgsql-verifier_auth-t does. + auto writeUser = [&](const Cell& c) -> bool { + std::string secret = c.password; + if (c.secret == Secret::VERIFIER) { + // A SCRAM verifier carries a RANDOM SALT, so generating one here would produce a + // verifier that does not match the backend's pg_authid.rolpassword and the backend + // leg could never succeed no matter how pass-through is implemented (that mismatch + // is what pgsql-verifier_passthrough-t asserts as its negative case). Pass-through + // requires the two to be byte-identical, so read the backend's own row. + PGConnPtr be = directBackendConn(); + if (!be || PQstatus(be.get()) != CONNECTION_OK) { + diag("cannot read rolpassword for '%s': no direct backend connection", c.role); + return false; + } + secret = scalar(be.get(), + std::string("SELECT rolpassword FROM pg_authid WHERE rolname='") + c.role + "'"); + if (secret.rfind("SCRAM-SHA-256$", 0) != 0) { + diag("backend role '%s' has no SCRAM verifier (rolpassword='%s')", + c.role, secret.c_str()); + return false; + } + } else if (c.secret == Secret::MD5HASH) { + // md5 has no salt: "md5"+hex(md5(password+role)) is deterministic, so the value + // generated here is byte-identical to the backend's stored hash. + char* enc = PQencryptPasswordConn(admin, c.password, c.role, "md5"); + if (enc == nullptr) { + diag("PQencryptPasswordConn(%s,md5) failed: %s", c.role, PQerrorMessage(admin)); + return false; + } + secret = enc; + PQfreemem(enc); + } + std::stringstream d, i; + d << "DELETE FROM pgsql_users WHERE username='" << c.role << "'"; + i << "INSERT INTO pgsql_users (username,password,active,default_hostgroup) VALUES ('" + << c.role << "','" << secret << "',1," << AUTH_HG << ")"; + return execAdmin(admin, d.str()) && execAdmin(admin, i.str()) && + execAdmin(admin, "LOAD PGSQL USERS TO RUNTIME"); + }; + + // ---- run the matrix ---------------------------------------------------- + for (const Run& run : runs) { + const Cell& c = *run.cell; + std::stringstream lbl; + lbl << c.id << " [backend_ssl=" << run.use_ssl + << " frontend_tls=" << (run.frontend_tls ? 1 : 0) << "]"; + const std::string label = lbl.str(); + + if (!setVar("pgsql-authentication_method", c.floor) || !writeUser(c)) { + diag("%s: setup failed, skipping to keep the plan aligned", label.c_str()); + } + // -------- native leg -------- + // setNative BEFORE applyServer: the flush is what makes the new mode take + // effect, so the variable must already be set when the pool is emptied. + setNative(true); + applyServer(run.use_ssl); + const Probe nat = probeThroughProxy(c, run.frontend_tls); + const bool nat_served = nat.conn_ok && nat.query_ok; + const std::string native_mode = nat_served ? poolNativeMode(admin, c.role) : std::string(); + + // Confirm the backend leg was genuinely encrypted, from OUTSIDE ProxySQL. + std::string ssl_detail = "not checked"; + bool ssl_confirmed = false; + if (nat_served && run.use_ssl) { + if (nat.backend_pid.empty()) { + ssl_detail = "backend pid not captured"; + } else { + PGConnPtr direct = directBackendConn(); + if (direct && PQstatus(direct.get()) == CONNECTION_OK) { + const std::string v = scalar(direct.get(), + "SELECT ssl::text FROM pg_stat_ssl WHERE pid=" + nat.backend_pid); + if (v.empty()) ssl_detail = "pid " + nat.backend_pid + " absent from pg_stat_ssl"; + else { ssl_confirmed = (v == "t" || v == "true"); + ssl_detail = "pg_stat_ssl.ssl=" + v + " for pid " + nat.backend_pid; } + } else { + ssl_detail = "no direct backend connection available"; + } + } + } + + // Sample the pool NOW, while only the native leg has run. Sampling after + // the libpq leg would count the connection libpq legitimately pools when + // it serves a cell native cannot (md5-hash and scram-verifier), and report + // it as a stranded native connection. + const int stranded_after_native = c.expect_ok ? 0 : poolCount(admin, c.role); + + // -------- libpq leg, for the differential diagnostic -------- + setNative(false); + applyServer(run.use_ssl); + const Probe lpq = probeThroughProxy(c, run.frontend_tls); + const bool lpq_served = lpq.conn_ok && lpq.query_ok; + + // -------- assertions -------- + ok(nat_served == c.expect_ok, + "%s: native %s, expected %s | libpq %s | %s", + label.c_str(), + nat_served ? "SERVED" : "FAILED", + c.expect_ok ? "SERVED" : "FAILED", + lpq_served ? "SERVED" : "FAILED", + nat_served ? "-" : nat.err.c_str()); + + if (nat_served != lpq_served) { + diag(" DIVERGENCE %s: native=%s libpq=%s -- %s", + label.c_str(), nat_served ? "served" : "failed", + lpq_served ? "served" : "failed", c.rationale); + if (!nat.err.empty()) diag(" native error: %s", nat.err.c_str()); + if (!lpq.err.empty()) diag(" libpq error: %s", lpq.err.c_str()); + } + + if (c.expect_ok) { + ok(native_mode == "true", + "%s: the pooled backend connection reports native_mode=true (got '%s')", + label.c_str(), native_mode.empty() ? "no free connection recorded" : native_mode.c_str()); + + // Pooled-TLS regression guard. Both queries ran on one client session; if the + // second was served by the same backend PID then the connection really + // was detached and re-attached between them -- the path on which a + // native TLS connection used to lose myds->encrypted and be read as + // plaintext. Different PIDs would mean a fresh connect served query 2 + // and the regression would go undetected. + ok(!nat.backend_pid.empty() && nat.backend_pid == nat.backend_pid2, + "%s: both queries served by the same pooled backend connection (pid '%s' then '%s')", + label.c_str(), + nat.backend_pid.empty() ? "-" : nat.backend_pid.c_str(), + nat.backend_pid2.empty() ? "-" : nat.backend_pid2.c_str()); + + if (run.use_ssl) { + ok(ssl_confirmed, + "%s: backend leg genuinely TLS-encrypted (%s)%s", + label.c_str(), ssl_detail.c_str(), + (c.secret == Secret::PLAINTEXT && strcmp(c.id, "scram") == 0) + ? " -- also proves SCRAM-SHA-256-PLUS channel binding, since a wrong " + "tls-server-end-point digest would have failed the login" + : ""); + } + } + + if (!c.expect_ok) { + // Failing is not enough: it has to fail for the reason the cell is about. A + // missing infra role or database fails too, and would turn this cell green while + // exercising nothing. + if (c.expect_err_substr) { + ok(!nat_served && nat.err.find(c.expect_err_substr) != std::string::npos, + "%s: native failed for the expected reason (looking for '%s' in: %s)", + label.c_str(), c.expect_err_substr, + nat.err.empty() ? "(no error text)" : nat.err.c_str()); + } + // The backend auth failed by design. A connection that never + // authenticated must be destroyed, not returned to the pool -- a + // stranded one is both a leak and, before that fix, the object that + // aborted the process on the next connect. + ok(stranded_after_native == 0, + "%s: failed auth left NO connection in the pool (found %d)", + label.c_str(), stranded_after_native); + } + + // Stop the moment the proxy dies. Without this the remaining cells emit a + // cascade of "no connection to the server" noise that buries the one fact + // that matters: WHICH cell killed it. + if (scalar(admin, "SELECT 1") != "1") { + BAIL_OUT("ProxySQL died during cell '%s' -- expect the assert at " + "lib/PgSQL_Connection.cpp:1075 (async_state_machine==ASYNC_IDLE) " + "in proxysql.log. That signature means a connection left behind " + "by a failed backend authentication was reused instead of " + "destroyed.", label.c_str()); + } + } + + // ---- the proxy survived the whole matrix ------------------------------- + // Three cells drive deliberate backend auth failures; one of them is a backend + // that answers an Authentication request with ErrorResponse. A crash there + // would take the process down mid-run, so prove it is still answering. + { + PGConnPtr a2 = adminConn(); + const bool alive = a2 && PQstatus(a2.get()) == CONNECTION_OK && + scalar(a2.get(), "SELECT 1") == "1"; + ok(alive, "ProxySQL still alive and answering admin queries after the full auth matrix"); + } + + restore(); + return exit_status(); +} diff --git a/test/tap/tests/pgsql-native_concurrency-t.cpp b/test/tap/tests/pgsql-native_concurrency-t.cpp new file mode 100644 index 0000000000..2d4aa16e7b --- /dev/null +++ b/test/tap/tests/pgsql-native_concurrency-t.cpp @@ -0,0 +1,431 @@ +/** + * @file pgsql-native_concurrency-t.cpp + * @brief Concurrent sessions competing for a small pool, plus abrupt client and + * backend disappearance. + * + * WHY THIS EXISTS + * --------------- + * Everything else written for the native backend protocol drives ONE session at + * a time. That leaves the pool itself untested: connections being handed between + * sessions, reset between users, returned mid-transaction, and reclaimed while + * another session is waiting for one. + * + * That gap matters because of finding F2 + * (docs/superpowers/specs/2026-08-03-pgsql-native-protocol-findings.md): ProxySQL + * aborted at PgSQL_Connection.cpp:1075 — the "not implemented yet" default arm of + * the connection state machine — reached through + * CONNECTING_SERVER -> get_connection(), i.e. while picking a POOLED connection + * for a new query. It was seen twice and could not be reproduced from any single + * sequential case, which points at concurrent sessions sharing the pool. This + * test is the deliberate attempt to reproduce it. + * + * It also covers the mirror image of finding F1. F1 is the BACKEND vanishing + * mid-result, which crashes the proxy. Nothing has ever tested the CLIENT + * vanishing while the backend is still streaming. + * + * WHAT IS ASSERTED + * ---------------- + * Robustness is necessary but not sufficient here, so correctness is checked + * too. Every worker thread stamps a value unique to itself, reads it back on the + * same connection, and fails if it sees anyone else's. A pool that hands a + * connection to the wrong session, or replays state across sessions, shows up as + * a token mismatch rather than as a vague "something broke". + * + * 1. no crash, and the admin interface still answers at the end + * 2. every worker sees its OWN data, never another worker's + * 3. queries either succeed or fail cleanly (no truncated/garbled results) + * 4. the pool returns to a sane size afterwards (no stranded connections) + * 5. the proxy still serves normal traffic when it is all over + * + * PHASES + * ------ + * P1 concurrent simple queries with per-thread token verification + * P2 concurrent transactions (BEGIN/INSERT/COMMIT and BEGIN/INSERT/ROLLBACK) + * P3 sessions abandoned mid-transaction (disconnect without COMMIT/ROLLBACK) + * P4 concurrent session-variable churn, which forces RESETTING_CONNECTION and + * is the path finding F6 concerns + * P5 clients that disappear mid-result (the F1 mirror) + * P6 the backend taken OFFLINE_HARD while queries are in flight + * + * The pool is deliberately smaller than the worker count so that sessions must + * queue for and reuse each other's connections; with a pool large enough for + * everyone, none of the interesting paths are taken. + * + * EXPECTATION + * ----------- + * Exploratory. If F2 reproduces here it will show as the proxy dying partway + * through, and the phase that was running localises it. + * + * INFRA: legacy-g1. Runtime state restored in memory only — never SAVE ... TO DISK. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libpq-fe.h" +#include "command_line.h" +#include "tap.h" +#include "utils.h" + +using PGConnPtr = std::unique_ptr; +CommandLine cl; + +static const int BACKEND_HG = 0; +static const int POOL_SIZE = 4; // deliberately smaller than WORKERS +static const int WORKERS = 16; +static const int ITERATIONS = 25; +static const char* CTAB = "native_concurrency_tab"; + +// ------------------------------------------------------------------ helpers + +static PGConnPtr openConn(const char* host, int port, const char* user, + const char* pass, const char* db) { + std::stringstream ss; + ss << "host=" << host << " port=" << port << " user=" << user << " password=" << pass; + if (db && *db) ss << " dbname=" << db; + ss << " sslmode=disable connect_timeout=10"; + return PGConnPtr(PQconnectdb(ss.str().c_str()), &PQfinish); +} +static PGConnPtr createAdminConn() { + return openConn(cl.pgsql_admin_host, cl.pgsql_admin_port, + cl.admin_username, cl.admin_password, nullptr); +} +static PGConnPtr createClientConn() { + return openConn(cl.pgsql_host, cl.pgsql_port, cl.pgsql_username, + cl.pgsql_password, cl.pgsql_username); +} +static PGConnPtr createBackendConn() { + // Must be the SAME database the workers reach through ProxySQL + // (createClientConn uses dbname=cl.pgsql_username): the fixture table is + // created on this connection, and a table in `postgres` is invisible to them. + return openConn(cl.pgsql_server_host, cl.pgsql_server_port, + cl.pgsql_root_username, cl.pgsql_root_password, cl.pgsql_username); +} +static bool execAdmin(PGconn* a, const std::string& q) { + PGresult* r = PQexec(a, q.c_str()); + ExecStatusType st = PQresultStatus(r); + bool good = (st == PGRES_COMMAND_OK || st == PGRES_TUPLES_OK); + if (!good) diag("admin failed: %s -- %s", q.c_str(), PQerrorMessage(a)); + PQclear(r); + return good; +} +static std::string adminScalar(PGconn* a, const std::string& q) { + PGresult* r = PQexec(a, q.c_str()); + std::string v; + if (PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) > 0 && !PQgetisnull(r, 0, 0)) + v = PQgetvalue(r, 0, 0); + PQclear(r); + return v; +} +static std::string scalar(PGconn* c, const std::string& q) { + PGresult* r = PQexec(c, q.c_str()); + std::string v; + if (PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) > 0 && !PQgetisnull(r, 0, 0)) + v = PQgetvalue(r, 0, 0); + PQclear(r); + return v; +} +static bool setNativeMode(PGconn* a, bool on) { + return execAdmin(a, std::string("SET pgsql-use_native_backend_protocol='") + + (on ? "true" : "false") + "'") && + execAdmin(a, "LOAD PGSQL VARIABLES TO RUNTIME"); +} +static int poolConns(PGconn* a) { + std::stringstream q; + q << "SELECT IFNULL(SUM(ConnUsed + ConnFree),0) FROM stats_pgsql_connection_pool " + << "WHERE hostgroup=" << BACKEND_HG; + const std::string v = adminScalar(a, q.str()); + return v.empty() ? 0 : atoi(v.c_str()); +} +static bool proxyAlive() { + auto a = createAdminConn(); + return a && PQstatus(a.get()) == CONNECTION_OK && adminScalar(a.get(), "SELECT 1") == "1"; +} + +// Shared failure accounting across worker threads. +struct Tally { + std::atomic ok{0}; + std::atomic conn_fail{0}; + std::atomic query_fail{0}; + std::atomic wrong_data{0}; // the serious one: someone else's value + std::mutex mtx; + std::string first_wrong; + void wrong(const std::string& what) { + wrong_data.fetch_add(1); + std::lock_guard g(mtx); + if (first_wrong.empty()) first_wrong = what; + } +}; + +// ------------------------------------------------------------------ phases + +// P1: each worker writes a token only it uses, then reads it back. +static void worker_tokens(int id, Tally& t) { + for (int i = 0; i < ITERATIONS; i++) { + auto c = createClientConn(); + if (!c || PQstatus(c.get()) != CONNECTION_OK) { t.conn_fail.fetch_add(1); continue; } + std::stringstream tok; + tok << "w" << id << "_i" << i; + const std::string q = "SELECT '" + tok.str() + "'::text"; + const std::string got = scalar(c.get(), q); + if (got.empty()) { t.query_fail.fetch_add(1); continue; } + if (got != tok.str()) { t.wrong("expected " + tok.str() + " got " + got); continue; } + t.ok.fetch_add(1); + } +} + +// P2: transactions that commit or roll back, each row tagged by worker. +static void worker_txn(int id, Tally& t) { + for (int i = 0; i < ITERATIONS; i++) { + auto c = createClientConn(); + if (!c || PQstatus(c.get()) != CONNECTION_OK) { t.conn_fail.fetch_add(1); continue; } + const bool commit = (i % 2) == 0; + const int key = id * 100000 + i; + std::stringstream ins; + ins << "INSERT INTO " << CTAB << " (id, owner) VALUES (" << key << ", " << id << ")"; + PGresult* r = PQexec(c.get(), "BEGIN"); + bool good = (PQresultStatus(r) == PGRES_COMMAND_OK); + PQclear(r); + if (!good) { t.query_fail.fetch_add(1); continue; } + r = PQexec(c.get(), ins.str().c_str()); + good = (PQresultStatus(r) == PGRES_COMMAND_OK); + PQclear(r); + if (!good) { t.query_fail.fetch_add(1); PQclear(PQexec(c.get(), "ROLLBACK")); continue; } + // Inside the transaction the row must be visible to US and tagged with + // OUR id; anything else means the session landed on foreign state. + std::stringstream sel; + sel << "SELECT owner FROM " << CTAB << " WHERE id=" << key; + const std::string owner = scalar(c.get(), sel.str()); + if (owner != std::to_string(id)) { + t.wrong("txn row owner expected " + std::to_string(id) + " got '" + owner + "'"); + } + PQclear(PQexec(c.get(), commit ? "COMMIT" : "ROLLBACK")); + t.ok.fetch_add(1); + } +} + +// P3: disconnect while a transaction is still open, without COMMIT or ROLLBACK. +static void worker_abandon_txn(int id, Tally& t) { + for (int i = 0; i < ITERATIONS; i++) { + auto c = createClientConn(); + if (!c || PQstatus(c.get()) != CONNECTION_OK) { t.conn_fail.fetch_add(1); continue; } + PQclear(PQexec(c.get(), "BEGIN")); + std::stringstream ins; + ins << "INSERT INTO " << CTAB << " (id, owner) VALUES (" + << (500000 + id * 1000 + i) << ", " << id << ")"; + { + // F2 needs a LIVE transaction abandoned, not an already-aborted one. + // Without this check a failing INSERT still counted as ok and the + // phase silently tested nothing. + PGresult* r = PQexec(c.get(), ins.str().c_str()); + const bool inserted = (PQresultStatus(r) == PGRES_COMMAND_OK); + PQclear(r); + if (!inserted) { t.query_fail.fetch_add(1); continue; } + } + t.ok.fetch_add(1); + // fall out of scope -> PQfinish mid-transaction + } +} + +// P4: session-variable churn. Different clients ask for different values, which +// is what drives requires_RESETTING_CONNECTION() and the reset path of F6. +static void worker_vars(int id, Tally& t) { + static const char* vals[] = { "hex", "escape" }; + for (int i = 0; i < ITERATIONS; i++) { + auto c = createClientConn(); + if (!c || PQstatus(c.get()) != CONNECTION_OK) { t.conn_fail.fetch_add(1); continue; } + const char* want = vals[(id + i) % 2]; + std::stringstream st; + st << "SET bytea_output = '" << want << "'"; + PGresult* r = PQexec(c.get(), st.str().c_str()); + const bool good = (PQresultStatus(r) == PGRES_COMMAND_OK); + PQclear(r); + if (!good) { t.query_fail.fetch_add(1); continue; } + const std::string got = scalar(c.get(), "SELECT current_setting('bytea_output')"); + if (got != want) { + t.wrong(std::string("bytea_output expected ") + want + " got '" + got + "'"); + continue; + } + t.ok.fetch_add(1); + } +} + +// P5: start a large result, then vanish without reading it (the F1 mirror). +static void worker_abort_midresult(int id, Tally& t) { + for (int i = 0; i < ITERATIONS; i++) { + auto c = createClientConn(); + if (!c || PQstatus(c.get()) != CONNECTION_OK) { t.conn_fail.fetch_add(1); continue; } + // Ask for a big result and deliberately do NOT drain it. + if (PQsendQuery(c.get(), + "SELECT g, repeat('x', 500) FROM generate_series(1,20000) g") == 0) { + t.query_fail.fetch_add(1); + continue; + } + PQconsumeInput(c.get()); // pull a little, then abandon + usleep(1000 * ((id % 5) + 1)); // stagger the abort point across workers + t.ok.fetch_add(1); + // PQfinish() mid-stream via scope exit + } +} + +static void runPhase(const char* label, void (*fn)(int, Tally&), Tally& t) { + std::vector th; + th.reserve(WORKERS); + for (int i = 0; i < WORKERS; i++) th.emplace_back(fn, i, std::ref(t)); + for (auto& x : th) x.join(); + diag("%s: ok=%d conn_fail=%d query_fail=%d wrong_data=%d", + label, t.ok.load(), t.conn_fail.load(), t.query_fail.load(), t.wrong_data.load()); +} + +int main(int, char**) { + // 5 phases x 2 assertions (no wrong data / proxy alive) + backend-removal + // phase + final pool + final traffic = 13 + plan(13); + + if (cl.getEnv()) return exit_status(); + + auto adminOwner = createAdminConn(); + if (!adminOwner || PQstatus(adminOwner.get()) != CONNECTION_OK) + BAIL_OUT("cannot proceed without an admin connection"); + PGconn* admin = adminOwner.get(); + + const std::string saved_native = adminScalar(admin, + "SELECT variable_value FROM global_variables WHERE variable_name='pgsql-use_native_backend_protocol'"); + std::string saved_maxconn; + { + std::stringstream q; + q << "SELECT max_connections FROM pgsql_servers WHERE hostgroup_id=" << BACKEND_HG << " LIMIT 1"; + saved_maxconn = adminScalar(admin, q.str()); + } + + auto restore = [&]() { + if (!saved_maxconn.empty()) { + std::stringstream u; + u << "UPDATE pgsql_servers SET max_connections=" << saved_maxconn + << ", status='ONLINE' WHERE hostgroup_id=" << BACKEND_HG; + execAdmin(admin, u.str()); + execAdmin(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + } + if (!saved_native.empty()) + setNativeMode(admin, saved_native == "true" || saved_native == "1"); + }; + + // fixture + { + auto be = createBackendConn(); + if (!be || PQstatus(be.get()) != CONNECTION_OK) { restore(); BAIL_OUT("no direct backend connection"); } + PQclear(PQexec(be.get(), (std::string("DROP TABLE IF EXISTS ") + CTAB).c_str())); + PQclear(PQexec(be.get(), (std::string("CREATE TABLE ") + CTAB + + " (id bigint primary key, owner int)").c_str())); + PQclear(PQexec(be.get(), (std::string("GRANT ALL ON ") + CTAB + " TO PUBLIC").c_str())); + } + + if (!setNativeMode(admin, true)) { restore(); BAIL_OUT("cannot enable the native backend protocol"); } + + // Squeeze the pool so sessions must share and reuse connections. + { + std::stringstream u; + u << "UPDATE pgsql_servers SET max_connections=" << POOL_SIZE + << " WHERE hostgroup_id=" << BACKEND_HG; + if (!execAdmin(admin, u.str()) || !execAdmin(admin, "LOAD PGSQL SERVERS TO RUNTIME")) { + restore(); BAIL_OUT("could not shrink the pool"); + } + } + usleep(300000); + diag("pool pinned to %d connections, %d workers, %d iterations each", + POOL_SIZE, WORKERS, ITERATIONS); + + struct { const char* label; void (*fn)(int, Tally&); } phases[] = { + { "P1 concurrent simple queries", worker_tokens }, + { "P2 concurrent transactions", worker_txn }, + { "P3 abandoned transactions", worker_abandon_txn }, + { "P4 session-variable churn", worker_vars }, + { "P5 clients aborting mid-result", worker_abort_midresult }, + }; + + for (const auto& ph : phases) { + Tally t; + runPhase(ph.label, ph.fn, t); + ok(t.wrong_data.load() == 0, + "%s: no session saw another session's data (%d violations%s%s)", + ph.label, t.wrong_data.load(), + t.first_wrong.empty() ? "" : "; first: ", + t.first_wrong.c_str()); + ok(proxyAlive(), "%s: ProxySQL still alive afterwards", ph.label); + if (!proxyAlive()) { + diag("ProxySQL died during %s -- this is where to look for F2", ph.label); + break; + } + } + + // P6: take the backend away while queries are in flight. + { + std::atomic stop{false}; + std::atomic completed{0}, failed{0}; + std::vector th; + for (int i = 0; i < 4; i++) { + th.emplace_back([&]() { + while (!stop.load()) { + auto c = createClientConn(); + if (!c || PQstatus(c.get()) != CONNECTION_OK) { failed.fetch_add(1); continue; } + PGresult* r = PQexec(c.get(), + "SELECT count(*) FROM generate_series(1,50000)"); + if (PQresultStatus(r) == PGRES_TUPLES_OK) completed.fetch_add(1); + else failed.fetch_add(1); + PQclear(r); + } + }); + } + usleep(400000); + std::stringstream off; + off << "UPDATE pgsql_servers SET status='OFFLINE_HARD' WHERE hostgroup_id=" << BACKEND_HG; + execAdmin(admin, off.str()); + execAdmin(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + usleep(600000); + std::stringstream on; + on << "UPDATE pgsql_servers SET status='ONLINE' WHERE hostgroup_id=" << BACKEND_HG; + execAdmin(admin, on.str()); + execAdmin(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + usleep(400000); + stop.store(true); + for (auto& x : th) x.join(); + diag("P6 backend removal: completed=%d failed=%d", completed.load(), failed.load()); + ok(proxyAlive(), "P6: ProxySQL survived the backend being taken OFFLINE_HARD mid-query"); + } + + // Settle, then check the pool is not left holding connections it cannot use. + usleep(2000000); + { + auto a2 = createAdminConn(); + const bool alive = a2 && PQstatus(a2.get()) == CONNECTION_OK; + const int left = alive ? poolConns(a2.get()) : -1; + ok(alive && left <= POOL_SIZE, + "pool holds no more than its %d-connection limit afterwards (found %d)", + POOL_SIZE, left); + } + + // And normal traffic still works. + { + auto c = createClientConn(); + const bool served = c && PQstatus(c.get()) == CONNECTION_OK && + scalar(c.get(), "SELECT 1") == "1"; + ok(served, "normal traffic still served after all concurrency phases"); + } + + // teardown + { + auto be = createBackendConn(); + if (be && PQstatus(be.get()) == CONNECTION_OK) + PQclear(PQexec(be.get(), (std::string("DROP TABLE IF EXISTS ") + CTAB).c_str())); + } + restore(); + return exit_status(); +} diff --git a/test/tap/tests/pgsql-native_copy_tls_differential-t.cpp b/test/tap/tests/pgsql-native_copy_tls_differential-t.cpp new file mode 100644 index 0000000000..4a620492b9 --- /dev/null +++ b/test/tap/tests/pgsql-native_copy_tls_differential-t.cpp @@ -0,0 +1,315 @@ +/** + * @file pgsql-native_copy_tls_differential-t.cpp + * @brief COPY through a TLS-encrypted backend connection, on both backend paths. + * + * COPY makes the session switch to fast forward, which takes the backend TLS + * session over from the connection. Existing COPY tests encrypt the CLIENT leg + * only (sslmode=require towards ProxySQL); this one turns on use_ssl for the + * SERVER leg, which is the leg that handover touches. libpq runs first as the + * reference, then the native path has to produce the same bytes and leave the + * connection usable afterwards. + */ + +#include +#include +#include +#include +#include + +#include "libpq-fe.h" +#include "command_line.h" +#include "tap.h" +#include "utils.h" + +CommandLine cl; + +using PGConnPtr = std::unique_ptr; + +static const int BACKEND_HG = 0; +static const std::string APP_NAME = "copytls_" + std::to_string(getpid()); +static const std::string TBL = "pgsql_copy_tls_" + std::to_string(getpid()); + +static PGConnPtr open_admin_conn() { + std::stringstream ss; + ss << "host=" << cl.pgsql_admin_host << " port=" << cl.pgsql_admin_port + << " user=" << cl.admin_username << " password=" << cl.admin_password + << " sslmode=disable"; + return PGConnPtr(PQconnectdb(ss.str().c_str()), &PQfinish); +} + +static PGConnPtr open_client_conn() { + std::stringstream ss; + ss << "host=" << cl.pgsql_host << " port=" << cl.pgsql_port + << " user=" << cl.pgsql_username << " password=" << cl.pgsql_password + << " dbname=" << cl.pgsql_username + << " application_name=" << APP_NAME + << " sslmode=disable"; + return PGConnPtr(PQconnectdb(ss.str().c_str()), &PQfinish); +} + +static bool execSQL(PGconn* c, const std::string& q) { + PGresult* res = PQexec(c, q.c_str()); + ExecStatusType st = PQresultStatus(res); + bool good = (st == PGRES_COMMAND_OK || st == PGRES_TUPLES_OK); + if (!good) diag("query failed: %s -- %s", q.c_str(), PQerrorMessage(c)); + PQclear(res); + return good; +} + +static std::string queryOneValue(PGconn* c, const std::string& q) { + PGresult* res = PQexec(c, q.c_str()); + std::string out; + if (PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) > 0 && !PQgetisnull(res, 0, 0)) + out = PQgetvalue(res, 0, 0); + PQclear(res); + return out; +} + +struct ServerRow { std::string hostname, port, max_connections, comment; }; + +static std::vector readServers(PGconn* admin) { + std::vector rows; + PGresult* res = PQexec(admin, + ("SELECT hostname, port, max_connections, COALESCE(comment,'') FROM pgsql_servers" + " WHERE hostgroup_id=" + std::to_string(BACKEND_HG)).c_str()); + if (PQresultStatus(res) == PGRES_TUPLES_OK) { + for (int i = 0; i < PQntuples(res); i++) + rows.push_back(ServerRow { PQgetvalue(res,i,0), PQgetvalue(res,i,1), + PQgetvalue(res,i,2), PQgetvalue(res,i,3) }); + } + PQclear(res); + return rows; +} + +// Deleting the servers drops every pooled connection to them, so the next +// query opens a fresh one under whatever use_ssl / native setting is current. +// Without this the phase would keep reusing connections made under the old one. +static bool reloadServers(PGconn* admin, const std::vector& rows, int use_ssl) { + if (rows.empty()) return false; + if (!execSQL(admin, "DELETE FROM pgsql_servers WHERE hostgroup_id=" + std::to_string(BACKEND_HG))) return false; + if (!execSQL(admin, "LOAD PGSQL SERVERS TO RUNTIME")) return false; + for (const auto& r : rows) { + std::string ins = "INSERT INTO pgsql_servers (hostgroup_id,hostname,port,max_connections,use_ssl,comment)" + " VALUES (" + std::to_string(BACKEND_HG) + ",'" + r.hostname + "'," + r.port + "," + + (r.max_connections.empty() ? std::string("1000") : r.max_connections) + "," + + std::to_string(use_ssl) + ",'" + r.comment + "')"; + if (!execSQL(admin, ins)) return false; + } + if (!execSQL(admin, "LOAD PGSQL SERVERS TO RUNTIME")) return false; + usleep(300000); + return true; +} + +static bool setNativeMode(PGconn* admin, bool on) { + return execSQL(admin, std::string("SET pgsql-use_native_backend_protocol='") + (on ? "true" : "false") + "'") + && execSQL(admin, "LOAD PGSQL VARIABLES TO RUNTIME"); +} + +// Asks the backend itself whether the connection carrying this session is +// encrypted. pg_backend_pid() is intercepted by ProxySQL and answers with a +// made-up pid, so the backend session is found by the text of this very query: +// the marker below appears in the query, so the row it matches is its own. +static bool backendLegIsEncrypted(PGconn* client) { + const std::string v = queryOneValue(client, + "SELECT s.ssl FROM pg_stat_ssl s JOIN pg_stat_activity a ON s.pid = a.pid" + " WHERE a.state = 'active' AND a.query LIKE '%copytls_self_marker%' LIMIT 1"); + diag("backend reports ssl=%s for the connection running this query", v.empty() ? "(none)" : v.c_str()); + return v == "t"; +} + +static std::string copyOut(PGconn* client, const std::string& sql, bool* ok) { + std::string out; + *ok = false; + PGresult* res = PQexec(client, sql.c_str()); + if (PQresultStatus(res) != PGRES_COPY_OUT) { + diag("COPY OUT did not start: %s -- %s", PQresStatus(PQresultStatus(res)), PQerrorMessage(client)); + PQclear(res); + return out; + } + PQclear(res); + char* buf = nullptr; + int n; + while ((n = PQgetCopyData(client, &buf, 0)) > 0) { + out.append(buf, n); + PQfreemem(buf); + buf = nullptr; + } + if (n == -2) { + diag("COPY OUT failed mid-stream: %s", PQerrorMessage(client)); + return out; + } + res = PQgetResult(client); + *ok = (res != nullptr && PQresultStatus(res) == PGRES_COMMAND_OK); + if (!*ok) diag("COPY OUT did not complete: %s", PQerrorMessage(client)); + PQclear(res); + while ((res = PQgetResult(client)) != nullptr) PQclear(res); + return out; +} + +static bool copyIn(PGconn* client, const std::string& table, const std::string& payload) { + PGresult* res = PQexec(client, ("COPY " + table + " FROM STDIN").c_str()); + if (PQresultStatus(res) != PGRES_COPY_IN) { + diag("COPY IN did not start: %s -- %s", PQresStatus(PQresultStatus(res)), PQerrorMessage(client)); + PQclear(res); + return false; + } + PQclear(res); + if (PQputCopyData(client, payload.data(), (int)payload.size()) != 1) { + diag("PQputCopyData failed: %s", PQerrorMessage(client)); + return false; + } + if (PQputCopyEnd(client, nullptr) != 1) { + diag("PQputCopyEnd failed: %s", PQerrorMessage(client)); + return false; + } + res = PQgetResult(client); + bool ok = (res != nullptr && PQresultStatus(res) == PGRES_COMMAND_OK); + if (!ok) diag("COPY IN did not complete: %s", PQerrorMessage(client)); + PQclear(res); + while ((res = PQgetResult(client)) != nullptr) PQclear(res); + return ok; +} + +static const char* COPY_OUT_SQL = + "COPY (SELECT g, 'row'||g FROM generate_series(1,5) g) TO STDOUT"; +static const char* COPY_IN_PAYLOAD = "1\tone\n2\ttwo\n3\tthree\n"; + +int main(int argc, char** argv) { + plan(15); + + if (cl.getEnv()) + return exit_status(); + + auto admin = open_admin_conn(); + ok(PQstatus(admin.get()) == CONNECTION_OK, "ADMIN connection created"); + if (PQstatus(admin.get()) != CONNECTION_OK) return exit_status(); + + const std::vector saved = readServers(admin.get()); + if (saved.empty()) BAIL_OUT("no servers in hostgroup 0 to work with"); + + const std::string saved_native = queryOneValue(admin.get(), + "SELECT variable_value FROM global_variables WHERE variable_name='pgsql-use_native_backend_protocol'"); + diag("saved: %lu server rows, native=%s", saved.size(), saved_native.c_str()); + + // ---------------- Phase 1: libpq, backend TLS on ---------------- + diag("---- Phase 1: libpq path, use_ssl=1 ----"); + setNativeMode(admin.get(), false); + ok(reloadServers(admin.get(), saved, 1), "backend servers set to use_ssl=1 and pool flushed"); + + std::string libpq_out; + bool libpq_copy_out_ok = false, libpq_copy_in_ok = false, libpq_tls = false; + { + auto client = open_client_conn(); + if (PQstatus(client.get()) != CONNECTION_OK) + BAIL_OUT("client connection failed: %s", PQerrorMessage(client.get())); + execSQL(client.get(), "DROP TABLE IF EXISTS " + TBL); + execSQL(client.get(), "CREATE TABLE " + TBL + " (id int, name text)"); + libpq_tls = backendLegIsEncrypted(client.get()); + libpq_out = copyOut(client.get(), COPY_OUT_SQL, &libpq_copy_out_ok); + libpq_copy_in_ok = copyIn(client.get(), TBL, COPY_IN_PAYLOAD); + } + ok(libpq_tls, "libpq phase really ran over an encrypted backend connection"); + ok(libpq_copy_out_ok && !libpq_out.empty(), "libpq: COPY TO STDOUT completed (%zu bytes)", libpq_out.size()); + ok(libpq_copy_in_ok, "libpq: COPY FROM STDIN completed"); + + // ---------------- Phase 2: native, backend TLS on ---------------- + diag("---- Phase 2: native path, use_ssl=1 ----"); + setNativeMode(admin.get(), true); + reloadServers(admin.get(), saved, 1); + + std::string native_out; + bool native_copy_out_ok = false, native_copy_in_ok = false, native_tls = false, still_usable = false; + { + auto client = open_client_conn(); + if (PQstatus(client.get()) != CONNECTION_OK) + BAIL_OUT("client connection failed in native phase: %s", PQerrorMessage(client.get())); + execSQL(client.get(), "TRUNCATE " + TBL); + native_tls = backendLegIsEncrypted(client.get()); + native_out = copyOut(client.get(), COPY_OUT_SQL, &native_copy_out_ok); + native_copy_in_ok = copyIn(client.get(), TBL, COPY_IN_PAYLOAD); + // The connection is handed back to the pool after the COPY; a plain + // query has to still work on it. + still_usable = (queryOneValue(client.get(), "SELECT 42") == "42"); + } + ok(native_tls, "native phase really ran over an encrypted backend connection"); + ok(native_copy_out_ok && !native_out.empty(), "native: COPY TO STDOUT completed (%zu bytes)", native_out.size()); + ok(native_out == libpq_out, "native COPY TO STDOUT bytes match libpq"); + ok(native_copy_in_ok, "native: COPY FROM STDIN completed"); + ok(still_usable, "native: the session still works after a COPY over a TLS backend connection"); + + // The bulk-load shape: connect, COPY, disconnect. Nothing there notices a + // broken connection; the next unrelated session is the one that gets it. + bool native_loader_pool_ok = false; + { + auto loader = open_client_conn(); + if (PQstatus(loader.get()) == CONNECTION_OK) copyIn(loader.get(), TBL, COPY_IN_PAYLOAD); + } + { + auto client = open_client_conn(); + if (PQstatus(client.get()) == CONNECTION_OK) + native_loader_pool_ok = (queryOneValue(client.get(), "SELECT 7") == "7"); + } + ok(native_loader_pool_ok, "native: a later session works on the connection a COPY-and-disconnect left pooled"); + + // adopt -> release -> adopt on one connection. + bool native_second_copy_ok = false; + { + auto client = open_client_conn(); + if (PQstatus(client.get()) == CONNECTION_OK) { + bool a = copyIn(client.get(), TBL, COPY_IN_PAYLOAD); + bool b = copyIn(client.get(), TBL, COPY_IN_PAYLOAD); + native_second_copy_ok = a && b && (queryOneValue(client.get(), "SELECT 5") == "5"); + } + } + ok(native_second_copy_ok, "native: two COPYs on one session, then a query, all succeed"); + + // Large enough to need many write cycles, so a partial write leaves buffered + // ciphertext rather than completing in one go. + bool native_bulk_ok = false; + { + std::string big; + big.reserve(600 * 1024); + for (int i = 0; big.size() < 512 * 1024; i++) + big += std::to_string(i) + "\tpadding-so-the-row-is-not-tiny-" + std::to_string(i) + "\n"; + auto client = open_client_conn(); + if (PQstatus(client.get()) == CONNECTION_OK) { + bool a = copyIn(client.get(), TBL, big); + native_bulk_ok = a && (queryOneValue(client.get(), "SELECT 9") == "9"); + } + } + ok(native_bulk_ok, "native: a half-megabyte COPY over TLS completes and leaves the session usable"); + + // Permanent fast forward reaches the other two places the backend TLS is + // borrowed: attach_connection() for a pooled connection joining a session that + // is already forwarding, and the connect path when one is opened for it. + bool native_ff_own = false, native_ff_pooled = false; + if (execSQL(admin.get(), "UPDATE pgsql_users SET fast_forward=1") + && execSQL(admin.get(), "LOAD PGSQL USERS TO RUNTIME")) { + reloadServers(admin.get(), saved, 1); // drop the pool: the first session must connect + { + auto client = open_client_conn(); + if (PQstatus(client.get()) == CONNECTION_OK) + native_ff_own = (queryOneValue(client.get(), "SELECT 13") == "13"); + } + { + auto client = open_client_conn(); + if (PQstatus(client.get()) == CONNECTION_OK) + native_ff_pooled = (queryOneValue(client.get(), "SELECT 17") == "17"); + } + execSQL(admin.get(), "UPDATE pgsql_users SET fast_forward=0"); + execSQL(admin.get(), "LOAD PGSQL USERS TO RUNTIME"); + } + ok(native_ff_own, "native: a forwarded session works on a TLS backend connection it opened itself"); + ok(native_ff_pooled, "native: a forwarded session works on a TLS backend connection taken from the pool"); + + // ---------------- restore ---------------- + { + auto client = open_client_conn(); + if (PQstatus(client.get()) == CONNECTION_OK) + execSQL(client.get(), "DROP TABLE IF EXISTS " + TBL); + } + setNativeMode(admin.get(), saved_native == "true"); + reloadServers(admin.get(), saved, 0); + + return exit_status(); +} diff --git a/test/tap/tests/pgsql-native_framer_retention-t.cpp b/test/tap/tests/pgsql-native_framer_retention-t.cpp new file mode 100644 index 0000000000..7434b69811 --- /dev/null +++ b/test/tap/tests/pgsql-native_framer_retention-t.cpp @@ -0,0 +1,277 @@ +/** + * @file pgsql-native_framer_retention-t.cpp + * @brief Drives the native backend path hard enough to trip the framer's + * consumed-byte assertion in PgSQL_Backend_Msg_Framer::feed(). + * + * The defect: feed() must slide the unread tail down and drop the consumed + * prefix. Without that the prefix accumulates for the whole result set. + * + * Detection lives in the framer itself, as an invariant that holds immediately + * after the compaction block: + * + * assert(pos == 0 || pos < len - pos); + * + * It is the exact negation of the compaction condition, so it cannot false-fire, + * and it trips on the first feed after a substantial drain. This test's only job + * is to make sure that code path actually RUNS: enable the native protocol, + * stream a result big enough to need many reads with drains in between, and + * confirm the proxy is still alive and serving afterwards. + * + * So a regression shows up as ProxySQL aborting mid-test, not as a numeric + * comparison here. Assertion 1 is the liveness check that catches it; the other + * two then fail as a consequence and say so. The assert message in the proxy log + * names the file, line and expression. + * + * REQUIRES a DEBUG build -- assert() is compiled out of the framer under + * #ifdef DEBUG, so on a release build this degrades to a plain smoke test. + * + * INFRA: legacy-g1. Runtime state restored in memory only -- never SAVE ... TO DISK. + */ +#include +#include +#include +#include +#include +#include + +#include "libpq-fe.h" +#include "command_line.h" +#include "tap.h" +#include "utils.h" + +using PGConnPtr = std::unique_ptr; +CommandLine cl; + +static const int BACKEND_HG = 0; + +// A DataRow carrying one text column of N chars is 11 + N bytes on the wire. +// The width is not load-bearing here (see the header) -- it only needs to be +// wide enough that ~58 MiB streams in a reasonable number of rows. +static const int PAYLOAD = 2038; + +// Deliberately fixed, not env-tunable. The stream must be large enough to need +// many reads with drains between them, which is what exercises the compaction +// path the assert guards. Change by reviewed edit, not env var. +static const int ROWS = 30000; // ~58.3 MiB streamed + +static PGConnPtr openConn(const char* host, int port, const char* user, + const char* pass, const char* db) { + std::stringstream ss; + ss << "host=" << host << " port=" << port << " user=" << user << " password=" << pass; + if (db && *db) ss << " dbname=" << db; + ss << " sslmode=disable connect_timeout=10"; + return PGConnPtr(PQconnectdb(ss.str().c_str()), &PQfinish); +} +static PGConnPtr createAdminConn() { + return openConn(cl.pgsql_admin_host, cl.pgsql_admin_port, + cl.admin_username, cl.admin_password, nullptr); +} +static PGConnPtr createClientConn() { + return openConn(cl.pgsql_host, cl.pgsql_port, cl.pgsql_username, + cl.pgsql_password, cl.pgsql_username); +} +static bool execAdmin(PGconn* a, const std::string& q) { + PGresult* r = PQexec(a, q.c_str()); + ExecStatusType st = PQresultStatus(r); + bool good = (st == PGRES_COMMAND_OK || st == PGRES_TUPLES_OK); + if (!good) diag("admin failed: %s -- %s", q.c_str(), PQerrorMessage(a)); + PQclear(r); + return good; +} +static std::string adminScalar(PGconn* a, const std::string& q) { + PGresult* r = PQexec(a, q.c_str()); + std::string v; + if (PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) > 0 && !PQgetisnull(r, 0, 0)) + v = PQgetvalue(r, 0, 0); + PQclear(r); + return v; +} +// ConnOK counts backend connections successfully created. Used to prove the +// stream really got a FRESH connection -- which, with native mode on at the +// time, is a connection that latched the native path. +static long long connOK(PGconn* a, int hg) { + std::stringstream q; + q << "SELECT IFNULL(SUM(ConnOK),0) FROM stats_pgsql_connection_pool WHERE hostgroup=" << hg; + const std::string v = adminScalar(a, q.str()); + return v.empty() ? -1 : atoll(v.c_str()); +} + +// Sets a runtime variable. Deliberately does NOT remember the old value: +// proxysql-tester.py reloads every config table FROM DISK before each test, so +// putting variables back is the harness's job, not the test's. +// Caller issues a single LOAD PGSQL VARIABLES TO RUNTIME afterwards. +static bool forceVar(PGconn* a, const std::string& name, const std::string& value) { + return execAdmin(a, "SET " + name + "='" + value + "'"); +} + +// native_mode is latched per backend connection on its first handler() call +// (lib/PgSQL_Connection.cpp), so connections pooled before the toggle keep the +// old path. Recycling pgsql_servers drops them and forces fresh ones. +struct ServerRow { std::string hostname, port, max_connections, comment; }; +static std::vector readServers(PGconn* a, int hg) { + std::vector rows; + std::stringstream q; + q << "SELECT hostname, port, max_connections, comment FROM pgsql_servers " + << "WHERE hostgroup_id=" << hg; + PGresult* r = PQexec(a, q.str().c_str()); + if (PQresultStatus(r) == PGRES_TUPLES_OK) { + for (int i = 0; i < PQntuples(r); i++) { + ServerRow s; + s.hostname = PQgetvalue(r, i, 0); + s.port = PQgetvalue(r, i, 1); + s.max_connections = PQgetvalue(r, i, 2); + s.comment = PQgetisnull(r, i, 3) ? "" : PQgetvalue(r, i, 3); + rows.push_back(std::move(s)); + } + } + PQclear(r); + return rows; +} +static bool flushBackendPool(PGconn* a, int hg, const std::vector& saved) { + if (saved.empty()) return false; + std::stringstream del; + del << "DELETE FROM pgsql_servers WHERE hostgroup_id=" << hg; + if (!execAdmin(a, del.str())) return false; + if (!execAdmin(a, "LOAD PGSQL SERVERS TO RUNTIME")) return false; + for (const auto& s : saved) { + std::stringstream ins; + ins << "INSERT INTO pgsql_servers (hostgroup_id,hostname,port,max_connections,comment) " + << "VALUES (" << hg << ",'" << s.hostname << "'," << s.port << "," + << (s.max_connections.empty() ? std::string("1000") : s.max_connections) + << ",'" << s.comment << "')"; + if (!execAdmin(a, ins.str())) return false; + } + if (!execAdmin(a, "LOAD PGSQL SERVERS TO RUNTIME")) return false; + usleep(300000); + return true; +} + +// Stream one wide result in single-row mode. Single-row mode keeps THIS process +// small; it does not change what ProxySQL does with the backend stream. +static bool streamWideResult(PGconn* c, long long* out_rows, long long* out_bytes) { + std::stringstream q; + // create_new_connection=1 forces a fresh backend connection. A connection + // latches native-vs-libpq when it is created, so one pooled before the + // toggle would serve this query on the old path. Needs query_digests on -- + // see main(). If it ever stops working the query runs on a pooled libpq + // connection, the framer is never fed, and the assert has nothing to guard. + q << "/* create_new_connection=1 */ " + << "SELECT repeat('x', " << PAYLOAD << ") FROM generate_series(1," << ROWS << ")"; + if (!PQsendQuery(c, q.str().c_str())) return false; + PQsetSingleRowMode(c); + long long rows = 0, bytes = 0; + bool ok = true; + PGresult* r; + while ((r = PQgetResult(c)) != nullptr) { + const ExecStatusType st = PQresultStatus(r); + if (st == PGRES_SINGLE_TUPLE) { rows++; bytes += PQgetlength(r, 0, 0); } + else if (st != PGRES_TUPLES_OK) ok = false; + PQclear(r); + } + *out_rows = rows; + *out_bytes = bytes; + return ok; +} + +int main(int, char**) { + // native path reachable + volume sanity + proxy survived + plan(3); + + if (cl.getEnv()) return exit_status(); + + auto adminOwner = createAdminConn(); + if (!adminOwner || PQstatus(adminOwner.get()) != CONNECTION_OK) + BAIL_OUT("cannot proceed without an admin connection"); + PGconn* admin = adminOwner.get(); + + const std::vector saved_servers = readServers(admin, BACKEND_HG); + if (saved_servers.empty()) BAIL_OUT("no pgsql_servers in hostgroup %d", BACKEND_HG); + + // NOT a variable restore (the harness handles those). This recycles the + // backend POOL, which the harness does not touch. + auto recyclePool = [&]() { + // Not redundant with create_new_connection=1: that hint gets us a native + // connection, it does not take it away. The connection returns to the + // pool still latched native, where a later test expecting libpq could be + // handed it. + flushBackendPool(admin, BACKEND_HG, saved_servers); + }; + + // Force the whole precondition set, not just the native switch. The + // create_new_connection hint is parsed from the query's first comment, and + // first_comment is only extracted inside `if (query_digests)` + // (Query_Processor.cpp, query_parser_init); first_comment_parsing=0 disables + // annotation parsing outright. With either off the hint silently does + // nothing. Set here because the harness reloads variables from disk before + // every test, so a group config could otherwise disarm it. + const bool prereq = + forceVar(admin, "pgsql-use_native_backend_protocol", "true") && + forceVar(admin, "pgsql-query_digests", "true") && + forceVar(admin, "pgsql-query_processor_first_comment_parsing", "3") && + execAdmin(admin, "LOAD PGSQL VARIABLES TO RUNTIME"); + if (!prereq) { recyclePool(); BAIL_OUT("cannot set the native-path preconditions"); } + + // No pool recycle here on purpose: create_new_connection=1 on the streaming + // query already gets a fresh connection, and recycling would add a window + // where pgsql_servers is empty for this hostgroup -- a crash inside it + // strands every later test with no backend. recyclePool() still recycles. + + auto c = createClientConn(); + if (!c || PQstatus(c.get()) != CONNECTION_OK) { + recyclePool(); + BAIL_OUT("client connection failed: %s", c ? PQerrorMessage(c.get()) : "null"); + } + + const long long conn_ok_before = connOK(admin, BACKEND_HG); + + long long rows = 0, bytes = 0; + const bool streamed = streamWideResult(c.get(), &rows, &bytes); + + // Liveness FIRST, and it is the headline. If the framer retained consumed + // bytes, the assert in feed() has already aborted ProxySQL, so a dead proxy + // here IS the signal -- not an infrastructure problem. Reporting it + // first matters: the crash also makes the two checks below fail, and without + // this line those read like a broken query rather than a memory defect. + auto probe = createClientConn(); + const bool alive = probe && PQstatus(probe.get()) == CONNECTION_OK && + adminScalar(probe.get(), "SELECT 1") == "1"; + probe.reset(); + + const long long want_bytes = (long long)ROWS * PAYLOAD; + + if (!alive) { + diag("ProxySQL is GONE after streaming. The framer invariant " + "assert(pos == 0 || pos < len - pos) in PgSQL_Backend_Msg_Framer::feed() " + "aborted the process: consumed bytes were not reclaimed."); + diag("Confirm with: docker logs | grep Assertion"); + diag("The two checks below could not be evaluated and fail as a consequence, " + "not as separate defects."); + } + + ok(alive, + "ProxySQL survived streaming %.1f MiB on the native path " + "[feed() must compact the consumed prefix; a crash here means it did not]", + (double)want_bytes / (1024.0 * 1024.0)); + + // Volume: a query that silently failed streams nothing, leaves the framer + // untouched, and would make the check above pass for the wrong reason. + ok(streamed && rows == (long long)ROWS && bytes == want_bytes, + "streamed the expected volume: %lld/%d rows, %lld/%lld payload bytes%s", + rows, ROWS, bytes, want_bytes, + alive ? "" : " <-- truncated because ProxySQL aborted mid-stream"); + + // The stream must have gone through the NATIVE path, or the framer was never + // fed and the assert had nothing to guard. create_new_connection=1 forces a + // fresh backend connection, so ConnOK must rise, and native mode was on when + // it was created, so that connection latched native. + const long long conn_ok_after = alive ? connOK(admin, BACKEND_HG) : -1; + ok(alive && conn_ok_before >= 0 && conn_ok_after > conn_ok_before, + "stream ran on a NEW backend connection created under native mode%s", + alive ? (std::string(" (ConnOK ") + std::to_string(conn_ok_before) + " -> " + + std::to_string(conn_ok_after) + "), so the framer was exercised").c_str() + : " <-- unreadable because ProxySQL aborted; admin is gone"); + + c.reset(); // close the client before touching runtime config + recyclePool(); + return exit_status(); +} diff --git a/test/tap/tests/pgsql-native_hostile_backend-t.cpp b/test/tap/tests/pgsql-native_hostile_backend-t.cpp new file mode 100644 index 0000000000..4c7a9f1e0f --- /dev/null +++ b/test/tap/tests/pgsql-native_hostile_backend-t.cpp @@ -0,0 +1,1048 @@ +/** + * @file pgsql-native_hostile_backend-t.cpp + * @brief Drives ProxySQL's native backend protocol against a deliberately + * hostile PostgreSQL server. + * + * PURPOSE + * ------- + * Every other native-path test uses a healthy PostgreSQL as its oracle. That + * leaves the entire error surface untested, because a real PostgreSQL never + * sends a malformed frame, never forges an authentication signature, and never + * disappears mid-result. This test supplies a backend that does all of those + * things (see pgsql_mock_backend.h) and asserts ProxySQL survives them. + * + * The bar for every case is the same three invariants, checked after each: + * 1. ProxySQL is still alive and its admin interface answers. + * 2. The proxy still serves the REAL backend — a hostile connection must not + * poison unrelated traffic. + * 3. No backend connections are left stranded in the mock's hostgroup pool. + * + * A case that produces a clean client error passes. A case that crashes, hangs, + * or leaks fails. What the SQLSTATE happens to be is mostly not asserted — the + * point is robustness, not a specific error string. + * + * ============================================================================ + * EXPECTATIONS AND HONESTY + * ============================================================================ + * Most cases here are EXPLORATORY. Predictions exist for the framing and + * bounds cases; the response to a forged SCRAM signature, a stray second + * ReadyForQuery, or an idle-pool NotificationResponse is not known in advance. + * Any failure is a NEW FINDING, to be investigated and reported — never + * normalised away or relabelled flaky. + * + * ONE CASE IS A PROBER, NOT A PROOF. Case R1 targets defect D4 + * (lib/PgSQL_Connection.cpp:2201 returns -1 on EOF while ignoring `got`, + * discarding an already-framed complete result; the TLS branch at :2181 + * correctly returns `got ? 1 : -1`). Triggering it requires the FINAL recv() + * to return exactly 16384 bytes, which a black-box test cannot arrange over + * TCP: ProxySQL wakes on the first readable segment (~1448 bytes over a Docker + * bridge), and any short read mid-stream destroys the alignment. R1 therefore + * retries a bounded number of times and reports + * "D4 not observed in N iterations" + * when it does not fire. That is NOT evidence the defect is absent — the same + * run against known-defective code can legitimately miss it. R1 must never be + * treated as a regression guard. + * + * INFRA + * ----- + * legacy-g1 (docker-pgsql16-single). The mock listens inside the test-runner + * container, which shares the Docker network with ProxySQL, and is registered + * in pgsql_servers by its runtime-discovered IP. + * + * PRECONDITIONS (without these, cases fail for unrelated reasons) + * --------------------------------------------------------------- + * A backend that breaks handshakes on purpose trips two mechanisms that would + * remove it from rotation before most cases run: + * - the Monitor shuns non-responsive servers (PgSQL_Monitor.cpp:1729), and it + * probes with libpq, which every hostile handshake defeats; + * - pgsql-shun_on_failures defaults to 5 (PgSQL_Thread.cpp:1042) and this + * file contains well over five deliberate failures. + * Both are adjusted in memory for the duration. They are deliberately NOT + * restored: proxysql-tester.py reloads every config table FROM DISK before each + * test, so the isolation lives in the harness. Never SAVE ... TO DISK, which is + * what would actually defeat it. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libpq-fe.h" +#include "command_line.h" +#include "tap.h" +#include "utils.h" + +#include "pgsql_mock_backend.h" + +using PGConnPtr = std::unique_ptr; +CommandLine cl; + +// Hostgroup and user dedicated to the mock; kept away from the real backend's +// hostgroup 0 so a shunned/broken mock cannot affect normal traffic. +static const int MOCK_HG = 47; +static const char* MOCK_USER = "hostile_mock_user"; +static const char* MOCK_PASS = "hostile_mock_pw"; + +// ------------------------------------------------------------------ helpers + +static PGConnPtr openConn(const char* host, int port, const char* user, + const char* pass, const char* db) { + std::stringstream ss; + ss << "host=" << host << " port=" << port << " user=" << user << " password=" << pass; + if (db && *db) ss << " dbname=" << db; + ss << " sslmode=disable connect_timeout=10"; + return PGConnPtr(PQconnectdb(ss.str().c_str()), &PQfinish); +} + +static PGConnPtr createAdminConn() { + return openConn(cl.pgsql_admin_host, cl.pgsql_admin_port, + cl.admin_username, cl.admin_password, nullptr); +} + +static bool execAdmin(PGconn* admin, const std::string& q) { + PGresult* r = PQexec(admin, q.c_str()); + ExecStatusType st = PQresultStatus(r); + bool good = (st == PGRES_COMMAND_OK || st == PGRES_TUPLES_OK); + if (!good) diag("admin failed: %s -- %s", q.c_str(), PQerrorMessage(admin)); + PQclear(r); + return good; +} + +static std::string adminScalar(PGconn* admin, const std::string& q) { + PGresult* r = PQexec(admin, q.c_str()); + std::string v; + if (PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) > 0 && !PQgetisnull(r, 0, 0)) + v = PQgetvalue(r, 0, 0); + PQclear(r); + return v; +} + +static bool setVar(PGconn* admin, const std::string& name, const std::string& val) { + return execAdmin(admin, "SET " + name + "='" + val + "'") && + execAdmin(admin, "LOAD PGSQL VARIABLES TO RUNTIME"); +} + +// Backend connections currently held for the mock's hostgroup. Used as the +// leak invariant: a hostile case must not strand connections in the pool. +static int mockPoolConnsNow(PGconn* admin) { + std::stringstream q; + q << "SELECT IFNULL(SUM(ConnUsed + ConnFree),0) FROM stats_pgsql_connection_pool " + << "WHERE hostgroup=" << MOCK_HG; + const std::string v = adminScalar(admin, q.str()); + return v.empty() ? 0 : atoi(v.c_str()); +} + +// Settle before judging. ProxySQL retries a failing backend many times (10-20 +// connects per hostile case were observed), and teardown of the last attempt +// can still be in flight when the client's error surfaces. Sampling the pool +// immediately reports connections that are on their way out as leaks. Poll to +// zero for a bounded window and report the last value seen. +// The pool drains ASYNCHRONOUSLY, and measurement showed the drain landing at +// ~2.0s -- which is exactly where the old 20*100ms budget expired. A case that +// drained on iteration 19 passed and one that needed iteration 21 failed, so the +// verdict flipped between runs of an identical binary (observed: R8/R14/R20). +// The budget must sit COMFORTABLY ABOVE the natural drain time, not on top of +// it. This does not weaken the assertion: a connection that is genuinely stranded +// still fails, it just takes longer to say so. `drain_ms` reports how long the +// drain actually took, so a future shift shows up as a number instead of a +// coin flip. +static int mockPoolConns(PGconn* admin, int* drain_ms = nullptr) { + int last = mockPoolConnsNow(admin); + int i = 0; + for (; i < 100 && last != 0; i++) { // up to ~10s + usleep(100000); + last = mockPoolConnsNow(admin); + } + if (drain_ms) *drain_ms = i * 100; + return last; +} + +// Drive one attempt through ProxySQL at the mock hostgroup. Returns true if the +// client got a usable answer; on failure `err` carries the client-visible error. +// Either outcome is acceptable for most cases — what matters is that this +// RETURNS AT ALL (no hang) and leaves the proxy healthy. +static bool queryThroughProxy(std::string& err, const char* query = "SELECT 1") { + auto c = openConn(cl.pgsql_host, cl.pgsql_port, MOCK_USER, MOCK_PASS, "postgres"); + if (!c || PQstatus(c.get()) != CONNECTION_OK) { + err = c ? PQerrorMessage(c.get()) : "null conn"; + return false; + } + PGresult* r = PQexec(c.get(), query); + const ExecStatusType st = PQresultStatus(r); + const bool good = (st == PGRES_TUPLES_OK || st == PGRES_COMMAND_OK); + if (!good) err = PQerrorMessage(c.get()); + PQclear(r); + return good; +} + +// Same idea as queryThroughProxy(), but it GIVES UP after `timeout_ms` instead +// of blocking forever. Every other case here can use a blocking PQexec because +// the proxy always answers something; F5 (R23) is the one case where the +// session wedges and never answers at all, and a blocking call there would hang +// the whole run instead of failing one assertion. +// +// `setup` is run first with an ordinary blocking PQexec: it is a tracked SET, +// which ProxySQL answers itself without touching a backend. +static bool queryThroughProxyBounded(std::string& err, const char* setup, const char* query, + int timeout_ms, bool& timed_out) { + timed_out = false; + auto c = openConn(cl.pgsql_host, cl.pgsql_port, MOCK_USER, MOCK_PASS, "postgres"); + if (!c || PQstatus(c.get()) != CONNECTION_OK) { + err = c ? PQerrorMessage(c.get()) : "null conn"; + return false; + } + if (setup && *setup) { + PGresult* r = PQexec(c.get(), setup); + if (PQresultStatus(r) != PGRES_COMMAND_OK && PQresultStatus(r) != PGRES_TUPLES_OK) { + err = std::string("setup failed: ") + PQerrorMessage(c.get()); + PQclear(r); + return false; + } + PQclear(r); + } + if (!PQsendQuery(c.get(), query)) { err = PQerrorMessage(c.get()); return false; } + + const int fd = PQsocket(c.get()); + struct timespec t0; + clock_gettime(CLOCK_MONOTONIC, &t0); + for (;;) { + if (!PQconsumeInput(c.get())) { err = PQerrorMessage(c.get()); break; } + if (!PQisBusy(c.get())) break; + if (fd < 0) { err = "no socket"; break; } // FD_SET(-1) is undefined behaviour + fd_set rf; + FD_ZERO(&rf); + FD_SET(fd, &rf); + struct timeval tv = { 0, 200000 }; + select(fd + 1, &rf, nullptr, nullptr, &tv); + struct timespec now; + clock_gettime(CLOCK_MONOTONIC, &now); + const long ms = (now.tv_sec - t0.tv_sec) * 1000 + (now.tv_nsec - t0.tv_nsec) / 1000000; + if (ms >= timeout_ms) { + timed_out = true; + err = "ProxySQL never answered"; + return false; + } + } + // Not busy: the first result is already buffered, so this cannot block. + PGresult* r = PQgetResult(c.get()); + const ExecStatusType st = r ? PQresultStatus(r) : PGRES_FATAL_ERROR; + const bool good = (st == PGRES_TUPLES_OK || st == PGRES_COMMAND_OK); + if (!good) err = PQerrorMessage(c.get()); + PQclear(r); + return good; +} + +// The three post-case invariants. Returns "" when healthy, else what broke. +static std::string checkInvariants(PGconn*& adminRef, PGConnPtr& adminOwner) { + // 1. ProxySQL alive / admin answering. Reconnect once: a crashed proxy + // fails the reconnect too, so this does not mask a crash. + if (adminScalar(adminRef, "SELECT 1") != "1") { + adminOwner = createAdminConn(); + if (!adminOwner || PQstatus(adminOwner.get()) != CONNECTION_OK) + return "ProxySQL admin unreachable (proxy down?)"; + adminRef = adminOwner.get(); + if (adminScalar(adminRef, "SELECT 1") != "1") + return "ProxySQL admin not answering"; + } + // 2. The real backend still serves traffic through the proxy. + { + auto c = openConn(cl.pgsql_host, cl.pgsql_port, cl.pgsql_username, + cl.pgsql_password, cl.pgsql_username); + if (!c || PQstatus(c.get()) != CONNECTION_OK) + return "real-backend traffic broken after hostile case"; + PGresult* r = PQexec(c.get(), "SELECT 1"); + const bool good = (PQresultStatus(r) == PGRES_TUPLES_OK); + PQclear(r); + if (!good) return "real-backend query failed after hostile case"; + } + return ""; +} + +// Drop every pooled backend connection for the mock hostgroup by removing and +// re-adding the server row. +// +// Without this, cases are not independent: a case whose handshake SUCCEEDS +// leaves a connection in the pool, and the next case may be served from it +// instead of opening a new one — so the next case's script never runs and its +// result describes the previous case's connection. Relying on the mock closing +// each connection would leave that to timing. +static bool resetMockPool(PGconn* admin, const std::string& ip, uint16_t port) { + std::stringstream del; + del << "DELETE FROM pgsql_servers WHERE hostgroup_id=" << MOCK_HG; + if (!execAdmin(admin, del.str()) || !execAdmin(admin, "LOAD PGSQL SERVERS TO RUNTIME")) + return false; + std::stringstream ins; + ins << "INSERT INTO pgsql_servers (hostgroup_id,hostname,port,max_connections,use_ssl,comment) " + << "VALUES (" << MOCK_HG << ",'" << ip << "'," << port << ",4,0,'hostile mock backend')"; + if (!execAdmin(admin, ins.str()) || !execAdmin(admin, "LOAD PGSQL SERVERS TO RUNTIME")) + return false; + usleep(150000); + return true; +} + +// Set by main() once the mock is listening, so runCase can flush the pool. +static std::string g_mock_ip; +static uint16_t g_mock_port = 0; + +// Run one hostile case end to end and emit exactly one TAP assertion. +static void runCase(PGconn*& admin, PGConnPtr& adminOwner, PgSQL_Mock_Backend& mock, + const char* label, const std::vector& script, + const char* query = "SELECT 1") { + resetMockPool(admin, g_mock_ip, g_mock_port); + mock.set_script(script); + mock.reset_stats(); + + std::string clierr; + const bool served = queryThroughProxy(clierr, query); // must simply RETURN + + const std::string broke = checkInvariants(admin, adminOwner); + int drain_ms = 0; + const int stranded = mockPoolConns(admin, &drain_ms); + + // Trim the client error to one line for readable TAP output. + std::string first_line = clierr.substr(0, clierr.find('\n')); + + ok(broke.empty() && stranded == 0, + "%s: proxy survived (client %s: %s; mock conns=%d; pool leftover=%d; drain=%dms)%s%s", + label, + served ? "served" : "errored", + first_line.empty() ? "-" : first_line.c_str(), + mock.connections_accepted(), stranded, drain_ms, + broke.empty() ? "" : " -- BROKE: ", broke.c_str()); +} + +// ------------------------------------------------------------- script pieces + +// Handshake prefix that gets the mock to a ready-for-query state. +static std::string acceptedHandshake() { + return pgmb_auth_ok() + + pgmb_parameter_status("server_version", "16.2") + + pgmb_parameter_status("client_encoding", "UTF8") + + pgmb_backend_key_data(4242, 987654321) + + pgmb_ready_for_query('I'); +} + +// ---------------------------------------------------------------- harness +// Six checks on the MOCK itself, using libpq as an independent oracle, before +// a single verdict is passed on ProxySQL. +// +// This has to come first. The cases below judge ProxySQL against fixtures this +// harness produces; if a fixture were wrong the verdict would be confident +// nonsense. A9 is the sharp example: it asserts ProxySQL REJECTS a forged SCRAM +// server signature, so if the mock's "forged" signature were accidentally +// VALID, A9 would pass no matter what ProxySQL did. libpq is the natural +// oracle -- an independent, widely deployed SCRAM client that verifies server +// signatures. Pointing it at the mock tests the mock, not the proxy. +// +// Folded in from the former pgsql-native_mock_selftest-t: a broken harness must +// surface before the results that depend on it, not in a separate test that may +// run later or not at all. +static PGconn* selftestConnect(uint16_t port, const char* pw) { + std::stringstream ss; + ss << "host=127.0.0.1 port=" << port << " user=mockuser password=" << pw + << " dbname=postgres sslmode=disable connect_timeout=5"; + return PQconnectdb(ss.str().c_str()); +} + +static void harness_selftest() { + PgSQL_Mock_Backend mock; + mock.set_scram_password("mockpw"); + if (!mock.start()) BAIL_OUT("harness self-test: mock backend failed to listen on loopback"); + diag("harness self-test: mock listening on 127.0.0.1:%u", mock.port()); + + // 1/2. Trust handshake plus a canned result: the framing, the startup + // exchange and the result builders are all wire-legal to a real client. + mock.set_script({ step_expect_startup(), step_send(acceptedHandshake()), + step_expect_message(), step_send(pgmb_simple_result("c", "1", 1)), + step_sleep(200) }); + { + PGconn* c = selftestConnect(mock.port(), "mockpw"); + const bool connected = (PQstatus(c) == CONNECTION_OK); + ok(connected, "harness: libpq completes the mock's trust handshake%s%s", + connected ? "" : ": ", connected ? "" : PQerrorMessage(c)); + if (connected) { + PGresult* r = PQexec(c, "SELECT 1"); + ok(PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) == 1 && PQnfields(r) == 1, + "harness: libpq parses the mock's canned RowDescription/DataRow/CommandComplete/ReadyForQuery"); + PQclear(r); + } else { + ok(false, "harness: skipped result parse -- handshake did not complete"); + } + PQfinish(c); + } + + // 3. Honest SCRAM-SHA-256: proves the mock's server-side key derivation. + mock.set_script({ step_expect_startup(), + step_send(pgmb_auth_sasl({ "SCRAM-SHA-256" })), + step_scram_server_first(false), + step_scram_server_final(false), + step_send(acceptedHandshake()), + step_sleep(200) }); + { + PGconn* c = selftestConnect(mock.port(), "mockpw"); + const bool connected = (PQstatus(c) == CONNECTION_OK); + ok(connected, "harness: libpq authenticates against the mock's honest SCRAM-SHA-256 exchange%s%s", + connected ? "" : ": ", connected ? "" : PQerrorMessage(c)); + PQfinish(c); + } + + // 4. Forged server signature. libpq MUST reject it -- this is what makes + // A9 below a real assertion rather than a formality. + mock.set_script({ step_expect_startup(), + step_send(pgmb_auth_sasl({ "SCRAM-SHA-256" })), + step_scram_server_first(false), + step_scram_server_final(true), // forged + step_send(acceptedHandshake()), // then pretend all is well + step_sleep(200) }); + { + PGconn* c = selftestConnect(mock.port(), "mockpw"); + const bool rejected = (PQstatus(c) != CONNECTION_OK); + const std::string why = rejected ? PQerrorMessage(c) : ""; + ok(rejected, + "harness: libpq REJECTS the mock's forged SCRAM server signature (makes A9 meaningful): %s", + rejected ? why.substr(0, why.find('\n')).c_str() + : "ACCEPTED -- forgery is not detectable, A9 is void"); + PQfinish(c); + } + + // 5. Server nonce that does not extend the client nonce (reference for A10). + mock.set_script({ step_expect_startup(), + step_send(pgmb_auth_sasl({ "SCRAM-SHA-256" })), + step_scram_server_first(true), // bad nonce + step_sleep(200) }); + { + PGconn* c = selftestConnect(mock.port(), "mockpw"); + const bool rejected = (PQstatus(c) != CONNECTION_OK); + ok(rejected, "harness: libpq REJECTS a server nonce that does not extend the client nonce (makes A10 meaningful)"); + PQfinish(c); + } + + // 6. The exact-size builder the R1 D4 prober depends on. Off by one byte + // and the prober can never hit its trigger condition. + { + std::string body; + const size_t target = 16384 * 4; + const bool built = pgmb_result_of_exact_size(body, target); + ok(built && body.size() == target, + "harness: exact-size result builder produces precisely %zu bytes (got %zu)", + target, body.size()); + } + + mock.stop(); +} + +int main(int, char**) { + // 6 harness self-test cases (the mock, judged by libpq) + // + 18 auth cases (A1-A18; A8 and A17 are positive controls) + // + 21 result cases (R1-R22, minus R17) + R18's recorded-row-count check + // + 1 final pool-cleanliness assertion + // + R23 (F5), which runs LAST -- see the comment on it. + plan(48); + + if (cl.getEnv()) return exit_status(); + + // Validate the harness before judging ProxySQL with it. A broken fixture + // would make every verdict below meaningless -- see harness_selftest(). + harness_selftest(); + + PGConnPtr adminOwner = createAdminConn(); + if (!adminOwner || PQstatus(adminOwner.get()) != CONNECTION_OK) + BAIL_OUT("cannot proceed without an admin connection"); + PGconn* admin = adminOwner.get(); + + // ---- preconditions ----------------------------------------------------- + // See the file header: without these the mock is shunned part-way through + // and the remaining cases never reach it. + if (!setVar(admin, "pgsql-monitor_enabled", "false")) BAIL_OUT("cannot disable the monitor"); + if (!setVar(admin, "pgsql-shun_on_failures", "10000")) BAIL_OUT("cannot raise shun_on_failures"); + // Bound the wait on cases where the mock deliberately stops responding, so + // a hang shows up as a failed case rather than a hung test run. + setVar(admin, "pgsql-connect_timeout_server_max", "5000"); + // The native path is what this suite exists to exercise. + // + // When investigating a failure it is worth running the SAME corpus against the + // libpq path -- flip pgsql-use_native_backend_protocol to 'false' below and diff + // the two runs. That answers "is this native-specific, or does the shipped libpq + // path do the same?", which is how #6109 and #6110 were established as + // pre-existing v3.0 defects rather than native regressions. It is a debugging + // technique, not something CI needs to toggle, so it is a one-line edit here + // rather than a knob. + if (!setVar(admin, "pgsql-use_native_backend_protocol", "true")) + BAIL_OUT("cannot enable the native backend protocol"); + + // ---- start the mock and point ProxySQL at it --------------------------- + PgSQL_Mock_Backend mock; + mock.set_scram_password(MOCK_PASS); + if (!mock.start()) BAIL_OUT("mock backend failed to listen"); + + const std::string myip = pgmb_local_ip_towards(cl.pgsql_host, cl.pgsql_port); + if (myip.empty()) { BAIL_OUT("could not discover this container's IP toward ProxySQL"); } + diag("mock backend listening on %s:%u (hostgroup %d)", myip.c_str(), mock.port(), MOCK_HG); + g_mock_ip = myip; + g_mock_port = mock.port(); + + { + std::stringstream q; + q << "DELETE FROM pgsql_servers WHERE hostgroup_id=" << MOCK_HG << ";"; + execAdmin(admin, q.str()); + std::stringstream ins; + ins << "INSERT INTO pgsql_servers (hostgroup_id,hostname,port,max_connections,use_ssl,comment) " + << "VALUES (" << MOCK_HG << ",'" << myip << "'," << mock.port() << ",4,0,'hostile mock backend')"; + if (!execAdmin(admin, ins.str()) || !execAdmin(admin, "LOAD PGSQL SERVERS TO RUNTIME")) { + BAIL_OUT("could not register the mock backend"); + } + std::stringstream u; + u << "INSERT OR REPLACE INTO pgsql_users (username,password,active,default_hostgroup) VALUES ('" + << MOCK_USER << "','" << MOCK_PASS << "',1," << MOCK_HG << ")"; + if (!execAdmin(admin, u.str()) || !execAdmin(admin, "LOAD PGSQL USERS TO RUNTIME")) { + BAIL_OUT("could not register the mock user"); + } + } + usleep(300000); + + // ====================================================================== + // AUTH-PHASE CASES + // ====================================================================== + + // A1: ErrorResponse where an Authentication message belongs. The backend's + // error fields should reach the client rather than a generic failure. + runCase(admin, adminOwner, mock, "A1 ErrorResponse instead of Authentication", + { step_expect_startup(), + step_send(pgmb_error_response("28000", "mock: no such role")), + step_close() }); + + // A2: a message type that has no meaning during authentication. + runCase(admin, adminOwner, mock, "A2 unexpected message type during auth", + { step_expect_startup(), + step_send(pgmb_command_complete("SELECT 1")), // 'C' out of nowhere + step_close() }); + + // A3: Authentication message too short to hold its own subtype code. + runCase(admin, adminOwner, mock, "A3 Authentication payload shorter than 4 bytes", + { step_expect_startup(), + step_send(std::string("R") + pgmb_be32(6) + "xx"), // len 6 => 2-byte payload + step_close() }); + + // A4: AuthenticationMD5Password promising a salt it does not supply. + runCase(admin, adminOwner, mock, "A4 md5 challenge with a truncated salt", + { step_expect_startup(), + step_send(pgmb_auth_raw(5, "ab")), // 2 salt bytes, needs 4 + step_close() }); + + // A5/A6: mechanisms the native path cannot do. Per design section 4 these + // are capability gaps: tear down, fall back to libpq, log once. libpq will + // also fail against the mock, so what is asserted is that ProxySQL handles + // it without crashing or hanging. + runCase(admin, adminOwner, mock, "A5 GSSAPI challenge (capability gap)", + { step_expect_startup(), step_send(pgmb_auth_raw(7, "")), step_close() }); + runCase(admin, adminOwner, mock, "A6 undefined authentication subtype", + { step_expect_startup(), step_send(pgmb_auth_raw(99, "")), step_close() }); + + // A7: SASL offered with an empty mechanism list. + runCase(admin, adminOwner, mock, "A7 SASL with an empty mechanism list", + { step_expect_startup(), step_send(pgmb_auth_sasl({})), step_close() }); + + // A8 (positive control): a full, HONEST SCRAM-SHA-256 exchange. This must + // succeed. Without it, A9's rejection could be caused by anything at all — + // this is what makes A9 a statement about signature verification. + { + std::vector s = { + step_expect_startup(), + step_send(pgmb_auth_sasl({ "SCRAM-SHA-256" })), + step_scram_server_first(false), + step_scram_server_final(false), + step_send(acceptedHandshake()), + step_expect_query(), // the Query + step_send(pgmb_simple_result("c", "1", 1)), + step_sleep(300) + }; + mock.set_script(s); + mock.reset_stats(); + std::string err; + const bool served = queryThroughProxy(err); + ok(served, "A8 control: honest SCRAM-SHA-256 exchange authenticates and serves a result%s%s", + served ? "" : " -- ", served ? "" : err.substr(0, err.find('\n')).c_str()); + } + + // A9 (SECURITY): the server returns a SCRAM final message whose signature it + // could not have computed without the shared secret. ProxySQL verifies it + // via pg_scram_verify_server_final() — the sole defence against a spoofed or + // MITM'd backend. Accepting this would mean authenticating to any server + // that merely claims to be the right one. + { + std::vector s = { + step_expect_startup(), + step_send(pgmb_auth_sasl({ "SCRAM-SHA-256" })), + step_scram_server_first(false), + step_scram_server_final(true), // forged signature + step_send(acceptedHandshake()), // pretend all is well + step_expect_query(), + step_send(pgmb_simple_result("c", "1", 1)), + step_sleep(300) + }; + mock.set_script(s); + mock.reset_stats(); + std::string err; + const bool served = queryThroughProxy(err); + ok(!served, + "A9 SECURITY: forged SCRAM server signature must be REJECTED " + "(served=%s) -- a served result means server impersonation succeeds", + served ? "YES (BAD)" : "no"); + } + + // A10: server nonce that does not extend the client nonce (RFC 5802 breach). + { + std::vector s = { + step_expect_startup(), + step_send(pgmb_auth_sasl({ "SCRAM-SHA-256" })), + step_scram_server_first(true), // bad nonce + step_sleep(300), + step_close() + }; + mock.set_script(s); + mock.reset_stats(); + std::string err; + const bool served = queryThroughProxy(err); + ok(!served, "A10 SECURITY: server nonce not extending the client nonce must be rejected (served=%s)", + served ? "YES (BAD)" : "no"); + } + + // A11/A12: disappearing mid-handshake. + runCase(admin, adminOwner, mock, "A11 FIN immediately after the startup packet", + { step_expect_startup(), step_close() }); + runCase(admin, adminOwner, mock, "A12 FIN after AuthenticationOk, before ReadyForQuery", + { step_expect_startup(), step_send(pgmb_auth_ok()), step_close() }); + + // A13: the whole handshake delivered one byte per write, so every length + // field is split across reads. + runCase(admin, adminOwner, mock, "A13 handshake delivered one byte at a time", + { step_expect_startup(), + step_send(acceptedHandshake(), /*chunk*/1, /*delay_us*/200), + step_expect_query(), + step_send(pgmb_simple_result("c", "1", 1), 1, 200), + step_sleep(300) }); + + // ---- A14-A18: branches of native_drive_auth() that a real PostgreSQL + // cannot produce, so only the mock can reach them. ---------------------- + + // A14: SCRAM-SHA-256-PLUS advertised as the ONLY mechanism, over a PLAINTEXT + // connection. Channel binding has no meaning without TLS, so mechanism + // selection (lib/PgSQL_Connection.cpp:2345) must take the capability-gap exit + // rather than try to derive a tls-server-end-point digest from a NULL SSL*. + // The libpq fallback that follows then fails against the mock, which is fine — + // what is under test is that the proxy chooses the gap and survives. + runCase(admin, adminOwner, mock, "A14 SCRAM-SHA-256-PLUS offered alone over plaintext", + { step_expect_startup(), + step_send(pgmb_auth_sasl({ "SCRAM-SHA-256-PLUS" })), + step_sleep(300), + step_close() }); + + // A15/A16: SASL continuation messages with no SASL exchange ever started, so + // native_scram is NULL. Both guards (:2410 and :2440) must reject cleanly + // rather than dereference it. + runCase(admin, adminOwner, mock, "A15 AuthenticationSASLContinue with no SASL exchange started", + { step_expect_startup(), + step_send(pgmb_auth_sasl_continue("r=nonce,s=c2FsdA==,i=4096")), + step_close() }); + runCase(admin, adminOwner, mock, "A16 AuthenticationSASLFinal with no SASL exchange started", + { step_expect_startup(), + step_send(pgmb_auth_sasl_final("v=bm90YXNpZ25hdHVyZQ==")), + step_close() }); + + // A17 (positive control): a NoticeResponse arriving mid-authentication must be + // IGNORED and the handshake must still complete (lib/PgSQL_Connection.cpp:2270). + // Asserting survival alone would pass even if the notice aborted the login, so + // this one asserts the query is actually SERVED. + { + // NoticeResponse payload: (field-type byte + NUL-terminated value)*, then a + // single 0 byte. Built here rather than with ProxySQL's own encoder, per + // the harness's independence rule. + std::string notice; + { + std::string payload; + payload += 'S'; payload += "NOTICE"; payload += '\0'; + payload += 'C'; payload += "00000"; payload += '\0'; + payload += 'M'; payload += "mid-auth notice"; payload += '\0'; + payload += '\0'; + pgmb_append_msg(notice, 'N', payload); + } + std::vector s = { + step_expect_startup(), + step_send(notice + acceptedHandshake()), + step_expect_query(), + step_send(pgmb_simple_result("c", "1", 1)), + step_sleep(300) + }; + resetMockPool(admin, g_mock_ip, g_mock_port); + mock.set_script(s); + mock.reset_stats(); + std::string err; + const bool served = queryThroughProxy(err); + ok(served, "A17 control: NoticeResponse during auth is ignored and the handshake completes%s%s", + served ? "" : " -- ", served ? "" : err.substr(0, err.find('\n')).c_str()); + } + + // A18: a mechanism list whose final name is NOT NUL-terminated — the payload + // simply ends mid-name. The scan at :2331 bounds each name with + // strnlen(mech, rest_len - i), so the walk must stop at the payload end + // instead of reading past it. Under ASAN a regression here is a heap + // buffer-overflow read, not merely a wrong answer. + runCase(admin, adminOwner, mock, "A18 SASL mechanism name not NUL-terminated", + { step_expect_startup(), + step_send(pgmb_auth_raw(10, "SCRAM-SHA-256")), // no trailing NUL, no list terminator + step_close() }); + + // ====================================================================== + // RESULT-PHASE CASES (handshake accepted, then hostile result bytes) + // ====================================================================== + + // R1: defect D4 prober. See the header — bounded retries, cannot prove + // absence. A large exact-multiple-of-16384 response followed by FIN. + { + // Every runCase() starts by flushing and re-registering the mock server; + // R1/R2 are hand-rolled and skipped it, so they inherited whatever state the + // ~18 hostile AUTH cases left -- including a SHUNNED server. The auth cases + // trip the shun threshold (which is min(shun_on_failures, + // connect_retries_on_failure + 1) = 11, so raising shun_on_failures alone + // cannot prevent it) and the shun lasts shun_recovery_time_sec = 10s. R1/R2 + // run inside that window and fail with "Hostgroup has no servers available", + // which reads exactly like a mid-result regression but is not one. + resetMockPool(admin, g_mock_ip, g_mock_port); + const size_t CHUNKSZ = 16384; + const size_t TARGET = CHUNKSZ * 64; // 1 MiB, exact multiple + std::string body; + bool built = pgmb_result_of_exact_size(body, TARGET); + int hits = 0, iterations = 0; + const int MAX_ITER = 50; + if (built) { + for (; iterations < MAX_ITER; iterations++) { + mock.set_script({ step_expect_startup(), + step_send(acceptedHandshake()), + step_expect_query(), + step_send(body), + step_close() }); + std::string err; + if (!queryThroughProxy(err)) { + if (err.find("backend closed during result fetch") != std::string::npos || + err.find("closed during result fetch") != std::string::npos) { + hits++; + diag("D4 HIT on iteration %d: %s", iterations, + err.substr(0, err.find('\n')).c_str()); + break; + } + } + } + } + if (!built) { + ok(false, "R1 D4 prober: could not build an exact-size response of %zu bytes", TARGET); + } else if (hits > 0) { + ok(false, + "R1 D4 CONFIRMED after %d iterations: a fully-delivered %zu-byte result was " + "reported as 'backend closed during result fetch' " + "[PgSQL_Connection.cpp:2201 returns -1 ignoring `got`; TLS branch at :2181 does not]", + iterations + 1, TARGET); + } else { + // Not a pass claim about the code — a statement about this run. + ok(true, "R1 D4 not observed in %d iterations (NOT evidence of absence; " + "the trigger needs the final recv() to return exactly %zu bytes, " + "which is not controllable over TCP)", MAX_ITER, CHUNKSZ); + } + } + + // R2 (control for R1): one byte over the multiple. Must be served cleanly. + { + // Same reason as R1 above: clear any shun left by the preceding cases. + resetMockPool(admin, g_mock_ip, g_mock_port); + std::string body; + if (pgmb_result_of_exact_size(body, 16384 * 64 + 1)) { + mock.set_script({ step_expect_startup(), step_send(acceptedHandshake()), + step_expect_query(), step_send(body), step_close() }); + std::string err; + const bool served = queryThroughProxy(err); + ok(served, "R2 control: non-multiple-sized result delivered before FIN%s%s", + served ? "" : " -- ", served ? "" : err.substr(0, err.find('\n')).c_str()); + } else { + ok(false, "R2 control: could not build the control response"); + } + } + + // R3: a DataRow that promises more bytes than the server ever sends. + runCase(admin, adminOwner, mock, "R3 truncated DataRow then FIN", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(pgmb_row_description_1col("c", 25) + + std::string("D") + pgmb_be32(1000) + "only-a-few-bytes"), + step_close() }); + + // R4: a message declaring nearly a gigabyte, then silence. Must not buffer + // toward the declared size, and must give up rather than wait forever. + runCase(admin, adminOwner, mock, "R4 900MB declared length, 100 bytes sent, then silence", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(std::string("D") + pgmb_be32(900u * 1024 * 1024) + std::string(100, 'x')), + step_sleep(6000), step_close() }); + + // R5: declared length below the 4-byte minimum, mid-result. + runCase(admin, adminOwner, mock, "R5 malformed frame (declared length 2) mid-result", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(pgmb_row_description_1col("c", 25) + + std::string("D") + pgmb_be32(2)), + step_sleep(300), step_close() }); + + // R6: a message type with no backend-direction meaning, mid-result. + runCase(admin, adminOwner, mock, "R6 unrecognised message type mid-result", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(pgmb_row_description_1col("c", 25) + + pgmb_data_row_1col("1")), + step_send(std::string("\x7f") + pgmb_be32(8) + "abcd"), // bogus type + step_send(pgmb_command_complete("SELECT 1") + pgmb_ready_for_query('I')), + step_sleep(300) }); + + // R7: a second ReadyForQuery after the result is complete. The drain stops + // at the first 'Z', so the stray bytes stay buffered on a connection that + // returns to the pool — the NEXT query on it is the real test. + runCase(admin, adminOwner, mock, "R7 stray extra ReadyForQuery poisoning the next query", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(pgmb_simple_result("c", "1", 1) + pgmb_ready_for_query('I')), + step_expect_query(), + step_send(pgmb_simple_result("c", "2", 1)), + step_sleep(300) }, + "SELECT 1"); + + // R8: ErrorResponse whose final field value has no NUL and no terminator. + runCase(admin, adminOwner, mock, "R8 ErrorResponse with an unterminated field value", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(pgmb_error_response_unterminated("42P01") + pgmb_ready_for_query('I')), + step_sleep(300) }); + + // R9: ParameterStatus with no NUL terminators — the mid-session tracking + // path parses these into native_params (PgSQL_Protocol.cpp:2939). + runCase(admin, adminOwner, mock, "R9 ParameterStatus with unterminated strings", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(std::string("S") + pgmb_be32(4 + 11) + "no_nul_here"), + step_send(pgmb_simple_result("c", "1", 1)), + step_sleep(300) }); + + // R10: an entire result delivered one byte per write. + runCase(admin, adminOwner, mock, "R10 result delivered one byte at a time", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(pgmb_simple_result("c", "hello", 3), 1, 100), + step_sleep(500) }); + + // R11: an asynchronous NotificationResponse arriving while the connection + // sits idle in the pool, followed by a query from a DIFFERENT client. The + // stray 'A' is buffered on the pooled connection; whether it reaches a + // client that never issued LISTEN is the question. + runCase(admin, adminOwner, mock, "R11 NotificationResponse while idle in the pool", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(pgmb_simple_result("c", "1", 1)), + step_send(pgmb_notification_response(4242, "unlistened_channel", "surprise")), + step_expect_query(), + step_send(pgmb_simple_result("c", "2", 1)), + step_sleep(300) }); + + // R12: the length field claims the message is longer than the framer's + // 1 GiB ceiling (PGSQL_MAX_BACKEND_MSG_LEN). Must be rejected as malformed + // rather than trusted into an enormous allocation. + runCase(admin, adminOwner, mock, "R12 declared length above the 1 GiB framer cap", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(std::string("D") + pgmb_be32(0x40000001u) + std::string(64, 'x')), + step_sleep(300), step_close() }); + + // R13: a DataRow claiming more columns than it supplies. The framing is + // valid; only the payload is internally inconsistent, so this reaches any + // code that parses row structure rather than just forwarding bytes. + runCase(admin, adminOwner, mock, "R13 DataRow claiming more columns than it carries", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(pgmb_row_description_1col("c", 25) + + std::string("D") + pgmb_be32(4 + 2 + 4) + pgmb_be16(99) + pgmb_be32(0xffffffffu) + + pgmb_command_complete("SELECT 1") + pgmb_ready_for_query('I')), + step_sleep(300) }); + + // R14: a field length that overruns its own message. A parser that trusts + // the field length without bounding it against payload_len over-reads here. + runCase(admin, adminOwner, mock, "R14 DataRow field length overruns the message", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(pgmb_row_description_1col("c", 25) + + std::string("D") + pgmb_be32(4 + 2 + 4 + 2) + pgmb_be16(1) + pgmb_be32(9999) + "ab" + + pgmb_command_complete("SELECT 1") + pgmb_ready_for_query('I')), + step_sleep(300) }); + + // R15: RowDescription announcing far more columns than it describes. + runCase(admin, adminOwner, mock, "R15 RowDescription with a lying column count", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(std::string("T") + pgmb_be32(4 + 2) + pgmb_be16(500) + + pgmb_command_complete("SELECT 0") + pgmb_ready_for_query('I')), + step_sleep(300) }); + + // R16: ReadyForQuery carrying an undefined transaction-status byte. Valid + // values are 'I', 'T', 'E'; the byte is cached as the connection's txn state + // and drives pooling decisions. + runCase(admin, adminOwner, mock, "R16 ReadyForQuery with an invalid status byte", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(pgmb_row_description_1col("c", 25) + pgmb_data_row_1col("1") + + pgmb_command_complete("SELECT 1")), + step_send(std::string("Z") + pgmb_be32(5) + "X"), + step_sleep(300) }); + + // R18: an unterminated CommandComplete tag. The trailing ParseComplete is + // there because its label '1' is a digit: a parse that runs off the end of the + // tag swallows it and reports 91 rows for a tag that says 9. The proxy survives + // either way and ASAN cannot see it, so the row count is the only verdict. + static const char* R18_QUERY = "SELECT 1 AS r18_probe"; + runCase(admin, adminOwner, mock, "R18 CommandComplete tag without a NUL terminator", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(std::string("C") + pgmb_be32(4 + 8) + "INSERT 9" + + std::string("1") + pgmb_be32(4) + // ParseComplete: label is a digit + pgmb_ready_for_query('I')), + step_sleep(300) }, + R18_QUERY); + { + // The alias survives digest normalisation, which isolates this row. Reading + // count_star first stops a probe that never reached the digest from passing + // as a row count of zero. + const std::string seen = adminScalar(admin, + "SELECT IFNULL(SUM(count_star),0) FROM stats_pgsql_query_digest " + "WHERE digest_text LIKE '%r18_probe%'"); + const std::string rows = adminScalar(admin, + "SELECT IFNULL(SUM(sum_rows_affected),0) FROM stats_pgsql_query_digest " + "WHERE digest_text LIKE '%r18_probe%'"); + ok(seen != "0" && seen != "" && rows == "0", + "R18 affected rows recorded from an unterminated tag: %s (query seen %s times); " + "expected 0 -- any value means the tag was parsed past the end of the message, " + "and 91 is the trailing ParseComplete label read as a digit", + rows.empty() ? "(no row)" : rows.c_str(), + seen.empty() ? "(no row)" : seen.c_str()); + } + + // R19: a CommandComplete tag whose trailing number is far wider than 64 + // bits, exercising the strtoull path that produces affected_rows. + runCase(admin, adminOwner, mock, "R19 CommandComplete with an absurd row count", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(pgmb_command_complete("INSERT 0 999999999999999999999999999999") + + pgmb_ready_for_query('I')), + step_sleep(300) }); + + // R20: CopyInResponse arriving where the native drive cannot supply + // CopyData. There is an explicit CopyFail safety net for this + // (PgSQL_Connection.cpp ~2830) that no test reaches with a real backend. + runCase(admin, adminOwner, mock, "R20 unexpected CopyInResponse (CopyFail safety net)", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(std::string("G") + pgmb_be32(4 + 1 + 2) + std::string(1, '\0') + pgmb_be16(0)), + step_expect_message(), // the CopyFail ProxySQL should send + step_send(pgmb_error_response("57014", "COPY aborted by client") + + pgmb_ready_for_query('I')), + step_sleep(300) }); + + // R21: an unsolicited NoticeResponse burst before the result. Notices are + // legal at any time and must not be mistaken for the result stream. + runCase(admin, adminOwner, mock, "R21 notice burst before the result", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(pgmb_error_response("00000", "notice one").replace(0, 1, "N") + + pgmb_error_response("00000", "notice two").replace(0, 1, "N") + + pgmb_simple_result("c", "1", 1)), + step_sleep(300) }); + + // R22: the backend answers a query with nothing but ReadyForQuery — no + // RowDescription, no CommandComplete. The result is "complete" by the + // drain's rule ('Z' seen) yet carries no command outcome. + runCase(admin, adminOwner, mock, "R22 bare ReadyForQuery as the whole result", + { step_expect_startup(), step_send(acceptedHandshake()), step_expect_query(), + step_send(pgmb_ready_for_query('I')), + step_sleep(300) }); + + // ---- final pool cleanliness ------------------------------------------- + // Every hostile connection above should have been torn down. Anything still + // held in the mock's hostgroup after all of it is a connection leak. + usleep(1000000); + { + const int leftover = mockPoolConns(admin); + ok(leftover == 0, + "no backend connections stranded in the mock hostgroup after all hostile cases (found %d)", + leftover); + } + + // ====================================================================== + // R23 -- F5: a RESULTSET answering ProxySQL's OWN housekeeping statement + // ====================================================================== + // + // WHY THIS RUNS LAST, AFTER the pool-cleanliness assertion above. + // On unfixed code this case does not merely fail: it wedges a worker thread + // in a tight loop that keeps logging after the client has gone (measured at + // 17.2M lines / 2.5 GB in under three minutes) and never releases the + // backend connection. Any case scheduled after it would be judged against a + // proxy in that state. It is the last thing this file does for that reason. + // + // THE DEFECT. async_send_simple_command() is what ProxySQL uses to configure + // a backend connection -- here, replaying a session variable the client set. + // If the reply carries a resultset it returns -2 WITHOUT + // clearing query_result (lib/PgSQL_Connection.cpp, the `return -2` after the + // PGSQL_QUERY_RESULT_TUPLE check). Its caller + // handler_again___status_SETTING_GENERIC_VARIABLE() branches on rc == 0 and + // rc == -1 only, so -2 lands in `else { // rc==1 , nothing to do for now }`: + // the session neither fails nor progresses, re-enters, re-detects the same + // uncleared resultset, and logs again. The sibling caller + // handler_again___status_SETTING_INIT_CONNECT() handles -1 and -2 together + // and refuses to retry on -2 -- one caller learned this, the other did not. + // + // THE FIXTURE. `SET bytea_output TO 'escape'` is intercepted by ProxySQL and + // never reaches a backend; it only records that the connection ProxySQL + // picks next must be told about it. `SELECT 1` then opens a backend + // connection, and ProxySQL sends its own `SET bytea_output TO 'escape'` + // before the query -- which is the statement the mock answers with a + // one-row resultset (step_expect_query(true) stops on it instead of + // acknowledging it). + // + // WHAT IS ASSERTED. Only that the client gets an ANSWER. An error is a fine + // outcome -- the backend did something ProxySQL cannot make sense of. What + // must not happen is silence, which is what the loop produces. + { + resetMockPool(admin, g_mock_ip, g_mock_port); + mock.set_script({ step_expect_startup(), step_send(acceptedHandshake()), + step_expect_query(/*stop_at_housekeeping*/ true), + step_send(pgmb_simple_result("c", "1", 1)), + step_sleep(3000) }); + mock.reset_stats(); + + std::string err; + bool timed_out = false; + const bool served = queryThroughProxyBounded( + err, "SET bytea_output TO 'escape'", "SELECT 1", 10000, timed_out); + const std::string first_line = err.substr(0, err.find('\n')); + + const std::string broke = checkInvariants(admin, adminOwner); + int drain_ms = 0; + const int stranded = mockPoolConns(admin, &drain_ms); + + // queries_observed() MUST be 0. The canned resultset is meant for the + // proxy's own SET, which step_expect_query(true) stops on; the client's + // SELECT never reaches the mock because the session is torn down during + // the replay. If ProxySQL ever stops replaying the variable (a change to + // ignore_vars or to the tracked-variable machinery), the first Q the mock + // sees is the client's SELECT instead, the canned result answers THAT -- + // which is perfectly legal -- and the case would pass while testing + // nothing. Asserting it turns that silent false pass into a failure. + const int client_queries_at_mock = mock.queries_observed(); + + ok(!timed_out && broke.empty() && stranded == 0 && client_queries_at_mock == 0, + "R23 F5: a resultset answering ProxySQL's own housekeeping SET must not wedge the " + "session (client %s: %s; mock conns=%d; client queries at mock=%d; pool leftover=%d; " + "drain=%dms)%s%s", + timed_out ? "GOT NOTHING (F5 loop)" : (served ? "served" : "errored"), + first_line.empty() ? "-" : first_line.c_str(), + mock.connections_accepted(), client_queries_at_mock, stranded, drain_ms, + broke.empty() ? "" : " -- BROKE: ", broke.c_str()); + if (client_queries_at_mock != 0) + diag("R23: the client's own query reached the mock -- the variable replay did not " + "happen, so the canned resultset answered the WRONG statement and this case " + "proved nothing. Check ignore_vars / tracked-variable handling for bytea_output."); + if (timed_out) + diag("R23: ProxySQL is very likely still looping and logging RIGHT NOW -- " + "check the size of proxysql.log and restart the proxy before trusting later runs"); + } + + mock.stop(); + return exit_status(); +} diff --git a/test/tap/tests/pgsql-native_pool_reset-t.cpp b/test/tap/tests/pgsql-native_pool_reset-t.cpp new file mode 100644 index 0000000000..356b530cdb --- /dev/null +++ b/test/tap/tests/pgsql-native_pool_reset-t.cpp @@ -0,0 +1,538 @@ +/** + * @file pgsql-native_pool_reset-t.cpp + * @brief Does a pooled backend connection really get reset before the next + * client reuses it, on the native backend protocol as well as libpq? + * + * ProxySQL resets a pooled connection that carries session state the incoming + * client never asked for. The reset is two commands: ROLLBACK when the + * connection is inside a transaction, DISCARD ALL otherwise -- DISCARD ALL is + * rejected by the backend inside a transaction block, so the order matters. + * + * Each scenario runs twice, once with the libpq backend and once with the native + * one, and the two runs must agree; libpq is the oracle. + * S1 (DISCARD ALL): client A sets bytea_output, disappears, client B must + * read back the default. + * S2 (ROLLBACK): client A opens a transaction, disappears, client B must not + * find itself inside one. + * S3 (ROLLBACK, failed transaction): A's statement errors first, so the connection + * goes back marked 'E' rather than 'T' -- a separate route, since a failed + * transaction still counts as reusable. B must be outside the transaction AND + * able to run a statement at all. + * + * A last check counts instead of comparing: a reset that does nothing still leaves + * each connection looking clean, because the broken one is thrown away. Only the + * number of connections opened per client gives that away. + * + * Both assertions are worthless unless B actually inherited A's connection, so + * every case proves it by backend pid and retries when it does not. The pid + * comes from a pg_stat_activity lookup, not from a bare SELECT pg_backend_pid() + * -- ProxySQL answers that one itself with its own session counter, which would + * make every case appear to reuse a connection that was never touched. + * + * bytea_output is the variable under test because ProxySQL tracks it as a + * dynamic variable, and that is what makes it hand the connection to the reset + * path in the first place. + * + * Two things have to be forced or the test silently measures nothing. + * + * A connection is built as either libpq or native once, at creation, and never + * converts. Flipping the variable therefore does nothing to a connection that + * is already pooled, and the "native" run happily reuses a libpq one. So the + * pool is emptied after every switch, and each run additionally asserts that + * its backend pid differs from the previous run's -- equal pids mean the flush + * did not take and the result proves nothing. + * + * The pool also prefers handing out a connection that needs no reset over one + * that does, so as long as a clean connection is available client B will get + * that one and the reset path is never entered. Emptying the pool first leaves + * client A's dirty connection as the only candidate. + * + * The pool is emptied by removing the hostgroup's servers and putting them back + * unchanged; ProxySQL closes a removed server's free connections. + * + * This assumes the hostgroup has ONE backend, which is what every group it is + * registered in provides. With several, ProxySQL picks a server before it picks a + * connection, so client B often opens a new connection to a different server + * instead of inheriting A's; the retries would run out and the run would report no + * verdict rather than a wrong one. + * + * INFRA: legacy-g1 (docker-pgsql16-single, scram-sha-256, no TLS). + * Runtime state is restored in memory at the end -- never SAVE ... TO DISK. + */ +#include +#include +#include +#include +#include +#include + +#include "libpq-fe.h" +#include "command_line.h" +#include "tap.h" +#include "utils.h" + +using PGConnPtr = std::unique_ptr; +CommandLine cl; + +static const int BACKEND_HG = 0; + +// The value client A leaves behind. The PostgreSQL default is 'hex', so seeing +// 'escape' from client B means A's state survived into B's session. +static const char* LEAKED_VALUE = "escape"; +static const char* DEFAULT_VALUE = "hex"; + +static PGConnPtr openConn(const char* host, int port, const char* user, + const char* pass, const char* db) { + std::stringstream ss; + ss << "host=" << host << " port=" << port << " user=" << user << " password=" << pass; + if (db && *db) ss << " dbname=" << db; + ss << " sslmode=disable connect_timeout=10"; + return PGConnPtr(PQconnectdb(ss.str().c_str()), &PQfinish); +} + +static PGConnPtr createAdminConn() { + return openConn(cl.pgsql_admin_host, cl.pgsql_admin_port, + cl.admin_username, cl.admin_password, nullptr); +} +static PGConnPtr createClientConn() { + return openConn(cl.pgsql_host, cl.pgsql_port, cl.pgsql_username, + cl.pgsql_password, cl.pgsql_username); +} + +static bool execAdmin(PGconn* admin, const std::string& q) { + PGresult* r = PQexec(admin, q.c_str()); + ExecStatusType st = PQresultStatus(r); + bool good = (st == PGRES_COMMAND_OK || st == PGRES_TUPLES_OK); + if (!good) diag("admin failed: %s -- %s", q.c_str(), PQerrorMessage(admin)); + PQclear(r); + return good; +} + +static std::string scalar(PGconn* c, const std::string& q) { + PGresult* r = PQexec(c, q.c_str()); + std::string v; + if (PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) > 0 && !PQgetisnull(r, 0, 0)) + v = PQgetvalue(r, 0, 0); + PQclear(r); + return v; +} + +// The real backend pid. A bare "SELECT pg_backend_pid()" never leaves ProxySQL +// -- it matches an intercepted digest prefix and is answered with the ProxySQL +// session id -- so it cannot tell us which backend connection served a client. +// Selecting through pg_stat_activity moves the digest past that prefix, so the +// query runs on the backend and the answer identifies the connection. +static std::string backendPid(PGconn* c) { + return scalar(c, "SELECT pid FROM pg_stat_activity WHERE pid = pg_backend_pid()"); +} + +static bool setNativeMode(PGconn* admin, bool enabled) { + return execAdmin(admin, std::string("SET pgsql-use_native_backend_protocol='") + + (enabled ? "true" : "false") + "'") && + execAdmin(admin, "LOAD PGSQL VARIABLES TO RUNTIME"); +} + +// Drop every pooled connection for the hostgroup. Only removing the server does +// that -- ProxySQL closes a removed server's free connections when the change is +// loaded. Setting status to OFFLINE_HARD is not enough: the server object and its +// pool survive it. So the rows are read, deleted, and put back exactly as they +// were, every column included, because anything left out of the re-insert would +// come back as a table default and stay that way at runtime. +static const char* SERVER_COLS = + "hostgroup_id,hostname,port,status,weight,compression,max_connections," + "max_replication_lag,use_ssl,max_latency_ms,comment"; + +static std::string sqlQuote(const std::string& v) { + std::string out = "'"; + for (char c : v) { if (c == '\'') out += "''"; else out += c; } + out += "'"; + return out; +} + +// Read the hostgroup's servers back as the INSERT statements that would recreate +// them. Kept as a snapshot at startup so the rows can be put back if a flush dies +// half way through -- between the DELETE and the re-INSERT the hostgroup has no +// servers at all, and leaving it that way would fail every later test in the group. +static std::vector snapshotServers(PGconn* admin, int hg) { + std::vector inserts; + std::stringstream sel; + sel << "SELECT " << SERVER_COLS << " FROM pgsql_servers WHERE hostgroup_id=" << hg; + PGresult* res = PQexec(admin, sel.str().c_str()); + if (PQresultStatus(res) != PGRES_TUPLES_OK) { + diag("snapshotServers: cannot read pgsql_servers for hg %d: %s", hg, PQerrorMessage(admin)); + PQclear(res); + return inserts; + } + const int rows = PQntuples(res), cols = PQnfields(res); + for (int r = 0; r < rows; r++) { + std::stringstream ins; + ins << "INSERT INTO pgsql_servers (" << SERVER_COLS << ") VALUES ("; + for (int c = 0; c < cols; c++) { + if (c) ins << ","; + if (PQgetisnull(res, r, c)) ins << "NULL"; + else ins << sqlQuote(PQgetvalue(res, r, c)); + } + ins << ")"; + inserts.push_back(ins.str()); + } + PQclear(res); + return inserts; +} + +static bool applyServers(PGconn* admin, const std::vector& inserts) { + for (const auto& q : inserts) { + if (!execAdmin(admin, q)) return false; + } + return execAdmin(admin, "LOAD PGSQL SERVERS TO RUNTIME"); +} + +static bool flushBackendPool(PGconn* admin, int hg) { + const std::vector inserts = snapshotServers(admin, hg); + if (inserts.empty()) { + diag("flushBackendPool: no servers in hg %d; refusing to flush", hg); + return false; + } + std::stringstream del; + del << "DELETE FROM pgsql_servers WHERE hostgroup_id=" << hg; + if (!execAdmin(admin, del.str())) return false; + if (!execAdmin(admin, "LOAD PGSQL SERVERS TO RUNTIME")) return false; // closes the free connections + if (!applyServers(admin, inserts)) return false; + usleep(200000); // let the servers come back online before anyone connects + return true; +} + +// Select a backend mode and guarantee the next connection is built under it. +static bool selectMode(PGconn* admin, bool native) { + return setNativeMode(admin, native) && flushBackendPool(admin, BACKEND_HG); +} + +// Result of one scenario run. +struct Outcome { + bool ran = false; // did B actually inherit A's backend connection? + bool usable = false; // did B's first statement on it succeed? + std::string observed; // what B read back + std::string a_pid, b_pid; +}; + +/** + * Scenario 1 — a plain session variable left behind. + * + * Retries until client B lands on the same backend connection client A used. + * The pool was emptied just before, so A's is the only one there to hand out. + */ +static Outcome runVariableLeakScenario(int max_attempts) { + Outcome out; + for (int attempt = 0; attempt < max_attempts; attempt++) { + std::string a_pid; + { + auto A = createClientConn(); + if (!A || PQstatus(A.get()) != CONNECTION_OK) { + diag("attempt %d: client A could not connect: %s", attempt, + A ? PQerrorMessage(A.get()) : "(null)"); + usleep(200000); continue; + } + a_pid = backendPid(A.get()); + if (a_pid.empty()) + diag("attempt %d: client A got no backend pid: %s", attempt, PQerrorMessage(A.get())); + PGresult* r = PQexec(A.get(), (std::string("SET bytea_output = '") + LEAKED_VALUE + "'").c_str()); + const bool set_ok = (PQresultStatus(r) == PGRES_COMMAND_OK); + PQclear(r); + if (!set_ok || a_pid.empty()) { usleep(200000); continue; } + // Confirm the backend really took it while A is still connected. + if (scalar(A.get(), "SELECT current_setting('bytea_output')") != LEAKED_VALUE) { + usleep(200000); continue; + } + } // A disconnects here; its backend connection returns to the pool + + usleep(300000); // let the connection settle back into the pool + + { + auto B = createClientConn(); + if (!B || PQstatus(B.get()) != CONNECTION_OK) { usleep(200000); continue; } + // Both values in one round trip. Read as two queries, multiplexing can + // answer them from different backend connections, and the pid check would + // then vouch for a connection that did not produce the value. + std::string b_pid, observed; + { + PGresult* r = PQexec(B.get(), + "SELECT (SELECT pid FROM pg_stat_activity WHERE pid = pg_backend_pid()), " + "current_setting('bytea_output')"); + if (PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) > 0) { + if (!PQgetisnull(r, 0, 0)) b_pid = PQgetvalue(r, 0, 0); + if (!PQgetisnull(r, 0, 1)) observed = PQgetvalue(r, 0, 1); + } + PQclear(r); + } + if (b_pid.empty() || b_pid != a_pid) { + diag("attempt %d: client B landed on backend pid %s, not A's %s; retrying", + attempt, b_pid.c_str(), a_pid.c_str()); + usleep(300000); + continue; + } + out.ran = true; + out.observed = observed; + out.a_pid = a_pid; + out.b_pid = b_pid; + return out; + } + } + return out; +} + +/** + * Scenario 2 — a connection returned while a transaction is still open. + * Scenario 3 — the same, but the transaction has already failed. + * + * A connection handed back with a transaction still open is rolled back before + * anyone else gets it, so client B must never find itself inside one. Read as B's + * transaction status: 'I' (idle) is clean, 'T' (in a transaction block) means A's + * transaction survived into B's session, 'E' means it survived and is broken. + * + * abort_txn picks which of the two. It matters because a failed transaction takes + * a different route: ProxySQL still counts the connection reusable, so it goes to + * the reset path rather than being destroyed. A reset that reports success without + * sending anything then pools a connection PostgreSQL will refuse every statement + * on, and the next client gets it. + */ +static Outcome runOpenTransactionScenario(int max_attempts, bool abort_txn) { + Outcome out; + for (int attempt = 0; attempt < max_attempts; attempt++) { + std::string a_pid; + { + auto A = createClientConn(); + if (!A || PQstatus(A.get()) != CONNECTION_OK) { + diag("attempt %d: client A could not connect: %s", attempt, + A ? PQerrorMessage(A.get()) : "(null)"); + usleep(200000); continue; + } + a_pid = backendPid(A.get()); + if (a_pid.empty()) + diag("attempt %d: client A got no backend pid: %s", attempt, PQerrorMessage(A.get())); + if (a_pid.empty()) { usleep(200000); continue; } + PGresult* r = PQexec(A.get(), "BEGIN"); + const bool began = (PQresultStatus(r) == PGRES_COMMAND_OK); + PQclear(r); + if (!began) { usleep(200000); continue; } + if (abort_txn) { + // Fail on purpose: the backend moves to 'E' and refuses everything + // until the transaction ends. Division by zero needs no fixture. + PGresult* bad = PQexec(A.get(), "SELECT 1/0"); + const bool did_fail = (PQresultStatus(bad) == PGRES_FATAL_ERROR); + PQclear(bad); + if (!did_fail) { usleep(200000); continue; } + if (PQtransactionStatus(A.get()) != PQTRANS_INERROR) { usleep(200000); continue; } + } else { + // Do real work inside the transaction so it is genuinely open. + PQclear(PQexec(A.get(), "CREATE TEMP TABLE IF NOT EXISTS pool_reset_probe(x int)")); + PQclear(PQexec(A.get(), "INSERT INTO pool_reset_probe VALUES (1)")); + } + } // A vanishes mid-transaction + + usleep(400000); + + { + auto B = createClientConn(); + if (!B || PQstatus(B.get()) != CONNECTION_OK) { usleep(200000); continue; } + const std::string b_pid = backendPid(B.get()); + if (b_pid.empty() || b_pid != a_pid) { usleep(300000); continue; } + // PQtransactionStatus reflects the last ReadyForQuery the client saw. + PGresult* first = PQexec(B.get(), "SELECT 1"); + const bool first_ok = (PQresultStatus(first) == PGRES_TUPLES_OK); + PQclear(first); + const PGTransactionStatusType ts = PQtransactionStatus(B.get()); + out.ran = true; + out.usable = first_ok; + out.observed = (ts == PQTRANS_IDLE) ? "I" + : (ts == PQTRANS_INTRANS) ? "T" + : (ts == PQTRANS_INERROR) ? "E" : "?"; + out.a_pid = a_pid; + out.b_pid = b_pid; + return out; + } + } + return out; +} + +// ConnOK for the hostgroup: backend connections ProxySQL has opened since it +// started. It only ever goes up, so the interesting number is the difference +// across a workload, not the value. +static int connOK(PGconn* admin, int hg) { + std::stringstream q; + q << "SELECT ConnOK FROM stats_pgsql_connection_pool WHERE hostgroup=" << hg; + const std::string v = scalar(admin, q.str()); + return v.empty() ? -1 : atoi(v.c_str()); +} + +// A reset that only claims to have run hands every client a connection stuck in +// the aborted transaction; its first statement is refused and the connection is +// thrown away, so the cost tracks the client count instead of staying flat. +// Measured here: 1-2 connections with the reset working, 16 without. +static int abandonBudget(PGconn* admin, int sessions) { + const int before = connOK(admin, BACKEND_HG); + if (before < 0) return -1; + for (int i = 0; i < sessions; i++) { + auto A = createClientConn(); + if (!A || PQstatus(A.get()) != CONNECTION_OK) continue; + PQclear(PQexec(A.get(), "BEGIN")); + PQclear(PQexec(A.get(), "SELECT 1/0")); + // A disconnects here, still inside the failed transaction. + } + usleep(500000); // let the last connection finish going back to the pool + const int after = connOK(admin, BACKEND_HG); + return (after < 0) ? -1 : (after - before); +} + +int main(int, char**) { + // Per scenario: libpq oracle ran, native ran, native used a fresh backend + // connection, native matches oracle. x3 scenarios = 12, plus one summary + // assertion naming the leak explicitly, plus S3's usability and connection + // budget checks. + plan(15); + + if (cl.getEnv()) return exit_status(); + + auto adminOwner = createAdminConn(); + if (!adminOwner || PQstatus(adminOwner.get()) != CONNECTION_OK) + BAIL_OUT("cannot proceed without an admin connection"); + PGconn* admin = adminOwner.get(); + + // ---- save runtime state ------------------------------------------------ + PGresult* sv = PQexec(admin, + "SELECT variable_value FROM global_variables WHERE variable_name='pgsql-use_native_backend_protocol'"); + std::string saved_native; + if (PQresultStatus(sv) == PGRES_TUPLES_OK && PQntuples(sv) > 0) + saved_native = PQgetvalue(sv, 0, 0); + PQclear(sv); + + const std::vector saved_servers = snapshotServers(admin, BACKEND_HG); + if (saved_servers.empty()) + BAIL_OUT("no pgsql_servers rows for the backend hostgroup; nothing to test against"); + + auto restore = [&]() { + // If a flush died between its DELETE and its re-INSERT the hostgroup is + // empty; put the startup snapshot back before anything else runs. + std::stringstream cnt; + cnt << "SELECT count(*) FROM pgsql_servers WHERE hostgroup_id=" << BACKEND_HG; + if (scalar(admin, cnt.str()) == "0") + applyServers(admin, saved_servers); + // Put the setting back, then empty the pool again. Without the second flush + // the pool keeps the connections built during the last phase, and the next + // test would run over native connections while the setting reads false -- + // a connection never changes mode after it is created. + if (!saved_native.empty()) { + setNativeMode(admin, saved_native == "true" || saved_native == "1"); + flushBackendPool(admin, BACKEND_HG); + } + }; + + usleep(500000); + + // Generous: each attempt is cheap, and a scenario that never establishes its + // precondition produces NO verdict at all — which is worse than a slow test. + const int ATTEMPTS = 15; + + // ================= Scenario 1: session variable ========================== + if (!selectMode(admin, false)) { restore(); BAIL_OUT("cannot select libpq mode"); } + const Outcome libpq_var = runVariableLeakScenario(ATTEMPTS); + ok(libpq_var.ran, + "S1 oracle: libpq run reused backend pid %s for both clients", + libpq_var.a_pid.c_str()); + + if (!selectMode(admin, true)) { restore(); BAIL_OUT("cannot select native mode"); } + const Outcome native_var = runVariableLeakScenario(ATTEMPTS); + ok(native_var.ran, + "S1 native: run reused backend pid %s for both clients", + native_var.a_pid.c_str()); + + ok(native_var.ran && libpq_var.ran && native_var.a_pid != libpq_var.a_pid, + "S1 native run is on a different backend connection than the libpq run " + "(libpq pid %s, native pid %s); equal pids mean the pool was not flushed " + "and the native run measured a libpq connection", + libpq_var.a_pid.c_str(), native_var.a_pid.c_str()); + + ok(libpq_var.ran && native_var.ran && libpq_var.observed == native_var.observed, + "S1 bytea_output after connection reuse: libpq='%s' native='%s' " + "(a mismatch means DISCARD ALL never reached the backend)", + libpq_var.observed.c_str(), native_var.observed.c_str()); + + // ================= Scenario 2: open transaction ========================== + if (!selectMode(admin, false)) { restore(); BAIL_OUT("cannot select libpq mode"); } + const Outcome libpq_txn = runOpenTransactionScenario(ATTEMPTS, false); + ok(libpq_txn.ran, "S2 oracle: libpq run reused backend pid %s for both clients", + libpq_txn.a_pid.c_str()); + + if (!selectMode(admin, true)) { restore(); BAIL_OUT("cannot select native mode"); } + const Outcome native_txn = runOpenTransactionScenario(ATTEMPTS, false); + ok(native_txn.ran, "S2 native: run reused backend pid %s for both clients", + native_txn.a_pid.c_str()); + + ok(native_txn.ran && libpq_txn.ran && native_txn.a_pid != libpq_txn.a_pid, + "S2 native run is on a different backend connection than the libpq run " + "(libpq pid %s, native pid %s); equal pids mean the pool was not flushed " + "and the native run measured a libpq connection", + libpq_txn.a_pid.c_str(), native_txn.a_pid.c_str()); + + ok(libpq_txn.ran && native_txn.ran && libpq_txn.observed == native_txn.observed, + "S2 transaction status inherited by the next client: libpq='%s' native='%s' " + "(a mismatch means ROLLBACK never reached the backend)", + libpq_txn.observed.c_str(), native_txn.observed.c_str()); + + // ================= Scenario 3: aborted transaction ======================= + if (!selectMode(admin, false)) { restore(); BAIL_OUT("cannot select libpq mode"); } + const Outcome libpq_abort = runOpenTransactionScenario(ATTEMPTS, true); + ok(libpq_abort.ran, "S3 oracle: libpq run reused backend pid %s for both clients", + libpq_abort.a_pid.c_str()); + + if (!selectMode(admin, true)) { restore(); BAIL_OUT("cannot select native mode"); } + const Outcome native_abort = runOpenTransactionScenario(ATTEMPTS, true); + ok(native_abort.ran, "S3 native: run reused backend pid %s for both clients", + native_abort.a_pid.c_str()); + + ok(native_abort.ran && libpq_abort.ran && native_abort.a_pid != libpq_abort.a_pid, + "S3 native run is on a different backend connection than the libpq run " + "(libpq pid %s, native pid %s); equal pids mean the pool was not flushed " + "and the native run measured a libpq connection", + libpq_abort.a_pid.c_str(), native_abort.a_pid.c_str()); + + ok(libpq_abort.ran && native_abort.ran && libpq_abort.observed == native_abort.observed, + "S3 transaction status inherited after an ABORTED transaction: libpq='%s' " + "native='%s' (a mismatch means the failed transaction was never rolled back)", + libpq_abort.observed.c_str(), native_abort.observed.c_str()); + + ok(native_abort.ran && native_abort.usable, + "S3 native: the next client can actually use the connection it was given%s", + (native_abort.ran && !native_abort.usable) + ? " -- ITS FIRST STATEMENT WAS REFUSED, the aborted transaction came with it" : ""); + + // ---- explicit statement of the leak ------------------------------------ + // Separate from the differential so a reader sees the concrete claim, not + // just "two strings differ". The oracle establishes what clean looks like. + { + const bool leaked = native_var.ran && native_var.observed == LEAKED_VALUE; + ok(!leaked, + "client B must not observe client A's session state: expected '%s', native gave '%s'%s", + DEFAULT_VALUE, native_var.observed.c_str(), + leaked ? " -- SESSION STATE CROSSED BETWEEN CLIENTS" : ""); + } + + + // ---- what the reset costs when it does not happen ----------------------- + // The differential above proves one connection is clean. This proves the pool + // as a whole is: a reset that silently does nothing still leaves each single + // connection looking fine after ProxySQL throws it away, and only the count of + // connections opened gives that away. + { + if (!selectMode(admin, true)) { restore(); BAIL_OUT("cannot select native mode"); } + const int SESSIONS = 30; + const int BUDGET = 10; // clean runs cost 1-2; a dead reset costs one per session + const int used = abandonBudget(admin, SESSIONS); + ok(used >= 0 && used <= BUDGET, + "%d clients abandoning a failed transaction opened %d backend connections " + "(budget %d); one per client means every reuse handed over a broken " + "connection and it was thrown away", + SESSIONS, used, BUDGET); + } + + restore(); + return exit_status(); +} diff --git a/test/tap/tests/pgsql-native_prepared-t.cpp b/test/tap/tests/pgsql-native_prepared-t.cpp index b99fde98bb..effbb80ac6 100644 --- a/test/tap/tests/pgsql-native_prepared-t.cpp +++ b/test/tap/tests/pgsql-native_prepared-t.cpp @@ -67,8 +67,11 @@ #include #include #include +#include #include #include +#include +#include #include "libpq-fe.h" #include "command_line.h" #include "tap.h" @@ -94,7 +97,7 @@ static PGConnPtr open_admin_conn() { return PGConnPtr(PQconnectdb(ss.str().c_str()), &PQfinish); } -static PGConnPtr open_client_conn() { +static PGConnPtr open_client_conn(const std::string& extra_opts = "") { std::stringstream ss; ss << "host=" << cl.pgsql_host << " port=" << cl.pgsql_port @@ -102,6 +105,7 @@ static PGConnPtr open_client_conn() { << " password=" << cl.pgsql_password << " dbname=" << cl.pgsql_username << " sslmode=disable"; + if (!extra_opts.empty()) ss << " " << extra_opts; return PGConnPtr(PQconnectdb(ss.str().c_str()), &PQfinish); } @@ -876,12 +880,472 @@ static ExtQCaseRunResult run_describe_cached(PGconn* admin, bool first_native, return {result_match, fell_back, det.str()}; } +// =========================================================================== +// DEALLOCATE-forwarding regression (ABSOLUTE, not differential). +// +// ProxySQL used to intercept every single-statement DEALLOCATE and resolve the +// name only against local_stmts -- which tracks extended-query (binary) +// prepares. A name from a SQL-level PREPARE is never in that map, so ProxySQL +// answered with a fabricated "prepared statement does not exist" and never +// forwarded the command, even though the statement was alive on the backend. +// +// The differential P0/P7 cases above cannot catch this: the interception lives +// in the protocol-independent client handler, so libpq-through-ProxySQL and +// native-through-ProxySQL are affected identically and still match each other. +// These checks assert the real-PostgreSQL outcome directly. Driven on the +// native path here; the libpq-path equivalent lives in +// pgsql-extended_query_protocol_test-t (test_deallocate_sql_prepared_via_simple_query). +// =========================================================================== +static const int N_DEALLOC_REG_PER_MODE = 10; + +static void run_dealloc_regression(PGconn* admin, bool native, + const std::vector& saved) { + const char* m = native ? "native" : "libpq"; + if (!setNativeMode(admin, native) || !flushBackendPool(admin, BACKEND_HG, saved)) { + for (int i = 0; i < N_DEALLOC_REG_PER_MODE; i++) + ok(false, "[%s] dealloc-regression: admin setup failed", m); + return; + } + PGConnPtr c = open_client_conn(); + if (!c || PQstatus(c.get()) != CONNECTION_OK) { + for (int i = 0; i < N_DEALLOC_REG_PER_MODE; i++) + ok(false, "[%s] dealloc-regression: client connect failed", m); + return; + } + PGconn* cc = c.get(); + + // 0. On a fresh, unpinned connection, a DEALLOCATE of an unknown name is + // answered locally (no SQL PREPARE happened, so it cannot exist) rather + // than acquiring a backend connection just to fail. + { PGresult* r = PQexec(cc, "DEALLOCATE dealloc_reg_unpinned"); + ok(PQresultStatus(r) == PGRES_FATAL_ERROR, + "[%s] DEALLOCATE of an unknown name on a fresh connection errors -> %s", + m, PQresStatus(PQresultStatus(r))); + PQclear(r); } + + // 1. A SQL-level PREPARE succeeds (forwarded to the backend as usual). + { PGresult* r = PQexec(cc, "PREPARE dealloc_reg AS SELECT 42"); + ok(PQresultStatus(r) == PGRES_COMMAND_OK, + "[%s] SQL PREPARE dealloc_reg -> %s", m, PQresStatus(PQresultStatus(r))); + PQclear(r); } + + // 2. EXECUTE returns the row: the statement is genuinely live on the backend. + { PGresult* r = PQexec(cc, "EXECUTE dealloc_reg"); + bool good = PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) == 1 + && std::string(PQgetvalue(r, 0, 0)) == "42"; + ok(good, "[%s] EXECUTE dealloc_reg returns 42", m); + PQclear(r); } + + // 3. THE FIX: DEALLOCATE of a SQL-prepared statement is forwarded and + // succeeds, instead of a fabricated "does not exist" error. + { PGresult* r = PQexec(cc, "DEALLOCATE dealloc_reg"); + ok(PQresultStatus(r) == PGRES_COMMAND_OK, + "[%s] DEALLOCATE dealloc_reg succeeds (forwarded, not fabricated) -> %s", + m, PQresStatus(PQresultStatus(r))); + PQclear(r); } + + // 4. It really was deallocated on the backend: a second EXECUTE now fails. + { PGresult* r = PQexec(cc, "EXECUTE dealloc_reg"); + ok(PQresultStatus(r) == PGRES_FATAL_ERROR, + "[%s] EXECUTE after DEALLOCATE fails, statement is gone -> %s", + m, PQresStatus(PQresultStatus(r))); + PQclear(r); } + + // 5. A mistyped/unknown name returns the backend's real error (not silent OK). + { PGresult* r = PQexec(cc, "DEALLOCATE dealloc_reg_never_prepared"); + ok(PQresultStatus(r) == PGRES_FATAL_ERROR, + "[%s] DEALLOCATE of an unknown name errors -> %s", + m, PQresStatus(PQresultStatus(r))); + PQclear(r); } + + // 6. ...and the session is still usable afterwards: a typo must not wedge it + // or lock the connection onto a hostgroup. + { PGresult* r = PQexec(cc, "SELECT 1"); + bool good = PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) == 1 + && std::string(PQgetvalue(r, 0, 0)) == "1"; + ok(good, "[%s] session still usable after a bogus DEALLOCATE", m); + PQclear(r); } + + // 7-9. ALL-prefix guard: a statement whose name starts with "all" must be + // treated as a normal DEALLOCATE (forwarded), not mistaken for + // DEALLOCATE ALL. Without the exact-match fix, DEALLOCATE all_users + // returns the tag "DEALLOCATE ALL" and never frees the statement. + { PGresult* r = PQexec(cc, "PREPARE all_users AS SELECT 7"); + ok(PQresultStatus(r) == PGRES_COMMAND_OK, "[%s] PREPARE all_users", m); + PQclear(r); } + { PGresult* r = PQexec(cc, "DEALLOCATE all_users"); + const char* tag = PQcmdStatus(r); + ok(PQresultStatus(r) == PGRES_COMMAND_OK && tag && strcmp(tag, "DEALLOCATE") == 0, + "[%s] DEALLOCATE all_users -> tag '%s' (a normal DEALLOCATE, not DEALLOCATE ALL)", + m, tag ? tag : ""); + PQclear(r); } + { PGresult* r = PQexec(cc, "EXECUTE all_users"); + ok(PQresultStatus(r) == PGRES_FATAL_ERROR, + "[%s] EXECUTE all_users after DEALLOCATE errors, so it was really deallocated -> %s", + m, PQresStatus(PQresultStatus(r))); + PQclear(r); } +} + +// =========================================================================== +// Cross-protocol DEALLOCATE (the tracked side of the same fix). +// +// A statement prepared via the EXTENDED (binary) protocol -- PQprepare -- is +// tracked by ProxySQL in local_stmts and RENAMED on the backend +// (proxysql_ps_). A SQL-text DEALLOCATE of its client name must therefore +// stay handled LOCALLY (client_close finds it) and must NOT be forwarded: +// forwarding the client name would fail on the backend, which knows it only by +// the renamed name. This guards that the DEALLOCATE-forwarding fix draws the +// line at the tracked/untracked boundary, not at "any DEALLOCATE". +// =========================================================================== +static const int N_DEALLOC_XPROTO_PER_MODE = 5; + +static void run_dealloc_xproto_regression(PGconn* admin, bool native, + const std::vector& saved) { + const char* m = native ? "native" : "libpq"; + if (!setNativeMode(admin, native) || !flushBackendPool(admin, BACKEND_HG, saved)) { + for (int i = 0; i < N_DEALLOC_XPROTO_PER_MODE; i++) + ok(false, "[%s] xproto-dealloc: admin setup failed", m); + return; + } + PGConnPtr c = open_client_conn(); + if (!c || PQstatus(c.get()) != CONNECTION_OK) { + for (int i = 0; i < N_DEALLOC_XPROTO_PER_MODE; i++) + ok(false, "[%s] xproto-dealloc: client connect failed", m); + return; + } + PGconn* cc = c.get(); + + // 1. Named binary prepare (extended protocol): ProxySQL tracks it and renames + // it on the backend. + { PGresult* r = PQprepare(cc, "xp_bp", "SELECT 77", 0, nullptr); + ok(PQresultStatus(r) == PGRES_COMMAND_OK, + "[%s] binary PQprepare xp_bp -> %s", m, PQresStatus(PQresultStatus(r))); + PQclear(r); } + + // 2. Binary execute returns the row. + { PGresult* r = PQexecPrepared(cc, "xp_bp", 0, nullptr, nullptr, nullptr, 0); + bool good = PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) == 1 + && std::string(PQgetvalue(r, 0, 0)) == "77"; + ok(good, "[%s] binary EXECUTE xp_bp returns 77", m); + PQclear(r); } + + // 3. SQL-text DEALLOCATE of the binary name is handled locally and succeeds + // -- it must NOT be forwarded (the backend name differs). + { PGresult* r = PQexec(cc, "DEALLOCATE xp_bp"); + ok(PQresultStatus(r) == PGRES_COMMAND_OK, + "[%s] SQL DEALLOCATE of a binary-prepared name succeeds (handled locally) -> %s", + m, PQresStatus(PQresultStatus(r))); + PQclear(r); } + + // 4. It is really gone: re-executing the binary statement now fails. + { PGresult* r = PQexecPrepared(cc, "xp_bp", 0, nullptr, nullptr, nullptr, 0); + ok(PQresultStatus(r) == PGRES_FATAL_ERROR, + "[%s] binary EXECUTE after DEALLOCATE fails, statement is gone -> %s", + m, PQresStatus(PQresultStatus(r))); + PQclear(r); } + + // 5. Session still usable. + { PGresult* r = PQexec(cc, "SELECT 1"); + bool good = PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) == 1 + && std::string(PQgetvalue(r, 0, 0)) == "1"; + ok(good, "[%s] session still usable after cross-protocol DEALLOCATE", m); + PQclear(r); } +} + +// =========================================================================== +// DEALLOCATE ALL matrix. +// +// DEALLOCATE ALL now forwards to the pinned backend and releases ProxySQL's +// backend-side statement bookkeeping (backend_close_all), so SQL-level PREPARE +// statements are actually freed while binary statements and the shared global +// statement cache stay consistent. Scenarios: +// S1 SQL-only S2 binary-only S3 mixed (SQL + binary) +// S4 nothing prepared S5 cross-connection isolation S6 repeated cycles +// S7 aborted-txn (DEALLOCATE ALL rejected -> statements survive, guard keeps tracking) +// =========================================================================== +static const int N_DALLALL_MATRIX = 36; + +static bool exec_ok(PGconn* c, const char* q) { + PGresult* r = PQexec(c, q); + bool good = PQresultStatus(r) == PGRES_COMMAND_OK || PQresultStatus(r) == PGRES_TUPLES_OK; + PQclear(r); + return good; +} +static bool val_is(PGresult* r, const char* v) { + return PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) == 1 + && std::string(PQgetvalue(r, 0, 0)) == v; +} + +static void run_dealloc_all_matrix(PGconn* admin, bool native, + const std::vector& saved) { + const char* m = native ? "native" : "libpq"; + auto fail = [&](int n, const char* why) { + for (int i = 0; i < n; i++) ok(false, "[%s] dealloc-all matrix: %s", m, why); + }; + if (!setNativeMode(admin, native) || !flushBackendPool(admin, BACKEND_HG, saved)) { + fail(N_DALLALL_MATRIX, "admin setup failed"); + return; + } + + // ---- S1: SQL-only. Pinned by SQL PREPARE -> DEALLOCATE ALL forwards; the + // statements are actually freed on the backend. ---- + { + PGConnPtr c = open_client_conn(); PGconn* cc = c.get(); + if (!c || PQstatus(cc) != CONNECTION_OK) { fail(5, "S1 conn failed"); } + else { + PGresult* r; + r = PQexec(cc, "PREPARE s1 AS SELECT 1"); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "[%s] S1 PREPARE s1", m); PQclear(r); + r = PQexec(cc, "PREPARE s2 AS SELECT 2"); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "[%s] S1 PREPARE s2", m); PQclear(r); + (void)exec_ok(cc, "DEALLOCATE ALL"); + r = PQexec(cc, "EXECUTE s1"); ok(PQresultStatus(r) == PGRES_FATAL_ERROR, "[%s] S1 EXECUTE s1 freed -> %s", m, PQresStatus(PQresultStatus(r))); PQclear(r); + r = PQexec(cc, "EXECUTE s2"); ok(PQresultStatus(r) == PGRES_FATAL_ERROR, "[%s] S1 EXECUTE s2 freed -> %s", m, PQresStatus(PQresultStatus(r))); PQclear(r); + r = PQexec(cc, "PREPARE s1 AS SELECT 1"); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "[%s] S1 re-PREPARE s1 (backend cleared) -> %s", m, PQresStatus(PQresultStatus(r))); PQclear(r); + } + } + + // ---- S2: binary-only. Not pinned -> DEALLOCATE ALL stays local; the client + // name is dropped, but the cached statement is reusable (no desync). ---- + { + PGConnPtr c = open_client_conn(); PGconn* cc = c.get(); + if (!c || PQstatus(cc) != CONNECTION_OK) { fail(5, "S2 conn failed"); } + else { + PGresult* r; + r = PQprepare(cc, "b1", "SELECT 88", 0, nullptr); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "[%s] S2 binary prepare b1", m); PQclear(r); + r = PQexecPrepared(cc, "b1", 0, nullptr, nullptr, nullptr, 0); ok(val_is(r, "88"), "[%s] S2 EXECUTE b1 = 88", m); PQclear(r); + (void)exec_ok(cc, "DEALLOCATE ALL"); + r = PQexecPrepared(cc, "b1", 0, nullptr, nullptr, nullptr, 0); ok(PQresultStatus(r) == PGRES_FATAL_ERROR, "[%s] S2 EXECUTE b1 after DEALLOCATE ALL fails -> %s", m, PQresStatus(PQresultStatus(r))); PQclear(r); + r = PQprepare(cc, "b2", "SELECT 88", 0, nullptr); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "[%s] S2 re-prepare same-hash b2 (no desync)", m); PQclear(r); + r = PQexecPrepared(cc, "b2", 0, nullptr, nullptr, nullptr, 0); ok(val_is(r, "88"), "[%s] S2 EXECUTE b2 = 88", m); PQclear(r); + } + } + + // ---- S3: mixed. SQL PREPARE pins the connection; a binary prepare then lands + // on it. DEALLOCATE ALL forwards + backend_close_all: the SQL stmt is + // freed and the binary bookkeeping stays consistent (same-hash reuse + // still works). ---- + { + PGConnPtr c = open_client_conn(); PGconn* cc = c.get(); + if (!c || PQstatus(cc) != CONNECTION_OK) { fail(7, "S3 conn failed"); } + else { + PGresult* r; + r = PQexec(cc, "PREPARE sp AS SELECT 5"); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "[%s] S3 SQL PREPARE sp", m); PQclear(r); + r = PQprepare(cc, "bp", "SELECT 88", 0, nullptr); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "[%s] S3 binary prepare bp", m); PQclear(r); + r = PQexecPrepared(cc, "bp", 0, nullptr, nullptr, nullptr, 0); ok(val_is(r, "88"), "[%s] S3 EXECUTE bp = 88", m); PQclear(r); + (void)exec_ok(cc, "DEALLOCATE ALL"); + r = PQexec(cc, "EXECUTE sp"); ok(PQresultStatus(r) == PGRES_FATAL_ERROR, "[%s] S3 EXECUTE sp freed -> %s", m, PQresStatus(PQresultStatus(r))); PQclear(r); + r = PQexec(cc, "PREPARE sp AS SELECT 5"); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "[%s] S3 re-PREPARE sp (backend cleared) -> %s", m, PQresStatus(PQresultStatus(r))); PQclear(r); + r = PQprepare(cc, "bp2", "SELECT 88", 0, nullptr); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "[%s] S3 re-prepare same-hash bp2 (no desync after backend_close_all)", m); PQclear(r); + r = PQexecPrepared(cc, "bp2", 0, nullptr, nullptr, nullptr, 0); ok(val_is(r, "88"), "[%s] S3 EXECUTE bp2 = 88", m); PQclear(r); + } + } + + // ---- S4: nothing prepared. DEALLOCATE ALL on a fresh connection is harmless + // and the session stays usable. ---- + { + PGConnPtr c = open_client_conn(); PGconn* cc = c.get(); + if (!c || PQstatus(cc) != CONNECTION_OK) { fail(2, "S4 conn failed"); } + else { + ok(exec_ok(cc, "DEALLOCATE ALL"), "[%s] S4 DEALLOCATE ALL on fresh connection ok", m); + PGresult* r = PQexec(cc, "SELECT 1"); ok(val_is(r, "1"), "[%s] S4 session usable after DEALLOCATE ALL", m); PQclear(r); + } + } + + // ---- S5: cross-connection isolation. connA holds a binary statement X; connB + // (mixed) does DEALLOCATE ALL, which forwards and releases connB's copy + // of X. connA's X must be untouched -- proof the shared cache/refcounts + // are not corrupted. ---- + { + PGConnPtr ca = open_client_conn(); PGconn* a = ca.get(); + PGConnPtr cb = open_client_conn(); PGconn* b = cb.get(); + if (!ca || PQstatus(a) != CONNECTION_OK || !cb || PQstatus(b) != CONNECTION_OK) { fail(5, "S5 conn failed"); } + else { + PGresult* r; + r = PQprepare(a, "X", "SELECT 42", 0, nullptr); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "[%s] S5 connA prepare X", m); PQclear(r); + r = PQexecPrepared(a, "X", 0, nullptr, nullptr, nullptr, 0); ok(val_is(r, "42"), "[%s] S5 connA EXECUTE X = 42", m); PQclear(r); + r = PQexec(b, "PREPARE spB AS SELECT 1"); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "[%s] S5 connB SQL PREPARE spB (pins)", m); PQclear(r); + r = PQprepare(b, "X", "SELECT 42", 0, nullptr); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "[%s] S5 connB prepare X (same hash)", m); PQclear(r); + (void)exec_ok(b, "DEALLOCATE ALL"); // connB forwards + backend_close_all + r = PQexecPrepared(a, "X", 0, nullptr, nullptr, nullptr, 0); ok(val_is(r, "42"), "[%s] S5 connA EXECUTE X still = 42 (no corruption) -> %s", m, PQresStatus(PQresultStatus(r))); PQclear(r); + } + } + + // ---- S6: repeated PREPARE + DEALLOCATE ALL cycles. Each cycle must re-prepare + // cleanly (no lingering 42P05), and the refcounts must stay balanced. ---- + { + PGConnPtr c = open_client_conn(); PGconn* cc = c.get(); + if (!c || PQstatus(cc) != CONNECTION_OK) { fail(4, "S6 conn failed"); } + else { + for (int i = 1; i <= 3; i++) { + PGresult* r = PQexec(cc, "PREPARE cyc AS SELECT 1"); + ok(PQresultStatus(r) == PGRES_COMMAND_OK, "[%s] S6 cycle %d PREPARE cyc -> %s", m, i, PQresStatus(PQresultStatus(r))); + PQclear(r); + (void)exec_ok(cc, "DEALLOCATE ALL"); + } + PGresult* r = PQexec(cc, "EXECUTE cyc"); ok(PQresultStatus(r) == PGRES_FATAL_ERROR, "[%s] S6 EXECUTE cyc after last DEALLOCATE ALL fails -> %s", m, PQresStatus(PQresultStatus(r))); PQclear(r); + } + } + + // ---- S7: aborted transaction. DEALLOCATE ALL inside an aborted txn is rejected + // by the backend, so every statement survives. The aborted-txn guard + // must keep our tracking intact (no optimistic client/backend clear) so + // both the SQL PREPARE and the binary prepare are still usable after + // ROLLBACK -- byte-for-byte what real PostgreSQL does. ---- + { + PGConnPtr c = open_client_conn(); PGconn* cc = c.get(); + if (!c || PQstatus(cc) != CONNECTION_OK) { fail(8, "S7 conn failed"); } + else { + PGresult* r; + r = PQexec(cc, "PREPARE sp AS SELECT 5"); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "[%s] S7 SQL PREPARE sp (pins)", m); PQclear(r); + r = PQprepare(cc, "bp", "SELECT 88", 0, nullptr); ok(PQresultStatus(r) == PGRES_COMMAND_OK, "[%s] S7 binary prepare bp", m); PQclear(r); + r = PQexecPrepared(cc, "bp", 0, nullptr, nullptr, nullptr, 0); ok(val_is(r, "88"), "[%s] S7 EXECUTE bp = 88", m); PQclear(r); + (void)exec_ok(cc, "BEGIN"); + r = PQexec(cc, "SELECT 1/0"); ok(PQresultStatus(r) == PGRES_FATAL_ERROR, "[%s] S7 SELECT 1/0 aborts txn -> %s", m, PQresStatus(PQresultStatus(r))); PQclear(r); + r = PQexec(cc, "DEALLOCATE ALL"); ok(PQresultStatus(r) == PGRES_FATAL_ERROR, "[%s] S7 DEALLOCATE ALL rejected in aborted txn -> %s", m, PQresStatus(PQresultStatus(r))); PQclear(r); + (void)exec_ok(cc, "ROLLBACK"); + r = PQexec(cc, "EXECUTE sp"); ok(val_is(r, "5"), "[%s] S7 SQL sp survives (guard kept tracking) -> %s", m, PQresStatus(PQresultStatus(r))); PQclear(r); + r = PQexecPrepared(cc, "bp", 0, nullptr, nullptr, nullptr, 0); ok(val_is(r, "88"), "[%s] S7 binary bp survives (guard kept tracking) -> %s", m, PQresStatus(PQresultStatus(r))); PQclear(r); + r = PQexec(cc, "SELECT 99"); ok(val_is(r, "99"), "[%s] S7 session usable after aborted-txn DEALLOCATE ALL", m); PQclear(r); + } + } + +} + +// =========================================================================== +// Variable-sync reuse regression (native path) +// =========================================================================== +// This case guards a hang that used to make a client session wait forever with no +// error and no timeout. The fix is the end-state pin in ASYNC_QUERY_START in +// lib/PgSQL_Connection.cpp, and the full write-up is in +// docs/superpowers/specs/2026-09-01-pgsql-native-varsync-reuse-hang.md. +// +// The bug worked like this. When a client used a prepared statement, the backend +// connection was left marked as ending in ASYNC_STMT_EXECUTE_END. Nothing cleared that +// mark when the connection went back into the pool, so the next session inherited it. +// If that next client happened to want a different client_encoding, ProxySQL sent it a +// "SET client_encoding" to bring the connection into line, and the reply to that SET +// was dispatched using the stale mark. The code driving the SET only ever accepted +// ASYNC_QUERY_END, so it decided the SET had not finished and kept waiting. +// +// The test therefore does two things in order. It opens a client that asks for LATIN1 +// and runs a prepared statement, which leaves the mark behind, then closes it so the +// connection returns to the pool. It then opens a second client asking for UTF8, which +// is what forces ProxySQL to issue the SET on that same pooled connection. +// +// Both details matter. If the second client asked for the same encoding as the first, +// ProxySQL would send no SET at all and the test would prove nothing. And the pool has +// to be flushed beforehand, otherwise the second client may be handed some other clean +// connection instead of the one this test just dirtied. +// +// Finally, this case brings its own deadline, built on libpq's async API and select(), +// rather than calling PQexec. The failure being tested for is an unbounded hang, and a +// plain PQexec would simply stop the whole TAP suite instead of reporting a failure. +// That is also why it runs before the DEALLOCATE blocks further down, none of which +// have a deadline of their own. +static const int N_VARSYNC_REUSE = 4; + +// Runs `sql` on `c` under a hard wall-clock deadline. +// Returns 1 = completed (result in *out, caller PQclears), 0 = deadline expired, +// -1 = transport/libpq error. Never blocks past `timeout_ms`. +static int exec_with_deadline(PGconn* c, const char* sql, int timeout_ms, PGresult** out) { + *out = nullptr; + if (PQsendQuery(c, sql) == 0) { + diag("varsync: PQsendQuery failed: %s", PQerrorMessage(c)); + return -1; + } + const int sock = PQsocket(c); + if (sock < 0) return -1; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms); + while (PQisBusy(c)) { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) return 0; + const long long left_us = + std::chrono::duration_cast(deadline - now).count(); + fd_set rfds; + FD_ZERO(&rfds); + FD_SET(sock, &rfds); + struct timeval tv; + tv.tv_sec = (time_t)(left_us / 1000000); + tv.tv_usec = (suseconds_t)(left_us % 1000000); + const int rc = select(sock + 1, &rfds, nullptr, nullptr, &tv); + if (rc < 0) { + if (errno == EINTR) continue; + diag("varsync: select() failed: %s", strerror(errno)); + return -1; + } + if (rc == 0) return 0; // deadline + if (PQconsumeInput(c) == 0) { + diag("varsync: PQconsumeInput failed: %s", PQerrorMessage(c)); + return -1; + } + } + *out = PQgetResult(c); + // Drain any trailing results, but only while libpq guarantees PQgetResult + // will not block -- never trade one hang for another. + while (!PQisBusy(c)) { + PGresult* extra = PQgetResult(c); + if (extra == nullptr) break; + PQclear(extra); + } + return 1; +} + +static void run_varsync_reuse_regression(PGconn* admin, const std::vector& saved) { + auto fail = [&](int n, const char* why) { + for (int i = 0; i < n; i++) ok(false, "[native] varsync-reuse: %s", why); + }; + if (!setNativeMode(admin, true) || !flushBackendPool(admin, BACKEND_HG, saved)) { + fail(N_VARSYNC_REUSE, "admin setup failed"); + return; + } + + // --- seed: an extended-query cycle leaves ASYNC_STMT_EXECUTE_END pinned on + // the backend connection, which then goes back to the pool. --- + { + PGConnPtr seed = open_client_conn("options='-c client_encoding=LATIN1'"); + if (!seed || PQstatus(seed.get()) != CONNECTION_OK) { + fail(N_VARSYNC_REUSE, "seed conn failed"); + return; + } + PGconn* sc = seed.get(); + PGresult* r = PQprepare(sc, "vsb", "SELECT 88", 0, nullptr); + bool prepared = PQresultStatus(r) == PGRES_COMMAND_OK; + PQclear(r); + r = PQexecPrepared(sc, "vsb", 0, nullptr, nullptr, nullptr, 0); + ok(prepared && val_is(r, "88"), + "[native] varsync-reuse: seed binary prepare+execute = 88 (pins stmt end state on the pooled conn)"); + PQclear(r); + } // PQfinish -> the dirty connection returns to the pool + + // --- reuse: a different client_encoding forces the variable-sync SET that + // used to wedge in SETTING_VARIABLE forever. --- + PGConnPtr reuse = open_client_conn("options='-c client_encoding=UTF8'"); + ok(reuse && PQstatus(reuse.get()) == CONNECTION_OK, + "[native] varsync-reuse: reusing client (client_encoding=UTF8) connected"); + if (!reuse || PQstatus(reuse.get()) != CONNECTION_OK) { + fail(2, "reuse conn failed"); + return; + } + + PGresult* res = nullptr; + const int rc = exec_with_deadline(reuse.get(), "SELECT 1", 10000, &res); + ok(rc == 1, + "[native] varsync-reuse: SELECT 1 on the reused conn completed within 10s (rc=%d; 0 = the SETTING_VARIABLE hang)", + rc); + ok(rc == 1 && val_is(res, "1"), + "[native] varsync-reuse: SELECT 1 returned 1"); + if (res) PQclear(res); +} + int main(int /*argc*/, char** /*argv*/) { auto sql_cases = build_sql_cases(); auto extq_cases = build_extq_cases(); const int n_extra_cases = 6; // EXT_MULTI_CYCLE, EXT_REUSE, EXT_GLOBAL_DEDUP, EXT_PARSE_ERR_MIDFRAME, 2x EXT_DESCRIBE_CACHED int n_cases = (int)(sql_cases.size() + extq_cases.size()) + n_extra_cases; - plan(n_cases + 1); + const int n_dealloc_reg = N_DEALLOC_REG_PER_MODE; // SQL DEALLOCATE forwarding, native path + const int n_dealloc_xproto = N_DEALLOC_XPROTO_PER_MODE; // binary-prepare + SQL DEALLOCATE, native path + const int n_dealloc_all = N_DALLALL_MATRIX; // DEALLOCATE ALL matrix, native path + const int n_varsync = N_VARSYNC_REUSE; // variable-sync reuse hang, native path + plan(n_cases + 1 + n_varsync + n_dealloc_reg + n_dealloc_xproto + n_dealloc_all); if (cl.getEnv()) return exit_status(); std::string log_path = get_env("REGULAR_INFRA_DATADIR") + "/proxysql.log"; @@ -986,5 +1450,22 @@ int main(int /*argc*/, char** /*argv*/) { } cov.emit_tap(); + + // Runs first among the absolute-check blocks: it is the only one with its own + // deadline, so a regression here reports a clean failure instead of letting the + // deadline-less DEALLOCATE cases wedge the whole run. + diag("=== Variable-sync reuse regression (native path; 10s deadline) ==="); + run_varsync_reuse_regression(admin.get(), saved); + + diag("=== DEALLOCATE-forwarding regression (native path; absolute checks) ==="); + run_dealloc_regression(admin.get(), /*native=*/true, saved); + + diag("=== Cross-protocol DEALLOCATE: binary prepare + SQL DEALLOCATE (native path) ==="); + run_dealloc_xproto_regression(admin.get(), /*native=*/true, saved); + + diag("=== DEALLOCATE ALL matrix (native path) ==="); + run_dealloc_all_matrix(admin.get(), /*native=*/true, saved); + setNativeMode(admin.get(), false); // leave the proxy in the default mode + return exit_status(); } diff --git a/test/tap/tests/pgsql-native_query_differential-t.cpp b/test/tap/tests/pgsql-native_query_differential-t.cpp index 4196aa22de..658d54e2d6 100644 --- a/test/tap/tests/pgsql-native_query_differential-t.cpp +++ b/test/tap/tests/pgsql-native_query_differential-t.cpp @@ -17,6 +17,18 @@ * Like the auth test, it ALSO asserts the native run actually used the native * path (no fallback warning in the proxy log). * + * PROXYSQL INTERNAL SESSION (the last 12 assertions) + * ------------------------------------------------- + * The corpus above compares RESULTS. The tail of main() applies the same + * two-phase method to `PROXYSQL INTERNAL SESSION`, which the corpus cannot + * carry: both paths answer it, but with legitimately DIFFERENT documents (they + * describe different backend connections), so the assertions are on whether the + * command is answered at all and on the SHAPE of the native document -- not on + * equality. It is a regression guard for a crash, not a fidelity check: four + * get_pg_*() accessors used to call libpq on the NULL PGconn of a native + * connection and the resulting NULL, assigned into a nlohmann::json, aborted the + * whole proxy process. See the comment block at that section for detail. + * * INFRA / SCENARIO COVERAGE * ------------------------- * Same legacy-g1 infra as the auth test (docker-pgsql16-single, scram-sha-256 @@ -30,12 +42,19 @@ #include #include #include +#include +#include +#include #include +#include #include "libpq-fe.h" #include "command_line.h" #include "tap.h" #include "utils.h" +#include "json.hpp" + +using nlohmann::json; CommandLine cl; @@ -215,6 +234,98 @@ static PGConnPtr createClientConn() { return PGConnPtr(PQconnectdb(ss.str().c_str()), &PQfinish); } +// Same as createClientConn(), plus the client-supplied `options` a real client passes +// as PGOPTIONS / options= in its conninfo. +static PGConnPtr createClientConnWithOptions(const std::string& options) { + std::stringstream ss; + ss << "host=" << cl.pgsql_host << " port=" << cl.pgsql_port + << " user=" << cl.pgsql_username << " password=" << cl.pgsql_password + << " dbname=" << cl.pgsql_username << " sslmode=disable"; + if (!options.empty()) ss << " options='" << options << "'"; + return PGConnPtr(PQconnectdb(ss.str().c_str()), &PQfinish); +} + +// Connects with `options` and reports what the BACKEND session ended up with, as one +// comparable string. A connection failure is folded into the same string rather than +// bailing, because failing to connect on one path and not the other is exactly the +// divergence this phase is here to catch. +static std::string options_outcome(const std::string& options, const std::string& probe) { + PGConnPtr c = createClientConnWithOptions(options); + if (!c || PQstatus(c.get()) != CONNECTION_OK) { + std::string e = c ? PQerrorMessage(c.get()) : "PQconnectdb returned null"; + for (char& ch : e) if (ch == '\n') ch = ' '; + return "CONNECT-FAILED: " + e; + } + QueryResult r = run_one_query(c.get(), probe); + if (!r.ok) return "QUERY-FAILED: sqlstate=" + r.err_sqlstate; + if (r.rows.empty() || r.rows[0].empty()) return "NO-ROWS"; + return r.rows[0][0]; +} + +// Runs `sql` under a hard wall-clock deadline using libpq's async API. +// Returns 1 = completed (result in *out, caller PQclears), 0 = deadline expired, +// -1 = transport error. Copied from pgsql-native_prepared-t.cpp, in line with this +// file's self-contained-helpers convention above. +// +// Needed because the failure this guards against is an unbounded hang: a plain +// PQexec would wedge the whole TAP suite instead of reporting `not ok`. +static int exec_with_deadline(PGconn* c, const char* sql, int timeout_ms, PGresult** out) { + *out = nullptr; + if (PQsendQuery(c, sql) == 0) return -1; + const int sock = PQsocket(c); + if (sock < 0) return -1; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms); + while (PQisBusy(c)) { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) return 0; + const long long left_us = + std::chrono::duration_cast(deadline - now).count(); + fd_set rfds; + FD_ZERO(&rfds); + FD_SET(sock, &rfds); + struct timeval tv; + tv.tv_sec = (time_t)(left_us / 1000000); + tv.tv_usec = (suseconds_t)(left_us % 1000000); + const int rc = select(sock + 1, &rfds, nullptr, nullptr, &tv); + if (rc < 0) { + if (errno == EINTR) continue; + return -1; + } + if (rc == 0) return 0; + if (PQconsumeInput(c) == 0) return -1; + } + *out = PQgetResult(c); + while (!PQisBusy(c)) { + PGresult* extra = PQgetResult(c); + if (extra == nullptr) break; + PQclear(extra); + } + return 1; +} + +// One step of the within-session variable-sync sequence, folded into a comparable +// string. Any hang shows up as "TIMEOUT()" rather than wedging the run. +static std::string deadline_step(PGconn* c, const char* step, const char* sql, bool want_rows) { + PGresult* r = nullptr; + const int rc = exec_with_deadline(c, sql, 10000, &r); + if (rc == 0) return std::string("TIMEOUT(") + step + ")"; + if (rc < 0) { if (r) PQclear(r); return std::string("SENDFAIL(") + step + ")"; } + const ExecStatusType st = PQresultStatus(r); + std::string outcome; + if (st == PGRES_TUPLES_OK && want_rows) { + outcome = (PQntuples(r) == 1 && PQnfields(r) == 1 && !PQgetisnull(r, 0, 0)) + ? std::string(PQgetvalue(r, 0, 0)) + : std::string("BAD-SHAPE"); + } else if (st == PGRES_COMMAND_OK || st == PGRES_TUPLES_OK) { + outcome = "OK"; + } else { + const char* ss = PQresultErrorField(r, PG_DIAG_SQLSTATE); + outcome = std::string("ERR(") + step + "," + (ss ? ss : "?") + ")"; + } + PQclear(r); + return outcome; +} + static std::fstream f_proxysql_log{}; static bool nativeFallbackObserved() { @@ -228,6 +339,104 @@ static void drainLogToNow() { get_matching_lines(f_proxysql_log, "__no_such_marker_line__"); } +// ------------------------------------------------- internal-session probe +// +// `PROXYSQL INTERNAL SESSION` is a ProxySQL command, so it has no direct- +// PostgreSQL oracle. Its oracle here is the OTHER ProxySQL path: the same +// command, on the same proxy, with pgsql-use_native_backend_protocol flipped. + +struct InternalSession { + bool answered = false; + json doc; +}; + +// Pin a freshly-built backend connection to a client session, then read the +// session-introspection document back out of it. +// +// BEGIN keeps the backend attached for the lifetime of the transaction, and the +// create_new_connection hint guarantees the attached connection was built under +// the CURRENT value of pgsql-use_native_backend_protocol rather than reused from +// the pool. Both matter: with no backend attached, "backends" is empty and not a +// single get_pg_*() accessor is called, so the probe would pass while testing +// nothing. +static InternalSession probe_internal_session() { + InternalSession out; + auto c = createClientConn(); + if (!c || PQstatus(c.get()) != CONNECTION_OK) { + diag("internal-session probe: client connection failed: %s", + c ? PQerrorMessage(c.get()) : "null conn"); + return out; + } + PQclear(PQexec(c.get(), "BEGIN")); + PQclear(PQexec(c.get(), "/* create_new_connection=1 */ SELECT 42")); + + PGresult* r = PQexec(c.get(), "PROXYSQL INTERNAL SESSION"); + if (PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) > 0 && !PQgetisnull(r, 0, 0)) { + try { + out.doc = json::parse(PQgetvalue(r, 0, 0)); + out.answered = true; + } catch (const std::exception& e) { + diag("internal-session probe: unparseable JSON: %s", e.what()); + } + } else { + diag("internal-session probe: PROXYSQL INTERNAL SESSION failed: %s", + PQerrorMessage(c.get())); + } + PQclear(r); + PQclear(PQexec(c.get(), "COMMIT")); + return out; +} + +// backends[0].conn.pgsql, or a null json when no backend is attached. +static json backend_pgsql(const json& j) { + try { + if (j.contains("backends") && j["backends"].is_array() && !j["backends"].empty()) { + const json& b = j["backends"][0]; + if (b.contains("conn") && b["conn"].contains("pgsql")) return b["conn"]["pgsql"]; + } + } catch (const std::exception&) {} + return json(); +} + +// One assertion that a reported field survived as a JSON string. A NULL returned +// by a libpq accessor cannot reach this point: assigning it into a nlohmann::json +// constructs a std::string from a null pointer and throws, taking the process +// with it. So this passes only once the accessor has a native branch returning a +// real C string. +static void ok_pgsql_string_field(const json& pg, const char* key) { + const bool present = pg.is_object() && pg.contains(key); + const std::string got = present ? pg[key].dump() : std::string(""); + ok(present && pg[key].is_string(), + "native: backends[0].conn.pgsql.%s is a string (got %s)", key, got.c_str()); +} + +// backends[0].conn.pgsql.address is PgSQL_Connection::get_pg_connection(), the libpq +// PGconn. A libpq connection always has one; a native connection never does. That is +// what tells the two legs apart, and each leg asserts its own so neither can be served +// by the wrong kind of pooled connection without the run saying so. +static std::string pgconn_address(const json& pg) { + if (pg.is_object() && pg.contains("address") && pg["address"].is_string()) + return pg[std::string("address")].get(); + return ""; +} +static bool is_null_pgconn(const std::string& addr) { + return addr == "(nil)" || addr == "0x0" || addr == "0"; +} + +// One assertion that a reported field is IDENTICAL on both paths. Used for values +// that describe the SERVER rather than the connection, so unlike host_addr / port / +// options they must not differ between libpq and native. Compared against the oracle +// rather than a literal, so the assertion does not pin the infra's PostgreSQL version. +static void ok_pgsql_field_matches(const json& opg, const json& npg, const char* key) { + const bool have = opg.is_object() && npg.is_object() && + opg.contains(key) && npg.contains(key); + const std::string o = have ? opg[key].dump() : std::string(""); + const std::string n = have ? npg[key].dump() : std::string(""); + ok(have && opg[key] == npg[key], + "native: backends[0].conn.pgsql.%s matches libpq (libpq=%s native=%s)", + key, o.c_str(), n.c_str()); +} + // One query: 2 assertions (result match, native path used). // On mismatch, log the diff to help diagnose. static void assert_query(const char* label, const std::vector& libpq_res, @@ -248,8 +457,10 @@ static void assert_query(const char* label, const std::vector& libp } int main(int /*argc*/, char** /*argv*/) { - // 15 query-result assertions + 1 native-path assertion = 16 lines. - plan(16); + // 15 query-result assertions + 1 native-path assertion + // + 12 PROXYSQL INTERNAL SESSION assertions + 3 client-options assertions + // + 2 within-session varsync assertions. + plan(33); if (cl.getEnv()) return exit_status(); @@ -369,6 +580,266 @@ int main(int /*argc*/, char** /*argv*/) { bool fell_back = nativeFallbackObserved(); ok(!fell_back, "native phase used native path (no libpq fallback in log)"); + // ---- PROXYSQL INTERNAL SESSION, libpq oracle vs native ------------------ + // + // Same two-phase method as the corpus above, on a query the corpus cannot + // carry: the two paths return legitimately DIFFERENT documents (they + // describe different backend connections), so what is compared is whether + // the command is answered at all, plus the shape of the native document. + // + // REGRESSION GUARD. generate_proxysql_internal_session_json() describes the + // attached backend through the get_pg_*() accessors. Four of them USED TO have + // no native branch and called libpq on the PGconn, which is NULL for a native + // connection: get_pg_hostaddr(), get_pg_port(), get_pg_password() and + // get_pg_options(). PQhostaddr(NULL) & co. return NULL, that NULL was assigned + // straight into a nlohmann::json, which constructs a std::string from a null + // pointer and throws std::logic_error, and nothing on the path catches it -- + // so three ordinary statements from any authenticated client took down the + // whole proxy process. An exception, not an assertion, so release builds died + // identically. Those four accessors now have native branches returning a real + // C string (include/PgSQL_Connection.h). The libpq branches are untouched: on a + // live PGconn none of the PQ*() calls can return NULL -- PQhost/PQhostaddr/ + // PQport/PQpass fall back to "" themselves, PQdb/PQuser are filled by + // connectOptions2() or the connect fails, and `options` has the compiled-in + // default DefaultOption "". Only a NULL PGconn produces NULL, which is exactly + // what a native connection has and a libpq one never does here. + // + // The oracle leg runs FIRST on purpose. Taken the other way round, a native + // leg that kills the proxy also takes the oracle leg down with it, and the + // run then reads as "libpq is broken too" -- the wrong diagnosis. + { + InternalSession oracle; + if (setNativeMode(admin.get(), false) && + flushBackendPool(admin.get(), BACKEND_HG, saved)) { + oracle = probe_internal_session(); + } else { + diag("internal session: could not return the proxy to libpq mode for the oracle leg"); + } + + InternalSession candidate; + if (setNativeMode(admin.get(), true) && + flushBackendPool(admin.get(), BACKEND_HG, saved)) { + candidate = probe_internal_session(); + } else { + diag("internal session: could not put the proxy into native mode for the candidate leg"); + } + + const json opg = backend_pgsql(oracle.doc); + const json npg = backend_pgsql(candidate.doc); + + ok(oracle.answered, "libpq: PROXYSQL INTERNAL SESSION answers"); + // Guards the probe recipe itself: if BEGIN + create_new_connection stops + // attaching a backend, every native assertion below would pass vacuously. + ok(opg.is_object(), "libpq: INTERNAL SESSION reports an attached backend (probe recipe works)"); + { + const std::string addr = pgconn_address(opg); + ok(opg.is_object() && addr != "" && !is_null_pgconn(addr), + "libpq: the oracle leg really is libpq, it has a PGconn (address=%s)", addr.c_str()); + } + + ok(candidate.answered, "native: PROXYSQL INTERNAL SESSION answers"); + + // Confirms the attached connection really is native: the reported address + // is PgSQL_Connection::get_pg_connection(), the libpq PGconn, which a + // native connection does not have. A real pointer here would mean the + // session fell back to libpq and the field assertions prove nothing. + { + const std::string addr = pgconn_address(npg); + ok(npg.is_object() && is_null_pgconn(addr), + "native: the attached backend has no libpq PGconn, i.e. it is native (address=%s)", + addr.c_str()); + } + + ok_pgsql_string_field(npg, "host_addr"); // was PQhostaddr(NULL) -> NULL + ok_pgsql_string_field(npg, "port"); // was PQport(NULL) -> NULL + ok_pgsql_string_field(npg, "options"); // was PQoptions(NULL) -> NULL + + // password is reported only by DEBUG builds (#ifdef DEBUG in + // generate_proxysql_internal_session_json), so its absence is not a + // failure -- but a missing document is, or this reads as "release build, + // nothing to check" when the truth is that the proxy died. + { + const bool have_doc = npg.is_object(); + const bool present = have_doc && npg.contains("password"); + ok(have_doc && (!present || npg["password"].is_string()), + "native: backends[0].conn.pgsql.password is a string when reported%s", + have_doc ? (present ? "" : " [absent: release build]") : " [no document]"); + } + + // Both of these describe the SERVER, so they must agree across paths. + // client_encoding came back -1 on native -- PQclientEncoding()'s error + // sentinel, returned because a native connection has no PGconn -- and + // server_version used the pre-PostgreSQL-10 numeric encoding, so a 16.14 + // backend reported "16.14.0" where libpq reports "16.0.14" from 160014. + ok_pgsql_field_matches(opg, npg, "client_encoding"); + ok_pgsql_field_matches(opg, npg, "server_version"); + + { + auto a2 = createAdminConn(); + bool alive = false; + if (a2 && PQstatus(a2.get()) == CONNECTION_OK) { + PGresult* r = PQexec(a2.get(), "SELECT 1"); + alive = (PQresultStatus(r) == PGRES_TUPLES_OK); + PQclear(r); + } + ok(alive, "ProxySQL still alive after the internal-session probes"); + } + } + + // ---- Client connection options, libpq oracle vs native ----------------- + // + // A client's `options` reach the backend inside the StartupMessage `options` + // parameter, which the backend splits on unescaped whitespace (pg_split_opts, + // postinit.c). A value that itself contains a space therefore has to arrive + // escaped, and at the right level: a conninfo value passes through libpq, which + // strips one level of backslashes before it reaches the wire, while the native + // path writes to the wire directly and must not carry that extra level. + // + // REGRESSION GUARD. Untracked options used to be stored already escaped for a + // conninfo and then handed to the native path verbatim, so the backend saw the + // over-escaped form: an option whose value held a space FAILED THE CONNECTION on + // native while libpq was fine, e.g. + // ERROR: invalid value for parameter "work_mem": "4\" + // The tracked variables carry the same hazard through DateStyle, whose default + // value "ISO, MDY" contains a space. + { + // The options string below is a CONNINFO value, and libpq strips one level of + // backslashes while parsing it. So a value that must reach the wire as `4\ MB` + // is written here as `4\\ MB`. Getting this level wrong makes both paths fail to + // connect, which would still compare equal and quietly assert nothing -- hence + // the explicit `expected` below rather than a bare libpq-vs-native comparison. + struct OptCase { + const char* label; + const char* options; // as a client passes it in its conninfo + const char* probe; // what to read back from the backend session + const char* expected; // what BOTH paths must report + }; + // work_mem, geqo and join_collapse_limit must stay OUT of pgsql_variable_name for + // these to exercise the untracked path -- note maintenance_work_mem IS tracked + // while work_mem is not. If one of them is ever added to that enum, the case + // silently starts testing the tracked path instead and still passes; pick a + // different GUC then rather than leaving it. + // Every expected value below MUST differ from the backend's own default, or the + // case cannot tell "the option arrived" from "the option was silently dropped" -- + // it would only ever catch a failure to connect. Defaults on PostgreSQL 16 are + // work_mem=4MB, DateStyle='ISO, MDY', geqo=on, join_collapse_limit=8. + const OptCase cases[] = { + {"untracked value containing a space", + "-c work_mem=8\\\\ MB", + "SELECT current_setting('work_mem')", + "8MB"}, + {"untracked values needing no escaping", + "-c geqo=off -c join_collapse_limit=3", + "SELECT current_setting('geqo')||' '||current_setting('join_collapse_limit')", + "off 3"}, + {"tracked value containing a space (DateStyle)", + "-c DateStyle=ISO,\\\\ DMY", + "SELECT current_setting('DateStyle')", + "ISO, DMY"}, + }; + for (const auto& oc : cases) { + setNativeMode(admin.get(), false); + flushBackendPool(admin.get(), BACKEND_HG, saved); + const std::string lp = options_outcome(oc.options, oc.probe); + + setNativeMode(admin.get(), true); + flushBackendPool(admin.get(), BACKEND_HG, saved); + const std::string nt = options_outcome(oc.options, oc.probe); + + ok(lp == oc.expected && nt == oc.expected, + "client options -- %s -- reach the backend on both paths " + "(expected='%s' libpq='%s' native='%s')", + oc.label, oc.expected, lp.c_str(), nt.c_str()); + } + } + + // ---- Within-session variable sync after an extended-query step --------- + // + // This guards the same hang as run_varsync_reuse_regression() in + // pgsql-native_prepared-t, but reaches it a different way. The fix is the end-state + // pin in ASYNC_QUERY_START in lib/PgSQL_Connection.cpp, and the full write-up is in + // docs/superpowers/specs/2026-09-01-pgsql-native-varsync-reuse-hang.md. + // + // That other test covers the pool boundary. A client dirties a backend connection by + // running a prepared statement, the connection goes back to the pool, and a second + // client picks it up and hangs. This test never lets the connection reach the pool + // at all. Everything happens on one session, holding the same backend connection + // throughout, first because a prepared statement keeps it attached and then, in the + // second sub-case, because an explicit transaction does. It is the same stale mark + // left by the prepared statement, but no pooling is involved in getting to it. + // + // The sequence is: prepare and execute a statement, send a SET client_encoding, then + // read the setting back. The deadline has to cover that last read, not just the SET. + // ProxySQL answers the SET to the client on its own and only passes it to the + // backend when the next query comes along, so that read is where the hang actually + // appears. A test that stopped after the SET would pass even with the bug present. + // + // Like everything else in this file the check is differential, with the libpq path + // as the oracle. libpq cannot hit this bug, because its flush never reports that it + // sent everything in one go, so a native-only regression shows up as the two paths + // disagreeing, and a break affecting both shows up as both disagreeing with the + // expected value. + { + struct VarSyncCase { + const char* label; + bool use_txn; // pin the backend connection with an explicit txn too + }; + const VarSyncCase cases[] = { + {"prepared-statement-pinned session", false}, + {"explicit transaction", true}, + }; + // The SET must change client_encoding to something the session does not + // already have, or ProxySQL issues no sync at all and the case asserts nothing. + const char* expected = "LATIN1"; + + for (const auto& vc : cases) { + std::string outcome[2]; + for (int native = 0; native <= 1; native++) { + setNativeMode(admin.get(), native != 0); + flushBackendPool(admin.get(), BACKEND_HG, saved); + + PGConnPtr c = createClientConn(); + if (!c || PQstatus(c.get()) != CONNECTION_OK) { + outcome[native] = "CONNECT-FAILED"; + continue; + } + PGconn* cc = c.get(); + + if (vc.use_txn) { + const std::string b = deadline_step(cc, "BEGIN", "BEGIN", false); + if (b != "OK") { outcome[native] = b; continue; } + } + + // Extended-query step: this is what pins ASYNC_STMT_EXECUTE_END. + PGresult* pr = PQprepare(cc, "vsd", "SELECT 88", 0, nullptr); + const bool prepared = PQresultStatus(pr) == PGRES_COMMAND_OK; + PQclear(pr); + if (!prepared) { outcome[native] = "PREPARE-FAILED"; continue; } + pr = PQexecPrepared(cc, "vsd", 0, nullptr, nullptr, nullptr, 0); + const bool executed = PQresultStatus(pr) == PGRES_TUPLES_OK; + PQclear(pr); + if (!executed) { outcome[native] = "EXECUTE-FAILED"; continue; } + + // The variable sync. This is the step that used to hang forever. + const std::string s1 = + deadline_step(cc, "SET", "SET client_encoding TO 'LATIN1'", false); + if (s1 != "OK") { outcome[native] = s1; continue; } + + outcome[native] = deadline_step( + cc, "probe", "SELECT current_setting('client_encoding')", true); + + if (vc.use_txn) { + const std::string cm = deadline_step(cc, "COMMIT", "COMMIT", false); + if (cm != "OK" && outcome[native] == expected) outcome[native] = cm; + } + } + ok(outcome[0] == expected && outcome[1] == expected, + "within-session varsync after extended query -- %s -- no hang, both paths agree " + "(expected='%s' libpq='%s' native='%s')", + vc.label, expected, outcome[0].c_str(), outcome[1].c_str()); + } + } + // Restore native mode to default (off) and flush the pool. setNativeMode(admin.get(), false); flushBackendPool(admin.get(), BACKEND_HG, saved); diff --git a/test/tap/tests/pgsql-native_ssl_pool_reuse-t.cpp b/test/tap/tests/pgsql-native_ssl_pool_reuse-t.cpp new file mode 100644 index 0000000000..a0cfc4cda3 --- /dev/null +++ b/test/tap/tests/pgsql-native_ssl_pool_reuse-t.cpp @@ -0,0 +1,788 @@ +/** + * @file pgsql-native_ssl_pool_reuse-t.cpp + * @brief Regression test: a pooled native TLS backend connection must keep its + * encryption when a DIFFERENT client session picks it up. + * + * THE BUG THIS GUARDS + * ------------------- + * The native path used to store its SSL and BIOs on PgSQL_Data_Stream. That + * object belongs to the SESSION and is destroyed when the session finishes with + * the backend (PgSQL_Session.cpp:1119/:1142/:1175); its destructor SSL_free()s + * the SSL (PgSQL_Data_Stream.cpp:376). But a PgSQL_Connection is POOLED -- it + * outlives any one session -- so pooling a TLS connection destroyed its TLS + * context while the socket stayed open and encrypted. The next session attached + * it to a fresh data stream with no SSL, read TLS records as plaintext, and + * reported "backend closed during result fetch". The fix moved the TLS state + * onto PgSQL_Connection, giving it the socket's lifetime. + * + * WHAT THE SCENARIOS COVER + * ------------------------ + * A (7 assertions x 4 combinations = 28) -- the pool-reuse guard, run under EVERY + * combination of backend use_ssl x client sslmode. Per combination: three + * separate client sessions, asserting all are served, that the backend PIDs + * MATCH, that the reused connection's real encryption state matches use_ssl, + * that the native path served it, and finally that EVERY connection left in + * the pool -- not merely the one a probe happened to land on -- is encrypted + * as use_ssl asked, cross-checked against ProxySQL's own using_ssl claim. + * + * The PID match is what makes this a regression test: if session 2 opened a + * fresh connection instead of reusing the pooled one, every other assertion + * would still pass while the bug went unexercised. + * + * The pool-wide check exists because every OTHER encryption assertion in this + * file is keyed on a pid the test learned from its own query, so it can only + * ever speak for connections the test personally used. It enumerates the pool + * from stats_pgsql_free_connections and verifies each entry against + * pg_stat_ssl. The trust boundary is preserved: ProxySQL supplies only the + * backend_pid -- an identifier -- while the encryption verdict still comes + * from PostgreSQL. ProxySQL's using_ssl is then compared against that verdict + * rather than trusted, so a proxy that MISREPORTS its own transport fails + * here. Both fields are reported for native connections as of + * lib/PgSQL_HostGroups_Manager.cpp:3082; against a binary predating that, the + * pid comes back absent and the assertion fails saying so rather than + * silently checking nothing. + * + * Running it per combination crosses the two axes that matter. The frontend + * leg (client <-> ProxySQL) and the backend leg (ProxySQL <-> PostgreSQL) are + * separate data streams with separate TLS state, so a fix that confused them, + * or that keyed the backend's transport off the client's, fails here. It also + * covers the case a real deployment actually runs -- BOTH legs encrypted + * across a pooled handoff -- which a single-combination test does not reach. + * + * B (3) -- configuration disagreeing with the connection. use_ssl is flipped to 0 + * while an encrypted connection is still warm in the pool. It must keep + * working: `encrypted` follows the connection's own SSL object, NOT config. A + * fix that re-derived it from use_ssl would fail here. + * + * (There is no Scenario C. It was removed, as was a Scenario E covering a + * multi-server hostgroup; the surviving labels were left alone rather than + * renumbered so that assertion numbers in older run logs still line up.) + * + * WHICH ASSERTIONS GUARD THE BUG ABOVE: A's first three, and B's second query. + * They fail because session 2 cannot READ the connection. The pool-wide checks + * (A's last two) deliberately do NOT catch that bug and must not be mistaken for + * it: when the SSL is freed on pool return the socket stays encrypted as far as + * PostgreSQL is concerned, so pg_stat_ssl still reports 't' and they pass. They + * cover a different property -- that no connection ANYWHERE in the pool has the + * wrong transport, and that ProxySQL's own account of it is truthful. + * + * D (4) -- churn. Several client sessions are held open at once and queried + * round-robin, so pooled TLS connections are detached and re-attached many + * times instead of once. A lifetime bug that only trips on the fifth or + * twelfth handoff is invisible to a scenario that hands off once; this is also + * the shape most likely to expose a use-after-free when run under ASAN. + * + * THE SINGLE-SERVER PRECONDITION IS VERIFIED, NOT ASSUMED: freshServer() bails + * out if the hostgroup does not hold exactly one server, because with several, + * two sessions legitimately land on different backends -- a pid mismatch would be + * correct behaviour reported as a bug, and a pid match would prove nothing. + * Every query about a server names the (hostname, port) it means, since + * "SELECT use_ssl ... WHERE hostgroup_id=N" returns an arbitrary row as soon as + * the hostgroup holds more than one. + * + * VALIDATED IN BOTH DIRECTIONS: with the bug deliberately reintroduced, Scenario + * A's reuse and encryption assertions fail and Scenario B's second query fails; + * with the fix in place the file passes in full. (That check was performed on the + * equivalent single-combination form of Scenario A -- parameterising it only + * widens the same assertions, so the specific assertion NUMBERS from that run no + * longer apply.) + * + * Backend encryption is read from pg_stat_ssl over a DIRECT connection to + * PostgreSQL, never from ProxySQL, which is the component under test. + * + * SCENARIO INDEPENDENCE: every scenario (and every Scenario A combination) begins + * with freshServer(), which rebuilds the server row and waits for the pool to + * drain. So none of them inherits the previous one's use_ssl or its pooled + * connections, and they may be reordered or run individually. Scenario B + * deliberately ends with use_ssl=0 -- without that rule the next scenario would + * silently start on a plaintext backend. + * + * Native path only (pgsql-use_native_backend_protocol=true). This test does NOT + * restore runtime state; it only ever uses LOAD ... TO RUNTIME and never writes + * to disk. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libpq-fe.h" +#include "command_line.h" +#include "tap.h" +#include "utils.h" + +using PGConnPtr = std::unique_ptr; +CommandLine cl; + +static const int HG = 92; // dedicated: never disturb other tests' pools + +// pg_backend_pid() alone is INTERCEPTED by ProxySQL (lib/PgSQL_Session.cpp:5247) +// and answered locally, never reaching a backend. Wrapping it in a catalog scan +// changes the digest so it executes server-side and returns the real pid. +// Scenario A runs once per row: backend use_ssl x client sslmode. At file scope +// so plan() is DERIVED from it -- adding a row must not silently desync the +// assertion count. +struct Combo { int backend_ssl; const char* client_sslmode; const char* want_backend; }; +static const Combo COMBOS[] = { + { 1, "require", "t" }, // both legs encrypted -- the production-like case + { 1, "disable", "t" }, // client plaintext, backend encrypted + { 0, "require", "f" }, // client encrypted, backend plaintext + { 0, "disable", "f" }, // neither encrypted +}; +static const size_t NCOMBOS = sizeof(COMBOS) / sizeof(COMBOS[0]); + +// Scenario D. Sessions are held open SIMULTANEOUSLY and queried round-robin: +// that is what forces repeated detach/re-attach against more than one pooled +// connection. Sequential sessions would hand the same single connection back and +// forth and never exercise a pool of them. +static const int CHURN_CLIENTS = 4; +static const int CHURN_ROUNDS = 8; + +// Assertions: A = 7 per combination, B = 3, D = 4, plus a final liveness check. +static const int PLAN_A_PER = 7, PLAN_B = 3, PLAN_D = 4, PLAN_FINAL = 1; + +// One pooled connection as ProxySQL describes it. Only the pid is taken on +// trust (it is an identifier, not a verdict); `claim` is ProxySQL's assertion +// about its own transport, which the test CHECKS rather than believes. +// One pooled connection as ProxySQL describes it. host/port are carried because +// a backend pid is only meaningful ON THE SERVER THAT ISSUED IT: pids are +// per-machine, so looking one up on the wrong PostgreSQL does not merely miss -- +// it can find an UNRELATED backend that happens to have the same pid number and +// report its encryption instead. Scenarios here pin the hostgroup to a single +// server, but the pool query is hostgroup-wide, so the lookup is aimed at the +// server the connection actually belongs to rather than at a global default. +struct PooledConn { std::string pid; std::string claim; std::string host; int port; }; + +// Minimal field reader for the pgsql_info JSON. This file links libpq and tap +// only -- there is no JSON parser available -- and poolNativeMode() already +// reads the same blob by substring search, so this stays consistent with it. +// nlohmann's dump() emits no spaces, hence the exact "key": prefix match. +static std::string jsonField(const std::string& blob, const std::string& key) { + const std::string k = "\"" + key + "\":"; + size_t p = blob.find(k); + if (p == std::string::npos) return ""; + p += k.size(); + if (p < blob.size() && blob[p] == '"') { // string value + const size_t e = blob.find('"', ++p); + return e == std::string::npos ? "" : blob.substr(p, e - p); + } + const size_t e = blob.find_first_of(",}", p); // numeric / bare value + return e == std::string::npos ? "" : blob.substr(p, e - p); +} + +static const char* PID_QUERY = + "SELECT pid FROM pg_stat_activity WHERE pid = pg_backend_pid()"; + +static PGConnPtr openConn(const char* h, int p, const char* u, const char* pw, + const char* db, const char* sslmode) { + std::stringstream ss; + ss << "host=" << h << " port=" << p << " user=" << u << " password=" << pw; + if (db && *db) ss << " dbname=" << db; + ss << " sslmode=" << sslmode << " connect_timeout=10"; + return PGConnPtr(PQconnectdb(ss.str().c_str()), &PQfinish); +} +// libpq errors arrive multi-line; TAP assertion text is one line per assertion. +static std::string oneline(std::string e) { + for (char& ch : e) if (ch == '\n' || ch == '\r') ch = ' '; + while (!e.empty() && e.back() == ' ') e.pop_back(); + return e; +} +static bool execAdmin(PGconn* a, const std::string& q) { + PGresult* r = PQexec(a, q.c_str()); + const ExecStatusType st = PQresultStatus(r); + const bool good = (st == PGRES_COMMAND_OK || st == PGRES_TUPLES_OK); + if (!good) diag("admin failed: %s -- %s", q.c_str(), PQerrorMessage(a)); + PQclear(r); + return good; +} +static std::string scalar(PGconn* c, const std::string& q) { + PGresult* r = PQexec(c, q.c_str()); + std::string v; + if (PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) > 0 && !PQgetisnull(r, 0, 0)) + v = PQgetvalue(r, 0, 0); + PQclear(r); + return v; +} + +int main(int, char**) { + plan((int)(NCOMBOS * PLAN_A_PER) + PLAN_B + PLAN_D + PLAN_FINAL); + if (cl.getEnv()) return exit_status(); + + PGConnPtr adminOwner = openConn(cl.pgsql_admin_host, cl.pgsql_admin_port, + cl.admin_username, cl.admin_password, nullptr, "disable"); + if (!adminOwner || PQstatus(adminOwner.get()) != CONNECTION_OK) + BAIL_OUT("cannot proceed without an admin connection"); + PGconn* admin = adminOwner.get(); + + + // NOTHING BELOW IS RESTORED, AND THAT IS CORRECT. The harness reconfigures + // ProxySQL before EVERY test: proxysql-tester.py:811 calls + // reconfigure_proxysql() (:370) inside the per-test loop, which issues + // LOAD PGSQL VARIABLES / USERS / SERVERS FROM DISK followed by TO RUNTIME. + // So the native flag, this user's default_hostgroup and the hostgroup rows + // are all reset before the next test sees them. + // + // That restore only works because this file uses LOAD ... TO RUNTIME and + // NEVER SAVE ... TO DISK: the on-disk config stays pristine, so reloading + // from disk genuinely undoes everything done here. Writing any of it to disk + // would defeat the harness for every test that follows. + + auto setVar = [&](const char* n, const std::string& v) { + return execAdmin(admin, std::string("SET ") + n + "='" + v + "'") && + execAdmin(admin, "LOAD PGSQL VARIABLES TO RUNTIME"); + }; + // Poll a condition instead of sleeping. Returns false if it never became true + // within the budget, so the caller can report it rather than silently continue. + auto waitUntil = [&](const std::function& cond) { + for (int waited = 0; waited <= 5000; waited += 50) { + if (cond()) return true; + usleep(50000); + } + return false; + }; + // A HOSTGROUP CAN HOLD MANY SERVERS. pgsql_servers is keyed on + // (hostgroup_id, hostname, port), so "the server in hostgroup N" is not a + // thing that exists: a hostgroup-wide "SELECT use_ssl FROM + // runtime_pgsql_servers WHERE hostgroup_id=N" returns an ARBITRARY row once + // there is more than one, and a hostgroup-wide pool count silently mixes + // servers together. Everything below that asks about a server therefore + // names the (hostname, port) it means. Only the drain check is deliberately + // hostgroup-wide, because "no connections left anywhere in this hostgroup" + // is genuinely the question it asks. + auto poolCountHG = [&]() { + std::stringstream q; + q << "SELECT count(*) FROM stats_pgsql_free_connections WHERE hostgroup=" << HG; + const std::string v = scalar(admin, q.str()); + return v.empty() ? -1 : atoi(v.c_str()); + }; + auto hgServerCount = [&]() { + std::stringstream q; + q << "SELECT count(*) FROM runtime_pgsql_servers WHERE hostgroup_id=" << HG; + const std::string v = scalar(admin, q.str()); + return v.empty() ? -1 : atoi(v.c_str()); + }; + auto srvPresent = [&](const std::string& host, int port) { + std::stringstream q; + q << "SELECT count(*) FROM runtime_pgsql_servers WHERE hostgroup_id=" << HG + << " AND hostname='" << host << "' AND port=" << port; + const std::string v = scalar(admin, q.str()); + return v.empty() ? -1 : atoi(v.c_str()); + }; + auto runtimeUseSslFor = [&](const std::string& host, int port) { + std::stringstream q; + q << "SELECT use_ssl FROM runtime_pgsql_servers WHERE hostgroup_id=" << HG + << " AND hostname='" << host << "' AND port=" << port; + const std::string v = scalar(admin, q.str()); + return v.empty() ? -1 : atoi(v.c_str()); + }; + // EVERY pooled connection in the hostgroup, as (backend_pid, using_ssl). + // This is what lets an assertion speak for the whole pool instead of only + // the connection a probe happened to land on: without a pid there is no way + // to correlate a pooled connection with pg_stat_ssl, and pg_stat_ssl is the + // only trustworthy source for what the transport really is. + auto poolConns = [&]() { + std::stringstream q; + q << "SELECT srv_host, srv_port, pgsql_info FROM stats_pgsql_free_connections " + << "WHERE hostgroup=" << HG; + std::vector out; + PGresult* r = PQexec(admin, q.str().c_str()); + if (PQresultStatus(r) == PGRES_TUPLES_OK) { + for (int i = 0; i < PQntuples(r); i++) { + const std::string host = PQgetvalue(r, i, 0); + const int port = atoi(PQgetvalue(r, i, 1)); + const std::string blob = PQgetvalue(r, i, 2); + out.push_back({ jsonField(blob, "backend_pid"), jsonField(blob, "using_ssl"), + host, port }); + } + } + PQclear(r); + return out; + }; + + auto freshServer = [&](int use_ssl) { + std::stringstream d, i; + d << "DELETE FROM pgsql_servers WHERE hostgroup_id=" << HG; + i << "INSERT INTO pgsql_servers (hostgroup_id,hostname,port,use_ssl) VALUES (" + << HG << ",'" << cl.pgsql_server_host << "'," << cl.pgsql_server_port << "," << use_ssl << ")"; + // TWO loads, deliberately. The first is what EVICTS the pool: connections + // are dropped only when the server actually disappears from the runtime + // table. Collapsing this into DELETE + INSERT + one LOAD evicts nothing -- + // measured: the pooled connection count stays at 1, because ProxySQL sees + // the same hostgroup/host/port still present and treats it as unchanged. + // That matters here because a connection fixes its TLS mode at creation, + // so a surviving use_ssl=0 connection would serve a use_ssl=1 cell and the + // test would blame the native path for a stale-pool artefact. + // + // Both waits POLL for the condition rather than sleeping a fixed span: a + // constant that is comfortable here is not necessarily comfortable on a + // loaded CI runner, and guessing high just makes every cell slower. + execAdmin(admin, d.str()); execAdmin(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + int seen = -1; + if (!waitUntil([&]{ seen = poolCountHG(); return seen == 0; })) { + // -1 means the admin query itself failed -- a different problem from + // "the pool still has connections", and worth saying so. + if (seen < 0) diag("could not read the pool for hostgroup %d (admin query failed)", HG); + else diag("hostgroup %d did not drain (%d connection(s) left); this cell " + "may be served by a stale connection", HG, seen); + } + + execAdmin(admin, i.str()); execAdmin(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + seen = -1; + if (!waitUntil([&]{ seen = srvPresent(cl.pgsql_server_host, cl.pgsql_server_port); return seen == 1; })) { + if (seen < 0) diag("could not read runtime_pgsql_servers (admin query failed)"); + else diag("server %s:%d did not come back online in hostgroup %d (found %d)", + cl.pgsql_server_host, cl.pgsql_server_port, HG, seen); + } + + // VERIFY the single-server precondition instead of assuming it. Scenarios + // A, B and D assert that two sessions land on the SAME backend pid; that + // only means anything when the hostgroup holds exactly one server. With + // several, ProxySQL load-balances and two sessions legitimately land on + // different backends, so a pid mismatch would be a correct result being + // reported as a bug -- and, worse, a pid MATCH would prove nothing. + const int nsrv = hgServerCount(); + if (nsrv != 1) { + BAIL_OUT("hostgroup %d holds %d servers, expected exactly 1 -- the reuse " + "assertions require a single-server hostgroup to be meaningful", + HG, nsrv); + } + }; + + if (!setVar("pgsql-use_native_backend_protocol", "true")) BAIL_OUT("cannot enable native mode"); + { + std::stringstream u; + u << "UPDATE pgsql_users SET default_hostgroup=" << HG << " WHERE username='" << cl.pgsql_username << "'"; + if (!execAdmin(admin, u.str()) || !execAdmin(admin, "LOAD PGSQL USERS TO RUNTIME")) { + BAIL_OUT("cannot point %s at hostgroup %d", cl.pgsql_username, HG); + } + } + + auto probeSsl = [&](std::string& pid, const char* client_sslmode) { + PGConnPtr c = openConn(cl.pgsql_host, cl.pgsql_port, cl.pgsql_username, cl.pgsql_password, cl.pgsql_username, client_sslmode); + if (!c || PQstatus(c.get()) != CONNECTION_OK) { + // Report the actual reason. Collapsing every connect failure to a + // fixed string hides exactly what a failing run needs: auth rejected, + // backend down, TLS refused and "remaining connection slots are + // reserved..." are very different problems. + const std::string e = oneline(c ? PQerrorMessage(c.get()) : "null connection"); + return e.empty() ? std::string("connect failed") : e; + } + PGresult* r = PQexec(c.get(), PID_QUERY); + std::string err; + if (PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) == 1) pid = PQgetvalue(r, 0, 0); + else err = oneline(PQerrorMessage(c.get())); + PQclear(r); + return err; + }; + auto probe = [&](std::string& pid) { return probeSsl(pid, "disable"); }; + // Positive proof that the NATIVE path served these queries. Without it the + // whole file passes just as happily if ProxySQL silently fell back to libpq, + // because every other check here (query served, pg_stat_ssl, matching pids) + // is satisfied by the libpq path too. Reads the per-connection + // "native_mode":true|false that stats_pgsql_free_connections exposes + // (lib/PgSQL_HostGroups_Manager.cpp:3077). Polls, because the return to the + // FREE list is not instantaneous on a loaded runner. + auto poolNativeMode = [&]() { + std::stringstream q; + q << "SELECT pgsql_info FROM stats_pgsql_free_connections WHERE hostgroup=" << HG; + std::string verdict; + // Reuses waitUntil so there is exactly ONE polling implementation here. + waitUntil([&]() { + PGresult* r = PQexec(admin, q.str().c_str()); + const int n = (PQresultStatus(r) == PGRES_TUPLES_OK) ? PQntuples(r) : 0; + if (n > 0) { + int native = 0, libpq = 0; + for (int i = 0; i < n; i++) { + const std::string info = PQgetvalue(r, i, 0); + if (info.find("\"native_mode\":true") != std::string::npos) native++; + else if (info.find("\"native_mode\":false") != std::string::npos) libpq++; + } + // Judge EVERY row: one libpq connection hiding behind a native one + // would otherwise pass. + if (libpq) { std::stringstream m; m << "false (native=" << native << " libpq=" << libpq << ")"; verdict = m.str(); } + else if (native) verdict = "true"; + else verdict = "unparsed"; + } + PQclear(r); + return !verdict.empty(); + }); + return verdict; // "" if no free connection ever appeared + }; + + // Ask a SPECIFIC PostgreSQL whether the backend with this pid is encrypted. + // The server must be named rather than assumed: pids are per-machine, so + // querying the wrong node can return an unrelated backend that happens to + // share the number, which is a wrong ANSWER rather than a missing one. + auto backendSslOn = [&](const std::string& host, int port, const std::string& pid) { + if (pid.empty() || host.empty()) return std::string(""); + // `pid` is BACKEND-CONTROLLED data -- it arrives as a text field through + // ProxySQL from the server -- and it is about to be concatenated into a + // query run as a SUPERUSER on the real PostgreSQL. This suite ships a + // scriptable mock server (pgsql_mock_backend.h) and a hostile-backend + // test, so a value like "1;DROP TABLE ..." is a reachable input, not a + // hypothetical. Refuse anything that is not a plain integer. + if (pid.find_first_not_of("0123456789") != std::string::npos) { + diag("refusing to query pg_stat_ssl with a non-numeric backend pid '%s'", pid.c_str()); + return std::string(""); + } + PGConnPtr d = openConn(host.c_str(), port, + cl.pgsql_server_username, cl.pgsql_server_password, "postgres", "disable"); + if (!d || PQstatus(d.get()) != CONNECTION_OK) { + // SAY SO when the ORACLE is what failed. This returns "" for the same + // reason "pid absent from pg_stat_ssl" does, and the callers turn both + // into "not found" -- which reads as the pooled-TLS path being broken + // when in fact the measuring instrument never connected. That is the + // exact misdiagnosis the rest of this file works to avoid, so the two + // causes must at least be distinguishable in the log. + diag("direct connection to %s:%d FAILED, so pg_stat_ssl could not be read for " + "pid %s -- the assertion below reports 'not found' for that reason, NOT " + "because the pooled connection is wrong: %s", + host.c_str(), port, pid.c_str(), + d ? oneline(PQerrorMessage(d.get())).c_str() : "null connection"); + return std::string(""); + } + return scalar(d.get(), "SELECT ssl::text FROM pg_stat_ssl WHERE pid=" + pid); + }; + // Scenarios A, B and D read a pid returned by a query they just ran through a + // hostgroup freshServer() has verified holds exactly ONE server, and that + // server is cl.pgsql_server_host by construction -- so naming it here is + // correct, not an assumption. + auto backendSsl = [&](const std::string& pid) { + return backendSslOn(cl.pgsql_server_host, cl.pgsql_server_port, pid); + }; + + // ====================================================================== + // SCENARIO A -- the pool-reuse regression guard, run under EVERY combination + // of backend use_ssl x client sslmode. Session 1 establishes a backend + // connection and pools it; session 2 must pick that same connection up and + // use it, and its real transport must still match what use_ssl asked for. + // + // This is the case that was broken: the TLS session used to live on the + // SESSION's backend data stream, which is destroyed when session 1 finishes + // (PgSQL_Session.cpp:1119/:1142/:1175) and whose destructor SSL_free()s the + // SSL (PgSQL_Data_Stream.cpp:376). The pooled connection was left with a live + // encrypted socket and no TLS context, so session 2 read ciphertext as + // plaintext and reported "backend closed during result fetch". The TLS state + // now lives on PgSQL_Connection, giving it the socket's lifetime. + // + // The use_ssl=0 rows are not filler: they are the control. They prove the + // encryption assertion is reading a real per-connection property rather than + // a constant that happens to say "encrypted" for every connection. + // ====================================================================== + for (const Combo& c : COMBOS) { + freshServer(c.backend_ssl); + const bool want_enc = (std::string(c.want_backend) == "t"); + std::stringstream tagss; + tagss << "use_ssl=" << c.backend_ssl << "/sslmode=" << c.client_sslmode; + const std::string tag = tagss.str(); + + std::string a1, a2; + const std::string ea1 = probeSsl(a1, c.client_sslmode); + // Session 1's backend must be back on the FREE list before session 2 + // asks for one. poolNativeMode() below already documents that this + // return is not instantaneous on a loaded runner; the pid match is this + // file's core assertion, so it waits for the same condition instead of + // relying on session 2's connect+SCRAM handshake taking long enough. + // Otherwise session 2 opens a FRESH connection and the reuse assertion + // fails for a reason that has nothing to do with TLS lifetime. + if (!waitUntil([&]{ return poolCountHG() >= 1; })) { + diag("A[%s]: nothing returned to the pool after session 1; session 2 may open " + "a fresh connection and the reuse assertion below would then fail for a " + "scheduling reason rather than a TLS one", tag.c_str()); + } + const std::string ea2 = probeSsl(a2, c.client_sslmode); + ok(ea1.empty() && ea2.empty() && !a1.empty() && !a2.empty(), + "A[%s]: two separate sessions were both served (pids '%s','%s')%s%s", + tag.c_str(), a1.empty() ? "-" : a1.c_str(), a2.empty() ? "-" : a2.c_str(), + (ea1.empty() && ea2.empty()) ? "" : " -- ", + ea1.empty() ? ea2.c_str() : ea1.c_str()); + + ok(!a1.empty() && a1 == a2, + "A[%s]: session 2 reused the SAME pooled connection (pid '%s' then '%s') " + "-- differing pids would mean the pooled-reuse path was never exercised", + tag.c_str(), a1.empty() ? "-" : a1.c_str(), a2.empty() ? "-" : a2.c_str()); + + // The BACKEND leg's real state, read from PostgreSQL itself. client + // sslmode=require is separately its own proof that the FRONTEND leg is + // encrypted -- libpq aborts the connection if TLS is not negotiated -- so + // a row where the two legs disagree is what proves they are independent. + const std::string assl = backendSsl(a2); + const bool enc = (assl == "t" || assl == "true"); + ok(!assl.empty() && enc == want_enc, + "A[%s]: the reused connection's backend encryption follows use_ssl and NOT " + "the client leg (pg_stat_ssl.ssl='%s', want '%s')", + tag.c_str(), assl.empty() ? "not found" : assl.c_str(), c.want_backend); + + std::string a3; + const std::string ea3 = probeSsl(a3, c.client_sslmode); + ok(ea3.empty() && !a3.empty(), + "A[%s]: a third session is served too (pid '%s')%s%s", tag.c_str(), + a3.empty() ? "-" : a3.c_str(), ea3.empty() ? "" : " -- ", ea3.c_str()); + + const std::string nm = poolNativeMode(); + ok(nm == "true", + "A[%s]: the pooled connection really used the NATIVE path, not a silent " + "libpq fallback (native_mode='%s')", + tag.c_str(), nm.empty() ? "no free connection recorded" : nm.c_str()); + + // ---- pool-wide: EVERY pooled connection, not just the probed one ----- + // Assertions above are keyed on pids this test learned from its own + // queries, so they can only speak for connections it personally used. + // Here the pool itself is enumerated and each entry checked against + // pg_stat_ssl, so a stray connection with the wrong transport -- one the + // probes never landed on -- is caught rather than ignored. + std::vector pooled; + waitUntil([&]{ pooled = poolConns(); return !pooled.empty(); }); + + // CROSS-CHECK ProxySQL's claim against the BACKEND's own report. Every + // assertion below is keyed on a pid ProxySQL ASSERTS, whereas a1/a2 came + // from the backend itself answering pg_stat_activity. Nothing otherwise + // ties the two together, so a wrong backend_pid would silently send the + // encryption checks to some OTHER connection's pg_stat_ssl row -- and on + // the use_ssl=0 rows, where 'f' is what is wanted, any plaintext + // connection on the server satisfies them. That is a real false pass, + // and it matters most precisely here: backend_pid is newly reported for + // native connections (lib/PgSQL_HostGroups_Manager.cpp), so this file is + // the first thing that exercises it. + bool pool_has_a2 = false; + for (const PooledConn& pc : pooled) { + if (!a2.empty() && pc.pid == a2) { pool_has_a2 = true; break; } + } + + int checked = 0, correct = 0, agree = 0, nopid = 0, noclaim = 0; + for (const PooledConn& pc : pooled) { + // No pid means the pool cannot be correlated with pg_stat_ssl at + // all. That is a real gap, not something to skip past, so it is + // counted and failed on below. + if (pc.pid.empty() || pc.pid == "0") { nopid++; continue; } + // Ask THIS connection's own server, not a global default. The pool + // query is hostgroup-wide and a hostgroup may hold many servers, so + // a fixed target would silently read the wrong machine -- where the + // same pid number can belong to an entirely different backend. + const std::string truth = backendSslOn(pc.host, pc.port, pc.pid); + if (truth.empty()) continue; // deliberately NOT counted in `checked` + checked++; + const bool e = (truth == "t" || truth == "true"); + if (e == want_enc) correct++; + // An ABSENT claim must not read as agreement. jsonField() returns "" + // for a missing field, and ("" == "YES") is false -- which happens to + // equal `e` for every use_ssl=0 row, so a proxy that stopped + // reporting using_ssl entirely would silently "agree" in exactly the + // combinations meant to be the control. Judge the claim only when + // there IS one, and count the rest as a missing measurement, the same + // way nopid does above. Testing against both literals rather than for + // emptiness also catches a re-encoding of the field (say to a JSON + // bool) instead of reading it as a plaintext claim. + if (pc.claim != "YES" && pc.claim != "NO") noclaim++; + else if ((pc.claim == "YES") == e) agree++; + } + + // `checked` must equal the pool size, not merely be non-zero: a + // connection that vanished from pg_stat_ssl between enumerating the pool + // and checking it must shrink the pass, never the denominator. + ok(!pooled.empty() && nopid == 0 && pool_has_a2 && + checked == (int)pooled.size() && correct == checked, + "A[%s]: EVERY connection in the pool is encrypted as use_ssl asked, not just " + "the probed one, and the pool really holds the connection session 2 used " + "(pid '%s' %s; %d pooled, %d verified against pg_stat_ssl, %d correct, " + "%d with no usable backend_pid)", + tag.c_str(), a2.empty() ? "-" : a2.c_str(), pool_has_a2 ? "present" : "MISSING", + (int)pooled.size(), checked, correct, nopid); + + // ProxySQL's own account of its transport, CHECKED against PostgreSQL's + // rather than trusted. A proxy that pooled a plaintext connection while + // reporting using_ssl=YES would satisfy every other assertion here. + ok(checked > 0 && noclaim == 0 && agree == checked, + "A[%s]: ProxySQL's using_ssl agrees with pg_stat_ssl for all %d verified " + "pooled connection(s) (%d agree, %d with no usable using_ssl claim)", + tag.c_str(), checked, agree, noclaim); + } + + // ====================================================================== + // SCENARIO B -- configuration disagreeing with the connection: turn use_ssl + // OFF while an encrypted connection is still warm in the pool. + // ====================================================================== + freshServer(1); + std::string pid1; + std::string err1 = probe(pid1); + ok(err1.empty() && !pid1.empty(), + "B: query 1 served over a use_ssl=1 backend (pid '%s')%s%s", + pid1.empty() ? "-" : pid1.c_str(), err1.empty() ? "" : " -- ", err1.c_str()); + + const std::string ssl1 = backendSsl(pid1); + ok(ssl1 == "t" || ssl1 == "true", + "B: that backend connection is genuinely TLS-encrypted (pg_stat_ssl.ssl='%s')", + ssl1.empty() ? "not found" : ssl1.c_str()); + + // ---- step 2: turn use_ssl OFF, WITHOUT draining the pool --------------- + { + std::stringstream u; + u << "UPDATE pgsql_servers SET use_ssl=0 WHERE hostgroup_id=" << HG + << " AND hostname='" << cl.pgsql_server_host << "' AND port=" << cl.pgsql_server_port; + execAdmin(admin, u.str()); + execAdmin(admin, "LOAD PGSQL SERVERS TO RUNTIME"); + + // Wait for the new setting to actually be visible in the runtime table, + // rather than sleeping a fixed span and hoping. Note this deliberately + // does NOT drain the pool -- the whole point of the scenario is that the + // already-established TLS connection stays warm across the change. + int seen = -1; + if (!waitUntil([&]{ seen = runtimeUseSslFor(cl.pgsql_server_host, cl.pgsql_server_port); + return seen == 0; })) { + if (seen < 0) diag("could not read runtime_pgsql_servers.use_ssl for %s:%d " + "(admin query failed)", cl.pgsql_server_host, cl.pgsql_server_port); + else diag("server %s:%d still advertises use_ssl=%d after LOAD; query 2 may not be " + "exercising the config-disagrees-with-connection case", + cl.pgsql_server_host, cl.pgsql_server_port, seen); + } + // Whether the warm connection survived is what makes the next probe + // meaningful, so report it instead of assuming it. + diag("after use_ssl=0 landed, hostgroup %d holds %d pooled connection(s)", HG, poolCountHG()); + } + + // ---- step 3: the warm TLS connection must still serve ------------------- + std::string pid2; + std::string err2 = probe(pid2); + ok(err2.empty() && !pid2.empty(), + "B: query 2 served after use_ssl was turned off with a warm TLS connection (pid '%s')%s%s", + pid2.empty() ? "-" : pid2.c_str(), err2.empty() ? "" : " -- ", err2.c_str()); + + // Diagnostic, not an assertion: whether the warm TLS connection was actually + // reused is ProxySQL's pooling decision, not this test's contract. What the + // test requires is that the query is SERVED either way. Reporting it keeps a + // "passed because a fresh plaintext connection was opened" result visible + // rather than silently counted as coverage. + if (!pid2.empty()) { + const std::string ssl2 = backendSsl(pid2); + diag("query 2: pid '%s' (%s), pg_stat_ssl.ssl='%s'", pid2.c_str(), + (pid1 == pid2) ? "SAME connection reused" : "new connection", + ssl2.empty() ? "not found" : ssl2.c_str()); + } + + // ====================================================================== + // SCENARIO D -- churn. Scenarios A and B each hand a connection over once. + // A lifetime bug that survives the first handoff and trips on a later one -- + // or that corrupts one connection out of several in the pool -- passes both. + // + // Here CHURN_CLIENTS sessions are held open at once and queried round-robin + // for CHURN_ROUNDS rounds, so every pooled TLS connection is detached and + // re-attached repeatedly and the pool holds several of them at a time. Under + // ASAN this is the shape that turns a stale SSL pointer into a reported + // use-after-free rather than a silent success. + // ====================================================================== + freshServer(1); + { + std::vector clients; + int connected = 0; + std::string connect_err; + for (int i = 0; i < CHURN_CLIENTS; i++) { + PGConnPtr c = openConn(cl.pgsql_host, cl.pgsql_port, cl.pgsql_username, + cl.pgsql_password, cl.pgsql_username, "require"); + if (c && PQstatus(c.get()) == CONNECTION_OK) connected++; + else if (connect_err.empty()) + connect_err = oneline(c ? PQerrorMessage(c.get()) : "null connection"); + clients.push_back(std::move(c)); + } + + int served = 0, failed = 0; + std::set pids; + std::string first_err; + for (int r = 0; r < CHURN_ROUNDS; r++) { + // Fire every query BEFORE collecting any of them, so they are in + // flight simultaneously. Issuing them synchronously would put only + // one query in flight at a time, and ProxySQL would multiplex all of + // them onto a SINGLE pooled connection -- measured: 32 queries, one + // backend pid. That still exercises repeated handoff, but never + // builds a pool holding several TLS connections at once, which is + // the state this scenario exists to cover. + std::vector sent(clients.size(), false); + for (size_t i = 0; i < clients.size(); i++) { + PGConnPtr& c = clients[i]; + if (!c || PQstatus(c.get()) != CONNECTION_OK) continue; + sent[i] = (PQsendQuery(c.get(), PID_QUERY) == 1); + } + for (size_t i = 0; i < clients.size(); i++) { + PGConnPtr& c = clients[i]; + if (!c || PQstatus(c.get()) != CONNECTION_OK || !sent[i]) { + failed++; + if (first_err.empty() && c) first_err = oneline(PQerrorMessage(c.get())); + continue; + } + // PQgetResult must be drained to NULL before the connection can + // be used again. + bool got = false; + while (PGresult* res = PQgetResult(c.get())) { + if (!got && PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) == 1) { + pids.insert(PQgetvalue(res, 0, 0)); + got = true; + } + PQclear(res); + } + if (got) served++; + else { + failed++; + if (first_err.empty()) first_err = oneline(PQerrorMessage(c.get())); + } + } + } + const int total = CHURN_CLIENTS * CHURN_ROUNDS; + + ok(connected == CHURN_CLIENTS, + "D: all %d concurrent client sessions connected over TLS (%d ok)%s%s", + CHURN_CLIENTS, connected, connect_err.empty() ? "" : " -- ", connect_err.c_str()); + + ok(served == total, + "D: all %d queries across %d interleaved sessions were served (%d ok, %d failed)%s%s", + total, CHURN_CLIENTS, served, failed, first_err.empty() ? "" : " -- ", + first_err.c_str()); + + // Reuse actually happened. If every query got its own backend connection + // there was no handoff to break, and the scenario would pass while + // testing nothing. At most CHURN_CLIENTS sessions are ever in flight, so + // more distinct pids than that means connections were destroyed and + // recreated rather than pooled. + std::string pidlist; + for (const std::string& p : pids) { if (!pidlist.empty()) pidlist += ","; pidlist += p; } + ok(!pids.empty() && (int)pids.size() <= CHURN_CLIENTS, + "D: %d queries were served by only %d distinct backend connection(s) [%s] " + "-- pooled reuse, not one connection per query", + total, (int)pids.size(), pidlist.empty() ? "-" : pidlist.c_str()); + + // Every connection that carried this traffic must STILL be encrypted -- + // not merely the last one sampled, which is all a single-handoff scenario + // can show. + int found = 0, encrypted = 0; + for (const std::string& p : pids) { + const std::string s = backendSsl(p); + if (s.empty()) continue; + found++; + if (s == "t" || s == "true") encrypted++; + } + // BOTH guards are required, and each closes a hole the other leaves. + // `found == pids.size()` stops a pid missing from pg_stat_ssl from + // shrinking the DENOMINATOR: judging "encrypted == found" alone lets + // three of four connections disappear and still reports success as + // "1 of 1 encrypted". `!pids.empty()` stops the whole thing passing + // VACUOUSLY at 0 == 0 when no connection was ever established -- measured: + // with the backend user misconfigured, every query failed and this + // assertion still reported "0 of 0 connections found, 0 encrypted" as a + // pass, which is the one outcome a TLS check must never call success. + ok(!pids.empty() && found == (int)pids.size() && encrypted == found, + "D: every backend connection used during churn is still TLS-encrypted " + "(%d of %d connections found in pg_stat_ssl, %d encrypted)", + found, (int)pids.size(), encrypted); + } + + { + PGConnPtr a2 = openConn(cl.pgsql_admin_host, cl.pgsql_admin_port, + cl.admin_username, cl.admin_password, nullptr, "disable"); + ok(a2 && PQstatus(a2.get()) == CONNECTION_OK && scalar(a2.get(), "SELECT 1") == "1", + "ProxySQL still alive afterwards"); + } + + return exit_status(); +} diff --git a/test/tap/tests/pgsql_mock_backend.cpp b/test/tap/tests/pgsql_mock_backend.cpp index a52ab31722..e1fdb81911 100644 --- a/test/tap/tests/pgsql_mock_backend.cpp +++ b/test/tap/tests/pgsql_mock_backend.cpp @@ -1,6 +1,6 @@ /** * @file pgsql_mock_backend.cpp - * @brief Implementation of the scriptable fake PostgreSQL backend. + * @brief Implementation of the scriptable hostile PostgreSQL backend. * * See pgsql_mock_backend.h for the rationale and usage. * @@ -23,6 +23,11 @@ #include #include +#include +#include +#include +#include + // ------------------------------------------------------------- byte builders std::string pgmb_be32(uint32_t v) { @@ -43,17 +48,42 @@ std::string pgmb_be16(uint16_t v) { void pgmb_append_msg(std::string& out, char type, const std::string& payload) { out.push_back(type); - out += pgmb_be32((uint32_t)payload.size() + 4); + out += pgmb_be32((uint32_t)(payload.size() + 4)); + out += payload; +} + +void pgmb_append_msg_raw_len(std::string& out, char type, uint32_t declared_len, + const std::string& payload) { + out.push_back(type); + out += pgmb_be32(declared_len); out += payload; } -// ------------------------------------------------------ backend-side messages +// ------------------------------------------------------------ auth messages -std::string pgmb_auth_ok() { +std::string pgmb_auth_raw(uint32_t auth_type, const std::string& rest) { std::string out; - pgmb_append_msg(out, 'R', pgmb_be32(0)); + pgmb_append_msg(out, 'R', pgmb_be32(auth_type) + rest); return out; } +std::string pgmb_auth_ok() { return pgmb_auth_raw(0, ""); } +std::string pgmb_auth_cleartext() { return pgmb_auth_raw(3, ""); } + +std::string pgmb_auth_md5(const unsigned char salt[4]) { + return pgmb_auth_raw(5, std::string((const char*)salt, 4)); +} + +std::string pgmb_auth_sasl(const std::vector& mechanisms) { + std::string body; + for (const auto& m : mechanisms) { body += m; body.push_back('\0'); } + body.push_back('\0'); // terminating empty name + return pgmb_auth_raw(10, body); +} + +std::string pgmb_auth_sasl_continue(const std::string& body) { return pgmb_auth_raw(11, body); } +std::string pgmb_auth_sasl_final(const std::string& body) { return pgmb_auth_raw(12, body); } + +// ------------------------------------------------------- steady-state messages std::string pgmb_parameter_status(const std::string& name, const std::string& value) { std::string payload = name; @@ -77,6 +107,31 @@ std::string pgmb_ready_for_query(char txn_state) { return out; } +std::string pgmb_error_response(const std::string& sqlstate, const std::string& message) { + std::string payload; + payload.push_back('S'); payload += "ERROR"; payload.push_back('\0'); + payload.push_back('V'); payload += "ERROR"; payload.push_back('\0'); + payload.push_back('C'); payload += sqlstate; payload.push_back('\0'); + payload.push_back('M'); payload += message; payload.push_back('\0'); + payload.push_back('\0'); // field-list terminator + std::string out; + pgmb_append_msg(out, 'E', payload); + return out; +} + +std::string pgmb_error_response_unterminated(const std::string& sqlstate) { + // Last field value runs to the end of the payload with no NUL and no + // field-list terminator. A parser that scans for NUL without bounding on + // payload_len walks off the end here. + std::string payload; + payload.push_back('S'); payload += "FATAL"; payload.push_back('\0'); + payload.push_back('C'); payload += sqlstate; payload.push_back('\0'); + payload.push_back('M'); payload += "unterminated message value"; // no NUL, no terminator + std::string out; + pgmb_append_msg(out, 'E', payload); + return out; +} + std::string pgmb_row_description_1col(const std::string& colname, uint32_t type_oid) { std::string payload = pgmb_be16(1); // one field payload += colname; payload.push_back('\0'); @@ -108,19 +163,13 @@ std::string pgmb_command_complete(const std::string& tag) { return out; } -std::string pgmb_copy_out_response(int ncols) { - std::string payload; - payload.push_back('\0'); // overall format: 0 = text - payload += pgmb_be16((uint16_t)ncols); - for (int i = 0; i < ncols; i++) payload += pgmb_be16(0); // per-column: text +std::string pgmb_notification_response(int32_t pid, const std::string& channel, + const std::string& payload_text) { + std::string payload = pgmb_be32((uint32_t)pid); + payload += channel; payload.push_back('\0'); + payload += payload_text; payload.push_back('\0'); std::string out; - pgmb_append_msg(out, 'H', payload); - return out; -} - -std::string pgmb_copy_data(const std::string& payload) { - std::string out; - pgmb_append_msg(out, 'd', payload); + pgmb_append_msg(out, 'A', payload); return out; } @@ -132,16 +181,98 @@ std::string pgmb_simple_result(const std::string& colname, const std::string& va return out; } +bool pgmb_result_of_exact_size(std::string& out, size_t target_bytes) { + // Fixed parts, then a single DataRow sized to absorb the remainder exactly. + const std::string rd = pgmb_row_description_1col("c", 25); + const std::string cc = pgmb_command_complete("SELECT 1"); + const std::string rfq = pgmb_ready_for_query('I'); + // DataRow overhead: 1 type + 4 length + 2 ncols + 4 value-length. + const size_t row_overhead = 11; + const size_t fixed = rd.size() + cc.size() + rfq.size() + row_overhead; + if (target_bytes < fixed) return false; + const size_t vallen = target_bytes - fixed; + + out.clear(); + out.reserve(target_bytes); + out += rd; + out += pgmb_data_row_1col(std::string(vallen, 'x')); + out += cc; + out += rfq; + return out.size() == target_bytes; +} + // ------------------------------------------------------------------- steps -Step step_send(const std::string& data) { +Step step_send(const std::string& data, size_t chunk_bytes, int chunk_delay_us) { Step s; s.kind = Step::SEND; s.data = data; + s.chunk_bytes = chunk_bytes; s.chunk_delay_us = chunk_delay_us; return s; } Step step_expect_startup() { Step s; s.kind = Step::EXPECT_STARTUP; return s; } -Step step_expect_query() { Step s; s.kind = Step::EXPECT_QUERY; return s; } +Step step_expect_message() { Step s; s.kind = Step::EXPECT_MESSAGE; return s; } +Step step_expect_query(bool stop_at_housekeeping) + { Step s; s.kind = Step::EXPECT_QUERY; s.stop_at_housekeeping = stop_at_housekeeping; return s; } Step step_close() { Step s; s.kind = Step::CLOSE; return s; } Step step_sleep(int ms) { Step s; s.kind = Step::SLEEP_MS; s.ms = ms; return s; } +Step step_scram_server_first(bool bad_nonce) { + Step s; s.kind = Step::SCRAM_SERVER_FIRST; s.bad_nonce = bad_nonce; return s; +} +Step step_scram_server_final(bool forge_signature) { + Step s; s.kind = Step::SCRAM_SERVER_FINAL; s.forge_signature = forge_signature; return s; +} + +std::vector pgmb_script_accept_trust() { + std::string post_auth = + pgmb_auth_ok() + + pgmb_parameter_status("server_version", "16.2") + + pgmb_parameter_status("client_encoding", "UTF8") + + pgmb_backend_key_data(4242, 987654321) + + pgmb_ready_for_query('I'); + return { step_expect_startup(), step_send(post_auth) }; +} + +// ---------------------------------------------------------------- crypto bits + +static std::string b64_encode(const std::string& in) { + if (in.empty()) return ""; + std::string out((((in.size() + 2) / 3) * 4) + 1, '\0'); + int n = EVP_EncodeBlock((unsigned char*)&out[0], (const unsigned char*)in.data(), + (int)in.size()); + if (n < 0) return ""; + out.resize((size_t)n); + return out; +} + +static std::string hmac_sha256(const std::string& key, const std::string& data) { + unsigned char md[EVP_MAX_MD_SIZE]; + unsigned int mdlen = 0; + HMAC(EVP_sha256(), key.data(), (int)key.size(), + (const unsigned char*)data.data(), data.size(), md, &mdlen); + return std::string((const char*)md, mdlen); +} + +// SCRAM-SHA-256 server-side key derivation (RFC 5802 / RFC 7677). +static std::string scram_server_key(const std::string& password, const std::string& salt, + int iterations) { + unsigned char salted[32]; + PKCS5_PBKDF2_HMAC(password.data(), (int)password.size(), + (const unsigned char*)salt.data(), (int)salt.size(), + iterations, EVP_sha256(), sizeof(salted), salted); + return hmac_sha256(std::string((const char*)salted, sizeof(salted)), "Server Key"); +} + +// Extract the value of `key=` from a comma-separated SCRAM attribute list. +static std::string scram_attr(const std::string& msg, char key) { + size_t i = 0; + while (i < msg.size()) { + size_t end = msg.find(',', i); + if (end == std::string::npos) end = msg.size(); + if (end - i >= 2 && msg[i] == key && msg[i + 1] == '=') + return msg.substr(i + 2, end - i - 2); + i = end + 1; + } + return ""; +} // --------------------------------------------------------------- socket I/O @@ -235,6 +366,11 @@ void PgSQL_Mock_Backend::set_script(const std::vector& steps) { script_ = steps; } +void PgSQL_Mock_Backend::set_scram_password(const std::string& pw) { + std::lock_guard g(script_mtx_); + scram_password_ = pw; +} + std::string PgSQL_Mock_Backend::last_error() { std::lock_guard g(err_mtx_); return last_error_; @@ -269,24 +405,34 @@ bool PgSQL_Mock_Backend::start() { void PgSQL_Mock_Backend::stop() { if (!running_.exchange(false)) return; - // shutdown() wakes the acceptor out of accept(); the fd is closed only after that - // thread has been joined. Closing first would let the acceptor call accept() on a - // descriptor number another thread could already have reused. - if (listen_fd_ >= 0) ::shutdown(listen_fd_, SHUT_RDWR); - if (acceptor_.joinable()) acceptor_.join(); - if (listen_fd_ >= 0) { ::close(listen_fd_); listen_fd_ = -1; } - // A worker can be blocked in recv() waiting for a startup packet or a query that - // a wedged proxy will never send. Joining it in that state hangs the run instead - // of letting the test report a failure, so wake them first. The fd is still open: - // handle_conn() removes itself from this list before closing, under the same lock. + if (listen_fd_ >= 0) { ::shutdown(listen_fd_, SHUT_RDWR); ::close(listen_fd_); listen_fd_ = -1; } + // Unblock workers parked in recv() on a proxy that never sent a query, so the + // join below cannot hang the whole run. { std::lock_guard g(conns_mtx_); for (int cfd : client_fds_) ::shutdown(cfd, SHUT_RDWR); } + if (acceptor_.joinable()) acceptor_.join(); for (auto& t : workers_) if (t.joinable()) t.join(); workers_.clear(); } +std::string pgmb_copy_out_response(int ncols) { + std::string payload; + payload.push_back('\0'); // overall format: 0 = text + payload += pgmb_be16((uint16_t)ncols); + for (int i = 0; i < ncols; i++) payload += pgmb_be16(0); // per-column: text + std::string out; + pgmb_append_msg(out, 'H', payload); + return out; +} + +std::string pgmb_copy_data(const std::string& payload) { + std::string out; + pgmb_append_msg(out, 'd', payload); + return out; +} + void PgSQL_Mock_Backend::accept_loop() { while (running_.load()) { int fd = ::accept(listen_fd_, nullptr, nullptr); @@ -297,10 +443,6 @@ void PgSQL_Mock_Backend::accept_loop() { int one = 1; setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); conns_accepted_.fetch_add(1); - { - std::lock_guard g(conns_mtx_); - client_fds_.push_back(fd); - } std::vector script; { std::lock_guard g(script_mtx_); @@ -311,6 +453,20 @@ void PgSQL_Mock_Backend::accept_loop() { } void PgSQL_Mock_Backend::handle_conn(int fd, std::vector script) { + { + std::lock_guard g(conns_mtx_); + client_fds_.push_back(fd); + } + std::string password; + { + std::lock_guard g(script_mtx_); + password = scram_password_; + } + + // SCRAM exchange state, carried across the two SCRAM steps. + std::string client_first_bare, server_first, salt; + const int iterations = 4096; + auto fail = [&](const char* what) { std::lock_guard g(err_mtx_); last_error_ = what; @@ -319,7 +475,17 @@ void PgSQL_Mock_Backend::handle_conn(int fd, std::vector script) { for (const Step& s : script) { switch (s.kind) { case Step::SEND: { - if (!write_all(fd, s.data.data(), s.data.size())) { fail("write failed"); goto done; } + if (s.chunk_bytes == 0) { + if (!write_all(fd, s.data.data(), s.data.size())) { fail("write failed"); goto done; } + } else { + size_t off = 0; + while (off < s.data.size()) { + const size_t n = std::min(s.chunk_bytes, s.data.size() - off); + if (!write_all(fd, s.data.data() + off, n)) { fail("chunked write failed"); goto done; } + off += n; + if (s.chunk_delay_us > 0) usleep((useconds_t)s.chunk_delay_us); + } + } break; } case Step::EXPECT_STARTUP: { @@ -327,6 +493,11 @@ void PgSQL_Mock_Backend::handle_conn(int fd, std::vector script) { if (!read_startup(fd, payload)) { fail("startup read failed"); goto done; } break; } + case Step::EXPECT_MESSAGE: { + char type = 0; std::string payload; + if (!read_frontend_msg(fd, &type, payload)) { fail("frontend message read failed"); goto done; } + break; + } case Step::EXPECT_QUERY: { // Swallow and generically acknowledge anything the proxy sends // ahead of the client's query, so a canned result is never handed @@ -336,8 +507,9 @@ void PgSQL_Mock_Backend::handle_conn(int fd, std::vector script) { // transaction control) arrives as a simple Query too, so stopping // at the first 'Q' is not enough — the caller's canned RESULTSET // would then answer a proxy SET. That is not a harmless mismatch: - // it drives ProxySQL into an unbounded error loop. Housekeeping is - // acknowledged with a COMMAND response and skipped. + // it drives ProxySQL into an unbounded error loop (see the + // async_send_simple_command finding). Housekeeping is acknowledged + // with a COMMAND response and skipped. for (;;) { char type = 0; std::string payload; if (!read_frontend_msg(fd, &type, payload)) { fail("query read failed"); goto done; } @@ -358,6 +530,7 @@ void PgSQL_Mock_Backend::handle_conn(int fd, std::vector script) { queries_observed_.fetch_add(1); break; } + if (s.stop_at_housekeeping) break; // leave it for the next step to answer const std::string ack = pgmb_command_complete("SET") + pgmb_ready_for_query('I'); if (!write_all(fd, ack.data(), ack.size())) { fail("housekeeping ack failed"); goto done; } @@ -371,14 +544,65 @@ void PgSQL_Mock_Backend::handle_conn(int fd, std::vector script) { case Step::CLOSE: goto done; case Step::SLEEP_MS: - // Slept in slices, watching running_, so a script that deliberately holds - // a socket open for seconds does not make stop() -- which joins these - // threads -- block for the remainder of it. - for (int slept = 0; slept < s.ms && running_.load(); slept += 100) { - usleep(100000); + usleep((useconds_t)s.ms * 1000); + break; + + case Step::SCRAM_SERVER_FIRST: { + // SASLInitialResponse: mechanism\0 | int32 len | client-first-message + char type = 0; std::string payload; + if (!read_frontend_msg(fd, &type, payload)) { fail("SASLInitialResponse read failed"); goto done; } + size_t z = payload.find('\0'); + if (z == std::string::npos || payload.size() < z + 5) { fail("malformed SASLInitialResponse"); goto done; } + const std::string client_first = payload.substr(z + 5); + // client-first-bare is everything after the gs2 header ("n,," / "y,," / "p=...,,"). + size_t bare = client_first.find(",,"); + client_first_bare = (bare == std::string::npos) ? client_first : client_first.substr(bare + 2); + const std::string client_nonce = scram_attr(client_first_bare, 'r'); + + unsigned char rnd[18]; + RAND_bytes(rnd, sizeof(rnd)); + const std::string server_nonce_part = b64_encode(std::string((const char*)rnd, sizeof(rnd))); + unsigned char saltb[16]; + RAND_bytes(saltb, sizeof(saltb)); + salt.assign((const char*)saltb, sizeof(saltb)); + + // RFC 5802: the server nonce MUST begin with the client nonce. With + // bad_nonce we deliberately violate that, which a correct client + // must detect and abort on. + const std::string combined_nonce = s.bad_nonce + ? server_nonce_part + : client_nonce + server_nonce_part; + + server_first = "r=" + combined_nonce + ",s=" + b64_encode(salt) + + ",i=" + std::to_string(iterations); + const std::string msg = pgmb_auth_sasl_continue(server_first); + if (!write_all(fd, msg.data(), msg.size())) { fail("server-first write failed"); goto done; } + break; + } + + case Step::SCRAM_SERVER_FINAL: { + char type = 0; std::string client_final; + if (!read_frontend_msg(fd, &type, client_final)) { fail("SASLResponse read failed"); goto done; } + // client-final-without-proof is everything before ",p=". + const size_t ppos = client_final.rfind(",p="); + const std::string cf_without_proof = (ppos == std::string::npos) + ? client_final : client_final.substr(0, ppos); + + const std::string auth_message = + client_first_bare + "," + server_first + "," + cf_without_proof; + std::string sig = hmac_sha256(scram_server_key(password, salt, iterations), auth_message); + + if (s.forge_signature) { + // Flip every bit. A client that verifies the server signature + // rejects this; a client that skips verification accepts a + // server that cannot prove it knows the secret. + for (char& c : sig) c = (char)(~(unsigned char)c); } + const std::string msg = pgmb_auth_sasl_final("v=" + b64_encode(sig)); + if (!write_all(fd, msg.data(), msg.size())) { fail("server-final write failed"); goto done; } break; } + } } // Script exhausted: hold the connection open briefly so ProxySQL observes @@ -387,6 +611,9 @@ void PgSQL_Mock_Backend::handle_conn(int fd, std::vector script) { usleep(200000); done: + // Deregister BEFORE closing: once this fd is closed the OS can hand the same + // number to a new socket, and stop() would then ::shutdown() an unrelated + // connection belonging to the test process. { std::lock_guard g(conns_mtx_); auto it = std::find(client_fds_.begin(), client_fds_.end(), fd); diff --git a/test/tap/tests/pgsql_mock_backend.h b/test/tap/tests/pgsql_mock_backend.h index fb3f1ab6bb..661f746c0b 100644 --- a/test/tap/tests/pgsql_mock_backend.h +++ b/test/tap/tests/pgsql_mock_backend.h @@ -1,51 +1,47 @@ /** * @file pgsql_mock_backend.h - * @brief A scriptable fake PostgreSQL backend for TAP tests. + * @brief A scriptable, deliberately hostile PostgreSQL backend for TAP tests. * * WHY THIS EXISTS * --------------- - * Some ProxySQL failure modes can only be reached from a backend that behaves - * in a way a real PostgreSQL never does. The one this file exists for is a - * server that VANISHES IN THE MIDDLE OF A RESULT — TCP close with no - * ErrorResponse, after rows have already gone out. + * ProxySQL's native backend protocol handles malformed framing, truncated + * messages, forged authentication, unexpected message types and mid-stream + * disconnects. None of those branches can be reached from a real PostgreSQL, + * because a real PostgreSQL does not emit those bytes. Every existing + * native-path test uses a healthy backend as its oracle, so the entire error + * surface of PgSQL_Connection.cpp and PgSQL_Backend_Protocol.cpp is untested. * - * A real PostgreSQL cannot be asked to do that. pg_terminate_backend() and a - * graceful shutdown both send a FATAL ErrorResponse before closing, which puts - * ProxySQL on its ordinary error path; the defect under test needs - * PQconsumeInput() to FAIL, which only happens on a raw transport close. The - * alternative — a real server behind a killable TCP relay — reproduces it but - * depends on killing at the exact instant rows are in flight, which is the kind - * of timing dependency that makes tests flaky. This harness hits the same state - * deterministically. + * This harness is the missing half: a listener that speaks just enough of the + * wire protocol to be accepted as a backend, then emits whatever the test tells + * it to. It is registered in `pgsql_servers` like any other backend, so + * ProxySQL reaches it through the ordinary connect/auth/query path. * * USAGE * ----- * PgSQL_Mock_Backend mock; * if (!mock.start()) BAIL_OUT("mock backend failed to listen"); - * mock.set_script({ step_expect_startup(), step_send(handshake), - * step_expect_query(), step_send(partial_result), - * step_close() }); - * // ... register pgmb_local_ip_towards(...) : mock.port() in pgsql_servers ... + * mock.set_script(script_that_sends_garbage_during_auth()); + * // ... register mock.host() : mock.port() in pgsql_servers, drive traffic ... * mock.stop(); * * A script is a list of Steps executed in order per accepted connection. The - * connection handler runs on its own thread, so several ProxySQL connections can - * be in flight at once; each gets a fresh copy of the script. + * connection handler runs on its own thread, so several ProxySQL connections + * can be in flight at once; each gets a fresh copy of the script. + * + * DELIVERY GRANULARITY + * -------------------- + * Step::chunk_bytes controls how the payload reaches the wire: 0 means one + * write(), N means N bytes per write() with a short pause between. That is how + * partial-message framing gets exercised — the framer must cope with a length + * field split across reads. * * WHAT THIS IS NOT * ---------------- - * Not a PostgreSQL implementation. It answers a startup packet with a trust-style - * AuthenticationOk and does not execute SQL — every result it returns is canned - * bytes. Plaintext only; it speaks no TLS and no MD5/SCRAM authentication. - * - * SCOPE NOTE - * ---------- - * This is a deliberately reduced version, carrying only what - * pgsql-midresult_disconnect-t needs. The fuller harness it came from also - * drives the SCRAM server side (including forged signatures and non-extending - * nonces), builds malformed frames with arbitrary declared lengths, and delivers - * payloads byte-by-byte — all of which exist to test a native backend protocol - * that this branch does not have. + * Not a PostgreSQL implementation. It answers a startup packet and can drive a + * real SCRAM-SHA-256 server side far enough to test ProxySQL's client side, but + * it does not execute SQL. Every result it returns is canned bytes. It is + * plaintext only — TLS backends are covered against the real server by + * pgsql-native_tls-t. */ #ifndef PGSQL_MOCK_BACKEND_H #define PGSQL_MOCK_BACKEND_H @@ -62,24 +58,49 @@ // Builders for the backend-direction messages a script needs. All lengths are // big-endian and include the 4-byte length field itself but not the type byte, // per the PostgreSQL protocol. Deliberately independent of ProxySQL's own -// encoders: a test that used the code under test to build its fixtures could not -// detect an encoding bug in it. +// encoders: a test that used the code under test to build its fixtures could +// not detect an encoding bug in it. // Append a framed message: type byte, int32 length, payload. void pgmb_append_msg(std::string& out, char type, const std::string& payload); +// Append a message with an ARBITRARY declared length, ignoring the real payload +// size. This is how truncation, over-long and under-long frames are built. +void pgmb_append_msg_raw_len(std::string& out, char type, uint32_t declared_len, + const std::string& payload); + std::string pgmb_be32(uint32_t v); std::string pgmb_be16(uint16_t v); -// AuthenticationOk -- the only authentication this harness performs. +// AuthenticationOk / AuthenticationCleartextPassword / AuthenticationMD5Password. std::string pgmb_auth_ok(); +std::string pgmb_auth_cleartext(); +std::string pgmb_auth_md5(const unsigned char salt[4]); +// AuthenticationSASL advertising the given mechanism list (may be empty). +std::string pgmb_auth_sasl(const std::vector& mechanisms); +// AuthenticationSASLContinue / AuthenticationSASLFinal with a raw body. +std::string pgmb_auth_sasl_continue(const std::string& body); +std::string pgmb_auth_sasl_final(const std::string& body); +// An Authentication message with an arbitrary subtype code (7=GSSAPI, 9=SSPI, +// or a value no PostgreSQL version defines). +std::string pgmb_auth_raw(uint32_t auth_type, const std::string& rest); std::string pgmb_parameter_status(const std::string& name, const std::string& value); std::string pgmb_backend_key_data(int32_t pid, int32_t secret); std::string pgmb_ready_for_query(char txn_state); // 'I' | 'T' | 'E' +std::string pgmb_error_response(const std::string& sqlstate, const std::string& message); +// ErrorResponse whose final field value is NOT NUL-terminated — the payload +// simply ends. Exercises the bounds checks in native_fill_error_from_E(). +std::string pgmb_error_response_unterminated(const std::string& sqlstate); std::string pgmb_row_description_1col(const std::string& colname, uint32_t type_oid); std::string pgmb_data_row_1col(const std::string& value); std::string pgmb_command_complete(const std::string& tag); +std::string pgmb_notification_response(int32_t pid, const std::string& channel, + const std::string& payload); + +// A complete, well-formed single-column result: RowDescription, `rows` DataRows +// each carrying `value`, CommandComplete, ReadyForQuery('I'). +std::string pgmb_simple_result(const std::string& colname, const std::string& value, int rows); // CopyOutResponse ('H') for a text-format copy with `ncols` columns, and one // CopyData ('d') payload. Used to drive ProxySQL's PGRES_COPY_OUT path, which is @@ -87,38 +108,74 @@ std::string pgmb_command_complete(const std::string& tag); std::string pgmb_copy_out_response(int ncols); std::string pgmb_copy_data(const std::string& payload); -// A complete, well-formed single-column result: RowDescription, `rows` DataRows -// each carrying `value`, CommandComplete, ReadyForQuery('I'). -std::string pgmb_simple_result(const std::string& colname, const std::string& value, int rows); +// A well-formed result padded with NoticeResponse messages until the total byte +// count is EXACTLY `target_bytes`. Used to hit the exact-multiple-of-16384 +// condition behind defect D4. Returns false if the target cannot be hit exactly +// (too small to fit the mandatory messages). +bool pgmb_result_of_exact_size(std::string& out, size_t target_bytes); // ---------------------------------------------------------------------- steps struct Step { enum Kind { SEND, // write `data` to the client + EXPECT_MESSAGE, // read and discard one frontend message (any type) // Read frontend messages until a simple Query ('Q') arrives, generically // acknowledging anything else along the way. ProxySQL may legitimately - // send its own statements (session-variable replay, transaction control) - // before the client's query; a step that stopped at the first message - // would answer one of those with the canned result and then leave the - // real query unanswered, which looks like a hang in the code under test - // rather than a scripting mistake in the fixture. + // send its own statements (session-variable replay, init_connect, a + // Sync) before the client's query; a fixed EXPECT_MESSAGE would answer + // one of those with the canned result and then leave the real query + // unanswered, which looks like a hang in the code under test rather + // than a scripting mistake in the fixture. EXPECT_QUERY, EXPECT_STARTUP, // read the startup packet (no type byte, length-prefixed) CLOSE, // close the connection immediately (FIN) - SLEEP_MS // pause, e.g. to let ProxySQL sit on an unfinished message + SLEEP_MS, // pause, e.g. to let ProxySQL park the connection + SCRAM_SERVER_FIRST,// read SASLInitialResponse, reply with a real server-first + SCRAM_SERVER_FINAL // read SASLResponse, reply with server-final (see forge_signature) }; Kind kind = SEND; std::string data; // SEND payload int ms = 0; // SLEEP_MS duration + size_t chunk_bytes = 0;// SEND granularity: 0 = one write, N = N bytes per write + int chunk_delay_us = 0;// pause between chunks when chunk_bytes > 0 + + // SCRAM_SERVER_FINAL only. When true the server signature sent to ProxySQL + // is deliberately wrong, simulating a server that cannot prove it knows the + // shared secret. ProxySQL MUST reject it: this is the sole defence against + // a spoofed or MITM'd backend. When false a correct signature is sent. + bool forge_signature = false; + + // SCRAM_SERVER_FIRST only. When true the server nonce does NOT extend the + // client nonce, violating RFC 5802. A correct client must abort. + bool bad_nonce = false; + + // EXPECT_QUERY only. Housekeeping statements ProxySQL issues on its own + // behalf (session-variable replay, DISCARD, transaction control) are + // normally acknowledged and skipped so the canned reply lands on the + // client's query. When this is set the step STOPS on the housekeeping + // statement instead and leaves it unanswered, so the NEXT step replies to + // it. That is the only way to script a response to a statement ProxySQL + // sent itself -- which is what F5 needs: a RESULTSET where ProxySQL + // expects nothing but a command acknowledgement. + bool stop_at_housekeeping = false; }; // Convenience constructors. -Step step_send(const std::string& data); +Step step_send(const std::string& data, size_t chunk_bytes = 0, int chunk_delay_us = 0); Step step_expect_startup(); -Step step_expect_query(); +Step step_expect_message(); +Step step_expect_query(bool stop_at_housekeeping = false); Step step_close(); Step step_sleep(int ms); +Step step_scram_server_first(bool bad_nonce = false); +Step step_scram_server_final(bool forge_signature = false); + +// Scripts for the handshake shapes tests reuse. +// +// A trust-style handshake: startup -> AuthenticationOk -> ParameterStatus x2 -> +// BackendKeyData -> ReadyForQuery('I'). Leaves the connection ready for a query. +std::vector pgmb_script_accept_trust(); // ------------------------------------------------------------------ the mock @@ -134,16 +191,23 @@ class PgSQL_Mock_Backend { bool start(); void stop(); - // The port ProxySQL should be pointed at. For the address, call - // pgmb_local_ip_towards() below: it discovers this container's routable IP at - // runtime rather than relying on Docker DNS, so the test does not depend on how - // the runner container's name or hostname is registered. + // The address ProxySQL should be pointed at. host() is this container's + // routable IP on the shared Docker network, discovered at runtime rather + // than via Docker DNS so the test does not depend on how the runner + // container's name or hostname is registered. + const std::string& host() const { return host_; } uint16_t port() const { return port_; } // Replace the script future connections will run. Connections already in // progress keep the script they started with. void set_script(const std::vector& steps); + // The password the SCRAM_SERVER_* steps derive keys from. Must match the + // password ProxySQL is configured to use for this backend, otherwise even + // the honest server-final is rejected and the forged-signature case proves + // nothing. + void set_scram_password(const std::string& pw); + // Connections accepted since the last reset_stats(). int connections_accepted() const { return conns_accepted_.load(); } @@ -164,6 +228,7 @@ class PgSQL_Mock_Backend { int listen_fd_ = -1; uint16_t port_ = 0; + std::string host_; std::thread acceptor_; std::vector workers_; std::atomic running_{false}; @@ -176,6 +241,7 @@ class PgSQL_Mock_Backend { std::vector client_fds_; std::mutex script_mtx_; std::vector script_; + std::string scram_password_ = "mockpw"; std::mutex err_mtx_; std::string last_error_; }; diff --git a/test/tap/tests/unit/Makefile b/test/tap/tests/unit/Makefile index f49625ff94..e463771acc 100644 --- a/test/tap/tests/unit/Makefile +++ b/test/tap/tests/unit/Makefile @@ -577,8 +577,10 @@ UNIT_TESTS := smoke_test-t vendored_openssl_version_unit-t \ gen_utils_unit-t \ proxy_protocol_unit-t \ pgsql_txn_state_unit-t \ + pgsql_conn_liveness_unit-t \ sqlite3db_unit-t \ pgsql_error_helper_unit-t \ + pgsql_native_params_unit-t \ charset_find_unit-t \ config_validation_unit-t \ config_write_unit-t \ @@ -605,8 +607,8 @@ UNIT_TESTS := smoke_test-t vendored_openssl_version_unit-t \ mysql_resultset_framer_traffic_unit-t \ pgsql_response_framer_unit-t \ pgsql_response_framer_traffic_unit-t \ - ffto_state_machine_unit-t \ pgsql_reconcile_unit-t \ + ffto_state_machine_unit-t \ pgsql_conninfo_credentials_unit-t \ re2_vendor_unit-t \ restapi_server_unit-t \ diff --git a/test/tap/tests/unit/pgsql_backend_auth-t.cpp b/test/tap/tests/unit/pgsql_backend_auth-t.cpp index d2a442a4fc..c4dcb8e377 100644 --- a/test/tap/tests/unit/pgsql_backend_auth-t.cpp +++ b/test/tap/tests/unit/pgsql_backend_auth-t.cpp @@ -10,8 +10,83 @@ #include "scram.h" // libscram: used to pin the RFC vector and to act as an independent SCRAM verifier #include "tap.h" + +// Walk a StartupMessage's key\0value\0...\0 body and return the value for `key`, +// or "" when the key is absent. Layout: int32 len, int32 protocol, then pairs. +static std::string startup_param(const unsigned char* sm, size_t smlen, const char* key) { + size_t off = 8; + while (off < smlen && sm[off] != 0) { + const char* k = (const char*)(sm + off); + off += strlen(k) + 1; + if (off >= smlen) break; + const char* v = (const char*)(sm + off); + off += strlen(v) + 1; + if (strcmp(k, key) == 0) return std::string(v); + } + return std::string(); +} + + +// RFC 7677 Section 3 SCRAM-SHA-256 vector (password "pencil", salt W22ZaJ0SNY7soEsUEjb6gQ==, +// i=4096), written as PostgreSQL stores it in pg_authid.rolpassword -- which is byte for byte +// what pgsql_users.password holds for a verifier-stored user. Using a published vector means +// the harvested-key assertions below have an external expected value rather than agreeing +// with whatever this code happens to compute. +static const char* const RFC7677_VERIFIER = + "SCRAM-SHA-256$4096:W22ZaJ0SNY7soEsUEjb6gQ==" + "$WG5d8oPm3OtcPnkdi4Uo7BkeZkBFzpcXkuLmtbsT4qY=:wfPLwcE6nTWhTAmQ7tl2KeoiWGPlZqQxSrmfPwDl2dU="; + +// ClientKey = HMAC(SaltedPassword,"Client Key") and ServerKey = HMAC(SaltedPassword,"Server Key") +// for that same vector. StoredKey = SHA256(ClientKey) is the value encoded in the verifier above. +static const uint8_t RFC7677_CLIENT_KEY[32] = { + 0xa6, 0x0f, 0xc9, 0x23, 0xd6, 0x7e, 0x86, 0x44, + 0xa9, 0x2d, 0x16, 0xb9, 0x6e, 0xda, 0x5e, 0xf4, + 0x65, 0x6b, 0x0c, 0x72, 0x5c, 0x48, 0x43, 0x74, + 0xbe, 0x25, 0x53, 0x55, 0x76, 0x99, 0x6e, 0x8b, +}; +static const uint8_t RFC7677_SERVER_KEY[32] = { + 0xc1, 0xf3, 0xcb, 0xc1, 0xc1, 0x3a, 0x9d, 0x35, + 0xa1, 0x4c, 0x09, 0x90, 0xee, 0xd9, 0x76, 0x29, + 0xea, 0x22, 0x58, 0x63, 0xe5, 0x66, 0xa4, 0x31, + 0x4a, 0xb9, 0x9f, 0x3f, 0x00, 0xe5, 0xd9, 0xd5, +}; + +// Drives the libscram SERVER side of one exchange: consumes a wrapper-produced client-first +// and returns the server-first message built from `stored_secret`. The returned string is +// owned by `srv` (build_server_first_message stores it as server_first_message and +// free_scram_state releases it) -- the caller must NOT free it. read_client_first_message +// mutates its input and hands back buffers the ScramState takes ownership of, so it gets a +// private copy. +static char* server_first_for(ScramState* srv, const char* client_first, const char* stored_secret) { + std::string copy(client_first); + char cbind_flag = 0; + char* cfmb = nullptr; + char* cnonce = nullptr; + if (!read_client_first_message(©[0], &cbind_flag, &cfmb, &cnonce)) return nullptr; + srv->cbind_flag = cbind_flag; + srv->client_first_message_bare = cfmb; // ownership transferred to srv + srv->client_nonce = cnonce; + return build_server_first_message(srv, "", stored_secret); +} + +// Server-side verification of a wrapper-produced client-final. Mirrors +// PgSQL_Protocol::scram_handle_client_final: read_client_final_message needs a pristine +// raw_input for the without-proof reconstruction AND a separate mutable buffer it fills +// with NULs, so the two copies must be distinct. +static bool server_accepts(ScramState* srv, const char* client_final) { + std::string raw(client_final); + std::string buf(client_final); + const char* nonce = nullptr; + char* proof = nullptr; + bool accepted = false; + if (read_client_final_message(srv, (const uint8_t*)raw.c_str(), &buf[0], &nonce, &proof)) + accepted = verify_final_nonce(srv, nonce) && verify_client_proof(srv, proof); + free(proof); + return accepted; +} + int main(int, char**) { - plan(15); + plan(29); // SSLRequest is a fixed 8 bytes: length=8, code=80877103 (0x04d2162f). unsigned char ssl[8]; @@ -21,10 +96,46 @@ int main(int, char**) { // Startup message: int32 length, int32 protocol 196608 (3.0), then key\0value\0... \0. unsigned char sm[256]; size_t smlen = 0; - pg_build_startup(sm, &smlen, sizeof(sm), "alice", "shop"); + pg_build_startup(sm, &smlen, sizeof(sm), "alice", "shop", nullptr, nullptr, nullptr); // protocol version at offset 4 must be 0x00030000 ok(sm[4]==0x00 && sm[5]==0x03 && sm[6]==0x00 && sm[7]==0x00, "startup protocol 3.0"); + // The startup message must be able to carry the session settings the libpq path + // sends as client_encoding=... and options='-c k=v ...'. Without them a client's + // connection options are silently dropped on the native path. application_name is + // what identifies the connection in pg_stat_activity and in log_line_prefix '%a'. + { + // Nothing optional supplied: none of the three keys may appear. + ok(startup_param(sm, smlen, "options").empty() && + startup_param(sm, smlen, "client_encoding").empty() && + startup_param(sm, smlen, "application_name").empty(), + "startup without options/client_encoding/application_name carries none of them"); + + unsigned char sm2[512]; size_t sm2len = 0; + const char* opts = "-c DateStyle=ISO -c geqo=off"; + bool built = pg_build_startup(sm2, &sm2len, sizeof(sm2), "alice", "shop", "UTF8", + opts, "proxysql"); + ok(built, "startup builds with client_encoding, options and application_name"); + ok(startup_param(sm2, sm2len, "options") == opts, + "startup carries the options string verbatim (got '%s')", + startup_param(sm2, sm2len, "options").c_str()); + ok(startup_param(sm2, sm2len, "application_name") == "proxysql", + "startup carries application_name, so the backend can identify the connection " + "in pg_stat_activity (got '%s')", + startup_param(sm2, sm2len, "application_name").c_str()); + ok(startup_param(sm2, sm2len, "client_encoding") == "UTF8" && + startup_param(sm2, sm2len, "user") == "alice" && + startup_param(sm2, sm2len, "database") == "shop", + "startup still carries user/database, plus client_encoding"); + + // A buffer one byte short of the encoded size must be rejected outright, with no + // partial write, now that application_name adds to that size. + unsigned char sm3[512]; size_t sm3len = 123; + ok(pg_build_startup(sm3, &sm3len, sm2len - 1, "alice", "shop", "UTF8", + opts, "proxysql") == false && sm3len == 0, + "startup refuses a buffer one byte too small and reports 0 bytes written"); + } + // AuthenticationMD5Password response: "md5" + hex(md5(hex(md5(pass+user))+salt)). // Known vector: user=postgres, password=postgres, salt={1,2,3,4} (independent python ref). char md5buf[36]; @@ -514,5 +625,190 @@ int main(int, char**) { len); } + // ================================================================== + // Credential pass-through on the BACKEND leg. + // + // pgsql_users.password may hold a plaintext, an md5 secret, or a SCRAM verifier. + // The first is the only one the primitives above can hash directly; for the other + // two the stored value IS already a derived secret and must be reused, never + // re-derived. Everything below pins that reuse, because getting it subtly wrong + // produces a handshake that is well-formed and simply always rejected. + // ------------------------------------------------------------------ + + // (22) md5 pass-through against the SAME reference vector pinned in (3). + // The stored secret is the inner hash: "md5" + hex(md5(password+user)). For + // user=postgres / password=postgres that inner hash is 3175bce1d3201d16594cebf9d7eb3f9d + // (md5 of the literal "postgrespostgres", computed independently of this code), so only + // the outer hash over (inner_hex || salt) is left to do and the response must come out + // byte-identical to the plaintext-derived one. + { + char out[36]; + memset(out, 0xAA, sizeof(out)); + const bool built = pg_build_md5_from_secret(out, "md53175bce1d3201d16594cebf9d7eb3f9d", salt); + ok(built && strcmp(out, "md568be9ed08db75f318087ab337aaea044") == 0, + "pg_build_md5_from_secret reproduces the plaintext-derived reference vector (built=%d, got: %s)", + (int)built, built ? out : "(not built)"); + } + + // (23) Equivalence at a second, unrelated salt, compared against pg_build_md5() itself. + // (22) alone could pass on a single coincidence; this pins the general property that + // for any salt the two entry points agree whenever the secret is that user's inner hash. + { + unsigned char salt2[4] = {0xde, 0xad, 0xbe, 0xef}; + char from_plain[36]; + char from_secret[36]; + pg_build_md5(from_plain, "postgres", "postgres", salt2); + const bool built = pg_build_md5_from_secret(from_secret, "md53175bce1d3201d16594cebf9d7eb3f9d", salt2); + ok(built && strcmp(from_plain, from_secret) == 0, + "pg_build_md5_from_secret == pg_build_md5 for the same credential at a second salt (plain=%s, secret=%s)", + from_plain, built ? from_secret : "(not built)"); + } + + // (24) Malformed secrets are refused BEFORE anything is written. A partially built + // response must never reach the wire, and a secret that is not exactly "md5" + 32 + // lowercase hex is not this user's inner hash -- PostgreSQL stores nothing else. + { + static const char* const bad[] = { + nullptr, + "", + "md5", // prefix only + "md53175bce1d3201d16594cebf9d7eb3f9", // 31 hex digits: one short + "md53175bce1d3201d16594cebf9d7eb3f9dd", // 33 hex digits: one long + "MD53175bce1d3201d16594cebf9d7eb3f9d", // prefix in the wrong case + "md53175BCE1D3201D16594CEBF9D7EB3F9D", // hex in the wrong case + "md53175bce1d3201d16594cebf9d7eb3f9z", // 'z' is not a hex digit + "3175bce1d3201d16594cebf9d7eb3f9dabc", // right length, no md5 prefix + "SCRAM-SHA-256$4096:c2FsdA==$c3Q=:c2s=", // a verifier, not an md5 secret + "plaintext-password", + }; + bool all_rejected = true; + bool out_untouched = true; + for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++) { + char out[36]; + memset(out, 0x5A, sizeof(out)); + if (pg_build_md5_from_secret(out, bad[i], salt)) { + all_rejected = false; + diag(" pg_build_md5_from_secret wrongly accepted: %s", bad[i] ? bad[i] : "(null)"); + } + for (size_t j = 0; j < sizeof(out); j++) { + if ((unsigned char)out[j] != 0x5A) { out_untouched = false; break; } + } + } + ok(all_rejected && out_untouched, + "pg_build_md5_from_secret rejects every malformed secret and leaves the output untouched (rejected=%d, untouched=%d)", + (int)all_rejected, (int)out_untouched); + } + + // (25) SCRAM verifier pass-through, end to end, in the exact shape production uses. + // + // Leg A is the FRONTEND login: the client presents the plaintext, ProxySQL answers from + // the stored verifier, and the ClientKey the client used is recovered from its proof + // (PgSQL_Protocol.cpp harvests scram_state->ClientKey at exactly this point). Leg B is + // the BACKEND login under test: a fresh exchange against the same verifier, driven with + // NO password at all -- only the harvested ClientKey and the verifier's ServerKey. + // + // The verifier is the RFC 7677 Section 3 vector (password "pencil", salt + // W22ZaJ0SNY7soEsUEjb6gQ==, i=4096) in pg_authid.rolpassword form, so the harvested keys + // have published expected values and leg A cannot pass by agreeing with itself. + { + ScramState* fe = scram_state_init(); + PgSQL_Scram_State* fe_client = pg_scram_new(); + const char* fe_cf = pg_scram_client_first(fe_client, /*channel_binding=*/false); + char* fe_sf = fe_cf ? server_first_for(fe, fe_cf, RFC7677_VERIFIER) : nullptr; + const char* fe_final = fe_sf + ? pg_scram_client_final(fe_client, "pencil", fe_sf, strlen(fe_sf)) : nullptr; + const bool fe_ok = fe_final && server_accepts(fe, fe_final); + + const bool harvested_ck = fe_ok && memcmp(fe->ClientKey, RFC7677_CLIENT_KEY, 32) == 0; + const bool verifier_sk = fe_ok && memcmp(fe->ServerKey, RFC7677_SERVER_KEY, 32) == 0; + ok(harvested_ck && verifier_sk, + "frontend SCRAM login against the RFC 7677 verifier yields the published ClientKey and ServerKey (login=%d, ck=%d, sk=%d)", + (int)fe_ok, (int)harvested_ck, (int)verifier_sk); + + // ---- leg B: the backend handshake, authenticating from the keys alone ---- + ScramState* be = scram_state_init(); + PgSQL_Scram_State* be_client = pg_scram_new(); + const bool keys_set = pg_scram_set_keys(be_client, fe->ClientKey, fe->ServerKey); + const char* be_cf = pg_scram_client_first(be_client, /*channel_binding=*/false); + char* be_sf = be_cf ? server_first_for(be, be_cf, RFC7677_VERIFIER) : nullptr; + // password == nullptr: the whole point. Nothing in this leg knows "pencil". + const char* be_final = be_sf + ? pg_scram_client_final(be_client, nullptr, be_sf, strlen(be_sf)) : nullptr; + const bool be_accepted = be_final && server_accepts(be, be_final); + + // (26) The injected ClientKey produces a proof the server accepts. + ok(keys_set && be_accepted, + "backend SCRAM leg authenticates from the injected ClientKey with NO password (keys_set=%d, accepted=%d)", + (int)keys_set, (int)be_accepted); + + // (27) Mutual authentication still happens: the client verifies the server's + // signature from the injected ServerKey, not from a SaltedPassword it never computed. + char* be_server_final = be_accepted ? build_server_final_message(be) : nullptr; + const bool mutual = be_server_final + && pg_scram_verify_server_final(be_client, be_server_final, strlen(be_server_final)); + ok(mutual, "backend SCRAM leg verifies the server signature from the injected ServerKey"); + + free(be_server_final); + pg_scram_free(be_client); + free_scram_state(be); + pg_scram_free(fe_client); + free_scram_state(fe); + } + + // (28) Mutual authentication is not silently skipped. With the correct ClientKey but a + // corrupted ServerKey the proof is still accepted -- the server has no way to tell -- + // yet the client MUST reject the server's signature. A pass-through that ignored the + // injected ServerKey, or fell back to a SaltedPassword it never computed, would let a + // backend impersonation through here. + { + uint8_t bad_sk[32]; + memcpy(bad_sk, RFC7677_SERVER_KEY, sizeof(bad_sk)); + bad_sk[0] ^= 0xff; + + ScramState* srv = scram_state_init(); + PgSQL_Scram_State* cli = pg_scram_new(); + const bool keys_set = pg_scram_set_keys(cli, RFC7677_CLIENT_KEY, bad_sk); + const char* cf = pg_scram_client_first(cli, /*channel_binding=*/false); + char* sf = cf ? server_first_for(srv, cf, RFC7677_VERIFIER) : nullptr; + const char* fin = sf ? pg_scram_client_final(cli, nullptr, sf, strlen(sf)) : nullptr; + const bool proof_accepted = fin && server_accepts(srv, fin); + char* server_final = proof_accepted ? build_server_final_message(srv) : nullptr; + const bool verified = server_final + && pg_scram_verify_server_final(cli, server_final, strlen(server_final)); + + ok(keys_set && proof_accepted && !verified, + "a wrong injected ServerKey still builds an accepted proof but FAILS server-signature verification (keys_set=%d, proof=%d, verified=%d)", + (int)keys_set, (int)proof_accepted, (int)verified); + + free(server_final); + pg_scram_free(cli); + free_scram_state(srv); + } + + // (29) A half-injection is refused outright. Accepting a ClientKey without a ServerKey + // would authenticate us to the backend while leaving nothing to check the backend with, + // which is exactly the silent downgrade (28) guards against -- so it is rejected at the + // door instead, and the state stays password-driven (proved by client-final still + // refusing a NULL password afterwards). + { + PgSQL_Scram_State* cli = pg_scram_new(); + const bool r_no_sk = pg_scram_set_keys(cli, RFC7677_CLIENT_KEY, nullptr); + const bool r_no_ck = pg_scram_set_keys(cli, nullptr, RFC7677_SERVER_KEY); + const bool r_no_both = pg_scram_set_keys(cli, nullptr, nullptr); + const bool r_no_state = pg_scram_set_keys(nullptr, RFC7677_CLIENT_KEY, RFC7677_SERVER_KEY); + + ScramState* srv = scram_state_init(); + const char* cf = pg_scram_client_first(cli, /*channel_binding=*/false); + char* sf = cf ? server_first_for(srv, cf, RFC7677_VERIFIER) : nullptr; + const char* fin = sf ? pg_scram_client_final(cli, nullptr, sf, strlen(sf)) : nullptr; + + ok(!r_no_sk && !r_no_ck && !r_no_both && !r_no_state && fin == nullptr, + "half-injected SCRAM keys are refused and leave the state password-driven (no_sk=%d, no_ck=%d, no_both=%d, no_state=%d, final=%s)", + (int)r_no_sk, (int)r_no_ck, (int)r_no_both, (int)r_no_state, fin ? "built" : "null"); + + free_scram_state(srv); + pg_scram_free(cli); + } + return exit_status(); } diff --git a/test/tap/tests/unit/pgsql_backend_framing-t.cpp b/test/tap/tests/unit/pgsql_backend_framing-t.cpp index 8645ac17be..8d0ad98fb9 100644 --- a/test/tap/tests/unit/pgsql_backend_framing-t.cpp +++ b/test/tap/tests/unit/pgsql_backend_framing-t.cpp @@ -1,7 +1,35 @@ +/** + * @file pgsql_backend_framing-t.cpp + * @brief Unit tests for PgSQL_Backend_Msg_Framer — the native backend protocol's + * message framer (lib/PgSQL_Backend_Protocol.cpp). + * + * The framer is the entry point for every byte the native backend path reads + * from a PostgreSQL server. It is fed possibly-partial socket reads and yields + * whole protocol messages. Everything downstream — auth, result streaming, the + * extended-query pipeline — trusts its framing and its bounds checks. + * + * ============================================================================ + * SCOPE: framing logic only + * ============================================================================ + * Framing, bounds and error-state behaviour, all deterministic and with no + * dependency on the host. + * + * Buffer RETENTION is deliberately NOT tested here. The framer's cap/len/pos + * are private, so a unit test could only infer growth from process RSS -- an + * indirect, noisy, Linux-only measurement that no other test in this suite + * uses. Retention is covered where it can be observed properly, end to end + * through the proxy and its admin stats: + * + * test/tap/tests/pgsql-native_framer_retention-t.cpp + * + * That is the regression guard for the compaction fix (03dd7983b); if buffer + * compaction is ever lost, that test is what fails. + */ #include "test_globals.h" #include "test_init.h" #include "PgSQL_Backend_Protocol.h" #include +#include #include "tap.h" // Build one backend message into buf, return total bytes written. @@ -14,30 +42,158 @@ static size_t put_msg(unsigned char* buf, char type, const char* payload, uint32 return 5 + plen; } +// Write just a 5-byte header with an arbitrary declared length, so length +// validation can be exercised without supplying a body. +static void put_header(unsigned char* buf, char type, uint32_t declared_len) { + buf[0] = (unsigned char)type; + buf[1] = (declared_len >> 24) & 0xff; buf[2] = (declared_len >> 16) & 0xff; + buf[3] = (declared_len >> 8) & 0xff; buf[4] = declared_len & 0xff; +} + +// Append a complete message of `payload` bytes to `s`. +static void append_msg(std::string& s, char type, size_t payload_len) { + s.push_back(type); + uint32_t L = (uint32_t)(payload_len + 4); + s.push_back((char)((L >> 24) & 0xff)); + s.push_back((char)((L >> 16) & 0xff)); + s.push_back((char)((L >> 8) & 0xff)); + s.push_back((char)(L & 0xff)); + s.append(payload_len, 'x'); +} + int main(int, char**) { - plan(7); - PgSQL_Backend_Msg_Framer f; - unsigned char buf[64]; + plan(21); - size_t n = put_msg(buf, 'Z', "I", 1); // ReadyForQuery, txn state 'I' - f.feed(buf, n); PgSQL_Backend_Msg m; - ok(f.next(m) == FRAME_OK, "complete message framed"); - ok(m.type == 'Z', "type is Z"); - ok(m.payload_len == 1 && m.payload[0] == 'I', "payload correct"); - ok(f.next(m) == FRAME_NEED_MORE, "buffer drained -> need more"); - - PgSQL_Backend_Msg_Framer f2; - f2.feed(buf, 3); // only 3 of 6 bytes - ok(f2.next(m) == FRAME_NEED_MORE, "partial message -> need more"); - f2.feed(buf + 3, n - 3); // rest arrives - ok(f2.next(m) == FRAME_OK && m.type == 'Z', "completes after remaining bytes"); - - // Oversized declared length is rejected as FRAME_ERROR (DoS guard), even with few bytes fed. - PgSQL_Backend_Msg_Framer f3; - unsigned char big[5] = { 'D', 0xff, 0xff, 0xff, 0xff }; // type 'D', length ~4GiB - f3.feed(big, 5); - ok(f3.next(m) == FRAME_ERROR, "oversized message length rejected"); + + // ---------------------------------------------------------------- basics + { + PgSQL_Backend_Msg_Framer f; + unsigned char buf[64]; + size_t n = put_msg(buf, 'Z', "I", 1); // ReadyForQuery, txn state 'I' + f.feed(buf, n); + ok(f.next(m) == FRAME_OK, "complete message framed"); + ok(m.type == 'Z', "type is Z"); + ok(m.payload_len == 1 && m.payload[0] == 'I', "payload correct"); + ok(f.next(m) == FRAME_NEED_MORE, "buffer drained -> need more"); + } + { + PgSQL_Backend_Msg_Framer f; + unsigned char buf[64]; + size_t n = put_msg(buf, 'Z', "I", 1); + f.feed(buf, 3); // only 3 of 6 bytes + ok(f.next(m) == FRAME_NEED_MORE, "partial message -> need more"); + f.feed(buf + 3, n - 3); // rest arrives + ok(f.next(m) == FRAME_OK && m.type == 'Z', "completes after remaining bytes"); + } + + // ------------------------------------------------- declared-length bounds + // The length field counts itself, so anything below 4 is malformed. A + // hostile or garbled backend reaching these branches must be rejected + // rather than trusted into a huge/negative payload_len. + for (uint32_t bad = 0; bad < 4; bad++) { + PgSQL_Backend_Msg_Framer f; + unsigned char hdr[5]; + put_header(hdr, 'D', bad); + f.feed(hdr, 5); + ok(f.next(m) == FRAME_ERROR, "declared length %u (< 4) rejected", bad); + } + { + // Length exactly 4 is legal and means an empty payload. + PgSQL_Backend_Msg_Framer f; + unsigned char hdr[5]; + put_header(hdr, 'n', 4); // NoData: no body + f.feed(hdr, 5); + PgSQL_Frame_Result r = f.next(m); + ok(r == FRAME_OK, "declared length 4 (empty payload) accepted"); + ok(m.type == 'n' && m.payload_len == 0, + "declared length 4 yields payload_len 0"); + } + { + // Exactly at the DoS ceiling: legal, so with only a header fed the + // framer must ask for more rather than reject. Distinguishes "at the + // cap" from "over the cap" — an off-by-one here would reject the + // largest legitimate message PostgreSQL can send. + PgSQL_Backend_Msg_Framer f; + unsigned char hdr[5]; + put_header(hdr, 'D', PGSQL_MAX_BACKEND_MSG_LEN); + f.feed(hdr, 5); + ok(f.next(m) == FRAME_NEED_MORE, "declared length == cap is accepted (awaits body)"); + } + { + PgSQL_Backend_Msg_Framer f; + unsigned char hdr[5]; + put_header(hdr, 'D', PGSQL_MAX_BACKEND_MSG_LEN + 1); + f.feed(hdr, 5); + ok(f.next(m) == FRAME_ERROR, "declared length == cap + 1 rejected"); + } + + // ------------------------------------------------ sticky failure + reset + // Once framing is lost the stream cannot be resynchronised, so the error + // must latch: a caller that keeps feeding must not be handed garbage that + // happens to parse. + { + PgSQL_Backend_Msg_Framer f; + unsigned char hdr[5]; + put_header(hdr, 'D', 2); // malformed + f.feed(hdr, 5); + ok(f.next(m) == FRAME_ERROR, "sticky: first next() reports the error"); + + unsigned char good[64]; + size_t n = put_msg(good, 'Z', "I", 1); // perfectly valid message + f.feed(good, n); + ok(f.next(m) == FRAME_ERROR, "sticky: valid bytes after an error do not clear it"); + + f.reset(); + f.feed(good, n); + ok(f.next(m) == FRAME_OK && m.type == 'Z', "reset() clears the error and framing resumes"); + } + + // ------------------------------------------- maximal read fragmentation + // A TCP read can split anywhere, including inside a length field. Feed + // three messages one byte at a time and require all three back in order. + { + std::string s; + append_msg(s, 'T', 7); + append_msg(s, 'D', 0); // zero-length payload mid-stream + append_msg(s, 'C', 11); + + PgSQL_Backend_Msg_Framer f; + char types[3] = { 0, 0, 0 }; + uint32_t lens[3] = { 0, 0, 0 }; + int got = 0; + for (size_t i = 0; i < s.size(); i++) { + f.feed((const unsigned char*)s.data() + i, 1); + for (;;) { + PgSQL_Frame_Result r = f.next(m); + if (r != FRAME_OK) break; + if (got < 3) { types[got] = m.type; lens[got] = m.payload_len; } + got++; + } + } + ok(got == 3, "byte-at-a-time: exactly 3 messages framed (got %d)", got); + ok(types[0] == 'T' && lens[0] == 7, "byte-at-a-time: message 1 intact"); + ok(types[1] == 'D' && lens[1] == 0 && types[2] == 'C' && lens[2] == 11, + "byte-at-a-time: zero-length message and message 3 intact"); + } + + // ------------------------------------------- many messages in one feed + { + std::string s; + const int N = 500; + for (int i = 0; i < N; i++) append_msg(s, 'D', (size_t)(i % 37)); + PgSQL_Backend_Msg_Framer f; + f.feed((const unsigned char*)s.data(), s.size()); + int got = 0; + bool all_good = true; + for (;;) { + PgSQL_Frame_Result r = f.next(m); + if (r != FRAME_OK) break; + if (m.type != 'D' || m.payload_len != (uint32_t)(got % 37)) all_good = false; + got++; + } + ok(got == N && all_good, "single feed of %d messages framed in order (got %d)", N, got); + } return exit_status(); } diff --git a/test/tap/tests/unit/pgsql_conn_liveness_unit-t.cpp b/test/tap/tests/unit/pgsql_conn_liveness_unit-t.cpp new file mode 100644 index 0000000000..4009205786 --- /dev/null +++ b/test/tap/tests/unit/pgsql_conn_liveness_unit-t.cpp @@ -0,0 +1,195 @@ +/** + * @file pgsql_conn_liveness_unit-t.cpp + * @brief A broken connection must never report itself as healthy. + * + * When a connection dies, the letter the backend last sent stays behind. The code + * used to read that letter and conclude the connection was idle and fine, so a + * dead connection went back into the pool and the next session got it. These + * tests build connections directly, with no server and no Docker. + */ + +#include "tap.h" +#include "test_globals.h" +#include "test_init.h" +#include "proxysql.h" +#include "PgSQL_Connection.h" + +#include +#include + +// Use a real file so the destructor's close() cannot hit some other file. +static int open_dummy_fd() { + int fd = ::open("/dev/null", O_RDONLY); + return fd; +} + +// A connection in the state a healthy pooled one is in: open socket, login done, +// backend last said it was idle. +static PgSQL_Connection* make_live_native_conn() { + PgSQL_Connection* c = new PgSQL_Connection(false); + c->native_mode = true; + c->fd = open_dummy_fd(); + c->native_st = PgSQL_Connection::PG_Native_Conn_St::DONE; + // native_connected is what marks the connection usable; native_st is set to + // match what a real completed login leaves behind. + c->native_connected = true; + c->set_ready_for_query_status('I'); + return c; +} + +// Do what the real teardown does: close the socket and mark it not connected. +// It leaves the transaction letter behind, and so do we -- that is the point. +static void simulate_teardown(PgSQL_Connection* c) { + if (c->fd >= 0) { ::close(c->fd); c->fd = -1; } + c->native_connected = false; +} + +static void test_live_native_conn_is_healthy() { + PgSQL_Connection* c = make_live_native_conn(); + ok(c->is_connected() == true, + "live native conn: is_connected() true"); + ok(c->get_pg_transaction_status() == PQTRANS_IDLE, + "live native conn: transaction status IDLE (from the 'I' byte)"); + ok(c->is_connection_in_reusable_state() == true, + "live native conn: reusable"); + simulate_teardown(c); + delete c; +} + +static void test_dead_native_conn_reports_dead() { + PgSQL_Connection* c = make_live_native_conn(); + simulate_teardown(c); + + ok(c->is_connected() == false, + "torn-down native conn: is_connected() false"); + ok(c->get_pg_connection_status() == CONNECTION_BAD, + "torn-down native conn: connection status BAD"); + ok(c->get_pg_transaction_status() == PQTRANS_UNKNOWN, + "torn-down native conn: transaction status UNKNOWN despite the stale 'I' byte " + "(F2/A1 regression -- reported PQTRANS_IDLE before the liveness gate)"); + ok(c->is_connection_in_reusable_state() == false, + "torn-down native conn: NOT reusable -- the pool must destroy it, not re-pool it"); + ok(c->IsKnownActiveTransaction() == false, + "torn-down native conn: holds no transaction (true here only because the " + "fixture's last ReadyForQuery byte is 'I' -- see " + "test_dead_conn_that_was_in_a_transaction_still_reports_one() for the 'T' case)"); + delete c; +} + +static void test_protocol_answer_stays_unqualified() { + PgSQL_Connection* c = make_live_native_conn(); + simulate_teardown(c); + ok(c->last_ready_for_query_status() == 'I', + "last_ready_for_query_status() still reports the raw byte after teardown -- " + "question 1 is deliberately unqualified; its callers carry their own guard"); + delete c; +} + +static void test_conn_that_died_after_handshake() { + PgSQL_Connection* c = make_live_native_conn(); + // The usual way a connection dies: it worked, then the socket went away in the + // middle of a result. Only the closed socket shows it, so that half of the + // check has to be doing its job. + if (c->fd >= 0) { ::close(c->fd); c->fd = -1; } + ok(c->native_st == PgSQL_Connection::PG_Native_Conn_St::DONE && + c->is_connection_in_reusable_state() == false, + "conn that died after a completed handshake (fd cleared, native_st still DONE) " + "is not reusable -- the fd half of the liveness gate is load-bearing"); + delete c; +} + +static void test_healthy_conn_mid_partial_send_is_live() { + PgSQL_Connection* c = make_live_native_conn(); + // A query too big to write in one go leaves the connection in a sending state. + // Nothing is wrong with it, so it must still count as usable. + c->native_st = PgSQL_Connection::PG_Native_Conn_St::SEND_STARTUP; + c->native_st_after_send = PgSQL_Connection::PG_Native_Conn_St::DONE; + ok(c->is_connected() == true && c->is_connection_in_reusable_state() == true, + "healthy conn parked at SEND_STARTUP by a partial send is still live and reusable"); + c->native_st = PgSQL_Connection::PG_Native_Conn_St::DONE; + simulate_teardown(c); + delete c; +} + +static void test_dead_conn_that_was_in_a_transaction_still_reports_one() { + PgSQL_Connection* c = make_live_native_conn(); + c->set_ready_for_query_status('T'); // backend last said "in transaction" + simulate_teardown(c); + // This connection died with a transaction open. If it claimed otherwise, the + // statement would be run again on a fresh connection, on its own, outside the + // transaction it belonged to. + ok(c->IsKnownActiveTransaction() == true, + "native conn torn down while in a transaction still reports an active " + "transaction, so the statement is not silently retried outside it"); + delete c; +} + +static void test_libpq_control() { + // The gate must be a no-op for libpq: PQstatus(NULL) is already CONNECTION_BAD + // and PQtransactionStatus(NULL) is already PQTRANS_UNKNOWN, so a dead libpq conn + // gave these answers before the change too. Checking it here means a future edit + // to the gate cannot quietly alter the shipped path. + PgSQL_Connection* c = new PgSQL_Connection(false); + c->native_mode = false; + // pgsql_conn stays NULL -- the constructor sets it so. + ok(c->is_connected() == false && c->get_pg_connection_status() == CONNECTION_BAD && + c->get_pg_transaction_status() == PQTRANS_UNKNOWN, + "libpq control: a dead libpq conn reports dead the same way it always did"); + // Record an error before asking the reusable question. libpq only reaches this + // state through a failure that sets one, and the check inside that function + // aborts a debug build if it ever sees a broken connection with no error -- a + // live invariant on the libpq path that this change must not switch off. + c->set_error("08006", "connection failure", true); + ok(c->is_error_present() == true && c->is_connection_in_reusable_state() == false, + "libpq control: a dead libpq conn with its error recorded is not reusable, " + "and the no-error check behind it is still armed"); + delete c; +} + +static void test_live_conn_in_failed_transaction_is_still_reusable() { + PgSQL_Connection* c = make_live_native_conn(); + // A statement failed but the backend answered and is waiting for ROLLBACK. The + // connection itself is fine, so a liveness check must not throw it away -- this + // is the case an over-eager gate would break. + c->set_ready_for_query_status('E'); + ok(c->get_pg_transaction_status() == PQTRANS_INERROR, + "live native conn in a failed transaction: status INERROR"); + ok(c->is_connection_in_reusable_state() == true, + "live native conn in a failed transaction is still reusable -- a query error " + "must not be mistaken for a broken connection"); + simulate_teardown(c); + delete c; +} + +static void test_conn_that_failed_login_is_not_live() { + PgSQL_Connection* c = make_live_native_conn(); + // Login failed: the socket is still open but the handshake never finished. This + // is the state an auth failure sits in before teardown runs, so the connected + // half of the check has to catch it on its own. + c->native_connected = false; + ok(c->fd >= 0 && c->is_connected() == false, + "conn with an open socket that never finished login is not live -- the " + "native_connected half of the gate is load-bearing"); + ok(c->is_connection_in_reusable_state() == false, + "conn that never finished login is not reusable"); + simulate_teardown(c); + delete c; +} + +int main() { + plan(18); + test_init_minimal(); + + test_live_native_conn_is_healthy(); // 3 + test_dead_native_conn_reports_dead(); // 5 + test_protocol_answer_stays_unqualified(); // 1 + test_conn_that_died_after_handshake(); // 1 + test_healthy_conn_mid_partial_send_is_live(); // 1 + test_dead_conn_that_was_in_a_transaction_still_reports_one(); // 1 + test_live_conn_in_failed_transaction_is_still_reusable(); // 2 + test_conn_that_failed_login_is_not_live(); // 2 + test_libpq_control(); // 2 + + test_cleanup_minimal(); + return exit_status(); +} diff --git a/test/tap/tests/unit/pgsql_native_params_unit-t.cpp b/test/tap/tests/unit/pgsql_native_params_unit-t.cpp new file mode 100644 index 0000000000..dea2550e64 --- /dev/null +++ b/test/tap/tests/unit/pgsql_native_params_unit-t.cpp @@ -0,0 +1,162 @@ +/** + * @file pgsql_native_params_unit-t.cpp + * @brief Unit tests for the native-mode branches of PgSQL_Connection::get_pg_server_version() + * and ::get_pg_client_encoding(). + * + * WHY A UNIT TEST AND NOT A TAP TEST + * ---------------------------------- + * Both accessors read what the backend announced by ParameterStatus during the native + * handshake (PgSQL_Connection::native_params). pgsql-native_query_differential-t already + * compares them against the libpq path on a LIVE backend, but a live backend only ever + * announces one thing: the infra runs PostgreSQL 16, so it says "16.14" and "UTF8". That + * reaches exactly one of the four routes through the version conversion, and one encoding + * name. Reaching the rest would mean standing up a PostgreSQL 9 server. + * + * The version conversion is worth pinning because native does NOT call libpq's converter -- + * it reimplements it. PostgreSQL changed the numeric encoding at version 10: from 10 onwards + * it is major*10000 + minor, before that major*10000 + minor*100 + revision. A second copy of + * those rules is somewhere a typo can hide where no PostgreSQL 16 test can find it, and + * applying the wrong one is precisely the bug these tests were written for -- it made a 16.14 + * backend report 161400 where libpq reports 160014. + * + * client_encoding cannot diverge the same way (both paths run the same char_to_encoding()), + * so its cases here cover the fallbacks rather than the mapping. + * + * The expected numbers are libpq's own, from pqSaveParameterStatus() + * (deps/postgresql/postgresql/src/interfaces/libpq/fe-exec.c). + */ + +#include +#include + +#include "tap.h" +#include "test_globals.h" +#include "test_init.h" +#include "proxysql.h" +#include "PgSQL_Connection.h" + +// A connection in the state a completed native handshake leaves behind, minus the socket: +// native_mode set, native_params carrying what the backend announced. The accessors under +// test read nothing else, and the destructor is inert here -- pgsql_conn is NULL, +// native_connected false and fd -1, so it only frees userinfo and local_stmts. +// Passing announced == nullptr models the parameter never arriving. +static int version_for(const char* announced) { + PgSQL_Connection c(false); + c.native_mode = true; + if (announced) c.native_params["server_version"] = announced; + return c.get_pg_server_version(); +} + +static std::string version_str_for(const char* announced) { + PgSQL_Connection c(false); + c.native_mode = true; + if (announced) c.native_params["server_version"] = announced; + char buf[64]; + return std::string(c.get_pg_server_version_str(buf, sizeof(buf))); +} + +static int encoding_for(const char* announced) { + PgSQL_Connection c(false); + c.native_mode = true; + if (announced) c.native_params["client_encoding"] = announced; + return c.get_pg_client_encoding(); +} + +int main() { + plan(15); + + if (test_init_minimal() != 0) + BAIL_OUT("test_init_minimal() failed"); + + // ---- server_version: every route through the conversion ----------------- + + // Modern: two numbers, major >= 10. The ONE route a live PostgreSQL 16 test reaches. + { + const int v = version_for("16.14"); + ok(v == 160014, "server_version '16.14' -> 160014 (major*10000 + minor) [got %d]", v); + } + // Same route, with the packager suffix a real Debian PostgreSQL actually announces. + // sscanf stops at the space, so the trailing text must not change the answer. + { + const int v = version_for("16.14 (Debian 16.14-1.pgdg13+1)"); + ok(v == 160014, "server_version with a packager suffix -> 160014 [got %d]", v); + } + // Old style: three numbers. Unreachable from the infra backend. + { + const int v = version_for("9.6.1"); + ok(v == 90601, "server_version '9.6.1' -> 90601 (major*10000 + minor*100 + rev) [got %d]", v); + } + // Old style without a revision: two numbers, major < 10. This is the route the + // pre-fix code applied to EVERYTHING, which is what broke modern versions. + { + const int v = version_for("9.6devel"); + ok(v == 90600, "server_version '9.6devel' -> 90600 (old style, no revision) [got %d]", v); + } + // Modern without a minor: one number. + { + const int v = version_for("10devel"); + ok(v == 100000, "server_version '10devel' -> 100000 (new style, no minor) [got %d]", v); + } + // Nothing parseable -> 0, libpq's "unknown" sentinel (fe-exec.c: conn->sversion = 0). + { + const int v = version_for(""); + ok(v == 0, "server_version '' -> 0 (unknown) [got %d]", v); + } + { + const int v = version_for("not-a-version"); + ok(v == 0, "server_version 'not-a-version' -> 0 (unknown) [got %d]", v); + } + // Never announced at all -- the state before the handshake completes. libpq reports 0 + // here too, because conn->sversion is only ever written when the parameter arrives. + { + const int v = version_for(nullptr); + ok(v == 0, "server_version absent -> 0 (unknown) [got %d]", v); + } + + // A backend announcing an absurd major version must not overflow the multiplies. + // libpq does not guard this; reporting unknown is the deliberate divergence. + { + const int v = version_for("99999999"); + ok(v == 0, "server_version '99999999' -> 0, no signed overflow [got %d]", v); + } + + // ---- the rendered string, which is what reaches the admin JSON ---------- + // get_pg_server_version_str() splits the integer /10000, /100%%100, %%100. That is + // shared with the libpq path, so these pin the pairing rather than the formatter: + // a modern version renders with the minor in the THIRD field, an old one round-trips. + { + const std::string s = version_str_for("16.14"); + ok(s == "16.0.14", "server_version '16.14' renders as '16.0.14' [got '%s']", s.c_str()); + } + { + const std::string s = version_str_for("9.6.1"); + ok(s == "9.6.1", "server_version '9.6.1' round-trips to '9.6.1' [got '%s']", s.c_str()); + } + + // ---- client_encoding: the mapping and its fallbacks --------------------- + // Compared against char_to_encoding() rather than a hard-coded id, so these assert the + // accessor routes the announced name through the converter -- not PostgreSQL's table. + { + const int enc = encoding_for("UTF8"); + const int want = PgSQL_Connection::char_to_encoding("UTF8"); + ok(enc == want && enc != 0, "client_encoding 'UTF8' resolves via char_to_encoding() [got %d want %d]", enc, want); + } + { + const int enc = encoding_for("LATIN1"); + const int want = PgSQL_Connection::char_to_encoding("LATIN1"); + ok(enc == want, "client_encoding 'LATIN1' resolves via char_to_encoding() [got %d want %d]", enc, want); + } + // Unrecognised name -> SQL_ASCII (0), the same fallback libpq applies. + { + const int enc = encoding_for("NO_SUCH_ENCODING"); + ok(enc == 0, "client_encoding 'NO_SUCH_ENCODING' falls back to SQL_ASCII [got %d]", enc); + } + // Never announced -> SQL_ASCII. Notably NOT -1: that is PQclientEncoding's error + // sentinel, and reporting it was the bug on the native path. + { + const int enc = encoding_for(nullptr); + ok(enc == 0, "client_encoding absent -> SQL_ASCII, not -1 [got %d]", enc); + } + + return exit_status(); +}