From b8868e363fdd9db1b5a706f584c64da532210465 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Fri, 31 Jul 2026 21:49:53 +0000 Subject: [PATCH 01/34] docs: add AWS Aurora blue green design specs --- ...ra-bgd-cluster-simulator-testing-design.md | 356 +++++++++++++ ...nfiguration-runtime-cluster-sync-design.md | 417 ++++++++++++++++ ...026-07-31-aurora-bgd-monitor-fsm-design.md | 468 ++++++++++++++++++ 3 files changed, 1241 insertions(+) create mode 100644 docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md create mode 100644 docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md create mode 100644 docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md new file mode 100644 index 0000000000..f3fda79484 --- /dev/null +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md @@ -0,0 +1,356 @@ +# Aurora BGD Cluster Simulator and Testing Design + +**Date:** 2026-07-31 + +**Branch:** `plan/aurora-bgd` + +**Status:** Design draft + +**Related designs:** + +- [Aurora BGD Configuration, Runtime Status, and Cluster Sync](2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md) +- [Aurora BGD Monitor Loop and FSM](2026-07-31-aurora-bgd-monitor-fsm-design.md) + +## Spec Boundary + +This specification owns the simulator capabilities and executable test coverage +needed to validate the other two specifications. It does not redefine public +configuration or FSM behavior. When a test expectation depends on those +contracts, the corresponding sibling specification is authoritative. + +## 1. Purpose + +Provide deterministic coverage for Aurora MySQL blue/green deployments without +requiring live AWS infrastructure. The test environment must be able to drive +the complete BGD observation sequence, publish target Aurora membership, +simulate member renaming, inject failures, observe monitor probes, and verify +ProxySQL runtime and routing effects. + +The design must preserve both existing regression suites: + +- the ordinary Aurora cluster simulator continues validating Aurora role, + membership, lag, failover, and autopurge behavior; +- the RDS Multi-AZ BGD suite continues validating its existing instance-based + mapping and reader-cleanup FSM. + +## 2. Architecture Decision + +Aurora BGD FSM tests extend the existing interactive RDS BGD simulator used by +`cluster_sim_rds_bgd-g1`. They do not add BGD sequencing to the legacy JSON +Aurora simulator. + +This is the preferred design because the RDS BGD simulator already provides: + +- per-backend `mysql.rds_topology` responses; +- empty, absent, and error topology modes; +- read-only controls; +- ordered probe logging; +- fixed AWS-style hostname-to-loopback mappings; +- TAP-driven, phase-by-phase transitions; +- the build and CI integration required for BGD tests. + +Aurora adds a target-cluster membership service and Aurora-specific TAP helpers +on top of that base. Shared topology, transaction, endpoint, TLS, read-only, +probe-wait, and cleanup behavior remains common. + +### Alternatives not selected + +1. Extending the JSON Aurora simulator with a long sequence of BGD states would + overload a batch-oriented two-state model and make late entry, retry, + rollback, and concurrent timing difficult to control. +2. Creating a third standalone simulator group would duplicate the existing RDS + BGD topology service, host maps, helpers, build flavor, and CI wiring. +3. Using only unit tests would not exercise monitor threads, DNS pins, + connection-pool drains, runtime status publication, or config reloads. + +## 3. Simulator Service Contract + +The `TEST_RDS_BGD` SQLite3-server flavor remains the executable backend for the +interactive suite. It is extended to recognize the Aurora membership query in +addition to the existing topology and read-only queries. + +All simulated responses are keyed by the backend address and port on which the +SQLite3 server accepted the monitor connection. This preserves isolation when +multiple deployments or multiple target endpoints are active concurrently. + +### 3.1 Existing topology service + +The existing service remains authoritative for: + +```text +mysql.rds_topology table presence +SOURCE and TARGET rows +deployment fingerprint fields +AWS status strings +configured topology errors +ordered table-check and metadata probe logs +``` + +Aurora TARGET rows contain a cluster endpoint. Multi-AZ TARGET rows continue to +contain an instance endpoint. The simulator must not infer deployment type from +test configuration; production detection consumes the endpoint shape and the +membership-query result. + +### 3.2 Aurora membership service + +The simulator adds per-backend control and row storage for +`INFORMATION_SCHEMA.REPLICA_HOST_STATUS`. + +Each membership row contains at least: + +```text +row_order +SERVER_ID +SESSION_ID +LAST_UPDATE_TIMESTAMP +IS_CURRENT +CPU +REPLICA_LAG_IN_MILLISECONDS +``` + +`SESSION_ID='MASTER_SESSION_ID'` identifies the writer. Other current rows are +readers. `row_order` makes response ordering deterministic while allowing tests +to prove that ProxySQL does not rely on writer-first or lexical membership +ordering. + +The control state supports: + +- a successful complete membership result; +- a successful writer-only result; +- an intentionally incomplete result; +- an empty result; +- table absence/error 1146; +- an arbitrary MySQL error code and message. + +Membership updates are atomic per supplied set of backends. A test must not +expose a partially rewritten snapshot unless it explicitly selects the +incomplete-result mode. + +### 3.3 Probe log + +The existing BGD probe log adds an Aurora-membership probe kind. Every topology +table check, topology metadata query, and target membership query records: + +```text +monotonic sequence +accepted backend IP +accepted backend port +probe kind +TLS state +``` + +Tests use the log to verify probe destination, ordering, cadence class, TLS, +probe-pin retention, and return to canonical probing. They must not use fixed +sleeps when an observable probe or runtime state can serve as the wait +condition. + +### 3.4 Error isolation + +Topology and membership errors are independent. A scenario can publish valid +topology with failed membership, or valid membership with failed topology. +Clearing one error source must not silently clear the other. + +Simulator cleanup removes topology, membership, read-only, and probe-log state +in one operation so each TAP binary begins from a known baseline. + +## 4. TAP Helper Model + +Aurora-specific helpers extend, rather than fork, the RDS BGD helper model. +They provide test-facing representations for: + +- a blue Aurora cluster and its canonical instance endpoints; +- a target cluster endpoint; +- one target writer and zero or more target readers; +- stable member session identities; +- pre-rename green `SERVER_ID` values; +- post-rename canonical `SERVER_ID` values; +- explicit and automatic Aurora hostgroup configuration; +- expected runtime `bgd_status` values. + +The helper API exposes operations equivalent to: + +```text +publish topology status +publish target membership snapshot +publish renamed membership snapshot +publish empty/absent topology +inject topology or membership error +record and wait for probes +query Aurora runtime row and bgd_status +query runtime server placement and status +query connection-pool state +open backend traffic and identify accepted simulator IP +clean up ProxySQL and simulator state +``` + +Helper methods perform control operations only. Individual test scenarios own +the sequence of AWS observations and all expected ProxySQL outcomes. + +## 5. Endpoint and DNS Model + +The `cluster_sim_rds_bgd-g1` fixed host map is extended with Aurora cluster +endpoints and member endpoints. Each hostname maps to a distinct loopback +address accepted by the same SQLite3 server. + +The map includes, per test cluster: + +- canonical blue writer and reader instance names; +- the green target cluster endpoint returned by `mysql.rds_topology`; +- green-suffixed target writer and reader instance names; +- canonical post-rename member names; +- a second target deployment for repeated-switchover coverage; +- additional clusters for concurrency coverage. + +Green and canonical names used for identity-renaming tests may resolve to the +same target-member loopback IP where that models AWS's post-processing rename. +Canonical blue DNS remains static in the container; DNS-cache pin behavior is +verified through accepted backend addresses and explicit pin removal, not by +claiming that the simulator reproduces mutable Route 53 propagation. + +## 6. Test Layers + +### 6.1 Schema and unit coverage + +Fast tests cover deterministic contracts that do not require monitor threads: + +- configuration and runtime table definitions and column order; +- paired-NULL and hostgroup-conflict validation; +- SQL NULL preservation in bind/extract helpers; +- config-file import and export; +- disk schema upgrade defaults; +- runtime `bgd_status` string mapping; +- configured-column cluster query and checksum projection; +- exclusion of `bgd_status` from SAVE and cluster synchronization; +- Aurora simulator JSON parsing of the two optional green hostgroups. + +### 6.2 Admin and cluster-sync integration coverage + +Admin/TAP tests cover: + +- memory-to-runtime and runtime-to-memory round trips; +- runtime-to-disk and config-file round trips; +- new runtime rows starting at `NONE`; +- reload preservation of `bgd_status` and active worker state; +- removal and deactivation behavior; +- peer synchronization of both green hostgroups, including NULL; +- proof that peers retain independent node-local `bgd_status` values; +- unchanged RDS Multi-AZ BGD cluster synchronization. + +### 6.3 Interactive Aurora BGD coverage + +The interactive suite covers the observable behavior of the monitor/FSM spec: + +1. `AVAILABLE` discovers exactly one writer and every reader, resolves each + member, and establishes only the probe state required by the design. +2. Automatic mode with NULL green hostgroups maps all target members without + generating green hostgroups or `mysql_servers` rows. +3. Explicit mode uses configured green hostgroups as staging pools but still + treats `REPLICA_HOST_STATUS` as membership truth. +4. Initiated and in-progress observations retain the complete snapshot and + engage BGD/read-only-monitor protection. +5. The first post-processing observation pins and drains each writer/reader pair + once; repeated observations retry only incomplete member actions. +6. Member `SERVER_ID` rename preserves reader identity through stable + `SESSION_ID` values and does not change cached target IPs. +7. The first TARGET `SWITCHOVER_COMPLETED` removes all traffic and probe pins, + performs cleanup once, and publishes `bgd_status='SWITCHOVER_COMPLETED'`. +8. Repeated completed rows are no-ops; successful topology drain changes the + runtime status to `NONE` and rearms discovery. +9. A different deployment fingerprint can rearm from the terminal latch without + inheriting stale members, pins, probes, or completion flags. + +### 6.4 Resilience and edge coverage + +Scenarios also cover: + +- membership row ordering and zero-reader target clusters; +- incomplete membership retaining the last complete snapshot; +- topology and membership query failures in every phase; +- rollback from initiated, in-progress, and post-processing observations; +- late entry at initiated, in-progress, post-processing, and completed states; +- config refresh and hostgroup refresh during active phases; +- worker restart/respawn with preserved fingerprint, members, IPs, pins, and + terminal latch; +- disabling automatic discovery during an active automatic deployment; +- deleting or deactivating an Aurora row during a switchover; +- explicit green-pool cleanup with ONLINE, SHUNNED, OFFLINE_SOFT, and + OFFLINE_HARD members; +- TLS selection for topology and membership probes; +- multiple concurrent Aurora deployments with independent state; +- simultaneous Aurora BGD and RDS Multi-AZ BGD deployments. + +## 7. Regression Coverage + +The existing `test_cluster_sim_aurora-t` JSON payload suite remains responsible +for ordinary Aurora behavior. Its schema accepts the two new green hostgroup +fields as optional values but old payloads remain valid and preserve their +existing results. + +The complete existing `test_rds_bgd_*` suite remains unchanged in semantics. +Aurora membership support is additive to the shared simulator and must not +alter instance TARGET handling, reader shun/unshun policy, writer fallback, or +Multi-AZ completion/drain behavior. + +Regression runs must include: + +```text +ordinary Aurora cluster simulator group +RDS Multi-AZ BGD simulator group +new Aurora BGD TAP binaries in cluster_sim_rds_bgd-g1 +configuration/unit tests +ProxySQL Cluster synchronization tests +``` + +## 8. Determinism and Timing + +Tests synchronize on observable state: runtime `bgd_status`, probe sequence, +server placement, pool counters, and successful simulator control commits. +Timeouts are derived from configured monitor intervals with bounded slack. + +Fixed sleeps are allowed only for negative assertions where no event can be +awaited directly, and must be shorter than the overall TAP timeout. Every wait +failure reports the last runtime row, relevant server/pool state, and probes +observed since the scenario checkpoint. + +Each scenario uses unique hostgroups or performs complete cleanup. Concurrent +tests use disjoint endpoints and deployment fingerprints. + +## 9. Build and CI Contract + +The existing `test_rds_bgd` build remains the focused local build. The combined +cluster-simulator build continues compiling both `TEST_AURORA` and +`TEST_RDS_BGD` support. + +New interactive TAP binaries register in `cluster_sim_rds_bgd-g1`, which keeps +the existing fixed-host injection, SQLite3-server startup, no-backend-infra +model, CI matrix discovery, and log collection. No new workflow or simulator +group is required. + +## 10. Non-Goals + +The simulator does not claim to validate: + +- AWS control-plane APIs or real AWS timing; +- mutable Route 53 propagation; +- application-level latency or packet loss; +- cross-process persistence of transient FSM state; +- behavior of Aurora versions that violate the AWS contract in the monitor/FSM + specification. + +Live-AWS evidence remains a separate validation layer for assumptions about +status ordering, member rename, writability, and DNS completion. + +## 11. Acceptance Criteria + +The simulator/testing design is satisfied when: + +1. Tests can independently control topology and target membership per backend. +2. The full Aurora BGD FSM can be driven without live AWS infrastructure. +3. Runtime status, routing, pool, DNS-pin, probe, reload, and cluster-sync + contracts have deterministic assertions. +4. Automatic and explicit modes both cover writer and all-reader membership. +5. Rename, rollback, late-entry, error, refresh, repeated, and concurrent paths + are covered. +6. Existing ordinary Aurora and RDS Multi-AZ BGD suites retain their semantics. +7. The suite runs through existing cluster-simulator build and CI plumbing with + no new infrastructure group. diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md new file mode 100644 index 0000000000..d2c4ac3c7f --- /dev/null +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md @@ -0,0 +1,417 @@ +# Aurora BGD Configuration, Runtime Status, and Cluster Sync Design + +**Date:** 2026-07-31 + +**Branch:** `plan/aurora-bgd` + +**Status:** Design draft; configuration decisions locked + +**Scope:** Configuration and runtime integration of Aurora MySQL blue/green +deployment handling with the existing `mysql_aws_aurora_hostgroups` subsystem. + +**Related designs:** + +- [Aurora BGD Monitor Loop and FSM](2026-07-31-aurora-bgd-monitor-fsm-design.md) +- [Aurora BGD Cluster Simulator and Testing](2026-07-31-aurora-bgd-cluster-simulator-testing-design.md) + +## Spec Boundary + +This specification owns the public configuration contract, the runtime-only +`bgd_status` contract, load/save and schema-upgrade behavior, and ProxySQL +Cluster synchronization. It treats the worker state machine as a consumer and +publisher through the interface defined here; membership discovery, routing +actions, and FSM internals belong to the monitor/FSM specification. + +## 1. Decision Summary + +Aurora blue/green handling is part of the existing Aurora monitor. It does not +require an entry in `mysql_aws_rds_bgd_hostgroups`. + +Two nullable columns are added to `mysql_aws_aurora_hostgroups` and +`runtime_mysql_aws_aurora_hostgroups`: + +```sql +green_writer_hostgroup INT DEFAULT NULL +green_reader_hostgroup INT DEFAULT NULL +``` + +The columns provide explicit staging hostgroups when configured. When both are +NULL, the existing global variable +`mysql-aws_blue_green_deployment_auto_discovery` controls automatic Aurora BGD +discovery. + +No separate Aurora BGD enable column is added. + +The runtime table adds one runtime-only observability column: + +```sql +bgd_status VARCHAR NOT NULL DEFAULT 'NONE' +``` + +It reports the Aurora worker's local BGD FSM state. It is not configuration and +is excluded from persistence and ProxySQL Cluster synchronization. + +## 2. Table Schema + +The configuration table uses the following column order: + +```text +writer_hostgroup +reader_hostgroup +green_writer_hostgroup +green_reader_hostgroup +active +aurora_port +domain_name +max_lag_ms +check_interval_ms +check_timeout_ms +writer_is_also_reader +new_reader_weight +add_lag_ms +min_lag_ms +lag_num_checks +autopurge_missing_checks +comment +``` + +The runtime table uses the same order and appends: + +```text +bgd_status +``` + +The new column definitions are: + +```sql +green_writer_hostgroup INT DEFAULT NULL + CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup >= 0), + +green_reader_hostgroup INT DEFAULT NULL + CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup >= 0) +``` + +Both columns are nullable in the configuration and runtime tables. Their NULL +state is meaningful and must be preserved by every load, save, export, import, +and cluster-synchronization path. + +The runtime table mirrors these configured fields and adds only `bgd_status`. +This design does not add an `auto_generated` column because the existing Aurora +row is the owner of both normal Aurora monitoring and BGD monitoring. It also +does not add a runtime `mode` column; effective mode remains derived from the +configured green hostgroups and the global auto-discovery variable. + +## 3. Configuration Modes + +The effective mode for an active Aurora row is determined as follows: + +| Green hostgroup columns | `aws_blue_green_deployment_auto_discovery` | Effective behavior | +|---|---:|---| +| Both non-NULL | `0` or `1` | Explicit Aurora BGD monitoring | +| Both NULL | `1` | Automatic Aurora BGD monitoring | +| Both NULL | `0` | Aurora monitoring without BGD discovery | +| Exactly one NULL | `0` or `1` | Invalid configuration | +| Any values with `active=0` | `0` or `1` | Neither normal Aurora nor BGD monitoring is active | + +### 3.1 Explicit mode + +A row is in explicit mode when both green hostgroups are non-NULL. + +In explicit mode: + +- BGD monitoring is enabled independently of the global auto-discovery value. +- The target cluster writer is associated with `green_writer_hostgroup`. +- All target cluster readers are associated with `green_reader_hostgroup`. +- Target membership still comes from the target cluster's + `REPLICA_HOST_STATUS`; configured green `mysql_servers` rows are not the + membership source of truth. +- The green hostgroups provide optional user-visible staging/routing pools in + addition to the internal BGD member map. + +### 3.2 Automatic mode + +A row is in automatic mode when both green hostgroups are NULL and +`mysql-aws_blue_green_deployment_auto_discovery=true` at runtime. + +In automatic mode: + +- The existing Aurora monitor detects BGD topology for the row. +- The monitor discovers the target writer and all target readers through + `REPLICA_HOST_STATUS`. +- The complete target member map and cached IPs remain internal. +- No green hostgroups or green `mysql_servers` rows are generated. +- Post-processing pinning covers the writer and every reader despite the green + hostgroup columns being NULL. + +Automatic mode therefore does not use the writer-only fallback or reader +shun/unshun policy from the Multi-AZ instance implementation. + +### 3.3 BGD disabled for the row + +When both green hostgroups are NULL and +`mysql-aws_blue_green_deployment_auto_discovery=false`, the worker continues its +existing Aurora role and lag monitoring but does not start discovery of a new +BGD deployment. + +Disabling global auto-discovery while an automatically discovered switchover is +already active must not abandon that switchover. The worker completes pin +cleanup and enters topology-drain wait before disabling discovery for the row. +The variable gates the start of new automatic BGD state machines, not safe +completion of one already in progress. + +## 4. Validation Rules + +For each `mysql_aws_aurora_hostgroups` row: + +1. `green_writer_hostgroup` and `green_reader_hostgroup` must either both be + NULL or both be non-NULL. +2. When non-NULL, all four hostgroups must be distinct: + + ```text + writer_hostgroup + reader_hostgroup + green_writer_hostgroup + green_reader_hostgroup + ``` + +3. A hostgroup assigned to one Aurora row must not conflict with any blue or + green role in another active Aurora row. +4. Existing writer-hostgroup primary-key and reader-hostgroup uniqueness rules + remain in effect. +5. Invalid rows must be rejected or excluded from runtime loading with a clear + admin error identifying the writer hostgroup and conflicting fields. + +Validation must occur before publishing the new Aurora monitor resultset so a +bad row cannot partially reconfigure a running monitor worker. + +## 5. Runtime Ownership + +`AWS_Aurora_Info`, keyed by `writer_hostgroup`, remains the runtime owner of the +cluster configuration. It is extended with optional integer fields: + +```text +green_writer_hostgroup = -1 when SQL NULL +green_reader_hostgroup = -1 when SQL NULL +``` + +The existing Aurora monitor worker remains one worker per active writer +hostgroup. The worker owns the BGD FSM and publishes each transition to the +runtime row's `bgd_status`. Its effective BGD mode is derived from: + +```text +AWS_Aurora_Info.active +green_writer_hostgroup +green_reader_hostgroup +mysql_thread___aws_blue_green_deployment_auto_discovery +``` + +No Aurora configuration is copied into `mysql_aws_rds_bgd_hostgroups`, and no +second BGD worker is started for the same Aurora writer hostgroup. + +Following RDS BGD, the Hostgroups Manager's internal Aurora table is the source +used to materialize `runtime_mysql_aws_aurora_hostgroups`. It stores the +configured fields plus `bgd_status`. Its runtime dump includes all of those +fields; paths that write back to configuration explicitly project away +`bgd_status`. + +## 6. LOAD Behavior + +`LOAD MYSQL SERVERS TO RUNTIME` must: + +1. Read both new columns from `mysql_aws_aurora_hostgroups`. +2. Preserve SQL NULL as the internal unset value rather than converting it to + hostgroup `0`. +3. Validate paired NULL/non-NULL and hostgroup-conflict rules. +4. Update or create the `AWS_Aurora_Info` entry. +5. Include both fields in the Aurora monitor resultset checksum. +6. Restart/refresh only the affected Aurora writer-hostgroup worker when either + green hostgroup changes. +7. Preserve an active BGD FSM safely across an unrelated configuration refresh. +8. Initialize `bgd_status` to `NONE` for a newly published runtime row. +9. Preserve the existing `bgd_status` when merging an existing + writer-hostgroup runtime row, following the RDS BGD table-reload behavior. + The Aurora worker separately preserves the underlying FSM state required by + the monitor/FSM specification. +10. Remove `bgd_status` with a removed runtime row; active-operation teardown + follows the existing RDS BGD worker-removal cleanup pattern. + +Changing the green hostgroups during an active switchover must not lose cached +member identities, applied DNS pins, or completion-latch state. The worker must +apply the refreshed explicit staging configuration without restarting the BGD +operation from `NONE`. + +`LOAD MYSQL VARIABLES TO RUNTIME` makes a change to +`aws_blue_green_deployment_auto_discovery` visible to Aurora monitor workers. +The variable controls admission of new automatic BGD operations as described in +Section 3.3. + +## 7. Runtime Table Behavior + +`runtime_mysql_aws_aurora_hostgroups` exposes both new columns in the same +positions as the configuration table and appends the runtime-only +`bgd_status` column. + +`bgd_status` exposes the internal Aurora BGD FSM state, not merely the last raw +AWS status string. Its values are: + +```text +NONE +AVAILABLE +SWITCHOVER_INITIATED +SWITCHOVER_IN_PROGRESS +SWITCHOVER_IN_POST_PROCESSING +SWITCHOVER_COMPLETED +``` + +The worker updates this column as its FSM changes, following the existing RDS +BGD runtime-status mechanism wherever applicable. `SWITCHOVER_COMPLETED` is the +terminal rearm latch: all routing cleanup has already completed, and the status +remains visible until `mysql.rds_topology` drains. A successful empty/absent +topology result then changes it to `NONE`. Query or connection errors while +latched do not reset the status or repeat cleanup. + +An Aurora row that is not handling a BGD deployment reports `NONE`, including +ordinary Aurora monitoring when automatic discovery is disabled. + +Examples: + +Explicit mode: + +```sql +INSERT INTO mysql_aws_aurora_hostgroups ( + writer_hostgroup, + reader_hostgroup, + green_writer_hostgroup, + green_reader_hostgroup, + domain_name +) VALUES (10, 20, 11, 21, '.cluster-example.eu-north-1.rds.amazonaws.com'); +``` + +Automatic mode: + +```sql +INSERT INTO mysql_aws_aurora_hostgroups ( + writer_hostgroup, + reader_hostgroup, + green_writer_hostgroup, + green_reader_hostgroup, + domain_name +) VALUES (10, 20, NULL, NULL, '.cluster-example.eu-north-1.rds.amazonaws.com'); + +SET mysql-aws_blue_green_deployment_auto_discovery = 'true'; +``` + +The BGD FSM state is operational monitor state, not user configuration. +`bgd_status` provides its runtime observability without changing the +explicit/automatic mode contract defined here. + +## 8. Persistence and Synchronization + +Both green hostgroup columns must be supported by: + +- admin-memory table creation; +- runtime table creation; +- disk database schema and online upgrade; +- `LOAD MYSQL SERVERS TO RUNTIME`; +- `SAVE MYSQL SERVERS FROM RUNTIME`; +- `SAVE MYSQL SERVERS TO DISK`; +- ProxySQL configuration-file import and export; +- ProxySQL Cluster fetch, insert, checksum, and conflict handling; +- `dump_table_mysql("mysql_aws_aurora_hostgroups")`; +- test/bootstrap table definitions. + +ProxySQL Cluster synchronization must include the two configured values. NULL +must remain NULL on the receiving peer. + +Every configuration-bearing path must use an explicit configured-column +projection rather than `SELECT *`. In particular: + +- `SAVE MYSQL SERVERS FROM RUNTIME` copies the configured Aurora columns and + excludes `bgd_status`; +- the ProxySQL Cluster fetch/query, checksum, and insert paths include the two + green hostgroups and exclude `bgd_status`; +- disk saves and configuration-file export never persist `bgd_status`. + +BGD's `bgd_status`, transient member map, cached IPs, and pins are node-local +runtime state and are not cluster-synced as configuration. + +## 9. Schema Upgrade + +A new Aurora hostgroups schema version must add the two nullable columns while +preserving every existing row and column value. + +Existing rows are migrated with: + +```text +green_writer_hostgroup = NULL +green_reader_hostgroup = NULL +``` + +After upgrade, existing active Aurora rows follow the global variable: + +- with auto-discovery enabled, they are eligible for automatic BGD discovery; +- with auto-discovery disabled, they retain existing Aurora monitoring only. + +This is backward-compatible at the table-data level because no green +hostgroups are invented during migration. + +`bgd_status` requires no disk-schema migration because it exists only in the +runtime table. The runtime table is created with the new column, and every row +starts at `NONE` until its local Aurora worker publishes another state. + +## 10. Non-Goals of This Configuration Change + +This configuration design does not: + +- add an Aurora-specific configuration table; +- create Aurora rows in `mysql_aws_rds_bgd_hostgroups`; +- require explicit green `mysql_servers` rows in automatic mode; +- add a second per-cluster monitor worker; +- add a separate `bgd_enabled` column; +- add a runtime `mode` or `auto_generated` column; +- persist transient BGD FSM state to disk; +- alter the existing RDS Multi-AZ BGD configuration contract. + +## 11. Required Configuration Tests + +1. Both green hostgroups NULL load successfully. +2. Both green hostgroups non-NULL load successfully. +3. Mixed NULL/non-NULL values are rejected. +4. Duplicate or overlapping blue/green hostgroups are rejected. +5. Explicit mode operates with global auto-discovery disabled. +6. Automatic mode starts only when global auto-discovery is enabled. +7. Disabling auto-discovery does not abort an active automatic switchover. +8. NULL values survive memory-to-runtime, runtime-to-memory, disk, config-file, + and cluster synchronization round trips. +9. Online upgrade preserves existing rows and initializes both new fields to + NULL. +10. Changing either green hostgroup refreshes only the affected Aurora worker. +11. Unrelated LOAD operations preserve active BGD pins and terminal-latch state. +12. New runtime rows initialize `bgd_status` to `NONE`. +13. Runtime `bgd_status` follows every Aurora FSM transition. +14. `SWITCHOVER_COMPLETED` remains visible until topology drain and then changes + to `NONE`. +15. Reloading an existing Aurora row preserves its `bgd_status` and active FSM + state. +16. `SAVE MYSQL SERVERS FROM RUNTIME`, disk/config export, and ProxySQL Cluster + synchronization exclude `bgd_status`. +17. ProxySQL Cluster peers retain their own node-local `bgd_status` values. +18. Existing Multi-AZ BGD configuration and tests remain unchanged. + +## 12. Acceptance Criteria + +The configuration/runtime integration is complete when: + +1. Aurora BGD explicit mode is configured solely by the two green hostgroup + columns on `mysql_aws_aurora_hostgroups`. +2. Aurora BGD automatic mode is controlled by the existing + `aws_blue_green_deployment_auto_discovery` variable when both columns are + NULL. +3. All load, save, disk, config-file, upgrade, and cluster-sync paths preserve + the fields and their NULL values. +4. Exactly one Aurora monitor worker owns normal Aurora and BGD handling for a + writer hostgroup. +5. No Aurora automatic-discovery row is generated in + `mysql_aws_rds_bgd_hostgroups`. +6. `runtime_mysql_aws_aurora_hostgroups.bgd_status` exposes the local Aurora BGD + FSM state without being saved, exported, or cluster-synchronized. diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md new file mode 100644 index 0000000000..abd7bccfd8 --- /dev/null +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md @@ -0,0 +1,468 @@ +# Aurora BGD Monitor Loop and FSM Design + +**Date:** 2026-07-31 + +**Branch:** `plan/aurora-bgd` + +**Status:** Design draft + +**Scope:** Aurora MySQL blue/green deployments exposed through +`mysql.rds_topology` and `INFORMATION_SCHEMA.REPLICA_HOST_STATUS`. + +**Evidence basis:** An observed Aurora switchover captured on 2026-07-30. The +relevant timings and behavioral conclusions are summarized in Section 3 so the +specification is self-contained. + +**Related designs:** + +- [Aurora BGD Configuration, Runtime Status, and Cluster Sync](2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md) +- [Aurora BGD Cluster Simulator and Testing](2026-07-31-aurora-bgd-cluster-simulator-testing-design.md) + +## Spec Boundary + +This specification owns the Aurora worker loop, topology and membership +discovery, cached member identity and IP state, DNS and pool actions, the BGD +FSM, rollback, and completion-latch behavior. Public table schemas, +persistence, and cluster synchronization belong to the configuration/runtime +specification. Simulator mechanics and the executable coverage matrix belong +to the simulator/testing specification. + +## 1. Purpose + +Provide a low-disruption ProxySQL switchover path for Aurora MySQL blue/green +deployments. ProxySQL discovers the complete target Aurora cluster before the +switchover, caches its member IPs, redirects the configured production +hostnames to those IPs during post-processing, and returns to normal DNS routing +as soon as AWS reports the target switchover complete. + +This is an Aurora-specific state machine. It does not replace the existing RDS +Multi-AZ instance behavior, where target `SWITCHOVER_COMPLETED` may describe only +the writer and reader cleanup must wait for a later topology-table drain. + +## 2. Terminology + +- **Blue:** The source/current production Aurora cluster before switchover. +- **Green:** The target Aurora cluster before promotion. +- **Canonical hostname/identifier:** The normal production instance name without + AWS's temporary `-green-` component. +- **Green hostname/identifier:** The target instance name containing + `-green-`. +- **Target cluster endpoint:** The TARGET endpoint returned by + `mysql.rds_topology`. For Aurora, this is a cluster endpoint, not an individual + writer endpoint. +- **Traffic pin:** A permanent ProxySQL DNS-cache mapping from a canonical blue + instance hostname to its cached green instance IP. +- **Probe pin:** The cached green writer/cluster IP used by the BGD monitor to + keep querying topology while canonical blue endpoints are unavailable. +- **Deployment fingerprint:** The TARGET topology identity retained after + cleanup to suppress repeated handling of the same completed row. At minimum, + it contains TARGET `id`, endpoint, and port. + +## 3. AWS/Aurora Behavioral Contract + +The Aurora implementation relies on these properties: + +1. `mysql.rds_topology` exposes SOURCE and TARGET cluster endpoints and a + monotonic status progression: + + ```text + AVAILABLE + -> SWITCHOVER_INITIATED + -> SWITCHOVER_IN_PROGRESS + -> SWITCHOVER_IN_POST_PROCESSING + -> TARGET-only SWITCHOVER_COMPLETED + ``` + +2. Querying `REPLICA_HOST_STATUS` through the target cluster endpoint returns + all current target members: exactly one writer and every Aurora reader. +3. The writer is the row whose `SESSION_ID` is `MASTER_SESSION_ID`; the other + current rows are readers. +4. During post-processing, target `SERVER_ID` values change from temporary + green identifiers to the canonical production identifiers. Reader + `SESSION_ID` values remain stable across the rename. +5. `SWITCHOVER_IN_POST_PROCESSING` is the AWS routing barrier. ProxySQL may + redirect traffic at this status without an additional target-writability + check. +6. TARGET `SWITCHOVER_COMPLETED` means canonical Aurora writer and reader DNS + cutover has completed. ProxySQL may remove all traffic pins without a + separate DNS-resolution verification. +7. The TARGET-only completed row can remain in `mysql.rds_topology` after + routing cleanup. Repeated completed rows describe the same completed + deployment and must be ignored until the table drains. + +The observed Aurora run supported these assumptions: + +- target writer writable: `T+14.438s`; +- target post-processing: `T+15.795s`; +- all target members had canonical `SERVER_ID`s: `T+21.380s`; +- canonical writer/readers accepted fresh connections: by `T+31.212s`; +- TARGET `SWITCHOVER_COMPLETED`: `T+40.655s`; +- topology table drained: `T+65.730s`. + +Thus, member cutover completed before TARGET completion; table drain was not a +reader-cutover barrier for Aurora. + +## 4. Aurora Detection + +The Aurora state machine must be selected only for an Aurora deployment. +Detection must establish both of the following: + +1. The TARGET topology endpoint is an Aurora cluster endpoint. +2. `REPLICA_HOST_STATUS` membership discovery succeeds through that endpoint. + +The existing Multi-AZ state machine remains active for non-Aurora TARGET +endpoints. + +## 5. Membership Discovery + +### 5.1 Source of truth + +`REPLICA_HOST_STATUS` is the source of truth for target Aurora membership. +Configured green hostgroups are not the source of membership and are not +required for automatic mode. + +The discovery query must include at least: + +```sql +SELECT + SERVER_ID, + SESSION_ID, + LAST_UPDATE_TIMESTAMP, + IS_CURRENT +FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS +ORDER BY SERVER_ID; +``` + +The implementation may retain the existing lag and CPU columns used by the +Aurora monitor. + +Only current rows are eligible for the active member snapshot. + +### 5.2 Constructing target instance hostnames + +The target cluster endpoint has the form: + +```text +.cluster- +``` + +A member hostname is formed as: + +```text +. +``` + +For example: + +```text +aurora-mysql-1-green-jkpanw.cluster-c1yqcg0ie39o.eu-north-1.rds.amazonaws.com + +aurora-mysql-1-writer-green-drodij.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com +``` + +### 5.3 Building blue/green pairs + +For every target member, maintain: + +```text +target SERVER_ID +SESSION_ID +role: writer or reader +green hostname +green IP +canonical blue hostname +port +configured blue server attributes +``` + +Before AWS renames the members, the canonical identifier is obtained by +removing the `-green-` component. After AWS renames them, the returned +`SERVER_ID` is already canonical. + +The writer pair is correlated by its unique writer role and cached IP. Reader +pairs are additionally correlated across renaming by their stable +`SESSION_ID`s. + +There must be exactly one writer pair. Every configured/current blue Aurora +reader must have a corresponding target reader pair before traffic pinning is +performed. + +### 5.4 Auto-generated configuration + +When `green_writer_hostgroup` and `green_reader_hostgroup` are `NULL`, the BGD +worker must still discover and map the target writer and all target readers in +memory. + +The absence of explicit green hostgroups means "no user-managed staging +hostgroups"; it does not mean "no green readers." + +The Aurora auto-generated path must not depend on green `mysql_servers` rows and +must not use the writer-only fallback as its normal reader policy. + +If the target cluster actually has no readers, the snapshot naturally contains +only the writer. If a membership query fails or yields a transient incomplete +snapshot for a cluster expected to have readers, retain the last complete +snapshot or defer the phase action. Do not reinterpret a failed/incomplete +query as a reader-less deployment. + +## 6. IP Resolution and Cached State + +During `AVAILABLE`, `SWITCHOVER_INITIATED`, and `SWITCHOVER_IN_PROGRESS`, the +worker must: + +1. Query target `REPLICA_HOST_STATUS`. +2. Build or refresh the complete member snapshot. +3. Resolve every green-suffixed member hostname. +4. Cache every member IP independently of DNS TTL expiry during the active + switchover. +5. Pin topology probing to the cached target writer/cluster IP so monitoring + survives the canonical endpoint outage and later retirement of green DNS. + +Discovery and identity refresh should continue during post-processing because +`SERVER_ID`s are renamed in that phase. Cached pre-rename IPs remain the traffic +pin values. + +Worker restart or configuration refresh must preserve enough state to avoid +losing: + +- the deployment fingerprint; +- the complete member snapshot; +- cached target IPs; +- applied traffic-pin flags; +- the probe pin; +- terminal completed/drain-wait state. + +## 7. Aurora State Machine + +### 7.1 States + +```text +NONE +AVAILABLE +SWITCHOVER_INITIATED +SWITCHOVER_IN_PROGRESS +SWITCHOVER_IN_POST_PROCESSING +SWITCHOVER_COMPLETED +``` + +TARGET `SWITCHOVER_COMPLETED` is an event that performs routing cleanup and +enters the internal `SWITCHOVER_COMPLETED` terminal latch; it is not a separate +reader-switchover phase. The runtime +`runtime_mysql_aws_aurora_hostgroups.bgd_status` column exposes these internal +states. + +### 7.2 State transitions and actions + +| Current observation | Required action | Polling cadence | +|---|---|---| +| No deployment | Remain `NONE`. | Normal discovery cadence | +| `AVAILABLE` | Build complete target membership, resolve all IPs, establish probe pin. | Normal/configured cadence | +| `SWITCHOVER_INITIATED` | Refresh membership/IPs, enable BGD in-progress protection. | Fast cadence | +| `SWITCHOVER_IN_PROGRESS` | Refresh membership/IPs and retain probe pin. | Fast cadence | +| First `SWITCHOVER_IN_POST_PROCESSING` | Apply all traffic pins and drain blue pools. | Fast cadence | +| Repeated `SWITCHOVER_IN_POST_PROCESSING` | Retry only unresolved/unapplied idempotent pair actions. | Fast cadence | +| First TARGET `SWITCHOVER_COMPLETED` | Remove all pins and perform immediate routing cleanup; enter the internal `SWITCHOVER_COMPLETED` latch. | Return to normal/slower cadence | +| Repeated same TARGET `SWITCHOVER_COMPLETED` while waiting | No-op. | Normal/slower cadence | +| Empty/absent table while waiting | Clear terminal fingerprint and return to `NONE`. | Normal discovery cadence | +| Different deployment fingerprint while waiting | Rearm and process the new deployment. | Appropriate new-state cadence | + +"Slower cadence" means a larger interval/lower polling frequency than the +active switchover cadence. + +## 8. Post-Processing Traffic Redirection + +On the first valid, monotonic TARGET +`SWITCHOVER_IN_POST_PROCESSING` observation: + +1. Require a complete cached writer/reader map with resolved target IPs. +2. Do **not** issue or wait for an additional writer `read_only` probe. +3. For every pair, pin: + + ```text + canonical blue hostname -> cached green IP + ``` + +4. Drain server connections for every pinned canonical blue hostname. +5. Purge free blue-host connection-pool entries so the next backend connection + uses the pinned target IP. +6. Keep mapped reader rows eligible in their existing reader hostgroup. + +Pinning, draining, and purging must be idempotent per member. A repeated +post-processing observation must not repeatedly drain an already transitioned +member. + +### 8.1 Reader policy + +The Aurora path must not shun and later unshun readers as part of a normal +switchover. All Aurora readers are discovered from `REPLICA_HOST_STATUS`, mapped, +and pinned together with the writer. + +The current writer-fallback behavior for missing green reader pairs remains a +Multi-AZ/instance policy; it is not the Aurora auto-configuration policy. + +The BGD in-progress marker must continue to suppress ordinary read-only monitor +actions that would conflict with this state machine. + +## 9. Completion and Immediate Cleanup + +On the first TARGET-only `SWITCHOVER_COMPLETED` observation, ProxySQL must trust +AWS's completion state and immediately: + +1. Remove the DNS-cache pin for every canonical writer and reader hostname. + `dns_cache->remove()` must invalidate the pinned/local entry so normal DNS + resolution resumes. +2. Remove the topology probe pin. +3. Stop fast switchover polling. +4. Clear BGD in-progress/read-only-monitor protection. +5. Drain obsolete pools belonging to explicit green hostgroups, subject to the + configured OFFLINE status preservation policy. +6. Preserve configured `mysql_servers` rows; cleanup concerns runtime routing + state and pools, not user configuration deletion. +7. Clear the active mapping/resolution state after retaining the minimal + terminal deployment fingerprint. +8. Enter the internal `SWITCHOVER_COMPLETED` latch. + +The completion path must **not**: + +- perform a new DNS-resolution verification before removing pins; +- wait for `mysql.rds_topology` to become empty; +- enter `READER_SWITCHOVER_IN_PROGRESS`; +- shun or unshun Aurora readers; +- process the same completed TARGET row more than once. + +## 10. `SWITCHOVER_COMPLETED` Terminal Latch + +The internal `SWITCHOVER_COMPLETED` state is a terminal latch, not an active +routing phase. All routing cleanup has already completed. It shares the AWS +event name intentionally but persists as ProxySQL runtime state while waiting +for topology drain. + +The worker retains only the deployment fingerprint and enough status to +recognize repeated results. While latched: + +- the same TARGET `SWITCHOVER_COMPLETED` result is ignored; +- query/connect errors do not cause rollback, repinning, or repeated cleanup; +- polling runs at the normal/slower cadence; +- successful empty results or confirmed table absence release the latch and + return the worker to `NONE`; +- a different deployment fingerprint may release/reinitialize the latch for a + new deployment. + +Table drain is therefore only the FSM rearm signal. It is not a writer or reader +availability signal for Aurora. + +## 11. Rollback Before Completion + +If AWS moves backward to `AVAILABLE`, cancels, or otherwise rolls back before +TARGET `SWITCHOVER_COMPLETED`, ProxySQL must undo only actions already applied: + +1. Remove applied traffic pins. +2. Remove the probe pin when target monitoring is no longer required. +3. Drain/purge affected canonical pools so subsequent connections follow the + restored canonical DNS path. +4. Clear BGD in-progress protection. +5. Restore normal hostgroup placement without deleting configured green rows. +6. Clear the active snapshot and rebuild it from the returned topology state. + +Rollback actions must be idempotent. Once the worker has entered the internal +`SWITCHOVER_COMPLETED` latch, transient query errors must not be treated as a +rollback. + +## 12. Separation from Multi-AZ Instance Logic + +The following existing behaviors remain valid for Multi-AZ instance BGD but +must not drive the Aurora branch: + +- treating TARGET `SWITCHOVER_COMPLETED` as writer-only completion; +- retaining reader pins until topology drains; +- inferring `READER_SWITCHOVER_IN_PROGRESS`; +- shunning unmapped blue readers; +- routing an empty reader hostgroup through the promoted writer as the normal + auto-generated fallback. + +Aurora has complete cluster membership through `REPLICA_HOST_STATUS`; its +writer and readers are switched and renamed as one cluster operation before the +TARGET completed state. + +## 13. Current Implementation Gaps + +The existing `aws_rds_bgd_build_map()` assumes the TARGET topology endpoint is +the green writer hostname and adds reader pairs only from an explicitly +configured green reader hostgroup. That assumption does not hold for Aurora: +its TARGET is a cluster endpoint. + +The Aurora implementation therefore needs a membership-driven map builder that: + +1. connects to the TARGET cluster endpoint; +2. queries `REPLICA_HOST_STATUS`; +3. constructs all member hostnames; +4. creates writer and reader pairs independently of green hostgroup + configuration; +5. retains stable reader identity across the post-processing rename; +6. applies the Aurora completion/terminal-latch FSM described above. + +The existing Multi-AZ builder and reader-cleanup FSM should remain available for +non-Aurora deployments. + +## 14. Required Tests + +### 14.1 Membership and mapping + +- Target cluster endpoint produces one writer and multiple reader pairs. +- Auto mode with NULL green hostgroups still maps all target members. +- Green-suffixed `SERVER_ID`s map to canonical blue hostnames. +- Reader `SESSION_ID`s preserve pair identity after canonical rename. +- A target cluster with no readers produces a valid writer-only snapshot. +- Failed/incomplete membership queries retain the previous complete snapshot or + defer actions. + +### 14.2 FSM + +- `AVAILABLE -> INITIATED -> IN_PROGRESS` resolves all target IPs without + changing traffic. +- First POST_PROCESSING pins and drains every pair exactly once. +- Repeated POST_PROCESSING retries only incomplete pairs. +- No additional writer-writability query gates pinning. +- Mapped readers remain eligible and are not shunned. +- First TARGET COMPLETED removes all pins and performs cleanup immediately. +- Completion does not perform a DNS verification. +- Repeated TARGET COMPLETED is a no-op while drain-wait is latched. +- Empty/absent topology releases the latch and returns to `NONE`. +- Query errors during drain-wait do not rollback or repin. +- A new deployment fingerprint can rearm discovery. + +### 14.3 Pool and DNS behavior + +- Each canonical blue hostname is pinned to the corresponding target IP. +- Pinning drains/purges old blue connections. +- Removing pins invalidates the local DNS-cache entry. +- Explicit green pools are drained at completion according to status policy. +- Configured rows are preserved. + +### 14.4 Resilience + +- Worker restart during each active phase preserves cached IPs, pins, and the + deployment fingerprint. +- Configuration refresh does not lose BGD in-progress protection. +- Rollback before completion removes all applied pins and restores normal + routing. +- Late entry at POST_PROCESSING builds a complete target snapshot before + pinning. +- Late entry at TARGET COMPLETED performs cleanup once and enters drain-wait + without first applying traffic pins. + +## 15. Acceptance Criteria + +The Aurora implementation is complete when: + +1. Auto mode discovers and maps every target Aurora member without configured + green hostgroups. +2. POST_PROCESSING redirects writer and reader traffic to cached target IPs + without reader shun/unshun actions. +3. TARGET COMPLETED removes all traffic/probe pins and completes routing cleanup + without waiting for topology drain. +4. Repeated completed rows cannot retrigger cleanup. +5. Empty/absent topology only rearms the FSM. +6. Multi-AZ instance behavior and tests remain unchanged. +7. TAP coverage verifies normal, repeated, rollback, late-entry, config-refresh, + and worker-restart paths. From 4876a87cf342283f264f491b5f384a16bfa4ebf4 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sat, 1 Aug 2026 21:37:18 +0000 Subject: [PATCH 02/34] docs: refine Aurora BGD monitor FSM design --- ...026-07-31-aurora-bgd-monitor-fsm-design.md | 854 +++++++++++------- 1 file changed, 514 insertions(+), 340 deletions(-) diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md index abd7bccfd8..20549dfa5c 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md @@ -4,14 +4,14 @@ **Branch:** `plan/aurora-bgd` -**Status:** Design draft +**Status:** Design approved; pending written-spec review **Scope:** Aurora MySQL blue/green deployments exposed through `mysql.rds_topology` and `INFORMATION_SCHEMA.REPLICA_HOST_STATUS`. -**Evidence basis:** An observed Aurora switchover captured on 2026-07-30. The -relevant timings and behavioral conclusions are summarized in Section 3 so the -specification is self-contained. +**Evidence basis:** The Aurora switchover observed on 2026-07-30 and recorded in +`aurora-bgd-switchover-analysis/RESULTS-20260730T073724Z.md`. Section 3 separates +observed behavior from design policy. **Related designs:** @@ -20,50 +20,77 @@ specification is self-contained. ## Spec Boundary -This specification owns the Aurora worker loop, topology and membership -discovery, cached member identity and IP state, DNS and pool actions, the BGD -FSM, rollback, and completion-latch behavior. Public table schemas, -persistence, and cluster synchronization belong to the configuration/runtime -specification. Simulator mechanics and the executable coverage matrix belong -to the simulator/testing specification. - -## 1. Purpose - -Provide a low-disruption ProxySQL switchover path for Aurora MySQL blue/green -deployments. ProxySQL discovers the complete target Aurora cluster before the -switchover, caches its member IPs, redirects the configured production -hostnames to those IPs during post-processing, and returns to normal DNS routing -as soon as AWS reports the target switchover complete. - -This is an Aurora-specific state machine. It does not replace the existing RDS -Multi-AZ instance behavior, where target `SWITCHOVER_COMPLETED` may describe only -the writer and reader cleanup must wait for a later topology-table drain. +This specification owns the Aurora worker loop, probe ownership and cadence, +target-membership discovery, cached member identity and IP state, DNS and pool +actions, the BGD FSM, rollback, reload continuity, and completion-latch +behavior. Public table schemas, persistence, and cluster synchronization belong +to the configuration/runtime specification. Simulator mechanics and the +executable coverage matrix belong to the simulator/testing specification. + +## 1. Decision Summary + +Aurora BGD is part of the existing Aurora monitor. Each active, user-created +`mysql_aws_aurora_hostgroups` row continues to own one worker keyed by +`writer_hostgroup`. Every Aurora configuration row is user-created, and the +design starts no second BGD worker for it. + +The worker owns three logically separate probes: + +1. the existing production Aurora membership/lag probe; +2. an Aurora BGD `mysql.rds_topology` probe; +3. a target `REPLICA_HOST_STATUS` membership probe. + +The worker follows existing RDS BGD parsing, status publication, DNS pinning, +connection draining, writer-placement, rollback, reload, and cleanup patterns +where their semantics match. Aurora-specific behavior is retained where RDS +Multi-AZ instance behavior does not apply: + +- membership comes from the target Aurora cluster; +- the topology probe rotates across reachable target members instead of being + pinned to the target writer IP; +- normal Aurora probing is suspended during the active switchover phases; +- every current writer and reader must be mapped before traffic changes; +- readers are not shunned or unshunned; +- TARGET completion cleans up writer and reader routing immediately; +- topology drain is only the terminal-latch rearm signal. + +An Aurora row either has both green hostgroups configured or has neither. This +document describes the distinction only as green hostgroups configured or not +configured. The existing +`mysql-aws_blue_green_deployment_auto_discovery` variable remains only an +admission gate for starting BGD discovery when green hostgroups are not +configured, as defined by the configuration/runtime specification. ## 2. Terminology -- **Blue:** The source/current production Aurora cluster before switchover. -- **Green:** The target Aurora cluster before promotion. -- **Canonical hostname/identifier:** The normal production instance name without - AWS's temporary `-green-` component. -- **Green hostname/identifier:** The target instance name containing - `-green-`. -- **Target cluster endpoint:** The TARGET endpoint returned by - `mysql.rds_topology`. For Aurora, this is a cluster endpoint, not an individual - writer endpoint. -- **Traffic pin:** A permanent ProxySQL DNS-cache mapping from a canonical blue - instance hostname to its cached green instance IP. -- **Probe pin:** The cached green writer/cluster IP used by the BGD monitor to - keep querying topology while canonical blue endpoints are unavailable. -- **Deployment fingerprint:** The TARGET topology identity retained after - cleanup to suppress repeated handling of the same completed row. At minimum, - it contains TARGET `id`, endpoint, and port. - -## 3. AWS/Aurora Behavioral Contract - -The Aurora implementation relies on these properties: - -1. `mysql.rds_topology` exposes SOURCE and TARGET cluster endpoints and a - monotonic status progression: +- **Production cluster:** The source/current Aurora cluster before switchover + and the promoted target after completion. +- **Target cluster:** The Aurora cluster identified by the TARGET row before + promotion. +- **Canonical identifier:** The original production `SERVER_ID`, without the + temporary AWS `-green-` component. +- **Target identifier:** A pre-promotion target `SERVER_ID` containing the + temporary `-green-` component. +- **Production hostname:** The canonical instance hostname already configured + in the production writer or reader hostgroup. +- **Target hostname:** A hostname constructed from a target `SERVER_ID` and the + RDS domain suffix. +- **Traffic pin:** A permanent ProxySQL DNS-cache mapping from a production + hostname to the cached IP of its target counterpart. +- **Complete target snapshot:** Exactly one current target writer plus a unique + target counterpart for every current production member, with all target IPs + resolved. +- **Deployment fingerprint:** TARGET topology identity retained after cleanup + to recognize repeated results. At minimum it contains TARGET `id`, endpoint, + and port. + +## 3. Evidence and AWS/Aurora Behavioral Contract + +### 3.1 Directly observed behavior + +The 2026-07-30 run observed: + +1. `mysql.rds_topology` progressed through: ```text AVAILABLE @@ -73,55 +100,133 @@ The Aurora implementation relies on these properties: -> TARGET-only SWITCHOVER_COMPLETED ``` -2. Querying `REPLICA_HOST_STATUS` through the target cluster endpoint returns - all current target members: exactly one writer and every Aurora reader. -3. The writer is the row whose `SESSION_ID` is `MASTER_SESSION_ID`; the other - current rows are readers. -4. During post-processing, target `SERVER_ID` values change from temporary - green identifiers to the canonical production identifiers. Reader - `SESSION_ID` values remain stable across the rename. -5. `SWITCHOVER_IN_POST_PROCESSING` is the AWS routing barrier. ProxySQL may - redirect traffic at this status without an additional target-writability - check. -6. TARGET `SWITCHOVER_COMPLETED` means canonical Aurora writer and reader DNS - cutover has completed. ProxySQL may remove all traffic pins without a - separate DNS-resolution verification. -7. The TARGET-only completed row can remain in `mysql.rds_topology` after - routing cleanup. Repeated completed rows describe the same completed - deployment and must be ignored until the table drains. - -The observed Aurora run supported these assumptions: - -- target writer writable: `T+14.438s`; -- target post-processing: `T+15.795s`; -- all target members had canonical `SERVER_ID`s: `T+21.380s`; -- canonical writer/readers accepted fresh connections: by `T+31.212s`; -- TARGET `SWITCHOVER_COMPLETED`: `T+40.655s`; -- topology table drained: `T+65.730s`. - -Thus, member cutover completed before TARGET completion; table drain was not a -reader-cutover barrier for Aurora. - -## 4. Aurora Detection - -The Aurora state machine must be selected only for an Aurora deployment. -Detection must establish both of the following: +2. The target cluster endpoint returned exactly one writer and all current + readers through `REPLICA_HOST_STATUS`. +3. The writer row used `SESSION_ID='MASTER_SESSION_ID'`; reader rows used + distinct session identifiers. +4. During post-processing, target `SERVER_ID` values changed from temporary + green identifiers to canonical production identifiers. +5. Reader `SESSION_ID` values remained unchanged across that rename. +6. The production topology became `SWITCHOVER_IN_PROGRESS` at `T+9.864s`; the + source writer became read-only at `T+10.398s`. +7. The target writer became writable at `T+14.438s`; target post-processing was + observed at `T+15.795s`. +8. Every target member had a canonical `SERVER_ID` by `T+21.380s`. +9. TARGET completion was visible at `T+40.655s`; the topology table drained at + `T+65.730s`. + +The implementation treats these observations as the evidence for the design, +not as a formal AWS compatibility guarantee. Tests must simulate the observed +changes, and unexpected or ambiguous metadata must fail closed. + +### 3.2 Design policy derived from the observation + +- POST_PROCESSING is the routing barrier; no additional target-writability + query gates traffic pinning. +- TARGET completion means Aurora writer and reader routing cleanup can occur + immediately. +- Table drain is not a reader-availability barrier for Aurora. +- A query error, incomplete membership result, or ambiguous identity does not + advance routing actions or cause rollback. + +## 4. Worker Ownership and Probe Model + +### 4.1 Single worker owner + +The existing Aurora worker for a `writer_hostgroup` is the sole owner of normal +Aurora monitoring and the BGD FSM. It owns all per-deployment state and +serializes the three probes and their actions. A separate Aurora BGD worker must +not be started for the same row because it would duplicate queries and race on +hostgroup state. + +### 4.2 Production Aurora probe + +This is the existing random-host `REPLICA_HOST_STATUS` probe used for Aurora +role, lag, and autopurge decisions. + +- It runs normally in `NONE` and `AVAILABLE`. +- It stops issuing queries in `SWITCHOVER_INITIATED`, + `SWITCHOVER_IN_PROGRESS`, and `SWITCHOVER_IN_POST_PROCESSING`. +- It resumes after completion cleanup enters `SWITCHOVER_COMPLETED`, or + immediately after rollback. + +Suspension means the query itself is skipped. Running the query while +discarding all role, lag, and autopurge actions provides no FSM input because +the target-membership probe owns target discovery. + +### 4.3 Topology probe + +In `NONE`, the topology probe uses the existing Aurora random-host selection: +choose a random reachable production member and fall back across the remaining +members when ping or connection setup fails. + +After the TARGET endpoint has bootstrapped a complete target member list, the +topology probe applies the same selection algorithm across the current target +members. It is not pinned to the target writer and does not use a dedicated +cached-IP probe pin. When `REPLICA_HOST_STATUS` reports renamed identifiers, +the target probe-host list is refreshed to the corresponding canonical +hostnames. + +After completion cleanup, topology probing returns to random reachable +canonical production members while the terminal latch waits for table drain. + +### 4.4 Target-membership probe + +The TARGET cluster endpoint from `mysql.rds_topology` bootstraps a separate +`REPLICA_HOST_STATUS` query. Its result is parsed only into the BGD target +snapshot. It must not be passed to normal Aurora evaluation before cutover, +because doing so could prematurely move or add target members in production +hostgroups. + +The target-membership probe continues through AVAILABLE, INITIATED, IN_PROGRESS, +and POST_PROCESSING. It stops after completion cleanup or rollback has removed +the need for target-side discovery. + +### 4.5 Cadence + +| FSM state | Topology probe | Target-membership probe | Production Aurora probe | +|---|---|---|---| +| `NONE` | Configured `check_interval_ms` when BGD discovery is admitted | Off | Configured interval | +| `AVAILABLE` | Configured `check_interval_ms` | Configured interval | Configured interval | +| `SWITCHOVER_INITIATED` | Fast, 100 ms | Fast, 100 ms | Suspended | +| `SWITCHOVER_IN_PROGRESS` | Fast, 100 ms | Fast, 100 ms | Suspended | +| `SWITCHOVER_IN_POST_PROCESSING` | Fast, 100 ms | Fast, 100 ms | Suspended | +| `SWITCHOVER_COMPLETED` | Configured interval | Off | Configured interval | + +The 100 ms active cadence follows the existing RDS BGD worker. Query duration +is accounted for so a slow query does not create an additional full-interval +sleep. + +## 5. Discovery Bootstrap and Admission + +For an eligible row in `NONE`: + +1. Probe `information_schema.TABLES` for `mysql.rds_topology`, following the + RDS BGD table-check state. +2. When the table exists, fetch and parse `SELECT * FROM mysql.rds_topology` so + AWS column-set differences remain tolerated by the shared parser. +3. Require a structurally valid BGD result with a TARGET row, non-empty TARGET + status, endpoint, and port. +4. Publish the observed TARGET status to `bgd_status`. +5. Use the TARGET cluster endpoint to bootstrap target membership. +6. After target membership is available, rotate topology probes across target + members. -1. The TARGET topology endpoint is an Aurora cluster endpoint. -2. `REPLICA_HOST_STATUS` membership discovery succeeds through that endpoint. +A membership failure does not undo a valid observed status. It prevents any +membership-dependent routing action and is retried at the cadence for the +current phase. Missing, empty, or malformed topology does not create a new +deployment. -The existing Multi-AZ state machine remains active for non-Aurora TARGET -endpoints. +If green hostgroups are configured, BGD discovery is admitted for the +user-created Aurora row. If neither is configured, the existing global +auto-discovery variable gates admission of a new deployment. Changing that +variable after a deployment has started does not abandon the active FSM. -## 5. Membership Discovery +## 6. Target Membership and Pair Mapping -### 5.1 Source of truth +### 6.1 Query and current rows -`REPLICA_HOST_STATUS` is the source of truth for target Aurora membership. -Configured green hostgroups are not the source of membership and are not -required for automatic mode. - -The discovery query must include at least: +The target query must include at least: ```sql SELECT @@ -133,108 +238,100 @@ FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS ORDER BY SERVER_ID; ``` -The implementation may retain the existing lag and CPU columns used by the -Aurora monitor. +The implementation may retain the existing lag and CPU columns. Only rows +identified as current are eligible. The writer is the sole current row whose +`SESSION_ID` is `MASTER_SESSION_ID`; all other eligible rows are readers. -Only current rows are eligible for the active member snapshot. +### 6.2 Complete-snapshot rule -### 5.2 Constructing target instance hostnames +A new snapshot replaces the last complete snapshot only when: -The target cluster endpoint has the form: +1. it contains exactly one current target writer; +2. each current production member has exactly one target counterpart; +3. no target member maps to more than one production member; +4. no production member maps to more than one target member; +5. every target hostname has a resolved IP. -```text -.cluster- -``` +A target writer-only snapshot is valid only when the current production +snapshot also contains no readers. Query errors, empty results, multiple +writers, missing readers, duplicate identities, and unresolved IPs do not +replace the last complete snapshot. They also must not be reinterpreted as a +reader-less deployment. -A member hostname is formed as: +POST_PROCESSING actions require a complete snapshot. If none exists, the +worker holds the current status, performs no partial traffic cutover, and +retries both active probes. -```text -. -``` +### 6.3 Constructing member hostnames -For example: +The TARGET cluster endpoint has the form: ```text -aurora-mysql-1-green-jkpanw.cluster-c1yqcg0ie39o.eu-north-1.rds.amazonaws.com - -aurora-mysql-1-writer-green-drodij.c1yqcg0ie39o.eu-north-1.rds.amazonaws.com +.cluster- ``` -### 5.3 Building blue/green pairs - -For every target member, maintain: +A target member hostname is: ```text -target SERVER_ID -SESSION_ID -role: writer or reader -green hostname -green IP -canonical blue hostname -port -configured blue server attributes +. ``` -Before AWS renames the members, the canonical identifier is obtained by -removing the `-green-` component. After AWS renames them, the returned -`SERVER_ID` is already canonical. - -The writer pair is correlated by its unique writer role and cached IP. Reader -pairs are additionally correlated across renaming by their stable -`SESSION_ID`s. +Each target member stores its role, current `SERVER_ID`, `SESSION_ID`, current +hostname, port, resolved IP, normalized canonical identifier, matching +production hostname, and per-action flags. -There must be exactly one writer pair. Every configured/current blue Aurora -reader must have a corresponding target reader pair before traffic pinning is -performed. +### 6.4 Identity across rename -### 5.4 Auto-generated configuration +Pair every target member to production by its normalized `SERVER_ID`: -When `green_writer_hostgroup` and `green_reader_hostgroup` are `NULL`, the BGD -worker must still discover and map the target writer and all target readers in -memory. +- before promotion, remove the AWS-added `-green-` component; +- after promotion, leave the already canonical identifier unchanged. -The absence of explicit green hostgroups means "no user-managed staging -hostgroups"; it does not mean "no green readers." +For readers, `SESSION_ID` is an additional continuity check. A reader whose +`SERVER_ID` changes but whose session identifier matches the cached pair is the +same target reader. A session mismatch or a conflict between normalized name +and cached session identity makes the new snapshot ambiguous. -The Aurora auto-generated path must not depend on green `mysql_servers` rows and -must not use the writer-only fallback as its normal reader policy. +For the writer, `MASTER_SESSION_ID` is only a role marker. It must not be used +as a unique instance identity. The sole writer is paired by normalized +`SERVER_ID`; no cached-IP writer identity rule is required. -If the target cluster actually has no readers, the snapshot naturally contains -only the writer. If a membership query fails or yields a transient incomplete -snapshot for a cluster expected to have readers, retain the last complete -snapshot or defer the phase action. Do not reinterpret a failed/incomplete -query as a reader-less deployment. +### 6.5 Cached IPs -## 6. IP Resolution and Cached State +During AVAILABLE and every active switchover phase, resolve all target member +hostnames and retain the last complete set of IPs independently of DNS TTL +expiry. Membership discovery continues through post-processing so renamed +identifiers refresh the host list, but pre-rename cached IPs remain valid +traffic-pin values. -During `AVAILABLE`, `SWITCHOVER_INITIATED`, and `SWITCHOVER_IN_PROGRESS`, the -worker must: +Cached target IPs are used for traffic redirection only. They do not pin the +topology probe to a particular member. -1. Query target `REPLICA_HOST_STATUS`. -2. Build or refresh the complete member snapshot. -3. Resolve every green-suffixed member hostname. -4. Cache every member IP independently of DNS TTL expiry during the active - switchover. -5. Pin topology probing to the cached target writer/cluster IP so monitoring - survives the canonical endpoint outage and later retirement of green DNS. +## 7. Worker-Owned State -Discovery and identity refresh should continue during post-processing because -`SERVER_ID`s are renamed in that phase. Cached pre-rename IPs remain the traffic -pin values. +The runtime owner for a writer hostgroup must retain: -Worker restart or configuration refresh must preserve enough state to avoid -losing: +```text +current FSM status +deployment fingerprint +last production membership snapshot +last complete target snapshot +normalized production/target member pairs +cached target IP per pair +traffic-pin-applied flag per pair +writer-demoted flag +production-probe-suspended flag +completion-cleanup-applied flag +configured green hostgroup identifiers, when present +``` -- the deployment fingerprint; -- the complete member snapshot; -- cached target IPs; -- applied traffic-pin flags; -- the probe pin; -- terminal completed/drain-wait state. +Per-member flags make repeated observations idempotent. The fingerprint keeps a +repeated completed row from starting cleanup again after the active map has +been released. -## 7. Aurora State Machine +## 8. State Machine -### 7.1 States +### 8.1 States ```text NONE @@ -245,224 +342,301 @@ SWITCHOVER_IN_POST_PROCESSING SWITCHOVER_COMPLETED ``` -TARGET `SWITCHOVER_COMPLETED` is an event that performs routing cleanup and -enters the internal `SWITCHOVER_COMPLETED` terminal latch; it is not a separate -reader-switchover phase. The runtime -`runtime_mysql_aws_aurora_hostgroups.bgd_status` column exposes these internal -states. +`runtime_mysql_aws_aurora_hostgroups.bgd_status` exposes these worker states. +The final state is a terminal rearm latch, not a separate reader-switchover +phase. -### 7.2 State transitions and actions +### 8.2 Transition summary -| Current observation | Required action | Polling cadence | -|---|---|---| -| No deployment | Remain `NONE`. | Normal discovery cadence | -| `AVAILABLE` | Build complete target membership, resolve all IPs, establish probe pin. | Normal/configured cadence | -| `SWITCHOVER_INITIATED` | Refresh membership/IPs, enable BGD in-progress protection. | Fast cadence | -| `SWITCHOVER_IN_PROGRESS` | Refresh membership/IPs and retain probe pin. | Fast cadence | -| First `SWITCHOVER_IN_POST_PROCESSING` | Apply all traffic pins and drain blue pools. | Fast cadence | -| Repeated `SWITCHOVER_IN_POST_PROCESSING` | Retry only unresolved/unapplied idempotent pair actions. | Fast cadence | -| First TARGET `SWITCHOVER_COMPLETED` | Remove all pins and perform immediate routing cleanup; enter the internal `SWITCHOVER_COMPLETED` latch. | Return to normal/slower cadence | -| Repeated same TARGET `SWITCHOVER_COMPLETED` while waiting | No-op. | Normal/slower cadence | -| Empty/absent table while waiting | Clear terminal fingerprint and return to `NONE`. | Normal discovery cadence | -| Different deployment fingerprint while waiting | Rearm and process the new deployment. | Appropriate new-state cadence | +| Valid observation | Required transition and action | +|---|---| +| No deployment in `NONE` | Remain `NONE`; continue admitted discovery at configured cadence. | +| `AVAILABLE` | Enter/retain `AVAILABLE`; build target membership and cache IPs without changing production routing. | +| `SWITCHOVER_INITIATED` | Suspend production Aurora probing, capture rollback state, and switch BGD probes to fast cadence. | +| `SWITCHOVER_IN_PROGRESS` | Move the production writer to the reader hostgroup; do not route to target yet. | +| First `SWITCHOVER_IN_POST_PROCESSING` | With a complete snapshot, pin every production member, drain old connections, and restore the canonical writer to the writer hostgroup. | +| Repeated POST_PROCESSING | Retry only incomplete idempotent member actions. | +| First TARGET-only `SWITCHOVER_COMPLETED` | Remove all pins, perform immediate cleanup, resume normal Aurora monitoring, and enter the completed latch. | +| Repeated same completed TARGET | No-op while latched. | +| Successful empty/absent topology while latched | Release the fingerprint and return to `NONE`. | +| Earlier valid status before completion | Run rollback, then enter the earlier state. | +| Successful empty/absent topology before completion | Treat as cancellation and run rollback. | +| Query/connect error in any state | Retain state and retry; never infer rollback or table drain. | +| Different valid fingerprint while latched | Rearm and process the new deployment. | -"Slower cadence" means a larger interval/lower polling frequency than the -active switchover cadence. +## 9. Phase Actions -## 8. Post-Processing Traffic Redirection +### 9.1 `AVAILABLE` -On the first valid, monotonic TARGET -`SWITCHOVER_IN_POST_PROCESSING` observation: - -1. Require a complete cached writer/reader map with resolved target IPs. -2. Do **not** issue or wait for an additional writer `read_only` probe. -3. For every pair, pin: - - ```text - canonical blue hostname -> cached green IP - ``` +On entry and repeated observations: -4. Drain server connections for every pinned canonical blue hostname. -5. Purge free blue-host connection-pool entries so the next backend connection - uses the pinned target IP. -6. Keep mapped reader rows eligible in their existing reader hostgroup. +1. Publish `bgd_status=AVAILABLE`. +2. Run normal production Aurora monitoring unchanged. +3. Build or refresh the last complete target snapshot. +4. Resolve and retain every target IP. +5. Keep all production DNS, pools, and hostgroup placement unchanged. -Pinning, draining, and purging must be idempotent per member. A repeated -post-processing observation must not repeatedly drain an already transitioned -member. +Configured green hostgroups, when present, remain user configuration and are +not the target-membership source of truth. When they are absent, the complete +target map remains worker-local. No Aurora configuration row is generated in +either case. -### 8.1 Reader policy +### 9.2 `SWITCHOVER_INITIATED` -The Aurora path must not shun and later unshun readers as part of a normal -switchover. All Aurora readers are discovered from `REPLICA_HOST_STATUS`, mapped, -and pinned together with the writer. +On entry: -The current writer-fallback behavior for missing green reader pairs remains a -Multi-AZ/instance policy; it is not the Aurora auto-configuration policy. +1. Publish `bgd_status=SWITCHOVER_INITIATED`. +2. Capture production writer placement and other state required for rollback. +3. Suspend the normal production Aurora probe. +4. Switch topology and target-membership probes to 100 ms. +5. Continue refreshing target membership and IPs. +6. Make no traffic-routing or hostgroup-placement change. -The BGD in-progress marker must continue to suppress ordinary read-only monitor -actions that would conflict with this state machine. +### 9.3 `SWITCHOVER_IN_PROGRESS` -## 9. Completion and Immediate Cleanup +On the first valid observation: -On the first TARGET-only `SWITCHOVER_COMPLETED` observation, ProxySQL must trust -AWS's completion state and immediately: +1. Publish `bgd_status=SWITCHOVER_IN_PROGRESS`. +2. Retain fast BGD probes and the suspended production probe. +3. Move the current production writer to the reader hostgroup using the same + writer-demotion behavior as RDS BGD. +4. Record that demotion so rollback can restore it. +5. Do not redirect any production hostname to target yet. -1. Remove the DNS-cache pin for every canonical writer and reader hostname. - `dns_cache->remove()` must invalidate the pinned/local entry so normal DNS - resolution resumes. -2. Remove the topology probe pin. -3. Stop fast switchover polling. -4. Clear BGD in-progress/read-only-monitor protection. -5. Drain obsolete pools belonging to explicit green hostgroups, subject to the - configured OFFLINE status preservation policy. -6. Preserve configured `mysql_servers` rows; cleanup concerns runtime routing - state and pools, not user configuration deletion. -7. Clear the active mapping/resolution state after retaining the minimal - terminal deployment fingerprint. -8. Enter the internal `SWITCHOVER_COMPLETED` latch. +The move is idempotent. A repeated observation does not repeat a completed +hostgroup action. -The completion path must **not**: +### 9.4 `SWITCHOVER_IN_POST_PROCESSING` -- perform a new DNS-resolution verification before removing pins; -- wait for `mysql.rds_topology` to become empty; -- enter `READER_SWITCHOVER_IN_PROGRESS`; -- shun or unshun Aurora readers; -- process the same completed TARGET row more than once. +POST_PROCESSING is the only pre-completion traffic-redirection boundary. -## 10. `SWITCHOVER_COMPLETED` Terminal Latch +Before applying any action, require a complete target snapshot. Then: -The internal `SWITCHOVER_COMPLETED` state is a terminal latch, not an active -routing phase. All routing cleanup has already completed. It shares the AWS -event name intentionally but persists as ProxySQL runtime state while waiting -for topology drain. +1. Publish `bgd_status=SWITCHOVER_IN_POST_PROCESSING`. +2. Do not issue an additional target-writability probe. +3. For every writer and reader pair, install: -The worker retains only the deployment fingerprint and enough status to -recognize repeated results. While latched: - -- the same TARGET `SWITCHOVER_COMPLETED` result is ignored; -- query/connect errors do not cause rollback, repinning, or repeated cleanup; -- polling runs at the normal/slower cadence; -- successful empty results or confirmed table absence release the latch and - return the worker to `NONE`; -- a different deployment fingerprint may release/reinitialize the latch for a - new deployment. - -Table drain is therefore only the FSM rearm signal. It is not a writer or reader -availability signal for Aurora. + ```text + production hostname -> cached target IP + ``` -## 11. Rollback Before Completion +4. Reuse the RDS BGD connection-drain behavior for each production hostname: + drop free backend connections immediately and mark used connections + unhealthy/non-reusable. +5. Purge the corresponding monitor connection-pool entries so new connections + resolve through the traffic pin. +6. After the writer pin is installed, restore the canonical writer to the + writer hostgroup. Its placement in the reader hostgroup follows the existing + `writer_is_also_reader` configuration. +7. Keep canonical readers eligible in the reader hostgroup. Do not shun or + unshun them. + +Every pair records whether pinning and draining completed. Repeated +POST_PROCESSING results retry only unapplied actions and do not repeatedly drain +an already transitioned member. + +### 9.5 `SWITCHOVER_COMPLETED` -If AWS moves backward to `AVAILABLE`, cancels, or otherwise rolls back before -TARGET `SWITCHOVER_COMPLETED`, ProxySQL must undo only actions already applied: +On the first TARGET-only completed result: + +1. Remove every production-hostname DNS pin immediately. `dns_cache->remove()` + must invalidate the local pinned entry so normal DNS resolution resumes. +2. Do not wait for topology drain or perform a separate DNS verification. +3. Preserve current production connections; they already point to promoted + members. Drain only obsolete pools belonging to configured green + hostgroups, subject to the configured OFFLINE-status preservation policy. +4. Preserve every user-created configuration and `mysql_servers` row. +5. Clear the switchover guard and resume normal Aurora monitoring. +6. Move topology probing back to random reachable canonical production members + and the configured interval. +7. Release the active member map after retaining the deployment fingerprint and + the completion-cleanup flag. +8. Publish and enter the internal `SWITCHOVER_COMPLETED` latch. + +While latched: + +- the same TARGET completed result is ignored; +- query/connect errors do not rollback, repin, or repeat cleanup; +- a successful empty result or confirmed table absence returns the worker to + `NONE`; +- a different valid deployment fingerprint rearms discovery. + +The topology drain is therefore only an FSM rearm signal. + +## 10. Rollback and Error Handling + +### 10.1 Rollback before completion + +A successful, structurally valid result for the same deployment with an +earlier status follows the existing RDS BGD backward-transition behavior. A +successful empty result or confirmed table absence before completion is treated +as cancellation. Either condition runs idempotent rollback: + +1. Remove every traffic pin that was applied. +2. Drain/purge affected production-hostname pools so subsequent connections use + restored canonical DNS. +3. Restore the production writer to the writer hostgroup when it was demoted. +4. Restore its reader placement according to `writer_is_also_reader`. +5. Resume normal production Aurora monitoring. +6. Return topology probing to production members and the appropriate cadence. +7. Preserve user-created green-hostgroup and server rows. +8. If AWS returned to AVAILABLE, rebuild target discovery in AVAILABLE; if the + topology disappeared, clear the deployment and return to `NONE`. + +Rollback does not drain configured green-hostgroup pools merely because an +attempt was cancelled. + +### 10.2 Errors and incomplete data + +- A topology or membership query/connect error retains the current state. +- An invalid or incomplete target snapshot retains the last complete snapshot. +- No error is interpreted as an empty table, reader-less cluster, cancellation, + or successful completion. +- No routing action runs without its state-specific prerequisites. +- Errors in the completed latch do not release or restart it. + +## 11. Late Entry, Reload, Removal, and Concurrency + +### 11.1 Late entry + +A worker that first observes INITIATED, IN_PROGRESS, or POST_PROCESSING rebuilds +all prerequisites before applying that phase's actions. In particular, +POST_PROCESSING cannot pin traffic until it has a complete target snapshot. + +A worker that first observes TARGET-only completion performs completion cleanup +once, even when no traffic pins were recorded, retains the fingerprint, and +enters the terminal latch. It must not manufacture or replay earlier phase +actions. + +### 11.2 Configuration and variable refresh + +An unrelated `LOAD MYSQL SERVERS TO RUNTIME` or variable refresh must preserve +the FSM status, deployment fingerprint, complete snapshots, cached IPs, applied +pin flags, writer-demotion state, probe-suspension state, and terminal latch. + +Changing configured green hostgroups refreshes the staging/pool references but +does not restart an active deployment from `NONE`. Disabling the global +auto-discovery variable prevents admission of a new no-green-hostgroup +deployment; it does not abort one already active. + +### 11.3 Row removal and worker exit + +Removing or deactivating the owning Aurora row follows the existing RDS BGD +worker-removal cleanup pattern: remove applied pins, restore safe production +writer placement where possible, release suspended monitoring state, and then +terminate the worker. No user configuration row is deleted as a side effect. -1. Remove applied traffic pins. -2. Remove the probe pin when target monitoring is no longer required. -3. Drain/purge affected canonical pools so subsequent connections follow the - restored canonical DNS path. -4. Clear BGD in-progress protection. -5. Restore normal hostgroup placement without deleting configured green rows. -6. Clear the active snapshot and rebuild it from the returned topology state. +### 11.4 Concurrent deployments -Rollback actions must be idempotent. Once the worker has entered the internal -`SWITCHOVER_COMPLETED` latch, transient query errors must not be treated as a -rollback. +All state is keyed by production `writer_hostgroup`. Probes, fingerprints, +snapshots, pins, writer placement, reloads, rollback, and completion for one +Aurora row must not mutate another row's state. -## 12. Separation from Multi-AZ Instance Logic +## 12. Separation from RDS Multi-AZ Instance Logic -The following existing behaviors remain valid for Multi-AZ instance BGD but -must not drive the Aurora branch: +The following RDS Multi-AZ instance behaviors must not drive the Aurora branch: -- treating TARGET `SWITCHOVER_COMPLETED` as writer-only completion; -- retaining reader pins until topology drains; -- inferring `READER_SWITCHOVER_IN_PROGRESS`; -- shunning unmapped blue readers; -- routing an empty reader hostgroup through the promoted writer as the normal - auto-generated fallback. +- pinning the topology probe to a single green writer IP; +- treating TARGET completion as writer-only completion; +- entering `READER_SWITCHOVER_IN_PROGRESS`; +- retaining reader pins until topology drain; +- shunning unmapped production readers; +- routing an empty reader hostgroup through the promoted writer as a missing-map + fallback. -Aurora has complete cluster membership through `REPLICA_HOST_STATUS`; its -writer and readers are switched and renamed as one cluster operation before the -TARGET completed state. +Aurora reuses the RDS machinery only where the behavior matches the contracts +in this specification. ## 13. Current Implementation Gaps -The existing `aws_rds_bgd_build_map()` assumes the TARGET topology endpoint is -the green writer hostname and adds reader pairs only from an explicitly -configured green reader hostgroup. That assumption does not hold for Aurora: -its TARGET is a cluster endpoint. +The existing RDS builder assumes the TARGET topology endpoint is an instance +writer and discovers readers from configured RDS green hostgroups. Aurora's +TARGET is a cluster endpoint and its complete membership comes from +`REPLICA_HOST_STATUS`. -The Aurora implementation therefore needs a membership-driven map builder that: +The Aurora implementation therefore requires: -1. connects to the TARGET cluster endpoint; -2. queries `REPLICA_HOST_STATUS`; -3. constructs all member hostnames; -4. creates writer and reader pairs independently of green hostgroup - configuration; -5. retains stable reader identity across the post-processing rename; -6. applies the Aurora completion/terminal-latch FSM described above. +1. three logical probes in the existing per-writer-hostgroup worker; +2. target-cluster membership parsing separate from normal Aurora evaluation; +3. normalized `SERVER_ID` pairing plus reader `SESSION_ID` continuity; +4. complete-snapshot validation and cached per-member IPs; +5. normal-probe suspension during the active switchover; +6. RDS-style writer demotion, DNS pinning, pool draining, rollback, and status + publication; +7. Aurora-specific all-member POST_PROCESSING and immediate completion cleanup; +8. completion-fingerprint retention until topology drain. -The existing Multi-AZ builder and reader-cleanup FSM should remain available for -non-Aurora deployments. +The existing RDS Multi-AZ builder and FSM remain unchanged. ## 14. Required Tests -### 14.1 Membership and mapping - -- Target cluster endpoint produces one writer and multiple reader pairs. -- Auto mode with NULL green hostgroups still maps all target members. -- Green-suffixed `SERVER_ID`s map to canonical blue hostnames. -- Reader `SESSION_ID`s preserve pair identity after canonical rename. -- A target cluster with no readers produces a valid writer-only snapshot. -- Failed/incomplete membership queries retain the previous complete snapshot or - defer actions. - -### 14.2 FSM - -- `AVAILABLE -> INITIATED -> IN_PROGRESS` resolves all target IPs without - changing traffic. -- First POST_PROCESSING pins and drains every pair exactly once. -- Repeated POST_PROCESSING retries only incomplete pairs. -- No additional writer-writability query gates pinning. -- Mapped readers remain eligible and are not shunned. -- First TARGET COMPLETED removes all pins and performs cleanup immediately. -- Completion does not perform a DNS verification. -- Repeated TARGET COMPLETED is a no-op while drain-wait is latched. +### 14.1 Probe ownership and cadence + +- One Aurora worker owns all three logical probes. +- NONE and AVAILABLE use configured cadence and continue normal Aurora probes. +- INITIATED, IN_PROGRESS, and POST_PROCESSING use 100 ms BGD probes and issue no + normal Aurora query. +- Topology selection rotates across target members and falls back when a member + is unreachable. +- No target-writer IP probe pin is installed. +- COMPLETED resumes normal probing and production-member topology selection. + +### 14.2 Membership and mapping + +- Target cluster membership yields one writer and multiple reader pairs. +- No configured green hostgroups still produces a complete in-memory map. +- Green-suffixed `SERVER_ID`s normalize to production identifiers. +- Reader `SESSION_ID`s preserve identity after canonical rename. +- The writer is paired by normalized `SERVER_ID`, not `MASTER_SESSION_ID` or IP. +- A genuine no-reader cluster accepts a writer-only snapshot. +- Empty, failed, duplicate-writer, ambiguous, unresolved, and incomplete results + retain the last complete snapshot or defer actions. + +### 14.3 FSM and routing + +- AVAILABLE discovers and resolves every member without routing changes. +- INITIATED suspends normal Aurora probing without moving servers. +- IN_PROGRESS moves the production writer to the reader hostgroup exactly once. +- POST_PROCESSING requires a complete snapshot. +- First POST_PROCESSING pins and drains every member and restores the writer. +- No target-writability query gates POST_PROCESSING. +- Readers remain eligible and are never shunned. +- Repeated POST_PROCESSING retries only unapplied work. +- First TARGET completion removes all pins and cleans up immediately. +- Completion performs no DNS verification and does not wait for table drain. +- Repeated completion is a no-op while latched. - Empty/absent topology releases the latch and returns to `NONE`. -- Query errors during drain-wait do not rollback or repin. -- A new deployment fingerprint can rearm discovery. -### 14.3 Pool and DNS behavior +### 14.4 Resilience and lifecycle -- Each canonical blue hostname is pinned to the corresponding target IP. -- Pinning drains/purges old blue connections. -- Removing pins invalidates the local DNS-cache entry. -- Explicit green pools are drained at completion according to status policy. -- Configured rows are preserved. - -### 14.4 Resilience - -- Worker restart during each active phase preserves cached IPs, pins, and the - deployment fingerprint. -- Configuration refresh does not lose BGD in-progress protection. -- Rollback before completion removes all applied pins and restores normal - routing. -- Late entry at POST_PROCESSING builds a complete target snapshot before - pinning. -- Late entry at TARGET COMPLETED performs cleanup once and enters drain-wait - without first applying traffic pins. +- Backward status and pre-completion topology disappearance run rollback. +- Query errors never run rollback or release the completed latch. +- Rollback removes pins, restores writer placement, and resumes normal probes. +- Late entry at every active state reconstructs prerequisites before acting. +- Reload preserves active and terminal state. +- Changing or removing the owning row runs safe, isolated cleanup. +- Disabling auto-discovery does not abort an admitted deployment. +- Concurrent writer hostgroups remain isolated. +- Configured green pools are drained only on successful completion, with user + rows and statuses preserved. ## 15. Acceptance Criteria -The Aurora implementation is complete when: - -1. Auto mode discovers and maps every target Aurora member without configured - green hostgroups. -2. POST_PROCESSING redirects writer and reader traffic to cached target IPs - without reader shun/unshun actions. -3. TARGET COMPLETED removes all traffic/probe pins and completes routing cleanup +The Aurora monitor/FSM design is satisfied when: + +1. One existing Aurora worker owns the three probes and all state for its + writer hostgroup. +2. Every production member is mapped from target `REPLICA_HOST_STATUS` before + traffic changes, whether or not green hostgroups are configured. +3. Normal Aurora queries stop during the three active switchover phases and + resume after completion or rollback. +4. IN_PROGRESS demotes the production writer using the RDS BGD behavior. +5. POST_PROCESSING redirects writer and reader traffic using cached target IPs, + drains old connections, and never shuns readers. +6. TARGET completion removes every traffic pin and finishes routing cleanup without waiting for topology drain. -4. Repeated completed rows cannot retrigger cleanup. -5. Empty/absent topology only rearms the FSM. -6. Multi-AZ instance behavior and tests remain unchanged. -7. TAP coverage verifies normal, repeated, rollback, late-entry, config-refresh, - and worker-restart paths. +7. Repeated results, errors, reloads, late entry, rollback, worker removal, and + concurrent deployments behave idempotently and remain isolated. +8. The terminal completed state persists only until a successful topology drain + observation rearms the FSM. +9. RDS Multi-AZ instance behavior and tests remain unchanged. From 9bf316dc1d886e9a3576020d9b426dab66d3f566 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 12 Aug 2026 15:21:47 +0000 Subject: [PATCH 03/34] docs: finalize Aurora BGD simulator testing design --- ...ra-bgd-cluster-simulator-testing-design.md | 605 ++++++++++-------- 1 file changed, 340 insertions(+), 265 deletions(-) diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md index f3fda79484..6796a97945 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md @@ -1,356 +1,431 @@ -# Aurora BGD Cluster Simulator and Testing Design +# AWS Aurora Blue/Green Cluster Simulator and Testing Design **Date:** 2026-07-31 -**Branch:** `plan/aurora-bgd` +**Branch:** `spec/aws-aurora-bgd` -**Status:** Design draft +**Status:** Design approved **Related designs:** - [Aurora BGD Configuration, Runtime Status, and Cluster Sync](2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md) - [Aurora BGD Monitor Loop and FSM](2026-07-31-aurora-bgd-monitor-fsm-design.md) -## Spec Boundary +## 1. Scope -This specification owns the simulator capabilities and executable test coverage -needed to validate the other two specifications. It does not redefine public -configuration or FSM behavior. When a test expectation depends on those -contracts, the corresponding sibling specification is authoritative. +This specification defines the simulator services, test controllers, endpoint +model, and executable coverage for AWS Aurora MySQL blue/green deployments. +The related designs remain authoritative for configuration, runtime status, +cluster synchronization, monitor scheduling, FSM transitions, and routing +behavior. -## 1. Purpose +The test environment drives the complete Aurora BGD lifecycle without live AWS +infrastructure and retains regression coverage for ordinary Aurora monitoring +and RDS Multi-AZ BGD. -Provide deterministic coverage for Aurora MySQL blue/green deployments without -requiring live AWS infrastructure. The test environment must be able to drive -the complete BGD observation sequence, publish target Aurora membership, -simulate member renaming, inject failures, observe monitor probes, and verify -ProxySQL runtime and routing effects. +## 2. Test Architecture -The design must preserve both existing regression suites: +The `cluster_sim_aurora` group contains two complementary test styles: -- the ordinary Aurora cluster simulator continues validating Aurora role, - membership, lag, failover, and autopurge behavior; -- the RDS Multi-AZ BGD suite continues validating its existing instance-based - mapping and reader-cleanup FSM. +- `test_cluster_sim_aurora-t` runs JSON-defined ordinary Aurora scenarios + through the standalone `cluster_simulator` controller; +- interactive TAP binaries drive Aurora BGD phases through the + `BGD_Simulator` helper. -## 2. Architecture Decision +RDS Multi-AZ BGD TAP binaries remain in `cluster_sim_rds_bgd`. A simulator +group is an execution and CI bucket; simulator capabilities come from the +ProxySQL build, SQLite-server handlers, control tables, and TAP helpers. +Only one test binary controls a given ProxySQL instance at a time, and every +binary owns setup and cleanup for its simulator state. -Aurora BGD FSM tests extend the existing interactive RDS BGD simulator used by -`cluster_sim_rds_bgd-g1`. They do not add BGD sequencing to the legacy JSON -Aurora simulator. +The build contract is: -This is the preferred design because the RDS BGD simulator already provides: +- `TEST_AURORA` enables ordinary Aurora and Aurora BGD simulation; +- `TEST_RDS_BGD` enables RDS Multi-AZ BGD simulation; +- AWS BGD topology services are available under either flag; +- Aurora replica services are available under `TEST_AURORA`; +- read-only simulation is available to Aurora BGD, RDS BGD, and focused + read-only simulator builds; +- no separate BGD simulator flag is used. -- per-backend `mysql.rds_topology` responses; -- empty, absent, and error topology modes; -- read-only controls; -- ordered probe logging; -- fixed AWS-style hostname-to-loopback mappings; -- TAP-driven, phase-by-phase transitions; -- the build and CI integration required for BGD tests. +Aurora monitor queries in simulator builds are the same production queries used +against AWS. The SQLite server identifies the simulated backend from the local +IP and port on which it accepted the monitor connection, then returns the state +assigned to that address. -Aurora adds a target-cluster membership service and Aurora-specific TAP helpers -on top of that base. Shared topology, transaction, endpoint, TLS, read-only, -probe-wait, and cleanup behavior remains common. +## 3. AWS BGD Topology Service -### Alternatives not selected +Aurora BGD and RDS Multi-AZ BGD use the same simulated +`mysql.rds_topology` service. -1. Extending the JSON Aurora simulator with a long sequence of BGD states would - overload a batch-oriented two-state model and make late entry, retry, - rollback, and concurrent timing difficult to control. -2. Creating a third standalone simulator group would duplicate the existing RDS - BGD topology service, host maps, helpers, build flavor, and CI wiring. -3. Using only unit tests would not exercise monitor threads, DNS pins, - connection-pool drains, runtime status publication, or config reloads. +### 3.1 `AWS_BGD_CONTROL` -## 3. Simulator Service Contract +One row controls topology behavior for one backend: -The `TEST_RDS_BGD` SQLite3-server flavor remains the executable backend for the -interactive suite. It is extended to recognize the Aurora membership query in -addition to the existing topology and read-only queries. +```text +backend_ip TEXT NOT NULL +backend_port INTEGER NOT NULL +topology_present INTEGER NOT NULL DEFAULT 0 +error_code INTEGER NOT NULL DEFAULT 0 +error_msg TEXT NOT NULL DEFAULT '' +PRIMARY KEY (backend_ip, backend_port) +``` -All simulated responses are keyed by the backend address and port on which the -SQLite3 server accepted the monitor connection. This preserves isolation when -multiple deployments or multiple target endpoints are active concurrently. +`topology_present` is restricted to zero or one. A missing control row and a +row with `topology_present=0` both represent an absent topology table. A +nonzero `error_code` returns the configured MySQL error. -### 3.1 Existing topology service +### 3.2 `AWS_BGD_TOPOLOGY` -The existing service remains authoritative for: +Successful topology results are stored as: ```text -mysql.rds_topology table presence -SOURCE and TARGET rows -deployment fingerprint fields -AWS status strings -configured topology errors -ordered table-check and metadata probe logs +backend_ip TEXT NOT NULL +backend_port INTEGER NOT NULL +row_order INTEGER NOT NULL +id TEXT NOT NULL +endpoint TEXT NOT NULL +topology_port INTEGER NOT NULL +role TEXT NOT NULL +status TEXT NOT NULL +PRIMARY KEY (backend_ip, backend_port, row_order) ``` -Aurora TARGET rows contain a cluster endpoint. Multi-AZ TARGET rows continue to -contain an instance endpoint. The simulator must not infer deployment type from -test configuration; production detection consumes the endpoint shape and the -membership-query result. +`row_order` makes the result deterministic. Aurora TARGET rows contain a +cluster endpoint; RDS Multi-AZ TARGET rows contain an instance endpoint. The +production monitor identifies the deployment type from the AWS metadata and +membership result rather than from simulator configuration. + +### 3.3 `AWS_BGD_PROBE_LOG` -### 3.2 Aurora membership service +Every topology table check and metadata query records: -The simulator adds per-backend control and row storage for -`INFORMATION_SCHEMA.REPLICA_HOST_STATUS`. +```text +sequence_id INTEGER PRIMARY KEY AUTOINCREMENT +backend_ip TEXT NOT NULL +backend_port INTEGER NOT NULL +probe_kind TEXT NOT NULL # table_check or metadata +encrypted INTEGER NOT NULL +``` -Each membership row contains at least: +`encrypted` is restricted to zero or one. The monotonic sequence supports +deterministic waits and ordering assertions. + +## 4. AWS Aurora Replica Service + +### 4.1 Membership sets + +`REPLICA_HOST_STATUS` stores each Aurora membership snapshot once: ```text -row_order -SERVER_ID -SESSION_ID -LAST_UPDATE_TIMESTAMP -IS_CURRENT -CPU -REPLICA_LAG_IN_MILLISECONDS +REPLICA_SET_ID TEXT NOT NULL +SERVER_ID VARCHAR NOT NULL +SESSION_ID VARCHAR NOT NULL +CPU REAL NOT NULL +LAST_UPDATE_TIMESTAMP VARCHAR NOT NULL +REPLICA_LAG_IN_MILLISECONDS REAL NOT NULL +IS_CURRENT INTEGER NOT NULL DEFAULT 1 +PRIMARY KEY (REPLICA_SET_ID, SERVER_ID) ``` -`SESSION_ID='MASTER_SESSION_ID'` identifies the writer. Other current rows are -readers. `row_order` makes response ordering deterministic while allowing tests -to prove that ProxySQL does not rely on writer-first or lexical membership -ordering. +`IS_CURRENT` is restricted to zero or one. It models the AWS column used by the +Aurora BGD membership query to exclude stale or decommissioned members. +Ordinary Aurora payloads default it to one. + +`REPLICA_SET_ID` is simulator-only and is never returned to ProxySQL. It groups +one membership snapshot independently of `DOMAIN_NAME`. The simulated table +does not contain `DOMAIN_NAME`; domain configuration remains in +`mysql_aws_aurora_hostgroups` and in the ordinary Aurora JSON input, where it +is used to construct member hostnames. -The control state supports: +Blue and green members share the same RDS domain suffix, so a BGD scenario uses +distinct replica-set identifiers for its blue and green snapshots. Multiple +backend addresses can map to the same set. A cluster with N members therefore +stores N membership rows plus lightweight backend mappings, rather than one +copy of the N rows per backend. -- a successful complete membership result; -- a successful writer-only result; -- an intentionally incomplete result; -- an empty result; -- table absence/error 1146; -- an arbitrary MySQL error code and message. +### 4.2 `AWS_AURORA_REPLICA_CONTROL` -Membership updates are atomic per supplied set of backends. A test must not -expose a partially rewritten snapshot unless it explicitly selects the -incomplete-result mode. +One row maps a backend address to its replica set and controls the query +response: -### 3.3 Probe log +```text +backend_ip TEXT NOT NULL +backend_port INTEGER NOT NULL +replica_set_id TEXT NOT NULL +replica_table_present INTEGER NOT NULL DEFAULT 0 +error_code INTEGER NOT NULL DEFAULT 0 +error_msg TEXT NOT NULL DEFAULT '' +PRIMARY KEY (backend_ip, backend_port) +``` + +`replica_table_present` is restricted to zero or one. Response behavior is: + +- no control row or `replica_table_present=0`: MySQL error 1146; +- nonzero `error_code`: the configured MySQL error; +- present table, no error, and matching rows: return that set; +- present table, no error, and no matching rows: return a successful empty + result. + +Writer-only and intentionally incomplete snapshots are represented directly by +the rows stored for the selected set. No response-mode column is required. +Topology and replica errors remain independent because their controls are in +separate tables. -The existing BGD probe log adds an Aurora-membership probe kind. Every topology -table check, topology metadata query, and target membership query records: +### 4.3 `AWS_AURORA_REPLICA_PROBE_LOG` + +Every intercepted production `REPLICA_HOST_STATUS` query records: ```text -monotonic sequence -accepted backend IP -accepted backend port -probe kind -TLS state +sequence_id INTEGER PRIMARY KEY AUTOINCREMENT +backend_ip TEXT NOT NULL +backend_port INTEGER NOT NULL +replica_set_id TEXT NULL +encrypted INTEGER NOT NULL ``` -Tests use the log to verify probe destination, ordering, cadence class, TLS, -probe-pin retention, and return to canonical probing. They must not use fixed -sleeps when an observable probe or runtime state can serve as the wait -condition. +The accepted address is logged even when no control mapping exists or the query +returns an error. `replica_set_id` is NULL when the backend has no mapping. + +### 4.4 Query handling + +Ordinary Aurora and Aurora BGD issue their production-shaped +`REPLICA_HOST_STATUS` queries under `TEST_AURORA`. For a recognized monitor +query, the SQLite handler: + +1. reads the accepted backend IP and port; +2. records the probe; +3. reads the matching `AWS_AURORA_REPLICA_CONTROL` row; +4. returns the configured error or rewrites the query to select the mapped + replica set; +5. preserves the production query's result columns, filtering, and ordering. -### 3.4 Error isolation +The ordinary Aurora query retains its timestamp, lag, and writer filtering. +The Aurora BGD query retains `IS_CURRENT` and `SERVER_ID` ordering. Other SQL +against the simulator tables executes normally so controllers can publish +state. -Topology and membership errors are independent. A scenario can publish valid -topology with failed membership, or valid membership with failed topology. -Clearing one error source must not silently clear the other. +## 5. State Publication -Simulator cleanup removes topology, membership, read-only, and probe-log state -in one operation so each TAP binary begins from a known baseline. +### 5.1 Ordinary Aurora controller -## 4. TAP Helper Model +Ordinary Aurora JSON files retain their current schema. For each Aurora cluster, +the standalone `cluster_simulator` controller: -Aurora-specific helpers extend, rather than fork, the RDS BGD helper model. -They provide test-facing representations for: +1. uses the JSON `DOMAIN_NAME` value as the replica-set identifier; +2. constructs each member hostname from `SERVER_ID` and `DOMAIN_NAME`; +3. reads `CLUSTER_SIM_HOST_FILE` to resolve those fixed simulator hostnames to + loopback IPs; +4. writes the membership rows with a current timestamp and `IS_CURRENT=1`; +5. maps every backend that can serve the cluster membership to the same set in + `AWS_AURORA_REPLICA_CONTROL`; +6. commits the membership replacement and complete backend mapping atomically. -- a blue Aurora cluster and its canonical instance endpoints; -- a target cluster endpoint; -- one target writer and zero or more target readers; -- stable member session identities; -- pre-rename green `SERVER_ID` values; -- post-rename canonical `SERVER_ID` values; -- explicit and automatic Aurora hostgroup configuration; -- expected runtime `bgd_status` values. +The group environment already supplies `CLUSTER_SIM_HOST_FILE`; test +infrastructure uses it to populate the ProxySQL container's host map, and the +controller uses the same file as the authoritative hostname-to-IP mapping. A +configured or discovered member missing from that map is a test-setup failure. -The helper API exposes operations equivalent to: +### 5.2 Aurora BGD controller + +Interactive Aurora BGD TAP tests publish state through `BGD_Simulator`. Each +membership publication supplies: + +- one stable replica-set identifier; +- one membership snapshot; +- every cluster endpoint or member backend that must return that snapshot. + +The helper replaces the set rows and complete backend mapping in one +transaction. Blue and green use separate identifiers. Tests do not store a +blue/green flag and do not infer environment identity from hostnames. + +The same helper publishes topology rows, topology errors, replica errors, +read-only state, and probe-log checkpoints. Non-1146 errors change only control +state, so clearing them exposes the retained data. An absent topology table +clears topology rows for the addressed backend, matching the topology service's +backend-scoped storage. An absent replica table preserves its shared membership +set because other backend mappings may still serve that set. + +### 5.3 Helper boundaries + +`BGD_Simulator` owns cross-service mechanics: SQL transactions, topology and +replica controls, read-only controls, probe-log reads and waits, endpoint +predicates, and cleanup. + +Deployment models remain engine-specific. `RDS_BGD_Cluster` and related RDS +types describe RDS instance topology. Aurora types describe cluster endpoints, +members, replica sets, rename identity, and expected runtime placement. + +### 5.4 Cleanup + +Set-scoped operations remove only the supplied replica set and backend +mappings. Full scenario cleanup clears: ```text -publish topology status -publish target membership snapshot -publish renamed membership snapshot -publish empty/absent topology -inject topology or membership error -record and wait for probes -query Aurora runtime row and bgd_status -query runtime server placement and status -query connection-pool state -open backend traffic and identify accepted simulator IP -clean up ProxySQL and simulator state +READONLY_STATUS +AWS_BGD_CONTROL +AWS_BGD_TOPOLOGY +AWS_BGD_PROBE_LOG +AWS_AURORA_REPLICA_CONTROL +AWS_AURORA_REPLICA_PROBE_LOG +REPLICA_HOST_STATUS ``` -Helper methods perform control operations only. Individual test scenarios own -the sequence of AWS observations and all expected ProxySQL outcomes. +Each TAP binary begins and ends with full cleanup. Sequential group execution +is expected, but correctness does not depend on a later test overwriting stale +state. -## 5. Endpoint and DNS Model +## 6. Endpoint Model -The `cluster_sim_rds_bgd-g1` fixed host map is extended with Aurora cluster -endpoints and member endpoints. Each hostname maps to a distinct loopback -address accepted by the same SQLite3 server. +`cluster_sim_aurora/add-hosts` contains fixed aliases for: -The map includes, per test cluster: - -- canonical blue writer and reader instance names; +- canonical blue cluster members; - the green target cluster endpoint returned by `mysql.rds_topology`; -- green-suffixed target writer and reader instance names; -- canonical post-rename member names; -- a second target deployment for repeated-switchover coverage; -- additional clusters for concurrency coverage. +- green target writer and reader members; +- canonical post-switchover member names; +- repeated deployments and concurrent test clusters. + +Independently controlled backends use distinct loopback IPs. Green and +canonical aliases may share an IP where the scenario represents AWS resource +renaming. Response selection always uses the accepted IP and port, not the +requested hostname. + +The fixed host map does not model mutable Route 53 propagation. DNS-related +tests assert cached-IP pins, accepted backend addresses, connection-pool +behavior, and explicit pin removal. + +## 7. Test Coverage + +### 7.1 Simulator contracts -Green and canonical names used for identity-renaming tests may resolve to the -same target-member loopback IP where that models AWS's post-processing rename. -Canonical blue DNS remains static in the container; DNS-cache pin behavior is -verified through accepted backend addresses and explicit pin removal, not by -claiming that the simulator reproduces mutable Route 53 propagation. +Focused tests cover: -## 6. Test Layers +- table schemas, primary keys, defaults, and value constraints; +- topology and replica routing by accepted backend address; +- several backend addresses sharing one replica set; +- independent blue and green sets with overlapping `SERVER_ID` values; +- production-query result shape, filtering, and ordering; +- atomic membership and mapping replacement; +- successful complete, writer-only, incomplete, and empty results; +- missing tables and arbitrary MySQL errors; +- independence of topology and replica errors; +- TLS state and ordered probe logging; +- `CLUSTER_SIM_HOST_FILE` parsing and missing-host failures; +- set-scoped deletion and full cleanup. -### 6.1 Schema and unit coverage +### 7.2 Configuration, runtime, and cluster synchronization -Fast tests cover deterministic contracts that do not require monitor threads: +Tests owned by the configuration design cover: -- configuration and runtime table definitions and column order; +- configuration and runtime table schemas and column order; - paired-NULL and hostgroup-conflict validation; -- SQL NULL preservation in bind/extract helpers; -- config-file import and export; -- disk schema upgrade defaults; -- runtime `bgd_status` string mapping; -- configured-column cluster query and checksum projection; +- config-file, memory, runtime, and disk round trips; +- runtime-only `bgd_status` initialization and preservation; - exclusion of `bgd_status` from SAVE and cluster synchronization; -- Aurora simulator JSON parsing of the two optional green hostgroups. - -### 6.2 Admin and cluster-sync integration coverage - -Admin/TAP tests cover: - -- memory-to-runtime and runtime-to-memory round trips; -- runtime-to-disk and config-file round trips; -- new runtime rows starting at `NONE`; -- reload preservation of `bgd_status` and active worker state; -- removal and deactivation behavior; -- peer synchronization of both green hostgroups, including NULL; -- proof that peers retain independent node-local `bgd_status` values; -- unchanged RDS Multi-AZ BGD cluster synchronization. - -### 6.3 Interactive Aurora BGD coverage - -The interactive suite covers the observable behavior of the monitor/FSM spec: - -1. `AVAILABLE` discovers exactly one writer and every reader, resolves each - member, and establishes only the probe state required by the design. -2. Automatic mode with NULL green hostgroups maps all target members without - generating green hostgroups or `mysql_servers` rows. -3. Explicit mode uses configured green hostgroups as staging pools but still - treats `REPLICA_HOST_STATUS` as membership truth. -4. Initiated and in-progress observations retain the complete snapshot and - engage BGD/read-only-monitor protection. -5. The first post-processing observation pins and drains each writer/reader pair - once; repeated observations retry only incomplete member actions. -6. Member `SERVER_ID` rename preserves reader identity through stable - `SESSION_ID` values and does not change cached target IPs. -7. The first TARGET `SWITCHOVER_COMPLETED` removes all traffic and probe pins, - performs cleanup once, and publishes `bgd_status='SWITCHOVER_COMPLETED'`. -8. Repeated completed rows are no-ops; successful topology drain changes the - runtime status to `NONE` and rearms discovery. -9. A different deployment fingerprint can rearm from the terminal latch without - inheriting stale members, pins, probes, or completion flags. - -### 6.4 Resilience and edge coverage - -Scenarios also cover: - -- membership row ordering and zero-reader target clusters; -- incomplete membership retaining the last complete snapshot; -- topology and membership query failures in every phase; -- rollback from initiated, in-progress, and post-processing observations; +- synchronization of the configured green hostgroups, including NULL; +- independent node-local runtime status on cluster peers. + +### 7.3 Interactive Aurora BGD scenarios + +Interactive TAP tests cover: + +1. `AVAILABLE` discovery of the target writer and all current readers through + a complete target membership snapshot. +2. Green hostgroups configured and green hostgroups not configured, with all + Aurora hostgroup rows created by the test. +3. Independent topology, target-membership, and ordinary Aurora probes with the + cadence and on/off behavior defined by the monitor/FSM design, including + random blue-member topology probes and random green-member target probes. +4. `SWITCHOVER_INITIATED`, `SWITCHOVER_IN_PROGRESS`, and + `SWITCHOVER_IN_POST_PROCESSING` state retention and retry behavior. +5. Source-writer read-only observation and the corresponding writer-to-reader + placement. +6. Target member `SERVER_ID` rename with stable reader `SESSION_ID` identity + and retained cached IPs. +7. Idempotent per-member pinning, placement, and pool-drain actions. +8. The first TARGET-only `SWITCHOVER_COMPLETED`, immediate cleanup, terminal + latch, and runtime `bgd_status='SWITCHOVER_COMPLETED'`. +9. Repeated completion observations, successful topology drain to `NONE`, and + rearming for a different deployment fingerprint. + +### 7.4 Resilience and lifecycle scenarios + +Coverage also includes: + +- zero-reader targets and membership row ordering; +- incomplete snapshots retaining the last complete target set; +- topology, replica, and read-only query failures in each relevant phase; +- rollback from initiated, in-progress, and post-processing states; - late entry at initiated, in-progress, post-processing, and completed states; -- config refresh and hostgroup refresh during active phases; -- worker restart/respawn with preserved fingerprint, members, IPs, pins, and - terminal latch; -- disabling automatic discovery during an active automatic deployment; -- deleting or deactivating an Aurora row during a switchover; -- explicit green-pool cleanup with ONLINE, SHUNNED, OFFLINE_SOFT, and - OFFLINE_HARD members; -- TLS selection for topology and membership probes; -- multiple concurrent Aurora deployments with independent state; -- simultaneous Aurora BGD and RDS Multi-AZ BGD deployments. - -## 7. Regression Coverage - -The existing `test_cluster_sim_aurora-t` JSON payload suite remains responsible -for ordinary Aurora behavior. Its schema accepts the two new green hostgroup -fields as optional values but old payloads remain valid and preserve their -existing results. - -The complete existing `test_rds_bgd_*` suite remains unchanged in semantics. -Aurora membership support is additive to the shared simulator and must not -alter instance TARGET handling, reader shun/unshun policy, writer fallback, or -Multi-AZ completion/drain behavior. - -Regression runs must include: +- configuration and hostgroup refresh during an active deployment; +- worker restart or respawn with retained deployment state; +- deleting or deactivating an Aurora row during switchover; +- ONLINE, SHUNNED, OFFLINE_SOFT, and OFFLINE_HARD staging-pool members; +- TLS selection for topology and replica probes; +- repeated and concurrent Aurora deployments with isolated state. -```text -ordinary Aurora cluster simulator group -RDS Multi-AZ BGD simulator group -new Aurora BGD TAP binaries in cluster_sim_rds_bgd-g1 -configuration/unit tests -ProxySQL Cluster synchronization tests -``` +### 7.5 Regression suites + +The complete ordinary Aurora JSON suite runs through the production query path +and retains its role, failover, lag, autodiscovery, and autopurge expectations. + +The RDS Multi-AZ BGD TAP suite retains its instance-target, reader policy, +writer fallback, completion, and topology-drain expectations while using the +shared `BGD_Simulator` and `AWS_BGD_*` services. -## 8. Determinism and Timing +## 8. Determinism and Diagnostics -Tests synchronize on observable state: runtime `bgd_status`, probe sequence, -server placement, pool counters, and successful simulator control commits. -Timeouts are derived from configured monitor intervals with bounded slack. +Tests synchronize on observable state: probe sequence, runtime `bgd_status`, +server placement, pool counters, and committed simulator controls. Timeouts are +derived from configured monitor intervals with bounded slack. -Fixed sleeps are allowed only for negative assertions where no event can be -awaited directly, and must be shorter than the overall TAP timeout. Every wait -failure reports the last runtime row, relevant server/pool state, and probes -observed since the scenario checkpoint. +Fixed sleeps are permitted only for bounded negative assertions with no event +that can be awaited directly. A failed wait reports the latest runtime row, +relevant server and pool state, and probes observed since the scenario +checkpoint. -Each scenario uses unique hostgroups or performs complete cleanup. Concurrent -tests use disjoint endpoints and deployment fingerprints. +Scenarios use disjoint hostgroups, endpoints, replica-set identifiers, and +deployment fingerprints, or perform full cleanup before reuse. ## 9. Build and CI Contract -The existing `test_rds_bgd` build remains the focused local build. The combined -cluster-simulator build continues compiling both `TEST_AURORA` and -`TEST_RDS_BGD` support. +`make testaurora` produces the focused Aurora simulator build, including the +Aurora replica, shared BGD topology, and read-only services. `make test_rds_bgd` +produces the focused RDS BGD build. The combined `testall` build enables both +families. -New interactive TAP binaries register in `cluster_sim_rds_bgd-g1`, which keeps -the existing fixed-host injection, SQLite3-server startup, no-backend-infra -model, CI matrix discovery, and log collection. No new workflow or simulator -group is required. +Ordinary Aurora and Aurora BGD binaries register in +`cluster_sim_aurora-g1`. RDS Multi-AZ BGD binaries register in +`cluster_sim_rds_bgd-g1`. Central simulator CI discovers groups and binaries +from `groups.json`, so no additional workflow, simulator group family, or CI +infrastructure is required. ## 10. Non-Goals -The simulator does not claim to validate: +The simulator does not validate: - AWS control-plane APIs or real AWS timing; - mutable Route 53 propagation; - application-level latency or packet loss; - cross-process persistence of transient FSM state; -- behavior of Aurora versions that violate the AWS contract in the monitor/FSM - specification. +- Aurora versions that violate the AWS contracts established by the live-AWS + analysis. -Live-AWS evidence remains a separate validation layer for assumptions about -status ordering, member rename, writability, and DNS completion. +Live-AWS evidence remains the validation layer for status ordering, member +rename, writability, and DNS completion. ## 11. Acceptance Criteria -The simulator/testing design is satisfied when: - -1. Tests can independently control topology and target membership per backend. -2. The full Aurora BGD FSM can be driven without live AWS infrastructure. -3. Runtime status, routing, pool, DNS-pin, probe, reload, and cluster-sync - contracts have deterministic assertions. -4. Automatic and explicit modes both cover writer and all-reader membership. -5. Rename, rollback, late-entry, error, refresh, repeated, and concurrent paths - are covered. -6. Existing ordinary Aurora and RDS Multi-AZ BGD suites retain their semantics. -7. The suite runs through existing cluster-simulator build and CI plumbing with - no new infrastructure group. +The design is satisfied when: + +1. Ordinary Aurora and Aurora BGD use production monitor queries and the same + backend-address-to-replica-set service. +2. Blue and green membership snapshots are stored once and returned from every + mapped backend. +3. The complete Aurora BGD FSM and its failure paths run deterministically + without live AWS infrastructure. +4. Runtime status, routing, pools, cached IPs, probes, refresh, and cluster-sync + contracts have observable assertions. +5. Ordinary Aurora and RDS Multi-AZ BGD regressions retain their semantics. +6. The suite uses the existing Aurora and RDS simulator CI groups without a + separate feature flag or simulator family. From 360e8eb76b8d62bce682285ea98d558c73906ecb Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 12 Aug 2026 15:37:00 +0000 Subject: [PATCH 04/34] docs: approve Aurora BGD configuration and FSM designs --- ...nfiguration-runtime-cluster-sync-design.md | 102 +++++++++--------- ...026-07-31-aurora-bgd-monitor-fsm-design.md | 4 +- 2 files changed, 50 insertions(+), 56 deletions(-) diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md index d2c4ac3c7f..c2ba431504 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md @@ -2,9 +2,9 @@ **Date:** 2026-07-31 -**Branch:** `plan/aurora-bgd` +**Branch:** `spec/aws-aurora-bgd` -**Status:** Design draft; configuration decisions locked +**Status:** Design approved **Scope:** Configuration and runtime integration of Aurora MySQL blue/green deployment handling with the existing `mysql_aws_aurora_hostgroups` subsystem. @@ -35,10 +35,9 @@ green_writer_hostgroup INT DEFAULT NULL green_reader_hostgroup INT DEFAULT NULL ``` -The columns provide explicit staging hostgroups when configured. When both are -NULL, the existing global variable -`mysql-aws_blue_green_deployment_auto_discovery` controls automatic Aurora BGD -discovery. +The columns provide staging hostgroups when configured. When both are NULL, the +existing global variable `mysql-aws_blue_green_deployment_auto_discovery` +controls whether the worker may start BGD discovery for the row. No separate Aurora BGD enable column is added. @@ -96,28 +95,24 @@ state is meaningful and must be preserved by every load, save, export, import, and cluster-synchronization path. The runtime table mirrors these configured fields and adds only `bgd_status`. -This design does not add an `auto_generated` column because the existing Aurora -row is the owner of both normal Aurora monitoring and BGD monitoring. It also -does not add a runtime `mode` column; effective mode remains derived from the -configured green hostgroups and the global auto-discovery variable. +Every Aurora row is user-created. There is no BGD mode or generated-row concept +in this table, so no `mode` or `auto_generated` column is present. -## 3. Configuration Modes +## 3. Green Hostgroup Configuration and BGD Admission -The effective mode for an active Aurora row is determined as follows: +An active Aurora row behaves as follows: -| Green hostgroup columns | `aws_blue_green_deployment_auto_discovery` | Effective behavior | +| Green hostgroup columns | `aws_blue_green_deployment_auto_discovery` | Behavior | |---|---:|---| -| Both non-NULL | `0` or `1` | Explicit Aurora BGD monitoring | -| Both NULL | `1` | Automatic Aurora BGD monitoring | +| Both non-NULL | `0` or `1` | BGD discovery is admitted | +| Both NULL | `1` | BGD discovery is admitted | | Both NULL | `0` | Aurora monitoring without BGD discovery | | Exactly one NULL | `0` or `1` | Invalid configuration | | Any values with `active=0` | `0` or `1` | Neither normal Aurora nor BGD monitoring is active | -### 3.1 Explicit mode +### 3.1 Green hostgroups configured -A row is in explicit mode when both green hostgroups are non-NULL. - -In explicit mode: +When both green hostgroups are non-NULL: - BGD monitoring is enabled independently of the global auto-discovery value. - The target cluster writer is associated with `green_writer_hostgroup`. @@ -128,12 +123,10 @@ In explicit mode: - The green hostgroups provide optional user-visible staging/routing pools in addition to the internal BGD member map. -### 3.2 Automatic mode - -A row is in automatic mode when both green hostgroups are NULL and -`mysql-aws_blue_green_deployment_auto_discovery=true` at runtime. +### 3.2 Green hostgroups not configured -In automatic mode: +When both green hostgroups are NULL and +`mysql-aws_blue_green_deployment_auto_discovery=true` at runtime: - The existing Aurora monitor detects BGD topology for the row. - The monitor discovers the target writer and all target readers through @@ -143,8 +136,8 @@ In automatic mode: - Post-processing pinning covers the writer and every reader despite the green hostgroup columns being NULL. -Automatic mode therefore does not use the writer-only fallback or reader -shun/unshun policy from the Multi-AZ instance implementation. +This path does not use the writer-only fallback or reader shun/unshun policy +from the Multi-AZ instance implementation. ### 3.3 BGD disabled for the row @@ -153,11 +146,10 @@ When both green hostgroups are NULL and existing Aurora role and lag monitoring but does not start discovery of a new BGD deployment. -Disabling global auto-discovery while an automatically discovered switchover is -already active must not abandon that switchover. The worker completes pin -cleanup and enters topology-drain wait before disabling discovery for the row. -The variable gates the start of new automatic BGD state machines, not safe -completion of one already in progress. +Disabling global auto-discovery after a switchover has started must not abandon +that switchover. The worker completes pin cleanup and enters the terminal latch +before disabling discovery for the row. The variable gates the start of a new +BGD state machine, not safe completion of one already in progress. ## 4. Validation Rules @@ -196,7 +188,8 @@ green_reader_hostgroup = -1 when SQL NULL The existing Aurora monitor worker remains one worker per active writer hostgroup. The worker owns the BGD FSM and publishes each transition to the -runtime row's `bgd_status`. Its effective BGD mode is derived from: +runtime row's `bgd_status`. BGD admission and configured staging references are +determined from: ```text AWS_Aurora_Info.active @@ -237,13 +230,13 @@ fields; paths that write back to configuration explicitly project away Changing the green hostgroups during an active switchover must not lose cached member identities, applied DNS pins, or completion-latch state. The worker must -apply the refreshed explicit staging configuration without restarting the BGD -operation from `NONE`. +apply the refreshed staging configuration without restarting the BGD operation +from `NONE`. `LOAD MYSQL VARIABLES TO RUNTIME` makes a change to `aws_blue_green_deployment_auto_discovery` visible to Aurora monitor workers. -The variable controls admission of new automatic BGD operations as described in -Section 3.3. +The variable controls admission of new BGD operations for rows without green +hostgroups, as described in Section 3.3. ## 7. Runtime Table Behavior @@ -271,11 +264,11 @@ topology result then changes it to `NONE`. Query or connection errors while latched do not reset the status or repeat cleanup. An Aurora row that is not handling a BGD deployment reports `NONE`, including -ordinary Aurora monitoring when automatic discovery is disabled. +ordinary Aurora monitoring when BGD discovery is not admitted. Examples: -Explicit mode: +Green hostgroups configured: ```sql INSERT INTO mysql_aws_aurora_hostgroups ( @@ -287,7 +280,7 @@ INSERT INTO mysql_aws_aurora_hostgroups ( ) VALUES (10, 20, 11, 21, '.cluster-example.eu-north-1.rds.amazonaws.com'); ``` -Automatic mode: +Green hostgroups not configured: ```sql INSERT INTO mysql_aws_aurora_hostgroups ( @@ -302,8 +295,8 @@ SET mysql-aws_blue_green_deployment_auto_discovery = 'true'; ``` The BGD FSM state is operational monitor state, not user configuration. -`bgd_status` provides its runtime observability without changing the -explicit/automatic mode contract defined here. +`bgd_status` provides its runtime observability without introducing a BGD mode +into the configuration contract. ## 8. Persistence and Synchronization @@ -349,7 +342,7 @@ green_reader_hostgroup = NULL After upgrade, existing active Aurora rows follow the global variable: -- with auto-discovery enabled, they are eligible for automatic BGD discovery; +- with auto-discovery enabled, they are eligible for BGD discovery; - with auto-discovery disabled, they retain existing Aurora monitoring only. This is backward-compatible at the table-data level because no green @@ -365,10 +358,10 @@ This configuration design does not: - add an Aurora-specific configuration table; - create Aurora rows in `mysql_aws_rds_bgd_hostgroups`; -- require explicit green `mysql_servers` rows in automatic mode; +- require green `mysql_servers` rows when green hostgroups are not configured; - add a second per-cluster monitor worker; - add a separate `bgd_enabled` column; -- add a runtime `mode` or `auto_generated` column; +- add a `mode` or `auto_generated` column; - persist transient BGD FSM state to disk; - alter the existing RDS Multi-AZ BGD configuration contract. @@ -378,9 +371,11 @@ This configuration design does not: 2. Both green hostgroups non-NULL load successfully. 3. Mixed NULL/non-NULL values are rejected. 4. Duplicate or overlapping blue/green hostgroups are rejected. -5. Explicit mode operates with global auto-discovery disabled. -6. Automatic mode starts only when global auto-discovery is enabled. -7. Disabling auto-discovery does not abort an active automatic switchover. +5. Configured green hostgroups admit BGD discovery with global auto-discovery + disabled. +6. A row without green hostgroups starts BGD discovery only when global + auto-discovery is enabled. +7. Disabling auto-discovery does not abort an active switchover. 8. NULL values survive memory-to-runtime, runtime-to-memory, disk, config-file, and cluster synchronization round trips. 9. Online upgrade preserves existing rows and initializes both new fields to @@ -402,16 +397,15 @@ This configuration design does not: The configuration/runtime integration is complete when: -1. Aurora BGD explicit mode is configured solely by the two green hostgroup - columns on `mysql_aws_aurora_hostgroups`. -2. Aurora BGD automatic mode is controlled by the existing - `aws_blue_green_deployment_auto_discovery` variable when both columns are - NULL. +1. Every Aurora configuration row is user-created, and its two green hostgroup + columns are either both configured or both NULL. +2. Configured green hostgroups admit BGD discovery independently of the global + variable; when both are NULL, the existing + `aws_blue_green_deployment_auto_discovery` variable controls admission. 3. All load, save, disk, config-file, upgrade, and cluster-sync paths preserve the fields and their NULL values. 4. Exactly one Aurora monitor worker owns normal Aurora and BGD handling for a writer hostgroup. -5. No Aurora automatic-discovery row is generated in - `mysql_aws_rds_bgd_hostgroups`. +5. No Aurora row is generated in `mysql_aws_rds_bgd_hostgroups`. 6. `runtime_mysql_aws_aurora_hostgroups.bgd_status` exposes the local Aurora BGD FSM state without being saved, exported, or cluster-synchronized. diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md index 20549dfa5c..1977ff1da8 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md @@ -2,9 +2,9 @@ **Date:** 2026-07-31 -**Branch:** `plan/aurora-bgd` +**Branch:** `spec/aws-aurora-bgd` -**Status:** Design approved; pending written-spec review +**Status:** Design approved **Scope:** Aurora MySQL blue/green deployments exposed through `mysql.rds_topology` and `INFORMATION_SCHEMA.REPLICA_HOST_STATUS`. From f6a83fe6001820fc49a6404301a2d20a5abd0910 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 12 Aug 2026 19:50:20 +0000 Subject: [PATCH 05/34] docs: decouple Aurora BGD spec from local evidence path --- .../2026-07-31-aurora-bgd-monitor-fsm-design.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md index 1977ff1da8..26df126da7 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md @@ -9,9 +9,8 @@ **Scope:** Aurora MySQL blue/green deployments exposed through `mysql.rds_topology` and `INFORMATION_SCHEMA.REPLICA_HOST_STATUS`. -**Evidence basis:** The Aurora switchover observed on 2026-07-30 and recorded in -`aurora-bgd-switchover-analysis/RESULTS-20260730T073724Z.md`. Section 3 separates -observed behavior from design policy. +**Evidence basis:** A timestamped live-Aurora switchover observation captured on +2026-07-30. Section 3 separates observed behavior from design policy. **Related designs:** From 05f9c43b21b9649f7ee292d115a6d3c7a0e87fd0 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Wed, 12 Aug 2026 20:28:09 +0000 Subject: [PATCH 06/34] docs: address Aurora BGD review feedback --- ...ra-bgd-cluster-simulator-testing-design.md | 52 ++++++++++-- ...nfiguration-runtime-cluster-sync-design.md | 83 +++++++++++++------ ...026-07-31-aurora-bgd-monitor-fsm-design.md | 58 +++++++++++-- 3 files changed, 156 insertions(+), 37 deletions(-) diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md index 6796a97945..32e8864b83 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md @@ -38,6 +38,12 @@ ProxySQL build, SQLite-server handlers, control tables, and TAP helpers. Only one test binary controls a given ProxySQL instance at a time, and every binary owns setup and cleanup for its simulator state. +The group runner enforces that ownership by starting one registered binary, +waiting for it to exit, and only then starting the next binary in sorted order. +Aurora BGD registration checks must retain that serial execution contract. If a +future runner executes binaries concurrently, each binary must receive an +isolated ProxySQL instance and simulator database. + The build contract is: - `TEST_AURORA` enables ordinary Aurora and Aurora BGD simulation; @@ -66,14 +72,28 @@ One row controls topology behavior for one backend: backend_ip TEXT NOT NULL backend_port INTEGER NOT NULL topology_present INTEGER NOT NULL DEFAULT 0 + CHECK (topology_present IN (0, 1)) error_code INTEGER NOT NULL DEFAULT 0 error_msg TEXT NOT NULL DEFAULT '' PRIMARY KEY (backend_ip, backend_port) ``` `topology_present` is restricted to zero or one. A missing control row and a -row with `topology_present=0` both represent an absent topology table. A -nonzero `error_code` returns the configured MySQL error. +row with `topology_present=0` both represent an absent topology table. Query +precedence is evaluated independently: + +| Query | Control state | Result | +|---|---|---| +| Table check | No row or `topology_present=0` | Successful empty result; `error_code` is ignored. | +| Table check | `topology_present=1` | Successful one-row result; `error_code` is ignored. | +| Metadata | No row | MySQL error 1146. | +| Metadata | `error_code!=0` | Configured MySQL error, including the defensive `topology_present=0` combination. | +| Metadata | `error_code=0`, `topology_present=0` | MySQL error 1146. | +| Metadata | `error_code=0`, `topology_present=1` | Ordered rows, including a successful empty result. | + +The helper normally publishes only consistent combinations: error 1146 marks +the table absent, and other errors retain a present table. The defensive matrix +keeps direct control-table writes deterministic. ### 3.2 `AWS_BGD_TOPOLOGY` @@ -106,6 +126,7 @@ backend_ip TEXT NOT NULL backend_port INTEGER NOT NULL probe_kind TEXT NOT NULL # table_check or metadata encrypted INTEGER NOT NULL + CHECK (encrypted IN (0, 1)) ``` `encrypted` is restricted to zero or one. The monotonic sequence supports @@ -125,6 +146,7 @@ CPU REAL NOT NULL LAST_UPDATE_TIMESTAMP VARCHAR NOT NULL REPLICA_LAG_IN_MILLISECONDS REAL NOT NULL IS_CURRENT INTEGER NOT NULL DEFAULT 1 + CHECK (IS_CURRENT IN (0, 1)) PRIMARY KEY (REPLICA_SET_ID, SERVER_ID) ``` @@ -154,15 +176,18 @@ backend_ip TEXT NOT NULL backend_port INTEGER NOT NULL replica_set_id TEXT NOT NULL replica_table_present INTEGER NOT NULL DEFAULT 0 + CHECK (replica_table_present IN (0, 1)) error_code INTEGER NOT NULL DEFAULT 0 error_msg TEXT NOT NULL DEFAULT '' PRIMARY KEY (backend_ip, backend_port) ``` -`replica_table_present` is restricted to zero or one. Response behavior is: +`replica_table_present` is restricted to zero or one. Absence takes precedence +over a configured error. Response behavior is: -- no control row or `replica_table_present=0`: MySQL error 1146; -- nonzero `error_code`: the configured MySQL error; +- no control row or `replica_table_present=0`: MySQL error 1146, regardless of + `error_code`; +- present table and nonzero `error_code`: the configured MySQL error; - present table, no error, and matching rows: return that set; - present table, no error, and no matching rows: return a successful empty result. @@ -182,6 +207,7 @@ backend_ip TEXT NOT NULL backend_port INTEGER NOT NULL replica_set_id TEXT NULL encrypted INTEGER NOT NULL + CHECK (encrypted IN (0, 1)) ``` The accepted address is logged even when no control mapping exists or the query @@ -308,6 +334,8 @@ Focused tests cover: - atomic membership and mapping replacement; - successful complete, writer-only, incomplete, and empty results; - missing tables and arbitrary MySQL errors; +- table-present/error precedence matrices for topology table-check, topology + metadata, and replica membership queries; - independence of topology and replica errors; - TLS state and ordered probe logging; - `CLUSTER_SIM_HOST_FILE` parsing and missing-host failures; @@ -358,7 +386,7 @@ Coverage also includes: - rollback from initiated, in-progress, and post-processing states; - late entry at initiated, in-progress, post-processing, and completed states; - configuration and hostgroup refresh during an active deployment; -- worker restart or respawn with retained deployment state; +- in-place worker configuration refresh with retained deployment state; - deleting or deactivating an Aurora row during switchover; - ONLINE, SHUNNED, OFFLINE_SOFT, and OFFLINE_HARD staging-pool members; - TLS selection for topology and replica probes; @@ -384,6 +412,13 @@ that can be awaited directly. A failed wait reports the latest runtime row, relevant server and pool state, and probes observed since the scenario checkpoint. +Random probe-selection tests do not assert one exact random order. They assert +that every observed destination belongs to the scenario's eligible set and +that fallback reaches another eligible member within a bounded number of +attempts. Failure diagnostics include the complete probe sequence since the +scenario checkpoint, so production randomness cannot make the expected result +order-dependent. + Scenarios use disjoint hostgroups, endpoints, replica-set identifiers, and deployment fingerprints, or perform full cleanup before reuse. @@ -400,6 +435,11 @@ Ordinary Aurora and Aurora BGD binaries register in from `groups.json`, so no additional workflow, simulator group family, or CI infrastructure is required. +CI must continue to invoke the group through the serial group runner described +in Section 2. Registration coverage verifies that every Aurora BGD binary is in +that group and is not scheduled simultaneously against the same ProxySQL +instance. + ## 10. Non-Goals The simulator does not validate: diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md index c2ba431504..5ef200ec6b 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md @@ -102,7 +102,7 @@ in this table, so no `mode` or `auto_generated` column is present. An active Aurora row behaves as follows: -| Green hostgroup columns | `aws_blue_green_deployment_auto_discovery` | Behavior | +| Green hostgroup columns | `mysql-aws_blue_green_deployment_auto_discovery` | Behavior | |---|---:|---| | Both non-NULL | `0` or `1` | BGD discovery is admitted | | Both NULL | `1` | BGD discovery is admitted | @@ -110,6 +110,13 @@ An active Aurora row behaves as follows: | Exactly one NULL | `0` or `1` | Invalid configuration | | Any values with `active=0` | `0` or `1` | Neither normal Aurora nor BGD monitoring is active | +Changing `active` from `1` to `0` uses the worker-removal cleanup contract in +the monitor/FSM specification before the worker exits: remove every applied +pin, drain and purge affected production-hostname pools, restore safe writer +placement, release suspended monitoring, clear any terminal latch and retained +fingerprint, and publish `bgd_status=NONE`. The configuration and +`mysql_servers` rows remain user-owned and are not deleted by this cleanup. + ### 3.1 Green hostgroups configured When both green hostgroups are non-NULL: @@ -170,11 +177,18 @@ For each `mysql_aws_aurora_hostgroups` row: green role in another active Aurora row. 4. Existing writer-hostgroup primary-key and reader-hostgroup uniqueness rules remain in effect. -5. Invalid rows must be rejected or excluded from runtime loading with a clear - admin error identifying the writer hostgroup and conflicting fields. - -Validation must occur before publishing the new Aurora monitor resultset so a -bad row cannot partially reconfigure a running monitor worker. +5. Invalid rows use per-row isolation: each invalid row is excluded from runtime + loading, while the valid rows from the same LOAD remain eligible to publish. + The admin error identifies the rejected writer hostgroup, the conflicting + fields and values, and the other writer hostgroup involved in a cross-row + conflict. + +Validation of the complete candidate result set must finish before publishing +the filtered valid result set. Valid rows publish together only after that +validation succeeds and atomically replace the prior runtime configuration. A +rejected row contributes no new fields; if it previously owned a runtime +worker, its absence from the replacement triggers the worker-removal cleanup +contract rather than applying a mixture of old and invalid fields. ## 5. Runtime Ownership @@ -207,6 +221,13 @@ configured fields plus `bgd_status`. Its runtime dump includes all of those fields; paths that write back to configuration explicitly project away `bgd_status`. +Configuration materialization and worker status publication use the same +Hostgroups Manager write-locked update path. Reload merges configured fields +into an existing writer-hostgroup row without writing its `bgd_status`; the +worker status API updates only `bgd_status` under that lock. A reload that began +from an older snapshot therefore cannot overwrite a transition published while +the reload is being applied. + ## 6. LOAD Behavior `LOAD MYSQL SERVERS TO RUNTIME` must: @@ -216,9 +237,10 @@ fields; paths that write back to configuration explicitly project away hostgroup `0`. 3. Validate paired NULL/non-NULL and hostgroup-conflict rules. 4. Update or create the `AWS_Aurora_Info` entry. -5. Include both fields in the Aurora monitor resultset checksum. -6. Restart/refresh only the affected Aurora writer-hostgroup worker when either - green hostgroup changes. +5. Include both fields in the Aurora monitor result set checksum. +6. Refresh the affected Aurora writer-hostgroup worker in place when either + green hostgroup changes. Apply only configuration-derived fields and staging + references; do not replace the worker-owned FSM object. 7. Preserve an active BGD FSM safely across an unrelated configuration refresh. 8. Initialize `bgd_status` to `NONE` for a newly published runtime row. 9. Preserve the existing `bgd_status` when merging an existing @@ -234,7 +256,8 @@ apply the refreshed staging configuration without restarting the BGD operation from `NONE`. `LOAD MYSQL VARIABLES TO RUNTIME` makes a change to -`aws_blue_green_deployment_auto_discovery` visible to Aurora monitor workers. +`mysql-aws_blue_green_deployment_auto_discovery` visible to Aurora monitor +workers. The variable controls admission of new BGD operations for rows without green hostgroups, as described in Section 3.3. @@ -371,27 +394,37 @@ This configuration design does not: 2. Both green hostgroups non-NULL load successfully. 3. Mixed NULL/non-NULL values are rejected. 4. Duplicate or overlapping blue/green hostgroups are rejected. -5. Configured green hostgroups admit BGD discovery with global auto-discovery +5. A LOAD containing valid and invalid rows excludes each invalid row, reports + its writer hostgroup and conflicting fields, and atomically publishes the + filtered valid result set. A previously active rejected row follows the safe + worker-removal cleanup path rather than receiving a partial update. +6. Configured green hostgroups admit BGD discovery with global auto-discovery disabled. -6. A row without green hostgroups starts BGD discovery only when global +7. A row without green hostgroups starts BGD discovery only when global auto-discovery is enabled. -7. Disabling auto-discovery does not abort an active switchover. -8. NULL values survive memory-to-runtime, runtime-to-memory, disk, config-file, +8. Disabling auto-discovery does not abort an active switchover. +9. Changing an active row to `active=0` removes pins, restores safe placement, + clears the latch and fingerprint, publishes `NONE`, and stops the worker + without deleting user configuration. +10. NULL values survive memory-to-runtime, runtime-to-memory, disk, config-file, and cluster synchronization round trips. -9. Online upgrade preserves existing rows and initializes both new fields to +11. Online upgrade preserves existing rows and initializes both new fields to NULL. -10. Changing either green hostgroup refreshes only the affected Aurora worker. -11. Unrelated LOAD operations preserve active BGD pins and terminal-latch state. -12. New runtime rows initialize `bgd_status` to `NONE`. -13. Runtime `bgd_status` follows every Aurora FSM transition. -14. `SWITCHOVER_COMPLETED` remains visible until topology drain and then changes +12. Changing either green hostgroup refreshes only the affected Aurora worker + in place. +13. Unrelated LOAD operations preserve active BGD pins and terminal-latch state. +14. A LOAD concurrent with an FSM transition cannot overwrite the newer + `bgd_status`. +15. New runtime rows initialize `bgd_status` to `NONE`. +16. Runtime `bgd_status` follows every Aurora FSM transition. +17. `SWITCHOVER_COMPLETED` remains visible until topology drain and then changes to `NONE`. -15. Reloading an existing Aurora row preserves its `bgd_status` and active FSM +18. Reloading an existing Aurora row preserves its `bgd_status` and active FSM state. -16. `SAVE MYSQL SERVERS FROM RUNTIME`, disk/config export, and ProxySQL Cluster +19. `SAVE MYSQL SERVERS FROM RUNTIME`, disk/config export, and ProxySQL Cluster synchronization exclude `bgd_status`. -17. ProxySQL Cluster peers retain their own node-local `bgd_status` values. -18. Existing Multi-AZ BGD configuration and tests remain unchanged. +20. ProxySQL Cluster peers retain their own node-local `bgd_status` values. +21. Existing Multi-AZ BGD configuration and tests remain unchanged. ## 12. Acceptance Criteria @@ -401,7 +434,7 @@ The configuration/runtime integration is complete when: columns are either both configured or both NULL. 2. Configured green hostgroups admit BGD discovery independently of the global variable; when both are NULL, the existing - `aws_blue_green_deployment_auto_discovery` variable controls admission. + `mysql-aws_blue_green_deployment_auto_discovery` variable controls admission. 3. All load, save, disk, config-file, upgrade, and cluster-sync paths preserve the fields and their NULL values. 4. Exactly one Aurora monitor worker owns normal Aurora and BGD handling for a diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md index 26df126da7..95e4a6771f 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md @@ -204,10 +204,13 @@ For an eligible row in `NONE`: RDS BGD table-check state. 2. When the table exists, fetch and parse `SELECT * FROM mysql.rds_topology` so AWS column-set differences remain tolerated by the shared parser. -3. Require a structurally valid BGD result with a TARGET row, non-empty TARGET - status, endpoint, and port. -4. Publish the observed TARGET status to `bgd_status`. -5. Use the TARGET cluster endpoint to bootstrap target membership. +3. Require a structurally valid BGD result with a TARGET row and non-empty + TARGET `id`, status, endpoint, and port. Construct the deployment fingerprint + from at least that validated `id`, endpoint, and port. +4. Map the observed TARGET status to an Aurora FSM state, publish it to + `bgd_status`, and leave `NONE` before starting membership discovery. +5. Use the TARGET cluster endpoint to bootstrap target membership after that + state transition. 6. After target membership is available, rotate topology probes across target members. @@ -216,6 +219,23 @@ membership-dependent routing action and is retried at the cadence for the current phase. Missing, empty, or malformed topology does not create a new deployment. +Raw topology statuses map to Aurora runtime states as follows: + +| Raw TARGET status | Aurora FSM and `bgd_status` value | +|---|---| +| `AVAILABLE` | `AVAILABLE` | +| `SWITCHOVER_INITIATED` | `SWITCHOVER_INITIATED` | +| `SWITCHOVER_IN_PROGRESS` | `SWITCHOVER_IN_PROGRESS` | +| `SWITCHOVER_IN_POST_PROCESSING` | `SWITCHOVER_IN_POST_PROCESSING` | +| `SWITCHOVER_COMPLETED` in a valid TARGET-only result | `SWITCHOVER_COMPLETED` terminal latch | + +`NONE` is an internal baseline and rearm state, not an arbitrary raw status. +An unsupported or unknown raw status is invalid metadata: a worker already in +an active or terminal state retains that state, and a worker in `NONE` remains +there. The value is never copied verbatim into `bgd_status`. Aurora does not use +the RDS Multi-AZ inferred `READER_SWITCHOVER_IN_PROGRESS` state; TARGET +completion maps directly to the Aurora terminal latch after cleanup. + If green hostgroups are configured, BGD discovery is admitted for the user-created Aurora row. If neither is configured, the existing global auto-discovery variable gates admission of a new deployment. Changing that @@ -345,7 +365,30 @@ SWITCHOVER_COMPLETED The final state is a terminal rearm latch, not a separate reader-switchover phase. -### 8.2 Transition summary +### 8.2 Topology row-shape predicates + +The worker validates the complete topology result before publishing a new +status or running an FSM action: + +- An active deployment result contains exactly one SOURCE row and one TARGET + row. Both rows have non-empty `id`, endpoint, role, status, and a valid port; + both expose the same supported pre-completion status. +- A completion result contains exactly one TARGET row, has the validated + deployment-fingerprint fields, and reports `SWITCHOVER_COMPLETED`. +- A successful empty result or confirmed table absence is the explicit + cancellation or terminal-drain observation described below. +- SOURCE-only results, TARGET-only pre-completion results, duplicate SOURCE or + TARGET rows, extra or unknown roles, mismatched statuses, missing required + fields, and unsupported statuses are invalid or incomplete observations. + +Invalid or incomplete observations do not publish a new status, trigger +cleanup, release pins, or resume suspended monitoring. A worker in `NONE` +remains there; an active or latched worker retains its existing state and +retries. This validation is distinct from a successful empty or absent +topology, which has the explicit state-dependent meaning in the transition +table. + +### 8.3 Transition summary | Valid observation | Required transition and action | |---|---| @@ -590,6 +633,9 @@ The existing RDS Multi-AZ builder and FSM remain unchanged. - A genuine no-reader cluster accepts a writer-only snapshot. - Empty, failed, duplicate-writer, ambiguous, unresolved, and incomplete results retain the last complete snapshot or defer actions. +- SOURCE-only, duplicate-role, mismatched-status, unknown-status, and + TARGET-only pre-completion topology results fail closed without publishing or + transitioning; only TARGET-only completion is accepted. ### 14.3 FSM and routing @@ -626,7 +672,7 @@ The Aurora monitor/FSM design is satisfied when: 1. One existing Aurora worker owns the three probes and all state for its writer hostgroup. 2. Every production member is mapped from target `REPLICA_HOST_STATUS` before - traffic changes, whether or not green hostgroups are configured. + traffic changes, regardless of whether green hostgroups are configured. 3. Normal Aurora queries stop during the three active switchover phases and resume after completion or rollback. 4. IN_PROGRESS demotes the production writer using the RDS BGD behavior. From ea8fb4681c9e1346ae7b2690b2eb8c4eb7ccf05c Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 04:49:11 +0000 Subject: [PATCH 07/34] docs: record Aurora BGD status contract --- ...ra-bgd-cluster-simulator-testing-design.md | 9 ++-- ...026-07-31-aurora-bgd-monitor-fsm-design.md | 42 +++++++++++++++---- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md index 32e8864b83..68dfe5ac9a 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md @@ -448,11 +448,12 @@ The simulator does not validate: - mutable Route 53 propagation; - application-level latency or packet loss; - cross-process persistence of transient FSM state; -- Aurora versions that violate the AWS contracts established by the live-AWS - analysis. +- Aurora versions that violate the AWS-provided topology-status contract. -Live-AWS evidence remains the validation layer for status ordering, member -rename, writability, and DNS completion. +The AWS-provided topology metadata semantics define the status-to-routing +contract. Live-AWS evidence remains the validation layer for observed status +ordering, member rename, DNS completion, and corroboration of the writability +transition. ## 11. Acceptance Criteria diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md index 95e4a6771f..a87f3ba463 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md @@ -9,8 +9,12 @@ **Scope:** Aurora MySQL blue/green deployments exposed through `mysql.rds_topology` and `INFORMATION_SCHEMA.REPLICA_HOST_STATUS`. -**Evidence basis:** A timestamped live-Aurora switchover observation captured on -2026-07-30. Section 3 separates observed behavior from design policy. +**Evidence basis:** The AWS-team-provided *RDS Topology metadata – Overview* +document defines the routing semantics of the switchover statuses. A +timestamped live-Aurora switchover observation captured on 2026-07-30 +corroborates those semantics and supplies the observed Aurora-specific details. +Section 3 separates the supplied status contract, observed behavior, and design +policy. **Related designs:** @@ -85,7 +89,24 @@ configured, as defined by the configuration/runtime specification. ## 3. Evidence and AWS/Aurora Behavioral Contract -### 3.1 Directly observed behavior +### 3.1 AWS-provided status semantics + +The AWS-team-provided *RDS Topology metadata – Overview* document defines these +traffic-routing semantics for `mysql.rds_topology` during switchover: + +| Status | AWS-provided meaning | Write traffic | Read traffic | +|---|---|---|---| +| `SWITCHOVER_INITIATED` | Switchover was triggered, but no modifications have occurred and rollback remains possible. | Source | Source | +| `SWITCHOVER_IN_PROGRESS` | Source writes are disabled while target replication catches up; rollback remains possible. | Nowhere | Source | +| `SWITCHOVER_IN_POST_PROCESSING` | The target is promoted and can receive writes; rollback is no longer possible. | Target | Target | +| `SWITCHOVER_COMPLETED` | DNS propagation is complete and the original source endpoint points to the target. | Target | Target | + +The implementation treats these meanings as the AWS-supplied behavioral +contract for routing decisions. In particular, POST_PROCESSING definitively +means that the promoted target can accept write traffic; target readiness is +not inferred solely from one observed run. + +### 3.2 Directly observed behavior The 2026-07-30 run observed: @@ -114,14 +135,17 @@ The 2026-07-30 run observed: 9. TARGET completion was visible at `T+40.655s`; the topology table drained at `T+65.730s`. -The implementation treats these observations as the evidence for the design, -not as a formal AWS compatibility guarantee. Tests must simulate the observed -changes, and unexpected or ambiguous metadata must fail closed. +The run corroborates the AWS-provided status semantics. Details learned only +from the observation, including timings, target-member rename behavior, and +table-drain timing, are implementation evidence rather than a formal AWS +compatibility guarantee. Tests must simulate the observed changes, and +unexpected or ambiguous metadata must fail closed. -### 3.2 Design policy derived from the observation +### 3.3 Design policy derived from the contract and observation -- POST_PROCESSING is the routing barrier; no additional target-writability - query gates traffic pinning. +- POST_PROCESSING is the routing barrier because the AWS-provided semantics say + that the promoted target can receive writes at that status; no additional + target-writability query gates traffic pinning. - TARGET completion means Aurora writer and reader routing cleanup can occur immediately. - Table drain is not a reader-availability barrier for Aurora. From 529295fd8695af850764f6129a4c338910100cae Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 08:34:07 +0000 Subject: [PATCH 08/34] docs: clarify Aurora BGD switchover lifecycle --- ...ra-bgd-cluster-simulator-testing-design.md | 14 ++- ...nfiguration-runtime-cluster-sync-design.md | 8 +- ...026-07-31-aurora-bgd-monitor-fsm-design.md | 101 +++++++++++++----- 3 files changed, 87 insertions(+), 36 deletions(-) diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md index 68dfe5ac9a..95e01ddba6 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md @@ -370,9 +370,11 @@ Interactive TAP tests cover: placement. 6. Target member `SERVER_ID` rename with stable reader `SESSION_ID` identity and retained cached IPs. -7. Idempotent per-member pinning, placement, and pool-drain actions. -8. The first TARGET-only `SWITCHOVER_COMPLETED`, immediate cleanup, terminal - latch, and runtime `bgd_status='SWITCHOVER_COMPLETED'`. +7. Idempotent per-member pinning, placement, and connection-retirement marking, + without treating asynchronous physical closure as an FSM prerequisite. +8. The first TARGET-only `SWITCHOVER_COMPLETED`, immediate effect-driven + cleanup, terminal latch, and runtime + `bgd_status='SWITCHOVER_COMPLETED'`. 9. Repeated completion observations, successful topology drain to `NONE`, and rearming for a different deployment fingerprint. @@ -385,6 +387,12 @@ Coverage also includes: - topology, replica, and read-only query failures in each relevant phase; - rollback from initiated, in-progress, and post-processing states; - late entry at initiated, in-progress, post-processing, and completed states; +- direct completion with an empty member map producing no production-member + routing or retirement actions; +- completion directly after in-progress restoring the writer without replaying + post-processing; +- completion after post-processing removing pins without reapplying connection + retirement or waiting for marked connections to close; - configuration and hostgroup refresh during an active deployment; - in-place worker configuration refresh with retained deployment state; - deleting or deactivating an Aurora row during switchover; diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md index 5ef200ec6b..fa1b7234bd 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md @@ -327,7 +327,7 @@ Both green hostgroup columns must be supported by: - admin-memory table creation; - runtime table creation; -- disk database schema and online upgrade; +- disk database schema and in-place migration on ProxySQL startup; - `LOAD MYSQL SERVERS TO RUNTIME`; - `SAVE MYSQL SERVERS FROM RUNTIME`; - `SAVE MYSQL SERVERS TO DISK`; @@ -351,7 +351,7 @@ projection rather than `SELECT *`. In particular: BGD's `bgd_status`, transient member map, cached IPs, and pins are node-local runtime state and are not cluster-synced as configuration. -## 9. Schema Upgrade +## 9. Disk Schema Migration A new Aurora hostgroups schema version must add the two nullable columns while preserving every existing row and column value. @@ -408,8 +408,8 @@ This configuration design does not: without deleting user configuration. 10. NULL values survive memory-to-runtime, runtime-to-memory, disk, config-file, and cluster synchronization round trips. -11. Online upgrade preserves existing rows and initializes both new fields to - NULL. +11. In-place disk schema migration on ProxySQL startup preserves existing rows + and initializes both new fields to NULL. 12. Changing either green hostgroup refreshes only the affected Aurora worker in place. 13. Unrelated LOAD operations preserve active BGD pins and terminal-latch state. diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md index a87f3ba463..aa9354a4cb 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md @@ -83,6 +83,9 @@ configured, as defined by the configuration/runtime specification. - **Complete target snapshot:** Exactly one current target writer plus a unique target counterpart for every current production member, with all target IPs resolved. +- **Effect-driven cleanup:** The idempotent RDS BGD pattern that reconciles + placement and removes transient routing state from the worker's existing + member map. A member-scoped operation is a no-op when its map input is absent. - **Deployment fingerprint:** TARGET topology identity retained after cleanup to recognize repeated results. At minimum it contains TARGET `id`, endpoint, and port. @@ -106,6 +109,13 @@ contract for routing decisions. In particular, POST_PROCESSING definitively means that the promoted target can accept write traffic; target readiness is not inferred solely from one observed run. +[AWS's Aurora switchover documentation](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/blue-green-deployments-switching.html) +also states that a DB cluster included in a switchover cannot be modified while +the switchover is running. The production and target member sets are therefore +fixed from INITIATED until the first COMPLETED observation. Membership can +change while the deployment is AVAILABLE, when the normal production probe +and target-membership probe remain active and refresh their snapshots. + ### 3.2 Directly observed behavior The 2026-07-30 run observed: @@ -146,6 +156,9 @@ unexpected or ambiguous metadata must fail closed. - POST_PROCESSING is the routing barrier because the AWS-provided semantics say that the promoted target can receive writes at that status; no additional target-writability query gates traffic pinning. +- Normal monitoring refreshes membership while the deployment is AVAILABLE. + INITIATED freezes the last complete production snapshot for the duration of + the switchover, matching AWS's cluster-modification restriction. - TARGET completion means Aurora writer and reader routing cleanup can occur immediately. - Table drain is not a reader-availability barrier for Aurora. @@ -301,6 +314,13 @@ writers, missing readers, duplicate identities, and unresolved IPs do not replace the last complete snapshot. They also must not be reinterpreted as a reader-less deployment. +The production snapshot is refreshed by normal Aurora monitoring throughout +AVAILABLE. When the worker first accepts INITIATED, that last complete snapshot +becomes the fixed production membership for the switchover. The normal +production probe remains suspended while AWS prevents changes to either +included cluster; the frozen snapshot and continuously refreshed target +snapshot therefore require no separate member-set generation. + POST_PROCESSING actions require a complete snapshot. If none exists, the worker holds the current status, performs no partial traffic cutover, and retries both active probes. @@ -362,9 +382,7 @@ last complete target snapshot normalized production/target member pairs cached target IP per pair traffic-pin-applied flag per pair -writer-demoted flag production-probe-suspended flag -completion-cleanup-applied flag configured green hostgroup identifiers, when present ``` @@ -422,7 +440,7 @@ table. | `SWITCHOVER_IN_PROGRESS` | Move the production writer to the reader hostgroup; do not route to target yet. | | First `SWITCHOVER_IN_POST_PROCESSING` | With a complete snapshot, pin every production member, drain old connections, and restore the canonical writer to the writer hostgroup. | | Repeated POST_PROCESSING | Retry only incomplete idempotent member actions. | -| First TARGET-only `SWITCHOVER_COMPLETED` | Remove all pins, perform immediate cleanup, resume normal Aurora monitoring, and enter the completed latch. | +| First TARGET-only `SWITCHOVER_COMPLETED` | Run immediate effect-driven cleanup, resume normal Aurora monitoring, and enter the completed latch. | | Repeated same completed TARGET | No-op while latched. | | Successful empty/absent topology while latched | Release the fingerprint and return to `NONE`. | | Earlier valid status before completion | Run rollback, then enter the earlier state. | @@ -466,7 +484,8 @@ On the first valid observation: 2. Retain fast BGD probes and the suspended production probe. 3. Move the current production writer to the reader hostgroup using the same writer-demotion behavior as RDS BGD. -4. Record that demotion so rollback can restore it. +4. Retain the member map for writer-placement reconciliation during completion + or rollback. 5. Do not redirect any production hostname to target yet. The move is idempotent. A repeated observation does not repeat a completed @@ -497,27 +516,41 @@ Before applying any action, require a complete target snapshot. Then: 7. Keep canonical readers eligible in the reader hostgroup. Do not shun or unshun them. -Every pair records whether pinning and draining completed. Repeated -POST_PROCESSING results retry only unapplied actions and do not repeatedly drain -an already transitioned member. +Every pair records that its pin-and-retirement action was applied after the DNS +pin, free-connection deletion, used-connection unhealthy/non-reusable marking, +and monitor-pool purge calls return. This flag does not mean that every used +connection has physically closed: destruction happens asynchronously when the +connection is released and is not a completion prerequisite. Repeated +POST_PROCESSING results retry only unapplied actions and do not reapply +retirement to an already transitioned member. ### 9.5 `SWITCHOVER_COMPLETED` -On the first TARGET-only completed result: - -1. Remove every production-hostname DNS pin immediately. `dns_cache->remove()` - must invalidate the local pinned entry so normal DNS resolution resumes. -2. Do not wait for topology drain or perform a separate DNS verification. -3. Preserve current production connections; they already point to promoted - members. Drain only obsolete pools belonging to configured green - hostgroups, subject to the configured OFFLINE-status preservation policy. -4. Preserve every user-created configuration and `mysql_servers` row. -5. Clear the switchover guard and resume normal Aurora monitoring. -6. Move topology probing back to random reachable canonical production members +On the first TARGET-only completed result, run effect-driven cleanup +immediately, before publishing the terminal latch: + +1. If the existing member map contains a writer, reconcile that writer into + the writer hostgroup and apply its reader placement according to + `writer_is_also_reader`. This restores a writer moved during IN_PROGRESS when + POST_PROCESSING was not observed, and is a no-op when the writer is already + restored or the map is empty. +2. For every mapped pair, remove the production-hostname DNS entry and purge + the corresponding monitor-pool entries so normal DNS resolution resumes. + Removing an absent pin or pool entry is a no-op. +3. Do not reapply connection retirement, wait for marked connections to close, + wait for topology drain, or perform a separate DNS verification. +4. Drain only obsolete pools belonging to configured green hostgroups, subject + to the configured OFFLINE-status preservation policy. +5. Preserve every user-created configuration and `mysql_servers` row. +6. Clear any installed switchover guard and resume normal Aurora monitoring. +7. Move topology probing back to random reachable canonical production members and the configured interval. -7. Release the active member map after retaining the deployment fingerprint and - the completion-cleanup flag. -8. Publish and enter the internal `SWITCHOVER_COMPLETED` latch. +8. Release the active member map after retaining the deployment fingerprint. +9. Publish and enter the internal `SWITCHOVER_COMPLETED` latch. + +The cleanup path is the same regardless of the prior phase. It acts only on the +available member map and worker state, so absent inputs naturally produce +no-ops. Completion never replays a skipped phase or waits for earlier actions. While latched: @@ -536,12 +569,13 @@ The topology drain is therefore only an FSM rearm signal. A successful, structurally valid result for the same deployment with an earlier status follows the existing RDS BGD backward-transition behavior. A successful empty result or confirmed table absence before completion is treated -as cancellation. Either condition runs idempotent rollback: +as cancellation. Either condition runs effect-driven cleanup in rollback mode: 1. Remove every traffic pin that was applied. 2. Drain/purge affected production-hostname pools so subsequent connections use restored canonical DNS. -3. Restore the production writer to the writer hostgroup when it was demoted. +3. Reconcile the mapped production writer into the writer hostgroup. This is a + no-op if the writer is already there or the map is empty. 4. Restore its reader placement according to `writer_is_also_reader`. 5. Resume normal production Aurora monitoring. 6. Return topology probing to production members and the appropriate cadence. @@ -569,16 +603,16 @@ A worker that first observes INITIATED, IN_PROGRESS, or POST_PROCESSING rebuilds all prerequisites before applying that phase's actions. In particular, POST_PROCESSING cannot pin traffic until it has a complete target snapshot. -A worker that first observes TARGET-only completion performs completion cleanup -once, even when no traffic pins were recorded, retains the fingerprint, and -enters the terminal latch. It must not manufacture or replay earlier phase -actions. +A worker that first observes TARGET-only completion has an empty member map. +Its member-scoped completion actions are consequently no-ops. The worker +retains the fingerprint and enters the terminal latch without manufacturing or +replaying earlier phase actions. ### 11.2 Configuration and variable refresh An unrelated `LOAD MYSQL SERVERS TO RUNTIME` or variable refresh must preserve the FSM status, deployment fingerprint, complete snapshots, cached IPs, applied -pin flags, writer-demotion state, probe-suspension state, and terminal latch. +pin flags, probe-suspension state, and terminal latch. Changing configured green hostgroups refreshes the staging/pool references but does not restart an active deployment from `NONE`. Disabling the global @@ -671,7 +705,16 @@ The existing RDS Multi-AZ builder and FSM remain unchanged. - No target-writability query gates POST_PROCESSING. - Readers remain eligible and are never shunned. - Repeated POST_PROCESSING retries only unapplied work. -- First TARGET completion removes all pins and cleans up immediately. +- The per-pair action flag records application of pinning and retirement + marking, not asynchronous physical connection closure. +- First TARGET completion runs immediate effect-driven cleanup using the + existing map and worker state. +- Direct completion with an empty map performs no production-member routing or + retirement action and enters the completed latch. +- Completion after IN_PROGRESS restores the writer even when POST_PROCESSING + was not observed. +- Completion after POST_PROCESSING removes applied pins without reapplying + connection retirement or waiting for marked connections to close. - Completion performs no DNS verification and does not wait for table drain. - Repeated completion is a no-op while latched. - Empty/absent topology releases the latch and returns to `NONE`. From e06fa1afb60a4026cde6396c773a26b202c572b9 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 09:45:03 +0000 Subject: [PATCH 09/34] feat: add Aurora BGD hostgroup schema --- include/MySQL_HostGroups_Manager.h | 18 +- include/ProxySQL_Admin_Tables_Definitions.h | 44 ++- lib/MySQL_HostGroups_Manager.cpp | 269 ++++++++++++++++-- lib/ProxySQL_Admin.cpp | 39 ++- lib/ProxySQL_Admin_Disk_Upgrade.cpp | 22 ++ test/tap/tests/unit/Makefile | 1 + .../tests/unit/admin_disk_upgrade_unit-t.cpp | 34 ++- .../tests/unit/aurora_bgd_config_unit-t.cpp | 242 ++++++++++++++++ 8 files changed, 611 insertions(+), 58 deletions(-) create mode 100644 test/tap/tests/unit/aurora_bgd_config_unit-t.cpp diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 6edc571723..fabdb6b5a6 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include "ev.h" #include "wqueue.h" @@ -52,6 +54,8 @@ #define MYHGM_MYSQL_GALERA_HOSTGROUPS "CREATE TABLE mysql_galera_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1,2)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" #define MYHGM_MYSQL_AWS_AURORA_HOSTGROUPS "CREATE TABLE mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ + "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , " \ + "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , " \ "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL DEFAULT '' , " \ "max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , " \ "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , " \ @@ -62,7 +66,7 @@ "min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , " \ "lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , " \ "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , " \ - "comment VARCHAR , UNIQUE (reader_hostgroup))" + "comment VARCHAR , bgd_status VARCHAR NOT NULL DEFAULT 'NONE' , UNIQUE (reader_hostgroup))" #define MYHGM_MYSQL_AWS_RDS_BGD_HOSTGROUPS "CREATE TABLE mysql_aws_rds_bgd_hostgroups ("\ "writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , "\ @@ -379,6 +383,18 @@ class AWS_Aurora_Info { AWS_Aurora_Info& operator=(const AWS_Aurora_Info&) = delete; }; +/** + * @brief Validate an Aurora hostgroup candidate and return its canonical configured projection. + * + * The returned result contains only valid rows, ordered like the Aurora Admin + * table. Legacy projections without green hostgroups are normalized with both + * green values set to SQL NULL. The caller owns the returned result. + */ +SQLite3_result* validate_and_filter_aws_aurora_hostgroups( + const SQLite3_result* candidate, + std::vector& errors +); + struct p_hg_counter { enum metric { servers_table_version = 0, diff --git a/include/ProxySQL_Admin_Tables_Definitions.h b/include/ProxySQL_Admin_Tables_Definitions.h index 4514853bd0..164ec01856 100644 --- a/include/ProxySQL_Admin_Tables_Definitions.h +++ b/include/ProxySQL_Admin_Tables_Definitions.h @@ -235,9 +235,47 @@ // V2_0_10: Adds autopurge_missing_checks column #define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_10 "CREATE TABLE mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup))" -#define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_10 - -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup))" +// V3_0_12: Adds nullable Aurora blue/green staging hostgroups. +#define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V3_0_12 "CREATE TABLE mysql_aws_aurora_hostgroups ("\ + "writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , "\ + "reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , "\ + "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , "\ + "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , "\ + "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , "\ + "aurora_port INT NOT NUlL DEFAULT 3306 , "\ + "domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , "\ + "max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , "\ + "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , "\ + "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , "\ + "writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , "\ + "new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , "\ + "add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , "\ + "min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , "\ + "lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , "\ + "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , "\ + "comment VARCHAR , UNIQUE (reader_hostgroup))" + +#define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V3_0_12 + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_aurora_hostgroups ("\ + "writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , "\ + "reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , "\ + "green_writer_hostgroup INT DEFAULT NULL CHECK (green_writer_hostgroup IS NULL OR green_writer_hostgroup>=0) , "\ + "green_reader_hostgroup INT DEFAULT NULL CHECK (green_reader_hostgroup IS NULL OR green_reader_hostgroup>=0) , "\ + "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , "\ + "aurora_port INT NOT NUlL DEFAULT 3306 , "\ + "domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , "\ + "max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , "\ + "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , "\ + "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , "\ + "writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , "\ + "new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , "\ + "add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , "\ + "min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , "\ + "lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , "\ + "autopurge_missing_checks INT NOT NULL CHECK (autopurge_missing_checks >= 0 AND autopurge_missing_checks <= 100) DEFAULT 0 , "\ + "comment VARCHAR , "\ + "bgd_status VARCHAR NOT NULL DEFAULT 'NONE' , UNIQUE (reader_hostgroup))" // AWS RDS hostgroups; adds blue/green (green_*_hostgroup) over aurora. // The runtime table carries one extra runtime-only column: auto_generated. diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 728550102a..be580d71c6 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -30,6 +30,7 @@ using json = nlohmann::json; #include #include +#include #include using std::function; @@ -58,6 +59,198 @@ class MyHGC; const int MYSQL_ERRORS_STATS_FIELD_NUM = 11; +namespace { + +constexpr int AWS_AURORA_CONFIG_COLUMN_COUNT = 17; + +const char* const AWS_AURORA_CONFIG_COLUMNS[AWS_AURORA_CONFIG_COLUMN_COUNT] = { + "writer_hostgroup", + "reader_hostgroup", + "green_writer_hostgroup", + "green_reader_hostgroup", + "active", + "aurora_port", + "domain_name", + "max_lag_ms", + "check_interval_ms", + "check_timeout_ms", + "writer_is_also_reader", + "new_reader_weight", + "add_lag_ms", + "min_lag_ms", + "lag_num_checks", + "autopurge_missing_checks", + "comment" +}; + +struct Aurora_Hostgroup_Role { + const char* field; + int value; +}; + +struct Aurora_Config_Row { + std::vector fields; + std::vector roles; + std::vector errors; + int writer_hostgroup; + bool active; + bool locally_valid; +}; + +int aws_aurora_column_index(const SQLite3_result* candidate, const char* name) { + for (size_t i = 0; i < candidate->column_definition.size(); ++i) { + if (strcasecmp(candidate->column_definition[i]->name, name) == 0) { + return static_cast(i); + } + } + return -1; +} + +std::string aws_aurora_nullable_value(const char* value) { + return value ? value : "NULL"; +} + +} // namespace + +SQLite3_result* validate_and_filter_aws_aurora_hostgroups( + const SQLite3_result* candidate, + std::vector& errors +) { + SQLite3_result* filtered = new SQLite3_result(AWS_AURORA_CONFIG_COLUMN_COUNT); + for (const char* column : AWS_AURORA_CONFIG_COLUMNS) { + filtered->add_column_definition(SQLITE_TEXT, column); + } + + if (candidate == nullptr) { + return filtered; + } + + std::vector source_indexes(AWS_AURORA_CONFIG_COLUMN_COUNT, -1); + for (int i = 0; i < AWS_AURORA_CONFIG_COLUMN_COUNT; ++i) { + source_indexes[i] = aws_aurora_column_index(candidate, AWS_AURORA_CONFIG_COLUMNS[i]); + } + + // Older peers send the pre-BGD 15-column projection. Treat the two absent + // green fields as SQL NULL until cluster synchronization is upgraded. + const bool green_writer_absent = source_indexes[2] == -1; + const bool green_reader_absent = source_indexes[3] == -1; + if (green_writer_absent != green_reader_absent) { + errors.emplace_back( + "mysql_aws_aurora_hostgroups rejected: candidate projection contains only one green hostgroup column" + ); + return filtered; + } + + for (int i = 0; i < AWS_AURORA_CONFIG_COLUMN_COUNT; ++i) { + if ((i == 2 || i == 3) && green_writer_absent) { + continue; + } + if (source_indexes[i] == -1) { + errors.emplace_back( + std::string("mysql_aws_aurora_hostgroups rejected: candidate projection is missing ") + + AWS_AURORA_CONFIG_COLUMNS[i] + ); + return filtered; + } + } + + std::vector rows; + rows.reserve(candidate->rows.size()); + + for (const SQLite3_row* source_row : candidate->rows) { + Aurora_Config_Row row; + row.fields.resize(AWS_AURORA_CONFIG_COLUMN_COUNT, nullptr); + for (int i = 0; i < AWS_AURORA_CONFIG_COLUMN_COUNT; ++i) { + if (source_indexes[i] != -1) { + row.fields[i] = source_row->fields[source_indexes[i]]; + } + } + + row.writer_hostgroup = row.fields[0] ? atoi(row.fields[0]) : -1; + row.active = row.fields[4] && atoi(row.fields[4]) != 0; + row.locally_valid = true; + + const bool has_green_writer = row.fields[2] != nullptr; + const bool has_green_reader = row.fields[3] != nullptr; + if (has_green_writer != has_green_reader) { + std::ostringstream message; + message << "green_writer_hostgroup=" << aws_aurora_nullable_value(row.fields[2]) + << " and green_reader_hostgroup=" << aws_aurora_nullable_value(row.fields[3]) + << " must both be NULL or both be non-NULL"; + row.errors.emplace_back(message.str()); + row.locally_valid = false; + } + + row.roles.push_back({"writer_hostgroup", row.writer_hostgroup}); + row.roles.push_back({"reader_hostgroup", row.fields[1] ? atoi(row.fields[1]) : -1}); + if (has_green_writer && has_green_reader) { + row.roles.push_back({"green_writer_hostgroup", atoi(row.fields[2])}); + row.roles.push_back({"green_reader_hostgroup", atoi(row.fields[3])}); + } + + if (row.locally_valid) { + for (size_t left = 0; left < row.roles.size(); ++left) { + for (size_t right = left + 1; right < row.roles.size(); ++right) { + if (row.roles[left].value == row.roles[right].value) { + std::ostringstream message; + message << "conflicting fields " << row.roles[left].field << '=' << row.roles[left].value + << " and " << row.roles[right].field << '=' << row.roles[right].value; + row.errors.emplace_back(message.str()); + row.locally_valid = false; + } + } + } + } + + rows.emplace_back(std::move(row)); + } + + // A locally invalid row is already absent from the replacement and must not + // make an otherwise valid row fail. Cross-row conflicts are directional: + // every row is checked against the roles owned by every other active row. + for (size_t row_index = 0; row_index < rows.size(); ++row_index) { + Aurora_Config_Row& row = rows[row_index]; + if (!row.locally_valid) { + continue; + } + + for (size_t other_index = 0; other_index < rows.size(); ++other_index) { + const Aurora_Config_Row& other = rows[other_index]; + if (row_index == other_index || !other.locally_valid || !other.active) { + continue; + } + + for (const Aurora_Hostgroup_Role& role : row.roles) { + for (const Aurora_Hostgroup_Role& other_role : other.roles) { + if (role.value == other_role.value) { + std::ostringstream message; + message << "conflicting field " << role.field << '=' << role.value + << " overlaps " << other_role.field << '=' << other_role.value + << " in writer_hostgroup=" << other.writer_hostgroup; + row.errors.emplace_back(message.str()); + } + } + } + } + } + + for (Aurora_Config_Row& row : rows) { + if (row.errors.empty()) { + filtered->add_row(row.fields.data()); + continue; + } + + for (const std::string& reason : row.errors) { + std::ostringstream message; + message << "mysql_aws_aurora_hostgroups writer_hostgroup=" << row.writer_hostgroup + << " rejected: " << reason; + errors.emplace_back(message.str()); + } + } + + return filtered; +} + struct ev_io * new_connect_watcher(char *address, uint16_t gtid_port, uint16_t mysql_port); void * GTID_syncer_run(); @@ -6422,9 +6615,9 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { } int rc; //sqlite3 *mydb3=mydb->get_db(); - char *query=(char *)"INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms," + char *query=(char *)"INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms," "check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES " - "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)"; + "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)"; auto [rc1, statement_unique] = mydb->prepare_v2(query); ASSERT_SQLITE_OK(rc1, mydb); sqlite3_stmt *statement = statement_unique.get(); @@ -6439,34 +6632,50 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { SQLite3_row *r=*it; int writer_hostgroup=atoi(r->fields[0]); int reader_hostgroup=atoi(r->fields[1]); - int active=atoi(r->fields[2]); - int aurora_port = atoi(r->fields[3]); - int max_lag_ms = atoi(r->fields[5]); - int check_interval_ms = atoi(r->fields[6]); - int check_timeout_ms = atoi(r->fields[7]); - int writer_is_also_reader = atoi(r->fields[8]); - int new_reader_weight = atoi(r->fields[9]); - int add_lag_ms = atoi(r->fields[10]); - int min_lag_ms = atoi(r->fields[11]); - int lag_num_checks = atoi(r->fields[12]); - int autopurge_missing_checks = atoi(r->fields[13]); - proxy_info("Loading AWS Aurora info for (%d,%d,%s,%d,\"%s\",%d,%d,%d,%d,%d,%d,%d,\"%s\")\n", writer_hostgroup,reader_hostgroup,(active ? "on" : "off"),aurora_port, - r->fields[4],max_lag_ms,add_lag_ms,min_lag_ms,lag_num_checks,check_interval_ms,check_timeout_ms,autopurge_missing_checks,r->fields[14]); + const char* green_writer_hostgroup = r->fields[2]; + const char* green_reader_hostgroup = r->fields[3]; + int active=atoi(r->fields[4]); + int aurora_port = atoi(r->fields[5]); + int max_lag_ms = atoi(r->fields[7]); + int check_interval_ms = atoi(r->fields[8]); + int check_timeout_ms = atoi(r->fields[9]); + int writer_is_also_reader = atoi(r->fields[10]); + int new_reader_weight = atoi(r->fields[11]); + int add_lag_ms = atoi(r->fields[12]); + int min_lag_ms = atoi(r->fields[13]); + int lag_num_checks = atoi(r->fields[14]); + int autopurge_missing_checks = atoi(r->fields[15]); + proxy_info("Loading AWS Aurora info for (%d,%d,%s,%s,%s,%d,\"%s\",%d,%d,%d,%d,%d,%d,%d,\"%s\")\n", writer_hostgroup,reader_hostgroup, + green_writer_hostgroup ? green_writer_hostgroup : "NULL", green_reader_hostgroup ? green_reader_hostgroup : "NULL", + (active ? "on" : "off"),aurora_port,r->fields[6],max_lag_ms,add_lag_ms,min_lag_ms,lag_num_checks, + check_interval_ms,check_timeout_ms,autopurge_missing_checks,r->fields[16]); rc=(*proxy_sqlite3_bind_int64)(statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); rc=(*proxy_sqlite3_bind_int64)(statement, 2, reader_hostgroup); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 3, active); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 4, aurora_port); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_text)(statement, 5, r->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 6, max_lag_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 7, check_interval_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 8, check_timeout_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 9, writer_is_also_reader); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 10, new_reader_weight); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 11, add_lag_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 12, min_lag_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 13, lag_num_checks); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 14, autopurge_missing_checks); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_text)(statement, 15, r->fields[14], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + if (green_writer_hostgroup) { + rc=(*proxy_sqlite3_bind_int64)(statement, 3, atoi(green_writer_hostgroup)); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 3); + } + ASSERT_SQLITE_OK(rc, mydb); + if (green_reader_hostgroup) { + rc=(*proxy_sqlite3_bind_int64)(statement, 4, atoi(green_reader_hostgroup)); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 4); + } + ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 5, active); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 6, aurora_port); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 7, r->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 8, max_lag_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 9, check_interval_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 10, check_timeout_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 11, writer_is_also_reader); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 12, new_reader_weight); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 13, add_lag_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 14, min_lag_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 15, lag_num_checks); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 16, autopurge_missing_checks); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 17, r->fields[16], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); SAFE_SQLITE3_STEP2(statement); rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mydb); @@ -6477,12 +6686,12 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { if (it2!=AWS_Aurora_Info_Map.end()) { info=it2->second; bool changed=false; - changed=info->update(reader_hostgroup, aurora_port, r->fields[4], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[14]); + changed=info->update(reader_hostgroup, aurora_port, r->fields[6], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[16]); if (changed) { //info->need_converge=true; } } else { - info=new AWS_Aurora_Info(writer_hostgroup, reader_hostgroup, aurora_port, r->fields[4], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[14]); + info=new AWS_Aurora_Info(writer_hostgroup, reader_hostgroup, aurora_port, r->fields[6], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[16]); //info->need_converge=true; AWS_Aurora_Info_Map.insert(AWS_Aurora_Info_Map.begin(), std::pair(writer_hostgroup,info)); } diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index b7a9ce5562..edf310dc6f 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -8110,27 +8110,11 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc } // support for AWS Aurora, table mysql_aws_aurora_hostgroups - - // look for invalid combinations - query=(char *)"SELECT a.* FROM mysql_aws_aurora_hostgroups a JOIN mysql_aws_aurora_hostgroups b ON a.writer_hostgroup=b.reader_hostgroup WHERE b.reader_hostgroup"; - proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - if (error) { - proxy_error("Error on %s : %s\n", query, error); - } else { - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - proxy_error("Incompatible entry in mysql_aws_aurora_hostgroups will be ignored : ( %s , %s , %s , %s )\n", r->fields[0], r->fields[1], r->fields[2], r->fields[3]); - } - } - if (resultset) delete resultset; - resultset=NULL; - -//#ifdef TEST_AURORA // temporary enabled only for testing purpose - query=(char *)"SELECT a.* FROM mysql_aws_aurora_hostgroups a LEFT JOIN mysql_aws_aurora_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup) WHERE b.reader_hostgroup IS NULL ORDER BY writer_hostgroup"; -//#else -// query=(char *)"SELECT a.* FROM mysql_aws_aurora_hostgroups a WHERE 1=0"; -//#endif + query=(char *) + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment " + "FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); if (incoming_aurora_hostgroups == nullptr) { admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_aws_aurora); @@ -8140,8 +8124,17 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc if (error) { proxy_error("Error on %s : %s\n", query, error); } else { - // Pass the resultset to MyHGM - MyHGM->save_incoming_mysql_table(resultset_aws_aurora,"mysql_aws_aurora_hostgroups"); + std::vector validation_errors; + SQLite3_result* filtered_aws_aurora = validate_and_filter_aws_aurora_hostgroups( + resultset_aws_aurora, + validation_errors + ); + for (const std::string& validation_error : validation_errors) { + proxy_error("%s\n", validation_error.c_str()); + } + delete resultset_aws_aurora; + resultset_aws_aurora = nullptr; + MyHGM->save_incoming_mysql_table(filtered_aws_aurora,"mysql_aws_aurora_hostgroups"); } // support for AWS RDS, table mysql_aws_rds_bgd_hostgroups diff --git a/lib/ProxySQL_Admin_Disk_Upgrade.cpp b/lib/ProxySQL_Admin_Disk_Upgrade.cpp index fbf3f2b638..b8cb964210 100644 --- a/lib/ProxySQL_Admin_Disk_Upgrade.cpp +++ b/lib/ProxySQL_Admin_Disk_Upgrade.cpp @@ -504,6 +504,28 @@ void ProxySQL_Admin::disk_upgrade_mysql_servers() { "add_lag_ms, min_lag_ms, lag_num_checks, comment FROM mysql_aws_aurora_hostgroups_v209"); } + // Upgrade mysql_aws_aurora_hostgroups from V2_0_10 to V3_0_12 + // (adds nullable green_writer_hostgroup and green_reader_hostgroup columns). + rci=configdb->check_table_structure("mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_10); + if (rci) { + proxy_warning("Detected version pre-v3.0.12 of mysql_aws_aurora_hostgroups\n"); + proxy_warning("ONLINE UPGRADE of table mysql_aws_aurora_hostgroups in progress\n"); + configdb->execute("DROP TABLE IF EXISTS mysql_aws_aurora_hostgroups_v2010"); + configdb->execute("ALTER TABLE mysql_aws_aurora_hostgroups RENAME TO mysql_aws_aurora_hostgroups_v2010"); + configdb->build_table("mysql_aws_aurora_hostgroups",ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS,false); + configdb->execute( + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, " + "check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, " + "add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment" + ") SELECT " + "writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, " + "check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, " + "add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment " + "FROM mysql_aws_aurora_hostgroups_v2010" + ); + } + // upgrade mysql_hostgroup_attributes rci=configdb->check_table_structure("mysql_hostgroup_attributes",ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_0); if (rci) { diff --git a/test/tap/tests/unit/Makefile b/test/tap/tests/unit/Makefile index 737a93b037..3bc9d2fce2 100644 --- a/test/tap/tests/unit/Makefile +++ b/test/tap/tests/unit/Makefile @@ -428,6 +428,7 @@ UNIT_TESTS := smoke_test-t query_cache_unit-t query_processor_unit-t \ gtid_set_unit-t \ gtid_server_data_unit-t \ admin_disk_upgrade_unit-t \ + aurora_bgd_config_unit-t \ glovars_unit-t \ pgsql_servers_ssl_params_unit-t \ connection_unhealthy_unit-t \ diff --git a/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp b/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp index f900625e6f..d223105327 100644 --- a/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp +++ b/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp @@ -382,6 +382,37 @@ static void test_mysql_servers_no_upgrade_needed() { ok(count == 1, "mysql_servers: data unchanged when no upgrade needed (got %d)", count); } +static void test_aurora_hostgroups_upgrade_from_v2_0_10() { + TestDiskUpgrade t; + SQLite3DB *db = t.db(); + + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_10); + db->execute( + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," + "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight," + "add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment" + ") VALUES (10,20,1,3307,'.cluster.example',125,1500,900,1,7,40,20,3,9,'existing Aurora row')" + ); + + t.upgrade_mysql_servers(); + + ok(table_matches_current(db, "mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS), + "mysql_aws_aurora_hostgroups: v2.0.10 upgrade produces current schema"); + ok(query_int(db, "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups") == 1, + "mysql_aws_aurora_hostgroups: v2.0.10 upgrade preserves the row"); + ok(query_int(db, "SELECT green_writer_hostgroup IS NULL FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=10") == 1, + "mysql_aws_aurora_hostgroups: migrated green writer is NULL"); + ok(query_int(db, "SELECT green_reader_hostgroup IS NULL FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=10") == 1, + "mysql_aws_aurora_hostgroups: migrated green reader is NULL"); + ok(query_string(db, "SELECT domain_name FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=10") == ".cluster.example", + "mysql_aws_aurora_hostgroups: migration preserves configured values"); + ok(query_int(db, "SELECT autopurge_missing_checks FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=10") == 9, + "mysql_aws_aurora_hostgroups: migration preserves autopurge_missing_checks"); + ok(query_int(db, "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='mysql_aws_aurora_hostgroups_v2010'") == 1, + "mysql_aws_aurora_hostgroups: v2.0.10 table is retained as v2010"); +} + // ============================================================================ // disk_upgrade_pgsql_replication_hostgroups() tests // ============================================================================ @@ -549,7 +580,7 @@ static void test_mysql_servers_upgrade_multiple_rows_with_fixes() { // ============================================================================ int main() { - plan(60); + plan(67); test_init_minimal(); // scheduler tests @@ -571,6 +602,7 @@ int main() { // mysql_servers tests test_mysql_servers_upgrade_from_v1_1_0(); test_mysql_servers_no_upgrade_needed(); + test_aurora_hostgroups_upgrade_from_v2_0_10(); // pgsql_replication_hostgroups tests test_pgsql_repl_hg_upgrade_from_v3_0_1(); diff --git a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp new file mode 100644 index 0000000000..ad078e4cd0 --- /dev/null +++ b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp @@ -0,0 +1,242 @@ +/** + * @file aurora_bgd_config_unit-t.cpp + * @brief Aurora blue/green hostgroup schema and LOAD validation contracts. + */ + +#include "tap.h" +#include "test_globals.h" +#include "test_init.h" + +#include "cpp.h" +#include "MySQL_HostGroups_Manager.h" +#include "ProxySQL_Admin_Tables_Definitions.h" + +#include +#include +#include + +static std::string query_string(SQLite3DB* db, const char* query) { + char* error = nullptr; + SQLite3_result* result = db->execute_statement(query, &error); + std::string value; + if (result && result->rows_count > 0 && result->rows[0]->fields[0]) { + value = result->rows[0]->fields[0]; + } + free(error); + delete result; + return value; +} + +static int query_int(SQLite3DB* db, const char* query) { + return db->return_one_int(query); +} + +static SQLite3DB* make_database() { + SQLite3DB* db = new SQLite3DB(); + db->open((char*)":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + return db; +} + +static const char* const AURORA_COLUMNS[] = { + "writer_hostgroup", "reader_hostgroup", "green_writer_hostgroup", "green_reader_hostgroup", + "active", "aurora_port", "domain_name", "max_lag_ms", "check_interval_ms", + "check_timeout_ms", "writer_is_also_reader", "new_reader_weight", "add_lag_ms", + "min_lag_ms", "lag_num_checks", "autopurge_missing_checks", "comment" +}; + +static SQLite3_result* make_candidate(bool legacy = false) { + SQLite3_result* result = new SQLite3_result(legacy ? 15 : 17); + for (size_t i = 0; i < 17; ++i) { + if (legacy && (i == 2 || i == 3)) { + continue; + } + result->add_column_definition(SQLITE_TEXT, AURORA_COLUMNS[i]); + } + return result; +} + +static void add_candidate_row( + SQLite3_result* result, + int writer, + int reader, + const char* green_writer, + const char* green_reader, + bool active, + const char* comment = "test" +) { + const std::string writer_value = std::to_string(writer); + const std::string reader_value = std::to_string(reader); + const char* fields[] = { + writer_value.c_str(), reader_value.c_str(), green_writer, green_reader, + active ? "1" : "0", "3306", ".cluster.example", "600000", "1000", "800", + "0", "1", "30", "30", "1", "0", comment + }; + result->add_row(fields); +} + +static void add_legacy_candidate_row(SQLite3_result* result, int writer, int reader) { + const std::string writer_value = std::to_string(writer); + const std::string reader_value = std::to_string(reader); + const char* fields[] = { + writer_value.c_str(), reader_value.c_str(), "1", "3306", ".legacy.example", "600000", + "1000", "800", "0", "1", "30", "30", "1", "0", "legacy" + }; + result->add_row(fields); +} + +static bool contains_error(const std::vector& errors, const std::string& first, const std::string& second = "") { + for (const std::string& error : errors) { + if (error.find(first) != std::string::npos && + (second.empty() || error.find(second) != std::string::npos)) { + return true; + } + } + return false; +} + +static bool contains_writer(SQLite3_result* result, int writer) { + for (SQLite3_row* row : result->rows) { + if (row->fields[0] && atoi(row->fields[0]) == writer) { + return true; + } + } + return false; +} + +static void test_schema_contract() { + SQLite3DB* db = make_database(); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS); + + const std::string configured_columns = + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment"; + const std::string runtime_columns = configured_columns + ",bgd_status"; + + ok(query_string(db, "SELECT GROUP_CONCAT(name, ',') FROM pragma_table_info('mysql_aws_aurora_hostgroups')") == configured_columns, + "Aurora configured columns follow the BGD contract order"); + ok(query_string(db, "SELECT GROUP_CONCAT(name, ',') FROM pragma_table_info('runtime_mysql_aws_aurora_hostgroups')") == runtime_columns, + "Aurora runtime columns append bgd_status"); + ok(query_string(db, "SELECT dflt_value FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='green_writer_hostgroup'") == "NULL", + "configured green_writer_hostgroup defaults to NULL"); + ok(query_string(db, "SELECT dflt_value FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='green_reader_hostgroup'") == "NULL", + "configured green_reader_hostgroup defaults to NULL"); + ok(query_string(db, "SELECT dflt_value FROM pragma_table_info('runtime_mysql_aws_aurora_hostgroups') WHERE name='green_writer_hostgroup'") == "NULL", + "runtime green_writer_hostgroup defaults to NULL"); + ok(query_string(db, "SELECT dflt_value FROM pragma_table_info('runtime_mysql_aws_aurora_hostgroups') WHERE name='green_reader_hostgroup'") == "NULL", + "runtime green_reader_hostgroup defaults to NULL"); + ok(query_string(db, "SELECT dflt_value FROM pragma_table_info('runtime_mysql_aws_aurora_hostgroups') WHERE name='bgd_status'") == "'NONE'", + "runtime bgd_status defaults to NONE"); + ok(query_int(db, "SELECT COUNT(*) FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='bgd_status'") == 0, + "bgd_status is absent from configured Aurora rows"); + ok(!db->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,domain_name) VALUES (1,2,-1,3,'.negative.example')"), + "configured schema rejects a negative green writer hostgroup"); + ok(!db->execute("INSERT INTO runtime_mysql_aws_aurora_hostgroups (writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,domain_name) VALUES (1,2,3,-1,'.negative.example')"), + "runtime schema rejects a negative green reader hostgroup"); + delete db; + + SQLite3DB* hgm_db = make_database(); + hgm_db->execute(MYHGM_MYSQL_AWS_AURORA_HOSTGROUPS); + ok(query_string(hgm_db, "SELECT GROUP_CONCAT(name, ',') FROM pragma_table_info('mysql_aws_aurora_hostgroups')") == runtime_columns, + "HGM Aurora columns match the runtime contract"); + ok(query_string(hgm_db, "SELECT dflt_value FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='bgd_status'") == "'NONE'", + "HGM bgd_status defaults to NONE"); + delete hgm_db; +} + +static void test_row_validation() { + SQLite3_result* candidate = make_candidate(); + add_candidate_row(candidate, 10, 20, nullptr, nullptr, true); + add_candidate_row(candidate, 30, 40, "31", "41", true); + add_candidate_row(candidate, 50, 60, "51", nullptr, true); + add_candidate_row(candidate, 70, 80, "70", "81", true); + add_candidate_row(candidate, 90, 100, "91", "101", true); + add_candidate_row(candidate, 110, 120, "111", "90", true); + add_candidate_row(candidate, 130, 140, nullptr, nullptr, true); + + std::vector errors; + SQLite3_result* filtered = validate_and_filter_aws_aurora_hostgroups(candidate, errors); + ok(filtered->rows_count == 3, "mixed LOAD publishes only its three valid Aurora rows"); + ok(contains_writer(filtered, 10), "paired NULL green hostgroups are valid"); + ok(contains_writer(filtered, 30), "four distinct configured hostgroups are valid"); + ok(contains_writer(filtered, 130), "an unrelated valid row survives other validation failures"); + ok(contains_error(errors, "writer_hostgroup=50", "green_reader_hostgroup=NULL"), + "mixed NULL validation identifies the rejected writer and fields"); + ok(contains_error(errors, "writer_hostgroup=70", "conflicting fields"), + "same-row overlap identifies the rejected writer and fields"); + ok(contains_error(errors, "writer_hostgroup=90", "writer_hostgroup=110"), + "first active row in a cross-row conflict identifies the other writer"); + ok(contains_error(errors, "writer_hostgroup=110", "writer_hostgroup=90"), + "second active row in a cross-row conflict identifies the other writer"); + delete filtered; + delete candidate; +} + +static void test_inactive_cross_row_validation() { + SQLite3_result* candidate = make_candidate(); + add_candidate_row(candidate, 150, 160, nullptr, nullptr, true); + add_candidate_row(candidate, 170, 180, "171", "150", false); + + std::vector errors; + SQLite3_result* filtered = validate_and_filter_aws_aurora_hostgroups(candidate, errors); + ok(filtered->rows_count == 1, "an inactive row conflicting with an active row is isolated"); + ok(contains_writer(filtered, 150), "the active owner survives an inactive conflicting row"); + ok(contains_error(errors, "writer_hostgroup=170", "writer_hostgroup=150"), + "inactive conflict reports the active owner"); + ok(!contains_error(errors, "mysql_aws_aurora_hostgroups writer_hostgroup=150 rejected:"), + "inactive roles do not invalidate the active owner"); + delete filtered; + delete candidate; +} + +static void test_legacy_projection_normalization() { + SQLite3_result* candidate = make_candidate(true); + add_legacy_candidate_row(candidate, 190, 200); + + std::vector errors; + SQLite3_result* filtered = validate_and_filter_aws_aurora_hostgroups(candidate, errors); + ok(filtered->columns == 17 && filtered->rows_count == 1, + "legacy Aurora projection is normalized to the configured BGD shape"); + ok(filtered->rows[0]->fields[2] == nullptr && filtered->rows[0]->fields[3] == nullptr, + "legacy projection normalizes both green hostgroups to SQL NULL"); + ok(errors.empty(), "legacy Aurora projection remains valid"); + delete filtered; + delete candidate; +} + +static void test_invalid_replacement_removes_previous_row() { + SQLite3_result* initial = make_candidate(); + add_candidate_row(initial, 210, 220, "211", "221", true); + std::vector initial_errors; + SQLite3_result* initial_filtered = validate_and_filter_aws_aurora_hostgroups(initial, initial_errors); + ok(initial_filtered->rows_count == 1, "initial valid Aurora row is publishable"); + + SQLite3_result* replacement = make_candidate(); + add_candidate_row(replacement, 210, 220, "211", nullptr, true); + std::vector replacement_errors; + SQLite3_result* replacement_filtered = validate_and_filter_aws_aurora_hostgroups(replacement, replacement_errors); + ok(replacement_filtered->rows_count == 0, + "invalid replacement omits a previously published writer from the atomic replacement"); + ok(contains_error(replacement_errors, "writer_hostgroup=210", "must both be NULL"), + "invalid replacement reports its former writer owner"); + + delete replacement_filtered; + delete replacement; + delete initial_filtered; + delete initial; +} + +int main() { + plan(30); + test_init_minimal(); + + test_schema_contract(); // 12 + test_row_validation(); // 8 + test_inactive_cross_row_validation(); // 4 + test_legacy_projection_normalization(); // 3 + test_invalid_replacement_removes_previous_row(); // 3 + + test_cleanup_minimal(); + return exit_status(); +} From 5a68861bfc8bc60820fbe1ed70bd88f9f6ba4e6b Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 10:05:47 +0000 Subject: [PATCH 10/34] feat: publish Aurora BGD runtime status --- include/MySQL_HostGroups_Manager.h | 14 +- lib/MySQL_HostGroups_Manager.cpp | 218 ++++++++++++++---- lib/ProxySQL_Admin.cpp | 29 ++- .../tests/unit/aurora_bgd_config_unit-t.cpp | 140 ++++++++++- 4 files changed, 347 insertions(+), 54 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index fabdb6b5a6..4d1c56aafd 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -360,6 +360,8 @@ class AWS_Aurora_Info { public: int writer_hostgroup; int reader_hostgroup; + int green_writer_hostgroup; + int green_reader_hostgroup; int aurora_port; int max_lag_ms; int add_lag_ms; @@ -376,8 +378,8 @@ class AWS_Aurora_Info { char * comment; bool active; bool active_; - AWS_Aurora_Info(int w, int r, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c); - bool update(int r, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c); + AWS_Aurora_Info(int w, int r, int gw, int gr, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c); + bool update(int r, int gw, int gr, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c); ~AWS_Aurora_Info(); AWS_Aurora_Info(const AWS_Aurora_Info&) = delete; AWS_Aurora_Info& operator=(const AWS_Aurora_Info&) = delete; @@ -786,6 +788,7 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { public: // Friend declaration for WebUI monitoring metrics collector friend class ProxySQL::Monitoring::MetricsCollector; + friend class TestAuroraBGDRuntime; std::mutex galera_set_writer_mutex; /** @@ -1120,6 +1123,13 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * @param status AWS_RDS_BGD_Status underlying value. */ void aws_rds_bgd_set_runtime_status(unsigned int writer_hg, int status); + /** + * @brief Publish the node-local Aurora BGD state for one runtime row. + * + * Invalid status strings and writer hostgroups not present at runtime are + * ignored. Configuration reloads do not write this column. + */ + void update_aws_aurora_bgd_status(int writer_hostgroup, const std::string& bgd_status); /** * @brief Aligns the runtime 'mysql_servers' table + checksums with the server state in MyHGM. * diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index be580d71c6..1a1c0b01c5 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -9,6 +9,7 @@ using json = nlohmann::json; #include "MySQL_PreparedStatement.h" #include "MySQL_Data_Stream.h" +#include #include #include #include @@ -1737,8 +1738,6 @@ bool MySQL_HostGroups_Manager::commit( // AWS Aurora if (incoming_aws_aurora_hostgroups) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_aws_aurora_hostgroups\n"); - mydb->execute("DELETE FROM mysql_aws_aurora_hostgroups"); generate_mysql_aws_aurora_hostgroups_table(); } @@ -2452,8 +2451,8 @@ void MySQL_HostGroups_Manager::update_table_mysql_servers_for_monitor(bool lock) SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql(const string& name) { char * query = (char *)""; if (name == "mysql_aws_aurora_hostgroups") { - query=(char *)"SELECT writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," - "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment FROM mysql_aws_aurora_hostgroups"; + query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," + "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment,bgd_status FROM mysql_aws_aurora_hostgroups"; } else if (name == "mysql_aws_rds_bgd_hostgroups") { query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,writer_is_also_reader," "check_interval_ms,check_timeout_ms,comment,auto_generated,status FROM mysql_aws_rds_bgd_hostgroups"; @@ -4181,6 +4180,38 @@ void MySQL_HostGroups_Manager::aws_rds_bgd_set_runtime_status(unsigned int write wrunlock(); } +void MySQL_HostGroups_Manager::update_aws_aurora_bgd_status(int writer_hostgroup, const std::string& bgd_status) { + static const std::array valid_statuses { + "NONE", + "AVAILABLE", + "SWITCHOVER_INITIATED", + "SWITCHOVER_IN_PROGRESS", + "SWITCHOVER_IN_POST_PROCESSING", + "SWITCHOVER_COMPLETED" + }; + + bool valid = std::any_of(valid_statuses.begin(), valid_statuses.end(), [&bgd_status](const char* status) { + return bgd_status == status; + }); + if (!valid) { + proxy_error("Invalid AWS Aurora BGD runtime status '%s' for writer hostgroup %d\n", + bgd_status.c_str(), writer_hostgroup); + return; + } + + wrlock(); + const char* query = "UPDATE mysql_aws_aurora_hostgroups SET bgd_status=?1 WHERE writer_hostgroup=?2"; + auto [prepare_rc, statement_unique] = mydb->prepare_v2(query); + ASSERT_SQLITE_OK(prepare_rc, mydb); + sqlite3_stmt* statement = statement_unique.get(); + int rc = (*proxy_sqlite3_bind_text)(statement, 1, bgd_status.c_str(), -1, SQLITE_TRANSIENT); + ASSERT_SQLITE_OK(rc, mydb); + rc = (*proxy_sqlite3_bind_int64)(statement, 2, writer_hostgroup); + ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(statement); + wrunlock(); +} + /** * @brief Aligns the runtime 'mysql_servers' table + checksums with the server state in MyHGM. * @@ -6223,13 +6254,15 @@ SQLite3_result * MySQL_HostGroups_Manager::get_mysql_errors(bool reset) { return result; } -AWS_Aurora_Info::AWS_Aurora_Info(int w, int r, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c) { +AWS_Aurora_Info::AWS_Aurora_Info(int w, int r, int gw, int gr, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c) { comment=NULL; if (c) { comment=strdup(c); } writer_hostgroup=w; reader_hostgroup=r; + green_writer_hostgroup=gw; + green_reader_hostgroup=gr; max_lag_ms=maxl; add_lag_ms=al; min_lag_ms=minl; @@ -6257,13 +6290,21 @@ AWS_Aurora_Info::~AWS_Aurora_Info() { } } -bool AWS_Aurora_Info::update(int r, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c) { +bool AWS_Aurora_Info::update(int r, int gw, int gr, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c) { bool ret=false; active_=true; if (reader_hostgroup!=r) { reader_hostgroup=r; ret=true; } + if (green_writer_hostgroup!=gw) { + green_writer_hostgroup=gw; + ret=true; + } + if (green_reader_hostgroup!=gr) { + green_reader_hostgroup=gr; + ret=true; + } if (max_lag_ms!=maxl) { max_lag_ms=maxl; ret=true; @@ -6613,14 +6654,81 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { if (incoming_aws_aurora_hostgroups==NULL) { return; } + + struct RuntimeRow { + int reader_hostgroup; + std::string bgd_status; + }; + + std::map runtime_rows; + std::map incoming_reader_hostgroups; + for (SQLite3_row* row : incoming_aws_aurora_hostgroups->rows) { + incoming_reader_hostgroups.emplace(atoi(row->fields[0]), atoi(row->fields[1])); + } + + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + const char* select_query = + "SELECT writer_hostgroup, reader_hostgroup, bgd_status FROM mysql_aws_aurora_hostgroups"; + mydb->execute_statement(select_query, &error, &cols, &affected_rows, &resultset); + if (error) { + proxy_error("Error on %s : %s\n", select_query, error); + free(error); + assert(0); + } + if (resultset) { + for (SQLite3_row* row : resultset->rows) { + runtime_rows.emplace( + atoi(row->fields[0]), + RuntimeRow {atoi(row->fields[1]), row->fields[2] ? row->fields[2] : "NONE"} + ); + } + delete resultset; + } + int rc; - //sqlite3 *mydb3=mydb->get_db(); - char *query=(char *)"INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms," - "check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES " - "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)"; - auto [rc1, statement_unique] = mydb->prepare_v2(query); - ASSERT_SQLITE_OK(rc1, mydb); - sqlite3_stmt *statement = statement_unique.get(); + const char* delete_query = "DELETE FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=?1"; + auto [delete_rc, delete_statement_unique] = mydb->prepare_v2(delete_query); + ASSERT_SQLITE_OK(delete_rc, mydb); + sqlite3_stmt* delete_statement = delete_statement_unique.get(); + + // Delete missing deployments and release a changed reader hostgroup before inserting its replacement. + for (const auto& [writer_hostgroup, runtime_row] : runtime_rows) { + auto incoming_it = incoming_reader_hostgroups.find(writer_hostgroup); + bool removed = incoming_it == incoming_reader_hostgroups.end(); + bool reader_changed = !removed && incoming_it->second != runtime_row.reader_hostgroup; + if (!removed && !reader_changed) { + continue; + } + rc=(*proxy_sqlite3_bind_int64)(delete_statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(delete_statement); + rc=(*proxy_sqlite3_clear_bindings)(delete_statement); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(delete_statement); ASSERT_SQLITE_OK(rc, mydb); + } + + const char* update_query = + "UPDATE mysql_aws_aurora_hostgroups SET " + "reader_hostgroup=?1, green_writer_hostgroup=?2, green_reader_hostgroup=?3, active=?4, " + "aurora_port=?5, domain_name=?6, max_lag_ms=?7, check_interval_ms=?8, check_timeout_ms=?9, " + "writer_is_also_reader=?10, new_reader_weight=?11, add_lag_ms=?12, min_lag_ms=?13, " + "lag_num_checks=?14, autopurge_missing_checks=?15, comment=?16 " + "WHERE writer_hostgroup=?17"; + auto [update_rc, update_statement_unique] = mydb->prepare_v2(update_query); + ASSERT_SQLITE_OK(update_rc, mydb); + sqlite3_stmt* update_statement = update_statement_unique.get(); + + const char* insert_query = + "INSERT INTO mysql_aws_aurora_hostgroups(" + "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, " + "aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, " + "new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment, bgd_status" + ") VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18)"; + auto [insert_rc, insert_statement_unique] = mydb->prepare_v2(insert_query); + ASSERT_SQLITE_OK(insert_rc, mydb); + sqlite3_stmt* insert_statement = insert_statement_unique.get(); + proxy_info("New mysql_aws_aurora_hostgroups table\n"); pthread_mutex_lock(&AWS_Aurora_Info_mutex); for (std::map::iterator it1 = AWS_Aurora_Info_Map.begin() ; it1 != AWS_Aurora_Info_Map.end(); ++it1) { @@ -6632,8 +6740,10 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { SQLite3_row *r=*it; int writer_hostgroup=atoi(r->fields[0]); int reader_hostgroup=atoi(r->fields[1]); - const char* green_writer_hostgroup = r->fields[2]; - const char* green_reader_hostgroup = r->fields[3]; + const char* green_writer_value = r->fields[2]; + const char* green_reader_value = r->fields[3]; + int green_writer_hostgroup = green_writer_value ? atoi(green_writer_value) : -1; + int green_reader_hostgroup = green_reader_value ? atoi(green_reader_value) : -1; int active=atoi(r->fields[4]); int aurora_port = atoi(r->fields[5]); int max_lag_ms = atoi(r->fields[7]); @@ -6645,37 +6755,54 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { int min_lag_ms = atoi(r->fields[13]); int lag_num_checks = atoi(r->fields[14]); int autopurge_missing_checks = atoi(r->fields[15]); - proxy_info("Loading AWS Aurora info for (%d,%d,%s,%s,%s,%d,\"%s\",%d,%d,%d,%d,%d,%d,%d,\"%s\")\n", writer_hostgroup,reader_hostgroup, - green_writer_hostgroup ? green_writer_hostgroup : "NULL", green_reader_hostgroup ? green_reader_hostgroup : "NULL", + proxy_info("Loading AWS Aurora info for (%d,%d,%d,%d,%s,%d,\"%s\",%d,%d,%d,%d,%d,%d,%d,\"%s\")\n", writer_hostgroup,reader_hostgroup, + green_writer_hostgroup, green_reader_hostgroup, (active ? "on" : "off"),aurora_port,r->fields[6],max_lag_ms,add_lag_ms,min_lag_ms,lag_num_checks, check_interval_ms,check_timeout_ms,autopurge_missing_checks,r->fields[16]); - rc=(*proxy_sqlite3_bind_int64)(statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 2, reader_hostgroup); ASSERT_SQLITE_OK(rc, mydb); - if (green_writer_hostgroup) { - rc=(*proxy_sqlite3_bind_int64)(statement, 3, atoi(green_writer_hostgroup)); + + auto runtime_it = runtime_rows.find(writer_hostgroup); + bool update_existing = + runtime_it != runtime_rows.end() && + runtime_it->second.reader_hostgroup == reader_hostgroup; + sqlite3_stmt* statement = update_existing ? update_statement : insert_statement; + int field_offset = update_existing ? 0 : 1; + if (!update_existing) { + rc=(*proxy_sqlite3_bind_int64)(statement, 1, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + } + rc=(*proxy_sqlite3_bind_int64)(statement, 1 + field_offset, reader_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + if (green_writer_hostgroup >= 0) { + rc=(*proxy_sqlite3_bind_int64)(statement, 2 + field_offset, green_writer_hostgroup); } else { - rc=(*proxy_sqlite3_bind_null)(statement, 3); + rc=(*proxy_sqlite3_bind_null)(statement, 2 + field_offset); } ASSERT_SQLITE_OK(rc, mydb); - if (green_reader_hostgroup) { - rc=(*proxy_sqlite3_bind_int64)(statement, 4, atoi(green_reader_hostgroup)); + if (green_reader_hostgroup >= 0) { + rc=(*proxy_sqlite3_bind_int64)(statement, 3 + field_offset, green_reader_hostgroup); } else { - rc=(*proxy_sqlite3_bind_null)(statement, 4); + rc=(*proxy_sqlite3_bind_null)(statement, 3 + field_offset); } ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 5, active); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 6, aurora_port); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_text)(statement, 7, r->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 8, max_lag_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 9, check_interval_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 10, check_timeout_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 11, writer_is_also_reader); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 12, new_reader_weight); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 13, add_lag_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 14, min_lag_ms); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 15, lag_num_checks); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_int64)(statement, 16, autopurge_missing_checks); ASSERT_SQLITE_OK(rc, mydb); - rc=(*proxy_sqlite3_bind_text)(statement, 17, r->fields[16], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 4 + field_offset, active); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 5 + field_offset, aurora_port); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 6 + field_offset, r->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 7 + field_offset, max_lag_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 8 + field_offset, check_interval_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 9 + field_offset, check_timeout_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 10 + field_offset, writer_is_also_reader); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 11 + field_offset, new_reader_weight); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 12 + field_offset, add_lag_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 13 + field_offset, min_lag_ms); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 14 + field_offset, lag_num_checks); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 15 + field_offset, autopurge_missing_checks); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 16 + field_offset, r->fields[16], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + if (update_existing) { + rc=(*proxy_sqlite3_bind_int64)(statement, 17, writer_hostgroup); ASSERT_SQLITE_OK(rc, mydb); + } else { + const std::string bgd_status = runtime_it == runtime_rows.end() + ? std::string("NONE") + : runtime_it->second.bgd_status; + rc=(*proxy_sqlite3_bind_text)(statement, 18, bgd_status.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + } SAFE_SQLITE3_STEP2(statement); rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mydb); @@ -6686,12 +6813,12 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { if (it2!=AWS_Aurora_Info_Map.end()) { info=it2->second; bool changed=false; - changed=info->update(reader_hostgroup, aurora_port, r->fields[6], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[16]); + changed=info->update(reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, aurora_port, r->fields[6], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[16]); if (changed) { //info->need_converge=true; } } else { - info=new AWS_Aurora_Info(writer_hostgroup, reader_hostgroup, aurora_port, r->fields[6], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[16]); + info=new AWS_Aurora_Info(writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, aurora_port, r->fields[6], max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks, check_interval_ms, check_timeout_ms, (bool)active, writer_is_also_reader, new_reader_weight, autopurge_missing_checks, r->fields[16]); //info->need_converge=true; AWS_Aurora_Info_Map.insert(AWS_Aurora_Info_Map.begin(), std::pair(writer_hostgroup,info)); } @@ -6713,9 +6840,11 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { // it is now time to build a new structure in Monitor - pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); - update_aws_aurora_hosts_monitor_resultset(false); - pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); + if (GloMyMon) { + pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); + update_aws_aurora_hosts_monitor_resultset(false); + pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); + } pthread_mutex_unlock(&AWS_Aurora_Info_mutex); } @@ -7414,7 +7543,8 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_reader(int _whid, int _rhid const char SELECT_AWS_AURORA_SERVERS_FOR_MONITOR[] { "SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl, max_lag_ms, check_interval_ms," - " check_timeout_ms, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, domain_name FROM mysql_servers" + " check_timeout_ms, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, domain_name," + " green_writer_hostgroup, green_reader_hostgroup FROM mysql_servers" " JOIN mysql_aws_aurora_hostgroups ON" " hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3)" " GROUP BY writer_hostgroup, hostname, port" diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index edf310dc6f..087b786d28 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -7539,9 +7539,9 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { char *query=NULL; if (_runtime) { - query=(char *)"INSERT INTO runtime_mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)"; + query=(char *)"INSERT INTO runtime_mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment,bgd_status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18)"; } else { - query=(char *)"INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)"; + query=(char *)"INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)"; } auto [rc1, statement_unique] = admindb->prepare_v2(query); @@ -7553,11 +7553,21 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { SQLite3_row *r=*it; rc=(*proxy_sqlite3_bind_int64)(statement, 1, atoi(r->fields[0])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 2, atoi(r->fields[1])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 3, atoi(r->fields[2])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 4, atoi(r->fields[3])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement, 5, r->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + if (r->fields[2]) { + rc=(*proxy_sqlite3_bind_int64)(statement, 3, atoi(r->fields[2])); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 3); + } + ASSERT_SQLITE_OK(rc, admindb); + if (r->fields[3]) { + rc=(*proxy_sqlite3_bind_int64)(statement, 4, atoi(r->fields[3])); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 4); + } + ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 5, atoi(r->fields[4])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 6, atoi(r->fields[5])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 7, atoi(r->fields[6])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement, 7, r->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 8, atoi(r->fields[7])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 9, atoi(r->fields[8])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 10, atoi(r->fields[9])); ASSERT_SQLITE_OK(rc, admindb); @@ -7565,7 +7575,12 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { rc=(*proxy_sqlite3_bind_int64)(statement, 12, atoi(r->fields[11])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 13, atoi(r->fields[12])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement, 14, atoi(r->fields[13])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement, 15, r->fields[14], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 15, atoi(r->fields[14])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 16, atoi(r->fields[15])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement, 17, r->fields[16], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + if (_runtime) { + rc=(*proxy_sqlite3_bind_text)(statement, 18, r->fields[17], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + } SAFE_SQLITE3_STEP2(statement); rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, admindb); diff --git a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp index ad078e4cd0..e4adee5c4f 100644 --- a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp +++ b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp @@ -13,8 +13,39 @@ #include #include +#include #include +extern MySQL_HostGroups_Manager* MyHGM; + +class TestAuroraBGDRuntime { +public: + static void reload(MySQL_HostGroups_Manager* hgm, SQLite3_result* candidate) { + hgm->wrlock(); + hgm->save_incoming_mysql_table(candidate, "mysql_aws_aurora_hostgroups"); + hgm->generate_mysql_aws_aurora_hostgroups_table(); + hgm->wrunlock(); + } + + static bool green_hostgroups( + MySQL_HostGroups_Manager* hgm, + int writer_hostgroup, + int& green_writer_hostgroup, + int& green_reader_hostgroup + ) { + pthread_mutex_lock(&hgm->AWS_Aurora_Info_mutex); + auto info_it = hgm->AWS_Aurora_Info_Map.find(writer_hostgroup); + if (info_it == hgm->AWS_Aurora_Info_Map.end()) { + pthread_mutex_unlock(&hgm->AWS_Aurora_Info_mutex); + return false; + } + green_writer_hostgroup = info_it->second->green_writer_hostgroup; + green_reader_hostgroup = info_it->second->green_reader_hostgroup; + pthread_mutex_unlock(&hgm->AWS_Aurora_Info_mutex); + return true; + } +}; + static std::string query_string(SQLite3DB* db, const char* query) { char* error = nullptr; SQLite3_result* result = db->execute_statement(query, &error); @@ -31,6 +62,23 @@ static int query_int(SQLite3DB* db, const char* query) { return db->return_one_int(query); } +static std::string hgm_query_string(const char* query) { + char* error = nullptr; + SQLite3_result* result = MyHGM->execute_query(const_cast(query), &error); + std::string value; + if (!error && result && result->rows_count > 0 && result->rows[0]->fields[0]) { + value = result->rows[0]->fields[0]; + } + free(error); + delete result; + return value; +} + +static int hgm_query_int(const char* query) { + const std::string value = hgm_query_string(query); + return value.empty() ? 0 : atoi(value.c_str()); +} + static SQLite3DB* make_database() { SQLite3DB* db = new SQLite3DB(); db->open((char*)":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); @@ -227,8 +275,94 @@ static void test_invalid_replacement_removes_previous_row() { delete initial; } +static void test_runtime_ownership_and_status() { + SQLite3_result* initial = make_candidate(); + add_candidate_row(initial, 300, 310, "301", "311", true); + add_candidate_row(initial, 320, 330, nullptr, nullptr, true); + TestAuroraBGDRuntime::reload(MyHGM, initial); + + int green_writer = 0; + int green_reader = 0; + ok(TestAuroraBGDRuntime::green_hostgroups(MyHGM, 300, green_writer, green_reader) && + green_writer == 301 && green_reader == 311, + "Aurora runtime info owns configured green hostgroups"); + ok(TestAuroraBGDRuntime::green_hostgroups(MyHGM, 320, green_writer, green_reader) && + green_writer == -1 && green_reader == -1, + "SQL NULL green hostgroups use the -1 runtime sentinel"); + ok(hgm_query_string( + "SELECT GROUP_CONCAT(bgd_status, ',') FROM " + "(SELECT bgd_status FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup)" + ) == "NONE,NONE", "new Aurora runtime rows start in NONE"); + + SQLite3_result* dump = MyHGM->dump_table_mysql("mysql_aws_aurora_hostgroups"); + ok(dump && dump->columns == 18, "Aurora runtime dump includes configured fields and bgd_status"); + delete dump; + + MyHGM->update_aws_aurora_bgd_status(300, "AVAILABLE"); + ok(hgm_query_string( + "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" + ) == "AVAILABLE", "Aurora BGD status API publishes an accepted state"); + const char* accepted_statuses[] = { + "NONE", + "AVAILABLE", + "SWITCHOVER_INITIATED", + "SWITCHOVER_IN_PROGRESS", + "SWITCHOVER_IN_POST_PROCESSING", + "SWITCHOVER_COMPLETED" + }; + bool accepted_all_statuses = true; + for (const char* status : accepted_statuses) { + MyHGM->update_aws_aurora_bgd_status(300, status); + accepted_all_statuses = accepted_all_statuses && hgm_query_string( + "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" + ) == status; + } + ok(accepted_all_statuses, "Aurora BGD status API accepts the complete state vocabulary"); + MyHGM->update_aws_aurora_bgd_status(300, "AVAILABLE"); + + SQLite3_result* unrelated_reload = make_candidate(); + add_candidate_row(unrelated_reload, 300, 310, "302", "312", true, "reloaded"); + TestAuroraBGDRuntime::reload(MyHGM, unrelated_reload); + ok(hgm_query_string( + "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" + ) == "AVAILABLE", "configuration reload preserves the runtime BGD status"); + ok(TestAuroraBGDRuntime::green_hostgroups(MyHGM, 300, green_writer, green_reader) && + green_writer == 302 && green_reader == 312, + "configuration reload updates runtime green hostgroups"); + + MyHGM->update_aws_aurora_bgd_status(300, "NOT_A_BGD_STATUS"); + ok(hgm_query_string( + "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" + ) == "AVAILABLE", "invalid Aurora BGD status leaves runtime state unchanged"); + + SQLite3_result* concurrent_reload = make_candidate(); + add_candidate_row(concurrent_reload, 300, 310, "303", "313", true, "concurrent"); + std::thread reload_thread([concurrent_reload]() { + TestAuroraBGDRuntime::reload(MyHGM, concurrent_reload); + }); + std::thread status_thread([]() { + MyHGM->update_aws_aurora_bgd_status(300, "SWITCHOVER_IN_PROGRESS"); + }); + reload_thread.join(); + status_thread.join(); + ok(hgm_query_string( + "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" + ) == "SWITCHOVER_IN_PROGRESS", "reload and status publication serialize without losing state"); + ok(TestAuroraBGDRuntime::green_hostgroups(MyHGM, 300, green_writer, green_reader) && + green_writer == 303 && green_reader == 313, + "serialized reload publishes its configured values"); + + SQLite3_result* empty_reload = make_candidate(); + TestAuroraBGDRuntime::reload(MyHGM, empty_reload); + ok(hgm_query_int( + "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" + ) == 0, "removing an Aurora deployment removes its runtime status row"); + ok(!TestAuroraBGDRuntime::green_hostgroups(MyHGM, 300, green_writer, green_reader), + "removing an Aurora deployment removes its runtime info"); +} + int main() { - plan(30); + plan(44); test_init_minimal(); test_schema_contract(); // 12 @@ -237,6 +371,10 @@ int main() { test_legacy_projection_normalization(); // 3 test_invalid_replacement_removes_previous_row(); // 3 + ok(test_init_hostgroups() == 0, "test_init_hostgroups() succeeds"); // 1 + test_runtime_ownership_and_status(); // 13 + test_cleanup_hostgroups(); + test_cleanup_minimal(); return exit_status(); } From e670d32777847e7a7807961b96c5b05828aec786 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 10:16:10 +0000 Subject: [PATCH 11/34] feat: persist Aurora BGD configuration fields --- include/proxysql_admin.h | 1 + lib/ProxySQL_Admin.cpp | 14 +++- lib/ProxySQL_Config.cpp | 52 ++++++++---- .../tests/unit/admin_disk_upgrade_unit-t.cpp | 84 ++++++++++++++++++- .../tests/unit/aurora_bgd_config_unit-t.cpp | 55 +++++++++++- test/tap/tests/unit/config_write_unit-t.cpp | 66 ++++++++++++++- 6 files changed, 252 insertions(+), 20 deletions(-) diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index d642b2756e..5529cb0d62 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -322,6 +322,7 @@ struct FlushVariableStats { class ProxySQL_Admin { friend class TestDiskUpgrade; + friend class TestAuroraBGDRuntime; private: volatile int main_shutdown; diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 087b786d28..f73dae3178 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -189,6 +189,11 @@ static unordered_map&> module_tablenames = { { "pgsql_query_rules", pgsql_query_rules_tablenames }, }; +static const string mysql_aws_aurora_config_columns = + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment"; + static void BQE1(SQLite3DB *db, const vector& tbs, const string& p1, const string& p2, const string& p3) { string query; for (auto it = tbs.begin(); it != tbs.end(); it++) { @@ -197,7 +202,14 @@ static void BQE1(SQLite3DB *db, const vector& tbs, const string& p1, con db->execute(query.c_str()); } if (p2 != "" && p3 != "") { - query = p2 + *it + p3 + *it; + const size_t wildcard_pos = p3.find('*'); + if (*it == "mysql_aws_aurora_hostgroups" && wildcard_pos != string::npos) { + string projected_select = p3; + projected_select.replace(wildcard_pos, 1, mysql_aws_aurora_config_columns); + query = p2 + *it + " (" + mysql_aws_aurora_config_columns + ")" + projected_select + *it; + } else { + query = p2 + *it + p3 + *it; + } db->execute(query.c_str()); } } diff --git a/lib/ProxySQL_Config.cpp b/lib/ProxySQL_Config.cpp index 560e70c4cf..b4c7e910fb 100644 --- a/lib/ProxySQL_Config.cpp +++ b/lib/ProxySQL_Config.cpp @@ -1282,7 +1282,10 @@ int ProxySQL_Config::Write_MySQL_Servers_to_configfile(std::string& data) { sqlite_resultset = NULL; } - query=(char *)"SELECT * FROM mysql_aws_aurora_hostgroups"; + query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment " + "FROM mysql_aws_aurora_hostgroups"; admindb->execute_statement(query, &error, &cols, &affected_rows, &sqlite_resultset); if (error) { // tolerate missing table (e.g. partial schemas in unit tests or old DBs) @@ -1298,18 +1301,21 @@ int ProxySQL_Config::Write_MySQL_Servers_to_configfile(std::string& data) { data += "\t{\n"; addField(data, "writer_hostgroup", r->fields[0], ""); addField(data, "reader_hostgroup", r->fields[1], ""); - addField(data, "active", r->fields[2], ""); - addField(data, "aurora_port", r->fields[3], ""); - addField(data, "domain_name", r->fields[4]); - addField(data, "max_lag_ms", r->fields[5], ""); - addField(data, "check_interval_ms", r->fields[6], ""); - addField(data, "check_timeout_ms", r->fields[7], ""); - addField(data, "writer_is_also_reader", r->fields[8], ""); - addField(data, "new_reader_weight", r->fields[9], ""); - addField(data, "add_lag_ms", r->fields[10], ""); - addField(data, "min_lag_ms", r->fields[11], ""); - addField(data, "lag_num_checks", r->fields[12], ""); - addField(data, "comment", r->fields[13]); + addField(data, "green_writer_hostgroup", r->fields[2], ""); + addField(data, "green_reader_hostgroup", r->fields[3], ""); + addField(data, "active", r->fields[4], ""); + addField(data, "aurora_port", r->fields[5], ""); + addField(data, "domain_name", r->fields[6]); + addField(data, "max_lag_ms", r->fields[7], ""); + addField(data, "check_interval_ms", r->fields[8], ""); + addField(data, "check_timeout_ms", r->fields[9], ""); + addField(data, "writer_is_also_reader", r->fields[10], ""); + addField(data, "new_reader_weight", r->fields[11], ""); + addField(data, "add_lag_ms", r->fields[12], ""); + addField(data, "min_lag_ms", r->fields[13], ""); + addField(data, "lag_num_checks", r->fields[14], ""); + addField(data, "autopurge_missing_checks", r->fields[15], ""); + addField(data, "comment", r->fields[16]); data += "\t}"; isNext = true; @@ -1727,11 +1733,14 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile(std::string& error) { if (root.exists("mysql_aws_aurora_hostgroups")==true) { const Setting &mysql_aws_aurora_hostgroups = root["mysql_aws_aurora_hostgroups"]; int count = mysql_aws_aurora_hostgroups.getLength(); - char *q=(char *)"INSERT OR REPLACE INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment ) VALUES (%d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, %d, '%s')"; + // Green hostgroups are nullable -> passed as %s ("NULL" or an integer). + char *q=(char *)"INSERT OR REPLACE INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment ) VALUES (%d, %d, %s, %s, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, %d, '%s')"; for (i=0; i< count; i++) { const Setting &line = mysql_aws_aurora_hostgroups[i]; int writer_hostgroup; int reader_hostgroup; + int green_writer_hostgroup; + int green_reader_hostgroup; int active=1; // default int aurora_port; int max_lag_ms; @@ -1753,6 +1762,19 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile(std::string& error) { proxy_error("Admin: detected a mysql_aws_aurora_hostgroups in config file without a mandatory reader_hostgroup\n"); continue; } + std::string green_writer_str; + std::string green_reader_str; + if (line.lookupValue("green_writer_hostgroup", green_writer_hostgroup)==false) { + green_writer_str = "NULL"; + } else { + green_writer_str = std::to_string(green_writer_hostgroup); + } + if (line.lookupValue("green_reader_hostgroup", green_reader_hostgroup)==false) { + green_reader_str = "NULL"; + } else { + green_reader_str = std::to_string(green_reader_hostgroup); + } + if (line.lookupValue("active", active)==false) active=1; if (line.lookupValue("aurora_port", aurora_port)==false) aurora_port=3306; if (line.lookupValue("max_lag_ms", max_lag_ms)==false) max_lag_ms=600000; if (line.lookupValue("check_interval_ms", check_interval_ms)==false) check_interval_ms=1000; @@ -1776,7 +1798,7 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile(std::string& error) { const size_t safe_domain_len = safe_strlen(safe_domain); const size_t query_len = query_base_len + safe_comment_len + safe_domain_len + 256; // 128 vs sizeof(int)*8 char *query=(char *)l_alloc(query_len); - format_query(query, query_len, q, writer_hostgroup, reader_hostgroup, active, aurora_port, safe_domain, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, safe_comment); + format_query(query, query_len, q, writer_hostgroup, reader_hostgroup, green_writer_str.c_str(), green_reader_str.c_str(), active, aurora_port, safe_domain, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, safe_comment); //fprintf(stderr, "%s\n", query); admindb->execute(query); if (o!=o1) free(o); diff --git a/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp b/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp index d223105327..1ac5f824dd 100644 --- a/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp +++ b/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp @@ -80,6 +80,9 @@ class TestDiskUpgrade { void upgrade_rest_api_routes() { admin->disk_upgrade_rest_api_routes(); } void upgrade_mysql_query_rules() { admin->disk_upgrade_mysql_query_rules(); } void upgrade_pgsql_replication_hostgroups() { admin->disk_upgrade_pgsql_replication_hostgroups(); } + void use_config_db_as_admin_db() { admin->admindb = admin->configdb; } + void save_mysql_servers_to_disk() { admin->__insert_or_replace_disktable_select_maintable(); } + void load_mysql_servers_from_disk() { admin->__insert_or_replace_maintable_select_disktable(); } }; // --------------------------------------------------------------------------- @@ -575,12 +578,90 @@ static void test_mysql_servers_upgrade_multiple_rows_with_fixes() { ok(w2 == 500, "mysql_servers: normal weight preserved (got %d)", w2); } +static void test_aurora_hostgroups_disk_roundtrip_uses_configured_projection() { + TestDiskUpgrade t; + SQLite3DB *db = t.db(); + t.use_config_db_as_admin_db(); + + struct TableDefinition { + const char *name; + const char *definition; + }; + const TableDefinition tables[] = { + { "mysql_servers", ADMIN_SQLITE_TABLE_MYSQL_SERVERS }, + { "mysql_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS }, + { "mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS }, + { "mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS }, + { "mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS }, + { "mysql_aws_rds_bgd_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS }, + { "mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES }, + { "mysql_servers_ssl_params", ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS }, + { "mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES }, + { "mysql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_FAST_ROUTING }, + { "mysql_users", ADMIN_SQLITE_TABLE_MYSQL_USERS }, + { "mysql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_USERS }, + { "mysql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_RULES }, + { "mysql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS }, + { "global_variables", ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES }, + { "scheduler", ADMIN_SQLITE_TABLE_SCHEDULER }, + { "restapi_routes", ADMIN_SQLITE_TABLE_RESTAPI_ROUTES }, + { "proxysql_servers", ADMIN_SQLITE_TABLE_PROXYSQL_SERVERS }, + { "pgsql_servers", ADMIN_SQLITE_TABLE_PGSQL_SERVERS }, + { "pgsql_replication_hostgroups", ADMIN_SQLITE_TABLE_PGSQL_REPLICATION_HOSTGROUPS }, + { "pgsql_hostgroup_attributes", ADMIN_SQLITE_TABLE_PGSQL_HOSTGROUP_ATTRIBUTES }, + { "pgsql_servers_ssl_params", ADMIN_SQLITE_TABLE_PGSQL_SERVERS_SSL_PARAMS }, + { "pgsql_query_rules", ADMIN_SQLITE_TABLE_PGSQL_QUERY_RULES }, + { "pgsql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_PGSQL_QUERY_RULES_FAST_ROUTING }, + { "pgsql_users", ADMIN_SQLITE_TABLE_PGSQL_USERS }, + { "pgsql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_USERS }, + { "pgsql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_RULES }, + { "pgsql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS } + }; + for (const TableDefinition& table : tables) { + db->execute(table.definition); + } + db->execute("ATTACH DATABASE ':memory:' AS disk"); + for (const TableDefinition& table : tables) { + std::string query = std::string("CREATE TABLE disk.") + table.name + + " AS SELECT * FROM main." + table.name + " WHERE 0"; + db->execute(query.c_str()); + } + + db->execute( + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,domain_name,comment" + ") VALUES (600,601,602,603,'.disk.example','disk round trip')" + ); + // Model a node-local runtime field on the source. The disk projection must ignore it. + db->execute("ALTER TABLE main.mysql_aws_aurora_hostgroups ADD COLUMN bgd_status VARCHAR NOT NULL DEFAULT 'LOCAL'"); + db->execute("UPDATE main.mysql_aws_aurora_hostgroups SET bgd_status='SWITCHOVER_IN_PROGRESS'"); + + t.save_mysql_servers_to_disk(); + ok(query_string(db, + "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " + "FROM disk.mysql_aws_aurora_hostgroups WHERE writer_hostgroup=600") == "602,603", + "Aurora disk SAVE preserves both configured green hostgroups"); + ok(query_int(db, + "SELECT COUNT(*) FROM pragma_table_info('mysql_aws_aurora_hostgroups','disk') WHERE name='bgd_status'") == 0, + "Aurora disk SAVE excludes node-local bgd_status"); + + db->execute("DELETE FROM main.mysql_aws_aurora_hostgroups"); + t.load_mysql_servers_from_disk(); + ok(query_string(db, + "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " + "FROM main.mysql_aws_aurora_hostgroups WHERE writer_hostgroup=600") == "602,603", + "Aurora disk LOAD restores both configured green hostgroups"); + ok(query_string(db, + "SELECT bgd_status FROM main.mysql_aws_aurora_hostgroups WHERE writer_hostgroup=600") == "LOCAL", + "Aurora disk LOAD leaves node-local bgd_status at its local default"); +} + // ============================================================================ // main // ============================================================================ int main() { - plan(67); + plan(71); test_init_minimal(); // scheduler tests @@ -615,6 +696,7 @@ int main() { // Multi-row tests test_scheduler_upgrade_preserves_multiple_rows(); test_mysql_servers_upgrade_multiple_rows_with_fixes(); + test_aurora_hostgroups_disk_roundtrip_uses_configured_projection(); test_cleanup_minimal(); return exit_status(); diff --git a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp index e4adee5c4f..104992bf75 100644 --- a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp +++ b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp @@ -44,6 +44,38 @@ class TestAuroraBGDRuntime { pthread_mutex_unlock(&hgm->AWS_Aurora_Info_mutex); return true; } + + static SQLite3DB* materialize_aurora_table(bool runtime) { + void* memory = calloc(1, sizeof(ProxySQL_Admin)); + ProxySQL_Admin* admin = reinterpret_cast(memory); + SQLite3DB* db = new SQLite3DB(); + db->open((char*)":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + admin->admindb = db; + + if (runtime) { + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_REPLICATION_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GROUP_REPLICATION_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GALERA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_RDS_BGD_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_HOSTGROUP_ATTRIBUTES); + db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS_SSL_PARAMS); + } else { + db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); + } + + admin->save_mysql_servers_runtime_to_database(runtime); + free(admin); + return db; + } }; static std::string query_string(SQLite3DB* db, const char* query) { @@ -319,6 +351,25 @@ static void test_runtime_ownership_and_status() { } ok(accepted_all_statuses, "Aurora BGD status API accepts the complete state vocabulary"); MyHGM->update_aws_aurora_bgd_status(300, "AVAILABLE"); + SQLite3DB* configured_db = TestAuroraBGDRuntime::materialize_aurora_table(false); + ok(query_string(configured_db, + "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " + "FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "301,311", + "SAVE from runtime preserves both configured green hostgroups"); + ok(query_int(configured_db, + "SELECT COUNT(*) FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='bgd_status'") == 0, + "SAVE from runtime excludes bgd_status from configuration"); + delete configured_db; + + SQLite3DB* runtime_db = TestAuroraBGDRuntime::materialize_aurora_table(true); + ok(query_string(runtime_db, + "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " + "FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "301,311", + "runtime materialization preserves both configured green hostgroups"); + ok(query_string(runtime_db, + "SELECT bgd_status FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "AVAILABLE", + "runtime materialization includes the node-local bgd_status"); + delete runtime_db; SQLite3_result* unrelated_reload = make_candidate(); add_candidate_row(unrelated_reload, 300, 310, "302", "312", true, "reloaded"); @@ -362,7 +413,7 @@ static void test_runtime_ownership_and_status() { } int main() { - plan(44); + plan(48); test_init_minimal(); test_schema_contract(); // 12 @@ -372,7 +423,7 @@ int main() { test_invalid_replacement_removes_previous_row(); // 3 ok(test_init_hostgroups() == 0, "test_init_hostgroups() succeeds"); // 1 - test_runtime_ownership_and_status(); // 13 + test_runtime_ownership_and_status(); // 17 test_cleanup_hostgroups(); test_cleanup_minimal(); diff --git a/test/tap/tests/unit/config_write_unit-t.cpp b/test/tap/tests/unit/config_write_unit-t.cpp index cda1fac9a0..1a12a85a7d 100644 --- a/test/tap/tests/unit/config_write_unit-t.cpp +++ b/test/tap/tests/unit/config_write_unit-t.cpp @@ -470,6 +470,69 @@ static void test_write_mysql_servers_replication_hostgroups() { delete db; } +static void test_roundtrip_mysql_aws_aurora_hostgroups() { + SQLite3DB* db = create_test_db(); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); + db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); + db->execute( + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment" + ") VALUES (500,501,502,503,0,3307,'.bgd.example',321,1200,900,1,7,40,20,3,9,'aurora bgd')" + ); + db->execute( + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,domain_name,comment" + ") VALUES (510,511,NULL,NULL,'.legacy.example','ordinary aurora')" + ); + + ProxySQL_Config cfg(db); + std::string data; + int write_rc = cfg.Write_MySQL_Servers_to_configfile(data); + + ok(write_rc == 0, "RT Aurora hostgroups: configuration export succeeds"); + ok(data.find("green_writer_hostgroup=502") != std::string::npos, + "RT Aurora hostgroups: green writer is exported"); + ok(data.find("green_reader_hostgroup=503") != std::string::npos, + "RT Aurora hostgroups: green reader is exported"); + ok(data.find("active=0") != std::string::npos && + data.find("autopurge_missing_checks=9") != std::string::npos, + "RT Aurora hostgroups: later configured columns retain their values"); + ok(data.find("bgd_status") == std::string::npos, + "RT Aurora hostgroups: runtime status is absent from configuration export"); + + db->execute("DELETE FROM mysql_aws_aurora_hostgroups"); + ProxySQL_ConfigFile* cf = load_config_from_string(data); + ProxySQL_ConfigFile* saved = GloVars.confFile; + GloVars.confFile = cf; + std::string error; + int rows = cfg.Read_MySQL_Servers_from_configfile(error); + GloVars.confFile = saved; + delete cf; + + ok(rows == 2, "RT Aurora hostgroups: configuration import restores both rows (got %d)", rows); + ok(db_select_string(db, "green_writer_hostgroup || ',' || green_reader_hostgroup", + "mysql_aws_aurora_hostgroups", "writer_hostgroup=500") == "502,503", + "RT Aurora hostgroups: both green hostgroups survive the round trip"); + ok(db_select_string(db, "green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL", + "mysql_aws_aurora_hostgroups", "writer_hostgroup=510") == "1", + "RT Aurora hostgroups: omitted green hostgroups remain NULL"); + ok(db_select_string(db, "active", "mysql_aws_aurora_hostgroups", "writer_hostgroup=500") == "0", + "RT Aurora hostgroups: inactive configuration survives the round trip"); + ok(db_select_string(db, "autopurge_missing_checks", "mysql_aws_aurora_hostgroups", + "writer_hostgroup=500") == "9", + "RT Aurora hostgroups: autopurge configuration survives the round trip"); + + delete db; +} + // ============================================================ // Write_Global_Variables_to_configfile() // ============================================================ @@ -1011,7 +1074,7 @@ static void test_roundtrip_pgsql_firewall() { // ============================================================ int main() { - plan(161); // matches exact number of ok() assertions in this file + plan(171); // matches exact number of ok() assertions in this file test_init_minimal(); // MySQL side - existing + new data-driven tests @@ -1040,6 +1103,7 @@ int main() { test_write_mysql_servers_empty(); test_write_mysql_servers_with_data(); test_write_mysql_servers_replication_hostgroups(); + test_roundtrip_mysql_aws_aurora_hostgroups(); test_write_global_variables_empty(); test_write_global_variables_single_prefix(); From d085604bcceca3f269566abe6cb22f20a9080fe0 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 10:45:06 +0000 Subject: [PATCH 12/34] feat: synchronize Aurora BGD configuration --- include/ProxySQL_Cluster.hpp | 2 +- lib/Admin_Handler.cpp | 6 +- lib/ProxySQL_Cluster.cpp | 81 ++++++---- test/tap/tests/test_cluster_sync-t.cpp | 197 ++++++++++++++++--------- 4 files changed, 183 insertions(+), 103 deletions(-) diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index eaccc4bbb6..6f1e07fb6c 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -70,7 +70,7 @@ #define CLUSTER_QUERY_PGSQL_SERVERS_SSL_PARAMS "PROXY_SELECT hostname, port, username, ssl_ca, ssl_cert, ssl_key, ssl_crl, ssl_crlpath, ssl_protocol_version_range, comment FROM runtime_pgsql_servers_ssl_params ORDER BY hostname, port, username" /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_aws_aurora_hostgroups'. See top comment for details. */ -#define CLUSTER_QUERY_MYSQL_AWS_AURORA "PROXY_SELECT writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment FROM runtime_mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup" +#define CLUSTER_QUERY_MYSQL_AWS_AURORA "PROXY_SELECT writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment FROM runtime_mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup" /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_aws_rds_bgd_hostgroups'. See top comment for details. */ #define CLUSTER_QUERY_MYSQL_AWS_RDS_BGD "PROXY_SELECT writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, active, writer_is_also_reader, check_interval_ms, check_timeout_ms, comment, auto_generated, status FROM runtime_mysql_aws_rds_bgd_hostgroups WHERE auto_generated=0 ORDER BY writer_hostgroup" diff --git a/lib/Admin_Handler.cpp b/lib/Admin_Handler.cpp index 7aaa379608..0566703047 100644 --- a/lib/Admin_Handler.cpp +++ b/lib/Admin_Handler.cpp @@ -4501,7 +4501,11 @@ void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt) { (query_no_space_length==sizeof("CHECKSUM MEM MYSQL AURORA HOSTGROUPS") - 1 && !strncasecmp("CHECKSUM MEM MYSQL AURORA HOSTGROUPS", query_no_space, query_no_space_length)) || (query_no_space_length==sizeof("CHECKSUM MYSQL AURORA HOSTGROUPS") - 1 && !strncasecmp("CHECKSUM MYSQL AURORA HOSTGROUPS", query_no_space, query_no_space_length))){ - char *q=(char *)"SELECT * FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"; + char *q=(char *) + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment " + "FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"; tablename=(char *)"MYSQL AURORA HOSTGROUPS"; SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); } diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index f82e571f45..bc223ec646 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -2389,41 +2389,62 @@ void ProxySQL_Cluster::pull_mysql_servers_v2_from_peer(const mysql_servers_v2_ch proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Writing mysql_aws_aurora_hostgroups table\n"); proxy_info("Cluster: Writing mysql_aws_aurora_hostgroups table\n"); GloAdmin->admindb->execute(SQLQueries::DELETE_MYSQL_AWS_AURORA_HOSTGROUPS); - q = (char*)"INSERT INTO mysql_aws_aurora_hostgroups ( " - "writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " - "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, comment) "; - while ((row = mysql_fetch_row(results[4]))) { - int l = 0; - for (int i = 0; i < 14; i++) { - l += strlen(row[i]); - } - char* o = nullptr; - char* query = nullptr; - std::string fqs = q; - - if (row[14] != nullptr) { - fqs += "VALUES (%s, %s, %s, %s, '%s', %s, %s, %s, %s, %s, %s, %s, %s, %s, '%s')"; - o = escape_string_single_quotes(row[14], false); - query = (char*)malloc(strlen(fqs.c_str()) + l + strlen(o) + 64); - sprintf(query, fqs.c_str(), row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10], row[11], row[12], row[13], o); - // free in case of 'o' being a copy - if (o != row[14]) { - free(o); + { + const char* insert_aurora = + "INSERT INTO mysql_aws_aurora_hostgroups (" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment" + ") VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)"; + auto [prepare_rc, statement_unique] = GloAdmin->admindb->prepare_v2(insert_aurora); + ASSERT_SQLITE_OK(prepare_rc, GloAdmin->admindb); + sqlite3_stmt* statement = statement_unique.get(); + + while ((row = mysql_fetch_row(results[4]))) { + int rc=(*proxy_sqlite3_bind_int64)(statement, 1, atol(row[0])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 2, atol(row[1])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + if (row[2]) { + rc=(*proxy_sqlite3_bind_int64)(statement, 3, atol(row[2])); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 3); } - } else { - // In case of comment being null, placeholder must not have '' - fqs += "VALUES (%s, %s, %s, %s, '%s', %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"; - o = const_cast("NULL"); - query = (char*)malloc(strlen(fqs.c_str()) + l + strlen("NULL") + 64); - sprintf(query, fqs.c_str(), row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10], row[11], row[12], row[13], o); + ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + if (row[3]) { + rc=(*proxy_sqlite3_bind_int64)(statement, 4, atol(row[3])); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 4); + } + ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 5, atol(row[4])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 6, atol(row[5])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_text)(statement, 7, row[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 8, atol(row[7])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 9, atol(row[8])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 10, atol(row[9])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 11, atol(row[10])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 12, atol(row[11])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 13, atol(row[12])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 14, atol(row[13])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 15, atol(row[14])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_bind_int64)(statement, 16, atol(row[15])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + if (row[16]) { + rc=(*proxy_sqlite3_bind_text)(statement, 17, row[16], -1, SQLITE_TRANSIENT); + } else { + rc=(*proxy_sqlite3_bind_null)(statement, 17); + } + ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + SAFE_SQLITE3_STEP2(statement); + rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); } - - GloAdmin->admindb->execute(query); - free(query); } proxy_debug(PROXY_DEBUG_CLUSTER, 5, "Dumping fetched 'mysql_aws_aurora_hostgroups'\n"); proxy_info("Dumping fetched 'mysql_aws_aurora_hostgroups'\n"); - GloAdmin->admindb->execute_statement((char*)"SELECT * FROM mysql_aws_aurora_hostgroups", &error, &cols, &affected_rows, &resultset); + GloAdmin->admindb->execute_statement((char*) + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active," + "aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader," + "new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment " + "FROM mysql_aws_aurora_hostgroups", &error, &cols, &affected_rows, &resultset); resultset->dump_to_stderr(); delete resultset; diff --git a/test/tap/tests/test_cluster_sync-t.cpp b/test/tap/tests/test_cluster_sync-t.cpp index 10b82af49d..1e85d03cd7 100644 --- a/test/tap/tests/test_cluster_sync-t.cpp +++ b/test/tap/tests/test_cluster_sync-t.cpp @@ -217,6 +217,25 @@ const std::string t_debug_query = "mysql -u%s -p%s -h %s -P%d -C -e \"%s\""; using mysql_server_tuple = tuple; +struct aurora_hostgroups_row_t { + int writer_hostgroup; + int reader_hostgroup; + int green_writer_hostgroup; + int green_reader_hostgroup; + int active; + int aurora_port; + const char* domain_name; + int max_lag_ms; + int check_interval_ms; + int check_timeout_ms; + int writer_is_also_reader; + int new_reader_weight; + int add_lag_ms; + int min_lag_ms; + int lag_num_checks; + const char* comment; +}; + int check_mysql_servers_sync( const CommandLine& cl, MYSQL* proxy_admin, MYSQL* r_proxy_admin, @@ -1213,7 +1232,7 @@ int main(int, char**) { plan( // Sync tests by values - 16 + + 17 + // Module checkums tests; enabled and disabled checksums check_modules_checksums_sync__tests + (cl.use_noise ? 3 : 0) @@ -2138,14 +2157,15 @@ int main(int, char**) { // Configure 'mysql_aws_aurora_hostgroups' and check sync const char* t_insert_mysql_aws_aurora_hostgroups = "INSERT INTO mysql_aws_aurora_hostgroups ( " - "writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " + "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, " + "active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks) " - "VALUES (%d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d)"; - std::vector> insert_aws_aurora_values { - std::make_tuple(1002, 1006, 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1), - std::make_tuple(1003, 1007, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1), - std::make_tuple(1000, 1004, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1), - std::make_tuple(1001, 1005, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1), + "VALUES (%d, %d, %d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d)"; + std::vector insert_aws_aurora_values { + { 1002, 1006, 1012, 1016, 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1, nullptr }, + { 1003, 1007, 1013, 1017, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1, nullptr }, + { 1000, 1004, 1010, 1014, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1, nullptr }, + { 1001, 1005, 1011, 1015, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1, nullptr }, }; std::vector insert_mysql_aws_aurora_hostgroup_queries {}; @@ -2154,26 +2174,29 @@ int main(int, char**) { string_format( t_insert_mysql_aws_aurora_hostgroups, insert_aws_aurora_hostgroup_query, - std::get<0>(values), - std::get<1>(values), - std::get<2>(values), - std::get<3>(values), - std::get<4>(values), - std::get<5>(values), - std::get<6>(values), - std::get<7>(values), - std::get<8>(values), - std::get<9>(values), - std::get<10>(values), - std::get<11>(values), - std::get<12>(values) + values.writer_hostgroup, + values.reader_hostgroup, + values.green_writer_hostgroup, + values.green_reader_hostgroup, + values.active, + values.aurora_port, + values.domain_name, + values.max_lag_ms, + values.check_interval_ms, + values.check_timeout_ms, + values.writer_is_also_reader, + values.new_reader_weight, + values.add_lag_ms, + values.min_lag_ms, + values.lag_num_checks ); insert_mysql_aws_aurora_hostgroup_queries.push_back(insert_aws_aurora_hostgroup_query); } const char* t_select_aws_aurora_inserted_entries = "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups WHERE " - "writer_hostgroup=%d AND reader_hostgroup=%d AND active=%d AND aurora_port=%d AND domain_name='%s' " + "writer_hostgroup=%d AND reader_hostgroup=%d AND green_writer_hostgroup=%d AND green_reader_hostgroup=%d " + "AND active=%d AND aurora_port=%d AND domain_name='%s' " "AND max_lag_ms=%d AND check_interval_ms=%d AND check_timeout_ms=%d AND writer_is_also_reader=%d " "AND new_reader_weight=%d AND add_lag_ms=%d AND min_lag_ms=%d AND lag_num_checks=%d AND comment IS NULL"; std::vector select_mysql_aws_aurora_hostgroup_queries {}; @@ -2183,19 +2206,21 @@ int main(int, char**) { string_format( t_select_aws_aurora_inserted_entries, select_aws_aurora_hostgroup_query, - std::get<0>(values), - std::get<1>(values), - std::get<2>(values), - std::get<3>(values), - std::get<4>(values), - std::get<5>(values), - std::get<6>(values), - std::get<7>(values), - std::get<8>(values), - std::get<9>(values), - std::get<10>(values), - std::get<11>(values), - std::get<12>(values) + values.writer_hostgroup, + values.reader_hostgroup, + values.green_writer_hostgroup, + values.green_reader_hostgroup, + values.active, + values.aurora_port, + values.domain_name, + values.max_lag_ms, + values.check_interval_ms, + values.check_timeout_ms, + values.writer_is_also_reader, + values.new_reader_weight, + values.add_lag_ms, + values.min_lag_ms, + values.lag_num_checks ); select_mysql_aws_aurora_hostgroup_queries.push_back(select_aws_aurora_hostgroup_query); } @@ -2265,14 +2290,15 @@ int main(int, char**) { // Configure 'mysql_aws_aurora_hostgroups' and check sync const char* t_insert_mysql_aws_aurora_hostgroups = "INSERT INTO mysql_aws_aurora_hostgroups ( " - "writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " + "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, " + "active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment) " - "VALUES (%d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, '%s')"; - std::vector> insert_aws_aurora_values { - std::make_tuple(1003, 1007, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), - std::make_tuple(1001, 1005, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), - std::make_tuple(1002, 1006, 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), - std::make_tuple(1000, 1004, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), + "VALUES (%d, %d, %d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, '%s')"; + std::vector insert_aws_aurora_values { + { 1003, 1007, 1013, 1017, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup" }, + { 1001, 1005, 1011, 1015, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup" }, + { 1002, 1006, 1012, 1016, 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup" }, + { 1000, 1004, 1010, 1014, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup" }, }; std::vector insert_mysql_aws_aurora_hostgroup_queries {}; @@ -2281,27 +2307,30 @@ int main(int, char**) { string_format( t_insert_mysql_aws_aurora_hostgroups, insert_aws_aurora_hostgroup_query, - std::get<0>(values), - std::get<1>(values), - std::get<2>(values), - std::get<3>(values), - std::get<4>(values), - std::get<5>(values), - std::get<6>(values), - std::get<7>(values), - std::get<8>(values), - std::get<9>(values), - std::get<10>(values), - std::get<11>(values), - std::get<12>(values), - std::get<13>(values) + values.writer_hostgroup, + values.reader_hostgroup, + values.green_writer_hostgroup, + values.green_reader_hostgroup, + values.active, + values.aurora_port, + values.domain_name, + values.max_lag_ms, + values.check_interval_ms, + values.check_timeout_ms, + values.writer_is_also_reader, + values.new_reader_weight, + values.add_lag_ms, + values.min_lag_ms, + values.lag_num_checks, + values.comment ); insert_mysql_aws_aurora_hostgroup_queries.push_back(insert_aws_aurora_hostgroup_query); } const char* t_select_aws_aurora_inserted_entries = "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups WHERE " - "writer_hostgroup=%d AND reader_hostgroup=%d AND active=%d AND aurora_port=%d AND domain_name='%s' " + "writer_hostgroup=%d AND reader_hostgroup=%d AND green_writer_hostgroup=%d AND green_reader_hostgroup=%d " + "AND active=%d AND aurora_port=%d AND domain_name='%s' " "AND max_lag_ms=%d AND check_interval_ms=%d AND check_timeout_ms=%d AND writer_is_also_reader=%d " "AND new_reader_weight=%d AND add_lag_ms=%d AND min_lag_ms=%d AND lag_num_checks=%d AND comment='%s'"; std::vector select_mysql_aws_aurora_hostgroup_queries {}; @@ -2311,20 +2340,22 @@ int main(int, char**) { string_format( t_select_aws_aurora_inserted_entries, select_aws_aurora_hostgroup_query, - std::get<0>(values), - std::get<1>(values), - std::get<2>(values), - std::get<3>(values), - std::get<4>(values), - std::get<5>(values), - std::get<6>(values), - std::get<7>(values), - std::get<8>(values), - std::get<9>(values), - std::get<10>(values), - std::get<11>(values), - std::get<12>(values), - std::get<13>(values) + values.writer_hostgroup, + values.reader_hostgroup, + values.green_writer_hostgroup, + values.green_reader_hostgroup, + values.active, + values.aurora_port, + values.domain_name, + values.max_lag_ms, + values.check_interval_ms, + values.check_timeout_ms, + values.writer_is_also_reader, + values.new_reader_weight, + values.add_lag_ms, + values.min_lag_ms, + values.lag_num_checks, + values.comment ); select_mysql_aws_aurora_hostgroup_queries.push_back(select_aws_aurora_hostgroup_query); } @@ -2371,6 +2402,30 @@ int main(int, char**) { system(print_replica_aws_aurora_hostgroups.c_str()); ok(not_synced_query == false, "'mysql_aws_aurora_hostgroups' should be synced."); + MYSQL_QUERY__(proxy_admin, + "CREATE VIEW aurora_bgd_local_status AS " + "SELECT writer_hostgroup,bgd_status FROM runtime_mysql_aws_aurora_hostgroups"); + MYSQL_QUERY__(r_proxy_admin, + "CREATE VIEW aurora_bgd_local_status AS " + "SELECT writer_hostgroup,bgd_status FROM runtime_mysql_aws_aurora_hostgroups"); + MYSQL_QUERY__(proxy_admin, + "UPDATE runtime_mysql_aws_aurora_hostgroups SET bgd_status='AVAILABLE' WHERE writer_hostgroup=1000"); + MYSQL_QUERY__(r_proxy_admin, + "UPDATE runtime_mysql_aws_aurora_hostgroups SET bgd_status='SWITCHOVER_IN_PROGRESS' WHERE writer_hostgroup=1000"); + sleep(2); + MYSQL_QUERY__(proxy_admin, + "SELECT COUNT(*) FROM aurora_bgd_local_status " + "WHERE writer_hostgroup=1000 AND bgd_status='AVAILABLE'"); + const bool master_status_is_local = fetch_single_int_res(proxy_admin) == 1; + MYSQL_QUERY__(r_proxy_admin, + "SELECT COUNT(*) FROM aurora_bgd_local_status " + "WHERE writer_hostgroup=1000 AND bgd_status='SWITCHOVER_IN_PROGRESS'"); + const bool replica_status_is_local = fetch_single_int_res(r_proxy_admin) == 1; + ok(master_status_is_local && replica_status_is_local, + "Aurora cluster sync leaves each node's bgd_status unchanged."); + MYSQL_QUERY__(proxy_admin, "DROP VIEW aurora_bgd_local_status"); + MYSQL_QUERY__(r_proxy_admin, "DROP VIEW aurora_bgd_local_status"); + // TEARDOWN CONFIG MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_aws_aurora_hostgroups"); MYSQL_QUERY__(proxy_admin, "INSERT INTO mysql_aws_aurora_hostgroups SELECT * FROM mysql_aws_aurora_hostgroups_sync_test_2687"); From 8e67581fb13ae8b9a3227e2c2c5929f13a5e8faa Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 10:59:15 +0000 Subject: [PATCH 13/34] fix: exclude Aurora BGD status from cluster checksum --- lib/MySQL_HostGroups_Manager.cpp | 11 ++++++++++- test/tap/groups/groups.json | 1 + test/tap/tests/unit/aurora_bgd_config_unit-t.cpp | 7 +++++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 1a1c0b01c5..468465f1a0 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1151,7 +1151,16 @@ void MySQL_HostGroups_Manager::CUCFT1( int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; - string query = "SELECT * FROM " + TableName + " ORDER BY " + ColumnName; + string query; + if (TableName == "mysql_aws_aurora_hostgroups") { + query = + "SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms," + "writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks," + "autopurge_missing_checks,comment FROM mysql_aws_aurora_hostgroups ORDER BY " + ColumnName; + } else { + query = "SELECT * FROM " + TableName + " ORDER BY " + ColumnName; + } mydb->execute_statement(query.c_str(), &error , &cols , &affected_rows , &resultset); if (resultset) { if (resultset->rows_count) { diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index cb972d6154..897b4fdc88 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -11,6 +11,7 @@ "ai_error_handling_edge_cases-t" : [ "ai-g1","@proxysql_min_version:4.0" ], "ai_llm_retry_scenarios-t" : [ "ai-g1","@proxysql_min_version:4.0" ], "ai_validation-t" : [ "ai-g1","@proxysql_min_version:4.0" ], + "aurora_bgd_config_unit-t" : [ "unit-tests-g1" ], "auth_unit-t" : [ "unit-tests-g1" ], "backend_sync_unit-t" : [ "unit-tests-g1" ], "basic-t" : [ "legacy-g1","mariadb10-galera-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql84-gr-g1","mysql90-g1","mysql90-gr-g1","mysql93-g1","mysql93-gr-g1","mysql95-g1","mysql95-gr-g1" ], diff --git a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp index 104992bf75..b1600a4215 100644 --- a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp +++ b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp @@ -325,6 +325,7 @@ static void test_runtime_ownership_and_status() { "SELECT GROUP_CONCAT(bgd_status, ',') FROM " "(SELECT bgd_status FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup)" ) == "NONE,NONE", "new Aurora runtime rows start in NONE"); + const std::string checksum_before_status = MyHGM->gen_global_mysql_servers_v2_checksum(0); SQLite3_result* dump = MyHGM->dump_table_mysql("mysql_aws_aurora_hostgroups"); ok(dump && dump->columns == 18, "Aurora runtime dump includes configured fields and bgd_status"); @@ -334,6 +335,8 @@ static void test_runtime_ownership_and_status() { ok(hgm_query_string( "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" ) == "AVAILABLE", "Aurora BGD status API publishes an accepted state"); + ok(MyHGM->gen_global_mysql_servers_v2_checksum(0) == checksum_before_status, + "node-local Aurora BGD status is excluded from the cluster checksum"); const char* accepted_statuses[] = { "NONE", "AVAILABLE", @@ -413,7 +416,7 @@ static void test_runtime_ownership_and_status() { } int main() { - plan(48); + plan(49); test_init_minimal(); test_schema_contract(); // 12 @@ -423,7 +426,7 @@ int main() { test_invalid_replacement_removes_previous_row(); // 3 ok(test_init_hostgroups() == 0, "test_init_hostgroups() succeeds"); // 1 - test_runtime_ownership_and_status(); // 17 + test_runtime_ownership_and_status(); // 18 test_cleanup_hostgroups(); test_cleanup_minimal(); From 7a7303575cb26a0cb78a75b944b903135f7bc193 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 21:42:25 +0000 Subject: [PATCH 14/34] refactor: simplify Aurora BGD configuration paths --- include/MySQL_HostGroups_Manager.h | 7 +- include/proxysql_admin.h | 20 +++- lib/MySQL_HostGroups_Manager.cpp | 82 ++++--------- lib/ProxySQL_Admin.cpp | 185 ++++++++++++++++++----------- 4 files changed, 162 insertions(+), 132 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 4d1c56aafd..a340e6fc10 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -388,9 +388,9 @@ class AWS_Aurora_Info { /** * @brief Validate an Aurora hostgroup candidate and return its canonical configured projection. * - * The returned result contains only valid rows, ordered like the Aurora Admin - * table. Legacy projections without green hostgroups are normalized with both - * green values set to SQL NULL. The caller owns the returned result. + * The candidate must exactly match the configured Aurora Admin table projection. + * The returned result contains only valid rows in that canonical order. The + * caller owns the returned result. */ SQLite3_result* validate_and_filter_aws_aurora_hostgroups( const SQLite3_result* candidate, @@ -998,6 +998,7 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { * @brief Creates a resultset with the current full content of the target table. * @param string The target table. Valid values are: * - "mysql_aws_aurora_hostgroups" + * - "runtime_mysql_aws_aurora_hostgroups" * - "mysql_galera_hostgroups" * - "mysql_group_replication_hostgroups" * - "mysql_replication_hostgroups" diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 5529cb0d62..4907c3245e 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -320,9 +320,27 @@ struct FlushVariableStats { int unknown = 0; }; +/** + * @brief Replace an Admin Aurora hostgroup table with an HGM runtime snapshot. + * + * @param db Destination Admin database. + * @param resultset Aurora rows returned by MySQL_HostGroups_Manager. + * @param runtime Selects runtime_mysql_aws_aurora_hostgroups when true and + * mysql_aws_aurora_hostgroups when false. + * @return true when the destination table was replaced. + */ +bool materialize_mysql_aws_aurora_hostgroups( + SQLite3DB* db, + const SQLite3_result* resultset, + bool runtime +); + +/** Copy the configured Aurora hostgroup projection between Admin and disk. */ +bool copy_mysql_aws_aurora_hostgroups_from_disk(SQLite3DB* db); +bool copy_mysql_aws_aurora_hostgroups_to_disk(SQLite3DB* db); + class ProxySQL_Admin { friend class TestDiskUpgrade; - friend class TestAuroraBGDRuntime; private: volatile int main_shutdown; diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 468465f1a0..27ce3332ec 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -9,7 +9,7 @@ using json = nlohmann::json; #include "MySQL_PreparedStatement.h" #include "MySQL_Data_Stream.h" -#include +#include #include #include #include @@ -98,15 +98,6 @@ struct Aurora_Config_Row { bool locally_valid; }; -int aws_aurora_column_index(const SQLite3_result* candidate, const char* name) { - for (size_t i = 0; i < candidate->column_definition.size(); ++i) { - if (strcasecmp(candidate->column_definition[i]->name, name) == 0) { - return static_cast(i); - } - } - return -1; -} - std::string aws_aurora_nullable_value(const char* value) { return value ? value : "NULL"; } @@ -126,35 +117,33 @@ SQLite3_result* validate_and_filter_aws_aurora_hostgroups( return filtered; } - std::vector source_indexes(AWS_AURORA_CONFIG_COLUMN_COUNT, -1); - for (int i = 0; i < AWS_AURORA_CONFIG_COLUMN_COUNT; ++i) { - source_indexes[i] = aws_aurora_column_index(candidate, AWS_AURORA_CONFIG_COLUMNS[i]); - } - - // Older peers send the pre-BGD 15-column projection. Treat the two absent - // green fields as SQL NULL until cluster synchronization is upgraded. - const bool green_writer_absent = source_indexes[2] == -1; - const bool green_reader_absent = source_indexes[3] == -1; - if (green_writer_absent != green_reader_absent) { + if (candidate->columns != static_cast(candidate->column_definition.size())) { errors.emplace_back( - "mysql_aws_aurora_hostgroups rejected: candidate projection contains only one green hostgroup column" + "mysql_aws_aurora_hostgroups rejected: candidate projection metadata is inconsistent" ); return filtered; } - for (int i = 0; i < AWS_AURORA_CONFIG_COLUMN_COUNT; ++i) { - if ((i == 2 || i == 3) && green_writer_absent) { - continue; - } - if (source_indexes[i] == -1) { - errors.emplace_back( - std::string("mysql_aws_aurora_hostgroups rejected: candidate projection is missing ") + - AWS_AURORA_CONFIG_COLUMNS[i] - ); + const int comparable_columns = std::min(candidate->columns, AWS_AURORA_CONFIG_COLUMN_COUNT); + for (int i = 0; i < comparable_columns; ++i) { + const char* actual_name = candidate->column_definition[i]->name; + if (strcasecmp(actual_name, AWS_AURORA_CONFIG_COLUMNS[i]) != 0) { + std::ostringstream message; + message << "mysql_aws_aurora_hostgroups rejected: candidate projection column " << i + << " must be " << AWS_AURORA_CONFIG_COLUMNS[i] + << ", got " << actual_name; + errors.emplace_back(message.str()); return filtered; } } + if (candidate->columns != AWS_AURORA_CONFIG_COLUMN_COUNT) { + errors.emplace_back( + "mysql_aws_aurora_hostgroups rejected: candidate projection must contain exactly 17 columns" + ); + return filtered; + } + std::vector rows; rows.reserve(candidate->rows.size()); @@ -162,9 +151,7 @@ SQLite3_result* validate_and_filter_aws_aurora_hostgroups( Aurora_Config_Row row; row.fields.resize(AWS_AURORA_CONFIG_COLUMN_COUNT, nullptr); for (int i = 0; i < AWS_AURORA_CONFIG_COLUMN_COUNT; ++i) { - if (source_indexes[i] != -1) { - row.fields[i] = source_row->fields[source_indexes[i]]; - } + row.fields[i] = source_row->fields[i]; } row.writer_hostgroup = row.fields[0] ? atoi(row.fields[0]) : -1; @@ -2460,6 +2447,9 @@ void MySQL_HostGroups_Manager::update_table_mysql_servers_for_monitor(bool lock) SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql(const string& name) { char * query = (char *)""; if (name == "mysql_aws_aurora_hostgroups") { + query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," + "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment FROM mysql_aws_aurora_hostgroups"; + } else if (name == "runtime_mysql_aws_aurora_hostgroups") { query=(char *)"SELECT writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment,bgd_status FROM mysql_aws_aurora_hostgroups"; } else if (name == "mysql_aws_rds_bgd_hostgroups") { @@ -4190,24 +4180,6 @@ void MySQL_HostGroups_Manager::aws_rds_bgd_set_runtime_status(unsigned int write } void MySQL_HostGroups_Manager::update_aws_aurora_bgd_status(int writer_hostgroup, const std::string& bgd_status) { - static const std::array valid_statuses { - "NONE", - "AVAILABLE", - "SWITCHOVER_INITIATED", - "SWITCHOVER_IN_PROGRESS", - "SWITCHOVER_IN_POST_PROCESSING", - "SWITCHOVER_COMPLETED" - }; - - bool valid = std::any_of(valid_statuses.begin(), valid_statuses.end(), [&bgd_status](const char* status) { - return bgd_status == status; - }); - if (!valid) { - proxy_error("Invalid AWS Aurora BGD runtime status '%s' for writer hostgroup %d\n", - bgd_status.c_str(), writer_hostgroup); - return; - } - wrlock(); const char* query = "UPDATE mysql_aws_aurora_hostgroups SET bgd_status=?1 WHERE writer_hostgroup=?2"; auto [prepare_rc, statement_unique] = mydb->prepare_v2(query); @@ -6849,11 +6821,9 @@ void MySQL_HostGroups_Manager::generate_mysql_aws_aurora_hostgroups_table() { // it is now time to build a new structure in Monitor - if (GloMyMon) { - pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); - update_aws_aurora_hosts_monitor_resultset(false); - pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); - } + pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); + update_aws_aurora_hosts_monitor_resultset(false); + pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); pthread_mutex_unlock(&AWS_Aurora_Info_mutex); } diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index f73dae3178..9c54baf9cd 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -143,7 +143,6 @@ static const vector mysql_servers_tablenames = { "mysql_replication_hostgroups", "mysql_group_replication_hostgroups", "mysql_galera_hostgroups", - "mysql_aws_aurora_hostgroups", "mysql_aws_rds_bgd_hostgroups", "mysql_hostgroup_attributes", "mysql_servers_ssl_params", @@ -202,19 +201,33 @@ static void BQE1(SQLite3DB *db, const vector& tbs, const string& p1, con db->execute(query.c_str()); } if (p2 != "" && p3 != "") { - const size_t wildcard_pos = p3.find('*'); - if (*it == "mysql_aws_aurora_hostgroups" && wildcard_pos != string::npos) { - string projected_select = p3; - projected_select.replace(wildcard_pos, 1, mysql_aws_aurora_config_columns); - query = p2 + *it + " (" + mysql_aws_aurora_config_columns + ")" + projected_select + *it; - } else { - query = p2 + *it + p3 + *it; - } + query = p2 + *it + p3 + *it; db->execute(query.c_str()); } } } +static bool copy_mysql_aws_aurora_hostgroups( + SQLite3DB* db, + const string& destination_schema, + const string& source_schema +) { + const string table_name = "mysql_aws_aurora_hostgroups"; + const string query = + "INSERT OR REPLACE INTO " + destination_schema + "." + table_name + + " (" + mysql_aws_aurora_config_columns + ") SELECT " + + mysql_aws_aurora_config_columns + " FROM " + source_schema + "." + table_name; + return db->execute(query.c_str()); +} + +bool copy_mysql_aws_aurora_hostgroups_from_disk(SQLite3DB* db) { + return copy_mysql_aws_aurora_hostgroups(db, "main", "disk"); +} + +bool copy_mysql_aws_aurora_hostgroups_to_disk(SQLite3DB* db) { + return copy_mysql_aws_aurora_hostgroups(db, "disk", "main"); +} + static int round_intv_to_time_interval(const char* name, int _intv) { int intv = _intv; @@ -5772,6 +5785,7 @@ int ProxySQL_Admin::flush_debug_levels_database_to_runtime(SQLite3DB *db) { void ProxySQL_Admin::__insert_or_replace_maintable_select_disktable() { admindb->execute("PRAGMA foreign_keys = OFF"); BQE1(admindb, mysql_servers_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); + copy_mysql_aws_aurora_hostgroups_from_disk(admindb); BQE1(admindb, mysql_query_rules_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); admindb->execute("INSERT OR REPLACE INTO main.mysql_users SELECT * FROM disk.mysql_users"); BQE1(admindb, mysql_firewall_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); @@ -5856,6 +5870,7 @@ void ProxySQL_Admin::__insert_or_replace_maintable_select_disktable() { void ProxySQL_Admin::__insert_or_replace_disktable_select_maintable() { BQE1(admindb, mysql_servers_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); + copy_mysql_aws_aurora_hostgroups_to_disk(admindb); BQE1(admindb, mysql_query_rules_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); admindb->execute("INSERT OR REPLACE INTO disk.mysql_users SELECT * FROM main.mysql_users"); BQE1(admindb, mysql_firewall_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); @@ -5964,8 +5979,16 @@ void ProxySQL_Admin::flush_GENERIC__from_to(const string& name, const string& di assert(it != module_tablenames.end()); if (direction == "disk_to_memory") { BQE1(admindb, it->second, "DELETE FROM main.", "INSERT INTO main.", " SELECT * FROM disk."); + if (name == "mysql_servers") { + admindb->execute("DELETE FROM main.mysql_aws_aurora_hostgroups"); + copy_mysql_aws_aurora_hostgroups_from_disk(admindb); + } } else if (direction == "memory_to_disk") { BQE1(admindb, it->second, "DELETE FROM disk.", "INSERT INTO disk.", " SELECT * FROM main."); + if (name == "mysql_servers") { + admindb->execute("DELETE FROM disk.mysql_aws_aurora_hostgroups"); + copy_mysql_aws_aurora_hostgroups_to_disk(admindb); + } } else { assert(0); } @@ -7323,6 +7346,83 @@ void ProxySQL_Admin::save_scheduler_runtime_to_database(bool _runtime) { free(args); } +bool materialize_mysql_aws_aurora_hostgroups( + SQLite3DB* db, + const SQLite3_result* resultset, + bool runtime +) { + if (db == nullptr) { + return false; + } + + const int expected_columns = runtime ? 18 : 17; + if (resultset != nullptr && resultset->columns < expected_columns) { + proxy_error( + "Cannot materialize mysql_aws_aurora_hostgroups: expected at least %d columns, got %d\n", + expected_columns, + resultset->columns + ); + return false; + } + + const char* delete_query = runtime + ? "DELETE FROM main.runtime_mysql_aws_aurora_hostgroups" + : "DELETE FROM main.mysql_aws_aurora_hostgroups"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", delete_query); + if (!db->execute(delete_query)) { + return false; + } + if (resultset == nullptr) { + return true; + } + + const char* insert_query = runtime + ? "INSERT INTO runtime_mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment,bgd_status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18)" + : "INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)"; + auto [prepare_rc, statement_unique] = db->prepare_v2(insert_query); + ASSERT_SQLITE_OK(prepare_rc, db); + sqlite3_stmt* statement = statement_unique.get(); + + for (const SQLite3_row* row : resultset->rows) { + int rc = (*proxy_sqlite3_bind_int64)(statement, 1, atoi(row->fields[0])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 2, atoi(row->fields[1])); ASSERT_SQLITE_OK(rc, db); + if (row->fields[2]) { + rc = (*proxy_sqlite3_bind_int64)(statement, 3, atoi(row->fields[2])); + } else { + rc = (*proxy_sqlite3_bind_null)(statement, 3); + } + ASSERT_SQLITE_OK(rc, db); + if (row->fields[3]) { + rc = (*proxy_sqlite3_bind_int64)(statement, 4, atoi(row->fields[3])); + } else { + rc = (*proxy_sqlite3_bind_null)(statement, 4); + } + ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 5, atoi(row->fields[4])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 6, atoi(row->fields[5])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_text)(statement, 7, row->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 8, atoi(row->fields[7])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 9, atoi(row->fields[8])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 10, atoi(row->fields[9])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 11, atoi(row->fields[10])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 12, atoi(row->fields[11])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 13, atoi(row->fields[12])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 14, atoi(row->fields[13])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 15, atoi(row->fields[14])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_int64)(statement, 16, atoi(row->fields[15])); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_text)(statement, 17, row->fields[16], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + if (runtime) { + rc = (*proxy_sqlite3_bind_text)(statement, 18, row->fields[17], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + } + + SAFE_SQLITE3_STEP2(statement); + rc = (*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, db); + } + + return true; +} + void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { // make sure that the caller has called mysql_servers_wrlock() char *query=NULL; @@ -7536,69 +7636,10 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { resultset = NULL; // dump mysql_aws_aurora_hostgroups - - if (_runtime) { - query=(char *)"DELETE FROM main.runtime_mysql_aws_aurora_hostgroups"; - } else { - query=(char *)"DELETE FROM main.mysql_aws_aurora_hostgroups"; - } - proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - admindb->execute(query); - resultset=MyHGM->dump_table_mysql("mysql_aws_aurora_hostgroups"); - if (resultset) { - int rc; - sqlite3_stmt *statement=NULL; - - char *query=NULL; - if (_runtime) { - query=(char *)"INSERT INTO runtime_mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment,bgd_status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18)"; - } else { - query=(char *)"INSERT INTO mysql_aws_aurora_hostgroups(writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,autopurge_missing_checks,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)"; - } - - auto [rc1, statement_unique] = admindb->prepare_v2(query); - rc = rc1; - statement = statement_unique.get(); - ASSERT_SQLITE_OK(rc, admindb); - - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - rc=(*proxy_sqlite3_bind_int64)(statement, 1, atoi(r->fields[0])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 2, atoi(r->fields[1])); ASSERT_SQLITE_OK(rc, admindb); - if (r->fields[2]) { - rc=(*proxy_sqlite3_bind_int64)(statement, 3, atoi(r->fields[2])); - } else { - rc=(*proxy_sqlite3_bind_null)(statement, 3); - } - ASSERT_SQLITE_OK(rc, admindb); - if (r->fields[3]) { - rc=(*proxy_sqlite3_bind_int64)(statement, 4, atoi(r->fields[3])); - } else { - rc=(*proxy_sqlite3_bind_null)(statement, 4); - } - ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 5, atoi(r->fields[4])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 6, atoi(r->fields[5])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement, 7, r->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 8, atoi(r->fields[7])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 9, atoi(r->fields[8])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 10, atoi(r->fields[9])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 11, atoi(r->fields[10])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 12, atoi(r->fields[11])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 13, atoi(r->fields[12])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 14, atoi(r->fields[13])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 15, atoi(r->fields[14])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement, 16, atoi(r->fields[15])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement, 17, r->fields[16], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - if (_runtime) { - rc=(*proxy_sqlite3_bind_text)(statement, 18, r->fields[17], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - } - - SAFE_SQLITE3_STEP2(statement); - rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, admindb); - } - } + resultset=MyHGM->dump_table_mysql( + _runtime ? "runtime_mysql_aws_aurora_hostgroups" : "mysql_aws_aurora_hostgroups" + ); + materialize_mysql_aws_aurora_hostgroups(admindb, resultset, _runtime); if(resultset) delete resultset; resultset=NULL; From 3d9c54a799faff8bae8a8e3cf57259a3ed4e0045 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 21:42:29 +0000 Subject: [PATCH 15/34] test: streamline Aurora BGD phase 1 coverage --- test/tap/test_helpers/test_init.cpp | 20 ++ test/tap/test_helpers/test_init.h | 13 ++ test/tap/tests/test_cluster_sync-t.cpp | 211 ++---------------- .../tests/unit/admin_disk_upgrade_unit-t.cpp | 109 ++++----- .../tests/unit/aurora_bgd_config_unit-t.cpp | 166 +++++++------- 5 files changed, 172 insertions(+), 347 deletions(-) diff --git a/test/tap/test_helpers/test_init.cpp b/test/tap/test_helpers/test_init.cpp index 08261b2efd..57192d5b00 100644 --- a/test/tap/test_helpers/test_init.cpp +++ b/test/tap/test_helpers/test_init.cpp @@ -20,6 +20,7 @@ #include "PgSQL_Query_Cache.h" #include "MySQL_Query_Processor.h" #include "PgSQL_Query_Processor.h" +#include "MySQL_Monitor.hpp" #include "test_globals.h" #include "test_init.h" @@ -32,6 +33,7 @@ extern MySQL_Query_Cache *GloMyQC; extern PgSQL_Query_Cache *GloPgQC; extern MySQL_Query_Processor *GloMyQPro; extern PgSQL_Query_Processor *GloPgQPro; +extern MySQL_Monitor *GloMyMon; // GloMTH is declared extern in proxysql_utils.h. // GloPTH has no extern declaration in any header, so we add one here. @@ -205,3 +207,21 @@ void test_cleanup_hostgroups() { PgHGM = nullptr; } } + +// ============================================================================ +// MySQL Monitor +// ============================================================================ + +int test_init_monitor() { + if (GloMyMon == nullptr) { + GloMyMon = new MySQL_Monitor(); + } + return 0; +} + +void test_cleanup_monitor() { + if (GloMyMon != nullptr) { + delete GloMyMon; + GloMyMon = nullptr; + } +} diff --git a/test/tap/test_helpers/test_init.h b/test/tap/test_helpers/test_init.h index 5818bde0ba..d623d06152 100644 --- a/test/tap/test_helpers/test_init.h +++ b/test/tap/test_helpers/test_init.h @@ -128,4 +128,17 @@ int test_init_hostgroups(); */ void test_cleanup_hostgroups(); +/** + * @brief Initialize the MySQL monitor without starting its worker threads. + * + * @pre test_init_minimal() must have been called. + * @return 0 on success, non-zero on failure. + */ +int test_init_monitor(); + +/** + * @brief Clean up resources allocated by test_init_monitor(). + */ +void test_cleanup_monitor(); + #endif /* TEST_INIT_H */ diff --git a/test/tap/tests/test_cluster_sync-t.cpp b/test/tap/tests/test_cluster_sync-t.cpp index 1e85d03cd7..f39fb62ef4 100644 --- a/test/tap/tests/test_cluster_sync-t.cpp +++ b/test/tap/tests/test_cluster_sync-t.cpp @@ -6,7 +6,7 @@ * - 'mysql_galera_hostgroups' with and without NULL comments. * - 'mysql_group_replication_hostgroups' with and without NULL comments. * - 'proxysql_servers' with new values and empty (exercising bug from '#3847'). - * - 'mysql_aws_aurora_hostgroups' with and without NULL comments. + * - 'mysql_aws_aurora_hostgroups' with configured and NULL green hostgroups/comments. * - 'mysql_variables'. * - 'admin_variables'. * @@ -220,8 +220,8 @@ using mysql_server_tuple = tuple insert_aws_aurora_values { - { 1002, 1006, 1012, 1016, 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1, nullptr }, - { 1003, 1007, 1013, 1017, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1, nullptr }, - { 1000, 1004, 1010, 1014, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1, nullptr }, - { 1001, 1005, 1011, 1015, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1, nullptr }, - }; - std::vector insert_mysql_aws_aurora_hostgroup_queries {}; - - for (auto const& values : insert_aws_aurora_values) { - std::string insert_aws_aurora_hostgroup_query = ""; - string_format( - t_insert_mysql_aws_aurora_hostgroups, - insert_aws_aurora_hostgroup_query, - values.writer_hostgroup, - values.reader_hostgroup, - values.green_writer_hostgroup, - values.green_reader_hostgroup, - values.active, - values.aurora_port, - values.domain_name, - values.max_lag_ms, - values.check_interval_ms, - values.check_timeout_ms, - values.writer_is_also_reader, - values.new_reader_weight, - values.add_lag_ms, - values.min_lag_ms, - values.lag_num_checks - ); - insert_mysql_aws_aurora_hostgroup_queries.push_back(insert_aws_aurora_hostgroup_query); - } - - const char* t_select_aws_aurora_inserted_entries = - "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups WHERE " - "writer_hostgroup=%d AND reader_hostgroup=%d AND green_writer_hostgroup=%d AND green_reader_hostgroup=%d " - "AND active=%d AND aurora_port=%d AND domain_name='%s' " - "AND max_lag_ms=%d AND check_interval_ms=%d AND check_timeout_ms=%d AND writer_is_also_reader=%d " - "AND new_reader_weight=%d AND add_lag_ms=%d AND min_lag_ms=%d AND lag_num_checks=%d AND comment IS NULL"; - std::vector select_mysql_aws_aurora_hostgroup_queries {}; - - for (auto const& values : insert_aws_aurora_values) { - std::string select_aws_aurora_hostgroup_query = ""; - string_format( - t_select_aws_aurora_inserted_entries, - select_aws_aurora_hostgroup_query, - values.writer_hostgroup, - values.reader_hostgroup, - values.green_writer_hostgroup, - values.green_reader_hostgroup, - values.active, - values.aurora_port, - values.domain_name, - values.max_lag_ms, - values.check_interval_ms, - values.check_timeout_ms, - values.writer_is_also_reader, - values.new_reader_weight, - values.add_lag_ms, - values.min_lag_ms, - values.lag_num_checks - ); - select_mysql_aws_aurora_hostgroup_queries.push_back(select_aws_aurora_hostgroup_query); - } - - // SETUP CONFIG - - // Backup current table - MYSQL_QUERY__(proxy_admin, "CREATE TABLE mysql_aws_aurora_hostgroups_sync_test_2687 AS SELECT * FROM mysql_aws_aurora_hostgroups"); - MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_aws_aurora_hostgroups"); - - // Insert the new aws_aurora hostgroups values - for (const auto& query : insert_mysql_aws_aurora_hostgroup_queries) { - MYSQL_QUERY__(proxy_admin, query.c_str()); - } - MYSQL_QUERY__(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); - std::cout << "MASTER TABLE BEFORE SYNC:" << std::endl; - system(print_master_aws_aurora_hostgroups.c_str()); - - // SYNCH CHECK - - uint waited = 0; - bool not_synced_query = false; - while (waited < SYNC_TIMEOUT) { - not_synced_query = false; - // Check that all the entries have been synced - for (const auto& query : select_mysql_aws_aurora_hostgroup_queries) { - MYSQL_QUERY__(r_proxy_admin, query.c_str()); - MYSQL_RES* aws_aurora_res = mysql_store_result(r_proxy_admin); - MYSQL_ROW row = mysql_fetch_row(aws_aurora_res); - int row_value = atoi(row[0]); - mysql_free_result(aws_aurora_res); - - if (row_value == 0) { - not_synced_query = true; - break; - } - } - - if (not_synced_query) { - waited += 1; - sleep(1); - } else { - break; - } - } - - std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; - system(print_replica_aws_aurora_hostgroups.c_str()); - ok(not_synced_query == false, "'mysql_aws_aurora_hostgroups' with NULL comments should be synced."); - - // TEARDOWN CONFIG - MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_aws_aurora_hostgroups"); - MYSQL_QUERY__(proxy_admin, "INSERT INTO mysql_aws_aurora_hostgroups SELECT * FROM mysql_aws_aurora_hostgroups_sync_test_2687"); - MYSQL_QUERY__(proxy_admin, "DROP TABLE mysql_aws_aurora_hostgroups_sync_test_2687"); - MYSQL_QUERY__(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); - } - - sleep(2); - - // Check 'mysql_aws_aurora_hostgroups' synchronization + // Check 'mysql_aws_aurora_hostgroups' synchronization for configured and NULL values { std::string print_master_aws_aurora_hostgroups = ""; string_format(t_debug_query, print_master_aws_aurora_hostgroups, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM runtime_mysql_aws_aurora_hostgroups"); @@ -2293,12 +2160,10 @@ int main(int, char**) { "writer_hostgroup, reader_hostgroup, green_writer_hostgroup, green_reader_hostgroup, " "active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment) " - "VALUES (%d, %d, %d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, '%s')"; + "VALUES (%d, %d, %s, %s, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, %s)"; std::vector insert_aws_aurora_values { - { 1003, 1007, 1013, 1017, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup" }, - { 1001, 1005, 1011, 1015, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup" }, - { 1002, 1006, 1012, 1016, 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup" }, - { 1000, 1004, 1010, 1014, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup" }, + { 1000, 1004, "1010", "1014", 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1, "'reader_writer_test_aws_aurora_hostgroup'" }, + { 1002, 1006, "NULL", "NULL", 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1, "NULL" }, }; std::vector insert_mysql_aws_aurora_hostgroup_queries {}; @@ -2329,10 +2194,10 @@ int main(int, char**) { const char* t_select_aws_aurora_inserted_entries = "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups WHERE " - "writer_hostgroup=%d AND reader_hostgroup=%d AND green_writer_hostgroup=%d AND green_reader_hostgroup=%d " + "writer_hostgroup=%d AND reader_hostgroup=%d AND green_writer_hostgroup IS %s AND green_reader_hostgroup IS %s " "AND active=%d AND aurora_port=%d AND domain_name='%s' " "AND max_lag_ms=%d AND check_interval_ms=%d AND check_timeout_ms=%d AND writer_is_also_reader=%d " - "AND new_reader_weight=%d AND add_lag_ms=%d AND min_lag_ms=%d AND lag_num_checks=%d AND comment='%s'"; + "AND new_reader_weight=%d AND add_lag_ms=%d AND min_lag_ms=%d AND lag_num_checks=%d AND comment IS %s"; std::vector select_mysql_aws_aurora_hostgroup_queries {}; for (auto const& values : insert_aws_aurora_values) { @@ -2361,6 +2226,7 @@ int main(int, char**) { } // Backup current table + MYSQL_QUERY__(proxy_admin, "DROP TABLE IF EXISTS mysql_aws_aurora_hostgroups_sync_test_2687"); MYSQL_QUERY__(proxy_admin, "CREATE TABLE mysql_aws_aurora_hostgroups_sync_test_2687 AS SELECT * FROM mysql_aws_aurora_hostgroups"); MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_aws_aurora_hostgroups"); @@ -2372,59 +2238,14 @@ int main(int, char**) { std::cout << "MASTER TABLE BEFORE SYNC:" << std::endl; system(print_master_aws_aurora_hostgroups.c_str()); - uint waited = 0; - bool not_synced_query = false; - while (waited < SYNC_TIMEOUT) { - not_synced_query = false; - // Check that all the entries have been synced - for (const auto& query : select_mysql_aws_aurora_hostgroup_queries) { - MYSQL_QUERY__(r_proxy_admin, query.c_str()); - MYSQL_RES* aws_aurora_res = mysql_store_result(r_proxy_admin); - MYSQL_ROW row = mysql_fetch_row(aws_aurora_res); - int row_value = atoi(row[0]); - mysql_free_result(aws_aurora_res); - - if (row_value == 0) { - not_synced_query = true; - break; - } - } - - if (not_synced_query) { - waited += 1; - sleep(1); - } else { - break; - } - } + const int wait_res = proc_wait_checks( + wait_for_conds(r_proxy_admin, select_mysql_aws_aurora_hostgroup_queries, SYNC_TIMEOUT) + ); std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; system(print_replica_aws_aurora_hostgroups.c_str()); - ok(not_synced_query == false, "'mysql_aws_aurora_hostgroups' should be synced."); - - MYSQL_QUERY__(proxy_admin, - "CREATE VIEW aurora_bgd_local_status AS " - "SELECT writer_hostgroup,bgd_status FROM runtime_mysql_aws_aurora_hostgroups"); - MYSQL_QUERY__(r_proxy_admin, - "CREATE VIEW aurora_bgd_local_status AS " - "SELECT writer_hostgroup,bgd_status FROM runtime_mysql_aws_aurora_hostgroups"); - MYSQL_QUERY__(proxy_admin, - "UPDATE runtime_mysql_aws_aurora_hostgroups SET bgd_status='AVAILABLE' WHERE writer_hostgroup=1000"); - MYSQL_QUERY__(r_proxy_admin, - "UPDATE runtime_mysql_aws_aurora_hostgroups SET bgd_status='SWITCHOVER_IN_PROGRESS' WHERE writer_hostgroup=1000"); - sleep(2); - MYSQL_QUERY__(proxy_admin, - "SELECT COUNT(*) FROM aurora_bgd_local_status " - "WHERE writer_hostgroup=1000 AND bgd_status='AVAILABLE'"); - const bool master_status_is_local = fetch_single_int_res(proxy_admin) == 1; - MYSQL_QUERY__(r_proxy_admin, - "SELECT COUNT(*) FROM aurora_bgd_local_status " - "WHERE writer_hostgroup=1000 AND bgd_status='SWITCHOVER_IN_PROGRESS'"); - const bool replica_status_is_local = fetch_single_int_res(r_proxy_admin) == 1; - ok(master_status_is_local && replica_status_is_local, - "Aurora cluster sync leaves each node's bgd_status unchanged."); - MYSQL_QUERY__(proxy_admin, "DROP VIEW aurora_bgd_local_status"); - MYSQL_QUERY__(r_proxy_admin, "DROP VIEW aurora_bgd_local_status"); + ok(wait_res == EXIT_SUCCESS, + "'mysql_aws_aurora_hostgroups' should sync configured and NULL values: '%d'", wait_res); // TEARDOWN CONFIG MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_aws_aurora_hostgroups"); diff --git a/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp b/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp index 1ac5f824dd..26d2d35cc8 100644 --- a/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp +++ b/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp @@ -80,9 +80,6 @@ class TestDiskUpgrade { void upgrade_rest_api_routes() { admin->disk_upgrade_rest_api_routes(); } void upgrade_mysql_query_rules() { admin->disk_upgrade_mysql_query_rules(); } void upgrade_pgsql_replication_hostgroups() { admin->disk_upgrade_pgsql_replication_hostgroups(); } - void use_config_db_as_admin_db() { admin->admindb = admin->configdb; } - void save_mysql_servers_to_disk() { admin->__insert_or_replace_disktable_select_maintable(); } - void load_mysql_servers_from_disk() { admin->__insert_or_replace_maintable_select_disktable(); } }; // --------------------------------------------------------------------------- @@ -116,6 +113,16 @@ static bool table_matches_current(SQLite3DB *db, const char *tbl, const char *de return db->check_table_structure(tbl, def) == 1; } +static bool create_table_in_schema(SQLite3DB* db, const char* schema, const char* definition) { + std::string query = definition; + const std::string prefix = "CREATE TABLE "; + if (query.compare(0, prefix.size(), prefix) != 0) { + return false; + } + query.insert(prefix.size(), std::string(schema) + "."); + return db->execute(query.c_str()); +} + // ============================================================================ // disk_upgrade_scheduler() tests // ============================================================================ @@ -579,81 +586,55 @@ static void test_mysql_servers_upgrade_multiple_rows_with_fixes() { } static void test_aurora_hostgroups_disk_roundtrip_uses_configured_projection() { - TestDiskUpgrade t; - SQLite3DB *db = t.db(); - t.use_config_db_as_admin_db(); - - struct TableDefinition { - const char *name; - const char *definition; - }; - const TableDefinition tables[] = { - { "mysql_servers", ADMIN_SQLITE_TABLE_MYSQL_SERVERS }, - { "mysql_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS }, - { "mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS }, - { "mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS }, - { "mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS }, - { "mysql_aws_rds_bgd_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS }, - { "mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES }, - { "mysql_servers_ssl_params", ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS }, - { "mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES }, - { "mysql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_FAST_ROUTING }, - { "mysql_users", ADMIN_SQLITE_TABLE_MYSQL_USERS }, - { "mysql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_USERS }, - { "mysql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_RULES }, - { "mysql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS }, - { "global_variables", ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES }, - { "scheduler", ADMIN_SQLITE_TABLE_SCHEDULER }, - { "restapi_routes", ADMIN_SQLITE_TABLE_RESTAPI_ROUTES }, - { "proxysql_servers", ADMIN_SQLITE_TABLE_PROXYSQL_SERVERS }, - { "pgsql_servers", ADMIN_SQLITE_TABLE_PGSQL_SERVERS }, - { "pgsql_replication_hostgroups", ADMIN_SQLITE_TABLE_PGSQL_REPLICATION_HOSTGROUPS }, - { "pgsql_hostgroup_attributes", ADMIN_SQLITE_TABLE_PGSQL_HOSTGROUP_ATTRIBUTES }, - { "pgsql_servers_ssl_params", ADMIN_SQLITE_TABLE_PGSQL_SERVERS_SSL_PARAMS }, - { "pgsql_query_rules", ADMIN_SQLITE_TABLE_PGSQL_QUERY_RULES }, - { "pgsql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_PGSQL_QUERY_RULES_FAST_ROUTING }, - { "pgsql_users", ADMIN_SQLITE_TABLE_PGSQL_USERS }, - { "pgsql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_USERS }, - { "pgsql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_RULES }, - { "pgsql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS } - }; - for (const TableDefinition& table : tables) { - db->execute(table.definition); - } - db->execute("ATTACH DATABASE ':memory:' AS disk"); - for (const TableDefinition& table : tables) { - std::string query = std::string("CREATE TABLE disk.") + table.name + - " AS SELECT * FROM main." + table.name + " WHERE 0"; - db->execute(query.c_str()); - } - - db->execute( + SQLite3DB* db = new SQLite3DB(); + db->open((char*)":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + const bool schema_ready = db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS) && + db->execute("ATTACH DATABASE ':memory:' AS disk") && + create_table_in_schema(db, "disk", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + ok(schema_ready, "Aurora disk round trip uses the current schemas in main and disk"); + + const bool rows_inserted = db->execute( "INSERT INTO mysql_aws_aurora_hostgroups (" "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,domain_name,comment" - ") VALUES (600,601,602,603,'.disk.example','disk round trip')" + ") VALUES " + "(600,601,602,603,'.disk.example','disk round trip')," + "(610,611,NULL,NULL,'.disk-null.example',NULL)" ); - // Model a node-local runtime field on the source. The disk projection must ignore it. - db->execute("ALTER TABLE main.mysql_aws_aurora_hostgroups ADD COLUMN bgd_status VARCHAR NOT NULL DEFAULT 'LOCAL'"); - db->execute("UPDATE main.mysql_aws_aurora_hostgroups SET bgd_status='SWITCHOVER_IN_PROGRESS'"); - - t.save_mysql_servers_to_disk(); + ok(rows_inserted && copy_mysql_aws_aurora_hostgroups_to_disk(db), + "Aurora disk SAVE copies the configured projection"); ok(query_string(db, "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " "FROM disk.mysql_aws_aurora_hostgroups WHERE writer_hostgroup=600") == "602,603", "Aurora disk SAVE preserves both configured green hostgroups"); + ok(query_int(db, + "SELECT COUNT(*) FROM disk.mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=610 AND green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL") == 1, + "Aurora disk SAVE preserves paired NULL green hostgroups"); + ok(query_int(db, + "SELECT COUNT(*) FROM disk.mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=610 AND comment IS NULL") == 1, + "Aurora disk SAVE preserves a NULL comment"); ok(query_int(db, "SELECT COUNT(*) FROM pragma_table_info('mysql_aws_aurora_hostgroups','disk') WHERE name='bgd_status'") == 0, "Aurora disk SAVE excludes node-local bgd_status"); - db->execute("DELETE FROM main.mysql_aws_aurora_hostgroups"); - t.load_mysql_servers_from_disk(); + ok(db->execute("DELETE FROM main.mysql_aws_aurora_hostgroups"), + "Aurora disk LOAD test clears the configured source table"); + ok(copy_mysql_aws_aurora_hostgroups_from_disk(db), + "Aurora disk LOAD copies the configured projection"); ok(query_string(db, "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " "FROM main.mysql_aws_aurora_hostgroups WHERE writer_hostgroup=600") == "602,603", "Aurora disk LOAD restores both configured green hostgroups"); - ok(query_string(db, - "SELECT bgd_status FROM main.mysql_aws_aurora_hostgroups WHERE writer_hostgroup=600") == "LOCAL", - "Aurora disk LOAD leaves node-local bgd_status at its local default"); + ok(query_int(db, + "SELECT COUNT(*) FROM main.mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=610 AND green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL") == 1, + "Aurora disk LOAD restores paired NULL green hostgroups"); + ok(query_int(db, + "SELECT COUNT(*) FROM main.mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=610 AND comment IS NULL") == 1, + "Aurora disk LOAD restores a NULL comment"); + delete db; } // ============================================================================ @@ -661,7 +642,7 @@ static void test_aurora_hostgroups_disk_roundtrip_uses_configured_projection() { // ============================================================================ int main() { - plan(71); + plan(78); test_init_minimal(); // scheduler tests diff --git a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp index b1600a4215..84e1401608 100644 --- a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp +++ b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp @@ -10,10 +10,10 @@ #include "cpp.h" #include "MySQL_HostGroups_Manager.h" #include "ProxySQL_Admin_Tables_Definitions.h" +#include "proxysql_admin.h" #include #include -#include #include extern MySQL_HostGroups_Manager* MyHGM; @@ -46,34 +46,21 @@ class TestAuroraBGDRuntime { } static SQLite3DB* materialize_aurora_table(bool runtime) { - void* memory = calloc(1, sizeof(ProxySQL_Admin)); - ProxySQL_Admin* admin = reinterpret_cast(memory); SQLite3DB* db = new SQLite3DB(); db->open((char*)":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); - admin->admindb = db; - - if (runtime) { - db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS); - db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_REPLICATION_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GROUP_REPLICATION_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GALERA_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_RDS_BGD_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_HOSTGROUP_ATTRIBUTES); - db->execute(ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS_SSL_PARAMS); - } else { - db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS); - db->execute(ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_RDS_BGD_HOSTGROUPS); - db->execute(ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); - db->execute(ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); + const bool created = db->execute(runtime + ? ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS + : ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + SQLite3_result* resultset = MyHGM->dump_table_mysql(runtime + ? "runtime_mysql_aws_aurora_hostgroups" + : "mysql_aws_aurora_hostgroups"); + const bool materialized = created && + materialize_mysql_aws_aurora_hostgroups(db, resultset, runtime); + delete resultset; + if (!materialized) { + delete db; + return nullptr; } - - admin->save_mysql_servers_runtime_to_database(runtime); - free(admin); return db; } }; @@ -270,17 +257,16 @@ static void test_inactive_cross_row_validation() { delete candidate; } -static void test_legacy_projection_normalization() { +static void test_noncanonical_projection_rejected() { SQLite3_result* candidate = make_candidate(true); add_legacy_candidate_row(candidate, 190, 200); std::vector errors; SQLite3_result* filtered = validate_and_filter_aws_aurora_hostgroups(candidate, errors); - ok(filtered->columns == 17 && filtered->rows_count == 1, - "legacy Aurora projection is normalized to the configured BGD shape"); - ok(filtered->rows[0]->fields[2] == nullptr && filtered->rows[0]->fields[3] == nullptr, - "legacy projection normalizes both green hostgroups to SQL NULL"); - ok(errors.empty(), "legacy Aurora projection remains valid"); + ok(filtered->columns == 17 && filtered->rows_count == 0, + "noncanonical Aurora projection publishes no rows"); + ok(contains_error(errors, "candidate projection", "green_writer_hostgroup"), + "noncanonical Aurora projection reports the first mismatched column"); delete filtered; delete candidate; } @@ -307,7 +293,7 @@ static void test_invalid_replacement_removes_previous_row() { delete initial; } -static void test_runtime_ownership_and_status() { +static void test_runtime_ownership() { SQLite3_result* initial = make_candidate(); add_candidate_row(initial, 300, 310, "301", "311", true); add_candidate_row(initial, 320, 330, nullptr, nullptr, true); @@ -325,55 +311,61 @@ static void test_runtime_ownership_and_status() { "SELECT GROUP_CONCAT(bgd_status, ',') FROM " "(SELECT bgd_status FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup)" ) == "NONE,NONE", "new Aurora runtime rows start in NONE"); - const std::string checksum_before_status = MyHGM->gen_global_mysql_servers_v2_checksum(0); - SQLite3_result* dump = MyHGM->dump_table_mysql("mysql_aws_aurora_hostgroups"); - ok(dump && dump->columns == 18, "Aurora runtime dump includes configured fields and bgd_status"); - delete dump; + SQLite3_result* configured_dump = MyHGM->dump_table_mysql("mysql_aws_aurora_hostgroups"); + ok(configured_dump && configured_dump->columns == 17, + "Aurora configured dump excludes bgd_status"); + delete configured_dump; + SQLite3_result* runtime_dump = MyHGM->dump_table_mysql("runtime_mysql_aws_aurora_hostgroups"); + ok(runtime_dump && runtime_dump->columns == 18, + "Aurora runtime dump includes configured fields and bgd_status"); + delete runtime_dump; +} + +static void test_status_and_materialization() { + const std::string checksum_before_status = MyHGM->gen_global_mysql_servers_v2_checksum(0); MyHGM->update_aws_aurora_bgd_status(300, "AVAILABLE"); ok(hgm_query_string( "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" ) == "AVAILABLE", "Aurora BGD status API publishes an accepted state"); ok(MyHGM->gen_global_mysql_servers_v2_checksum(0) == checksum_before_status, "node-local Aurora BGD status is excluded from the cluster checksum"); - const char* accepted_statuses[] = { - "NONE", - "AVAILABLE", - "SWITCHOVER_INITIATED", - "SWITCHOVER_IN_PROGRESS", - "SWITCHOVER_IN_POST_PROCESSING", - "SWITCHOVER_COMPLETED" - }; - bool accepted_all_statuses = true; - for (const char* status : accepted_statuses) { - MyHGM->update_aws_aurora_bgd_status(300, status); - accepted_all_statuses = accepted_all_statuses && hgm_query_string( - "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" - ) == status; - } - ok(accepted_all_statuses, "Aurora BGD status API accepts the complete state vocabulary"); - MyHGM->update_aws_aurora_bgd_status(300, "AVAILABLE"); + SQLite3DB* configured_db = TestAuroraBGDRuntime::materialize_aurora_table(false); - ok(query_string(configured_db, - "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " - "FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "301,311", - "SAVE from runtime preserves both configured green hostgroups"); - ok(query_int(configured_db, - "SELECT COUNT(*) FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='bgd_status'") == 0, - "SAVE from runtime excludes bgd_status from configuration"); + ok(configured_db != nullptr, "configured Aurora table materializes without Admin object emulation"); + if (configured_db) { + ok(query_string(configured_db, + "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " + "FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "301,311", + "SAVE from runtime preserves both configured green hostgroups"); + ok(query_int(configured_db, + "SELECT COUNT(*) FROM pragma_table_info('mysql_aws_aurora_hostgroups') WHERE name='bgd_status'") == 0, + "SAVE from runtime excludes bgd_status from configuration"); + } else { + skip(2, "configured Aurora materialization failed"); + } delete configured_db; SQLite3DB* runtime_db = TestAuroraBGDRuntime::materialize_aurora_table(true); - ok(query_string(runtime_db, - "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " - "FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "301,311", - "runtime materialization preserves both configured green hostgroups"); - ok(query_string(runtime_db, - "SELECT bgd_status FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "AVAILABLE", - "runtime materialization includes the node-local bgd_status"); + ok(runtime_db != nullptr, "runtime Aurora table materializes without Admin object emulation"); + if (runtime_db) { + ok(query_string(runtime_db, + "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " + "FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "301,311", + "runtime materialization preserves both configured green hostgroups"); + ok(query_string(runtime_db, + "SELECT bgd_status FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300") == "AVAILABLE", + "runtime materialization includes the node-local bgd_status"); + } else { + skip(2, "runtime Aurora materialization failed"); + } delete runtime_db; +} +static void test_reload_and_status_ordering() { + int green_writer = 0; + int green_reader = 0; SQLite3_result* unrelated_reload = make_candidate(); add_candidate_row(unrelated_reload, 300, 310, "302", "312", true, "reloaded"); TestAuroraBGDRuntime::reload(MyHGM, unrelated_reload); @@ -384,28 +376,21 @@ static void test_runtime_ownership_and_status() { green_writer == 302 && green_reader == 312, "configuration reload updates runtime green hostgroups"); - MyHGM->update_aws_aurora_bgd_status(300, "NOT_A_BGD_STATUS"); + SQLite3_result* status_after_reload = make_candidate(); + add_candidate_row(status_after_reload, 300, 310, "303", "313", true, "status after reload"); + TestAuroraBGDRuntime::reload(MyHGM, status_after_reload); + MyHGM->update_aws_aurora_bgd_status(300, "SWITCHOVER_IN_PROGRESS"); ok(hgm_query_string( "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" - ) == "AVAILABLE", "invalid Aurora BGD status leaves runtime state unchanged"); - - SQLite3_result* concurrent_reload = make_candidate(); - add_candidate_row(concurrent_reload, 300, 310, "303", "313", true, "concurrent"); - std::thread reload_thread([concurrent_reload]() { - TestAuroraBGDRuntime::reload(MyHGM, concurrent_reload); - }); - std::thread status_thread([]() { - MyHGM->update_aws_aurora_bgd_status(300, "SWITCHOVER_IN_PROGRESS"); - }); - reload_thread.join(); - status_thread.join(); - ok(hgm_query_string( - "SELECT bgd_status FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=300" - ) == "SWITCHOVER_IN_PROGRESS", "reload and status publication serialize without losing state"); + ) == "SWITCHOVER_IN_PROGRESS", "status publication after reload updates the reloaded row"); ok(TestAuroraBGDRuntime::green_hostgroups(MyHGM, 300, green_writer, green_reader) && green_writer == 303 && green_reader == 313, - "serialized reload publishes its configured values"); + "status publication after reload keeps its configured values"); +} +static void test_runtime_removal() { + int green_writer = 0; + int green_reader = 0; SQLite3_result* empty_reload = make_candidate(); TestAuroraBGDRuntime::reload(MyHGM, empty_reload); ok(hgm_query_int( @@ -416,17 +401,22 @@ static void test_runtime_ownership_and_status() { } int main() { - plan(49); + plan(50); test_init_minimal(); test_schema_contract(); // 12 test_row_validation(); // 8 test_inactive_cross_row_validation(); // 4 - test_legacy_projection_normalization(); // 3 + test_noncanonical_projection_rejected(); // 2 test_invalid_replacement_removes_previous_row(); // 3 ok(test_init_hostgroups() == 0, "test_init_hostgroups() succeeds"); // 1 - test_runtime_ownership_and_status(); // 18 + ok(test_init_monitor() == 0, "test_init_monitor() succeeds"); // 1 + test_runtime_ownership(); // 4 + test_status_and_materialization(); // 8 + test_reload_and_status_ordering(); // 4 + test_runtime_removal(); // 2 + test_cleanup_monitor(); test_cleanup_hostgroups(); test_cleanup_minimal(); From f98563eb0a77c831ca90f39066ac4aaac2fe64c3 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 22:50:40 +0000 Subject: [PATCH 16/34] refactor: share AWS BGD simulator topology --- doc/AWS_Blue_Green/RDS_BGD_Simulator.md | 34 +-- include/SQLite3_Server.h | 14 +- src/SQLite3_Server.cpp | 84 +++--- test/tap/tap/Makefile | 9 +- test/tap/tap/bgd_simulator.cpp | 223 ++++++++++++++++ test/tap/tap/bgd_simulator.h | 60 +++++ test/tap/tap/rds_bgd_simulator.cpp | 220 +--------------- test/tap/tap/rds_bgd_simulator.h | 244 +----------------- test/tap/tap/rds_bgd_tap.h | 40 +-- .../test_rds_bgd_automatic_discovery-t.cpp | 22 +- .../test_rds_bgd_concurrent_isolation-t.cpp | 42 +-- ...nfig_refresh_after_writer_completion-t.cpp | 28 +- ...st_rds_bgd_configuration_persistence-t.cpp | 22 +- ...st_rds_bgd_disable_during_switchover-t.cpp | 14 +- .../tests/test_rds_bgd_explicit_startup-t.cpp | 18 +- ...st_rds_bgd_green_membership_ordering-t.cpp | 22 +- .../test_rds_bgd_green_pool_cleanup-t.cpp | 28 +- .../test_rds_bgd_late_entry_completed-t.cpp | 22 +- ...est_rds_bgd_late_entry_writer_phases-t.cpp | 26 +- test/tap/tests/test_rds_bgd_probe_tls-t.cpp | 38 +-- .../tests/test_rds_bgd_reader_policy-t.cpp | 24 +- ...st_rds_bgd_reader_switchover_cleanup-t.cpp | 36 +-- ...est_rds_bgd_remove_during_switchover-t.cpp | 14 +- .../test_rds_bgd_repeated_deployment-t.cpp | 38 +-- test/tap/tests/test_rds_bgd_rollback-t.cpp | 24 +- test/tap/tests/test_rds_bgd_smoke-t.cpp | 14 +- .../test_rds_bgd_topology_empty_absent-t.cpp | 44 ++-- .../tests/test_rds_bgd_topology_errors-t.cpp | 50 ++-- .../test_rds_bgd_worker_config_refresh-t.cpp | 32 +-- ...est_rds_bgd_worker_hostgroup_refresh-t.cpp | 22 +- .../test_rds_bgd_writer_switchover-t.cpp | 34 +-- 31 files changed, 694 insertions(+), 848 deletions(-) create mode 100644 test/tap/tap/bgd_simulator.cpp create mode 100644 test/tap/tap/bgd_simulator.h diff --git a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md index 7c228a93bc..01777d12d8 100644 --- a/doc/AWS_Blue_Green/RDS_BGD_Simulator.md +++ b/doc/AWS_Blue_Green/RDS_BGD_Simulator.md @@ -78,7 +78,7 @@ Create the following tables in `SQLite3_Server::init()` and store them in the existing persistent `GloVars.sqlite3serverdb` database: ```sql -CREATE TABLE RDS_BGD_CONTROL ( +CREATE TABLE AWS_BGD_CONTROL ( backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, topology_present INTEGER NOT NULL DEFAULT 0 CHECK (topology_present IN (0,1)), @@ -87,7 +87,7 @@ CREATE TABLE RDS_BGD_CONTROL ( PRIMARY KEY (backend_ip, backend_port) ); -CREATE TABLE RDS_BGD_TOPOLOGY ( +CREATE TABLE AWS_BGD_TOPOLOGY ( backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, row_order INTEGER NOT NULL, @@ -99,7 +99,7 @@ CREATE TABLE RDS_BGD_TOPOLOGY ( PRIMARY KEY (backend_ip, backend_port, row_order) ); -CREATE TABLE RDS_BGD_PROBE_LOG ( +CREATE TABLE AWS_BGD_PROBE_LOG ( sequence_id INTEGER PRIMARY KEY AUTOINCREMENT, backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, @@ -141,7 +141,7 @@ The TAP helper publishes control and topology changes atomically. Monitor probes read committed simulator state without holding a cross-query snapshot. The supported states are: -| `RDS_BGD_CONTROL` state | Topology rows | Meaning | +| `AWS_BGD_CONTROL` state | Topology rows | Meaning | |---|---|---| | No backend row | None | Backend is unconfigured; topology is absent. | | `topology_present=1`, `error_code=0` | One or more | Return the configured topology. | @@ -180,7 +180,7 @@ configured simulator errors may use `HY000` unless a test requires a specific mapping. Every handled topology-table check or metadata query appends one row to -`RDS_BGD_PROBE_LOG`, including empty and error responses. `sequence_id` +`AWS_BGD_PROBE_LOG`, including empty and error responses. `sequence_id` preserves order, `probe_kind` identifies the query, `backend_ip, backend_port` identify the destination, and `encrypted` records the accepted stream's TLS state. @@ -205,7 +205,7 @@ returns one `read_only` column. Table writes refresh the cache, and a missing entry returns the safe default `read_only=1`. BGD topology tasks send the production topology queries unchanged. Read-only -handling does not consult `RDS_BGD_CONTROL` or write `RDS_BGD_PROBE_LOG`. +handling does not consult `AWS_BGD_CONTROL` or write `AWS_BGD_PROBE_LOG`. ## TAP Helper API @@ -246,7 +246,7 @@ API; the helper closes it when destroyed. `read_only_update()` changes the ### Topology and Host Types ```cpp -struct RDS_BGD_Topology_Row { +struct BGD_Topology_Row { std::string id; std::string endpoint; int port; @@ -264,7 +264,7 @@ struct RDS_BGD_Host { }; ``` -`RDS_BGD_Topology_Row` represents one `mysql.rds_topology` row using +`BGD_Topology_Row` represents one `mysql.rds_topology` row using C++11-compatible field types. `RDS_BGD_Host` keeps the ProxySQL-facing hostname and simulator-facing IP together. @@ -282,7 +282,7 @@ public: std::vector get_blue_endpoints(); std::vector get_green_endpoints(); std::vector get_endpoints(); - std::vector get_topology(std::string status); + std::vector get_topology(std::string status); }; ``` @@ -300,7 +300,7 @@ reader mapping. ```cpp int topology_update( std::vector backends, - std::vector rows); + std::vector rows); int topology_delete(std::vector backends); @@ -330,26 +330,26 @@ not inherit simulator state from an earlier binary. ### Probe-Log Operations ```cpp -enum class RDS_BGD_Probe_Kind { +enum class BGD_Probe_Kind { table_check, metadata, }; -struct RDS_BGD_Probe_Log { +struct BGD_Probe_Log { uint64_t sequence_id; Endpoint backend; - RDS_BGD_Probe_Kind probe_kind; + BGD_Probe_Kind probe_kind; bool encrypted; }; rc_t probe_log_last_sequence(); -rc_t> probe_log_since(uint64_t sequence_id); +rc_t> probe_log_since(uint64_t sequence_id); -rc_t wait_for_probe_log( +rc_t wait_for_probe_log( uint64_t sequence_id, Endpoint backend, - RDS_BGD_Probe_Kind probe_kind, + BGD_Probe_Kind probe_kind, uint32_t timeout_ms, int encrypted = -1); ``` @@ -366,7 +366,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator simulator {}; + BGD_Simulator simulator {}; if (setup(cl, admin, simulator) != EXIT_SUCCESS) return exit_status(); diff --git a/include/SQLite3_Server.h b/include/SQLite3_Server.h index a2ceaabe2a..20b5f30ab5 100644 --- a/include/SQLite3_Server.h +++ b/include/SQLite3_Server.h @@ -51,13 +51,13 @@ class SQLite3_Server { std::unordered_map grouprep_map; std::vector *tables_defs_grouprep; #endif // TEST_GROUPREP -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) std::vector *tables_defs_readonly; std::unordered_map readonly_map; -#endif // TEST_READONLY || TEST_RDS_BGD -#ifdef TEST_RDS_BGD - std::vector *tables_defs_rds_bgd; -#endif // TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD +#if defined(TEST_AURORA) || defined(TEST_RDS_BGD) + std::vector *tables_defs_aws_bgd; +#endif // TEST_AURORA || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG std::unordered_map> replicationlag_map; std::vector* tables_defs_replicationlag; @@ -97,14 +97,14 @@ class SQLite3_Server { void init_grouprep_ifaces_string(std::string& s); group_rep_status grouprep_test_value(const std::string& srv_addr); #endif // TEST_GROUPREP -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) pthread_mutex_t test_readonly_mutex; void load_readonly_table(MySQL_Session *sess); int readonly_test_value(char *p); int readonly_map_size() { return readonly_map.size(); } -#endif // TEST_READONLY || TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG pthread_mutex_t test_replicationlag_mutex; void load_replicationlag_table(MySQL_Session* sess); diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index d2e7d3414f..093b284045 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -11,7 +11,7 @@ #include "proxysql_utils.h" #include "MySQL_Query_Processor.h" #include "SQLite3_Server.h" -#ifdef TEST_RDS_BGD +#if defined(TEST_AURORA) || defined(TEST_RDS_BGD) #include "MySQL_Monitor.hpp" #endif @@ -285,7 +285,7 @@ class sqlite3server_main_loop_listeners { static sqlite3server_main_loop_listeners S_amll; -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) static void ensure_readonly_table(SQLite3_Server *server, MySQL_Session *sess) { if (server->readonly_map_size() == 0) { server->load_readonly_table(sess); @@ -854,17 +854,17 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p if (run_query) { #if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG) || defined(TEST_RDS_BGD) if (strncasecmp("SELECT",query_no_space,6)==0) { -#ifdef TEST_RDS_BGD - const bool rds_bgd_table_check = +#if defined(TEST_AURORA) || defined(TEST_RDS_BGD) + const bool aws_bgd_table_check = strcasecmp(query_no_space, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK) == 0; - const bool rds_bgd_metadata = + const bool aws_bgd_metadata = strcasecmp(query_no_space, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY) == 0; - if (rds_bgd_table_check || rds_bgd_metadata) { + if (aws_bgd_table_check || aws_bgd_metadata) { if (sess->client_myds->proxy_addr.addr == NULL || sess->client_myds->proxy_addr.port <= 0) { GloSQLite3Server->send_MySQL_ERR( &sess->client_myds->myprot, 1105, - "RDS BGD simulator could not identify the accepted backend address"); + "AWS BGD simulator could not identify the accepted backend address"); run_query=false; } else { SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; @@ -875,16 +875,16 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p std::to_string(backend_port) }; const std::string log_query { - "INSERT INTO RDS_BGD_PROBE_LOG" + "INSERT INTO AWS_BGD_PROBE_LOG" "(backend_ip,backend_port,probe_kind,encrypted) VALUES ('" + backend_ip + "'," + std::to_string(backend_port) + ",'" + - (rds_bgd_table_check ? "table_check" : "metadata") + "'," + + (aws_bgd_table_check ? "table_check" : "metadata") + "'," + (sess->client_myds->encrypted ? "1" : "0") + ")" }; if (!sqlite_sess->sessdb->execute(log_query.c_str())) { GloSQLite3Server->send_MySQL_ERR( &sess->client_myds->myprot, 1105, - "RDS BGD simulator failed to record the topology probe"); + "AWS BGD simulator failed to record the topology probe"); run_query=false; } else { char *control_error=NULL; @@ -892,7 +892,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p int control_affected_rows=0; SQLite3_result *control_result=NULL; const std::string control_query { - "SELECT topology_present,error_code,error_msg FROM RDS_BGD_CONTROL WHERE " + + "SELECT topology_present,error_code,error_msg FROM AWS_BGD_CONTROL WHERE " + predicate }; sqlite_sess->sessdb->execute_statement( @@ -917,7 +917,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } delete control_result; - if (run_query && rds_bgd_table_check) { + if (run_query && aws_bgd_table_check) { const char* topology_sql = topology_present ? "SELECT 1" : "SELECT 1 WHERE 0"; static constexpr size_t topology_sql_len = sizeof("SELECT 1") - 1; @@ -940,7 +940,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } else if (run_query) { const std::string topology_query { "SELECT id,endpoint,topology_port AS port,role,status " - "FROM RDS_BGD_TOPOLOGY WHERE " + predicate + + "FROM AWS_BGD_TOPOLOGY WHERE " + predicate + " ORDER BY row_order" }; l_free(query_length,query); @@ -951,7 +951,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } -#endif // TEST_RDS_BGD +#endif // TEST_AURORA || TEST_RDS_BGD #ifdef TEST_AURORA if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { pthread_mutex_lock(&GloSQLite3Server->aurora_mutex); @@ -1034,7 +1034,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } #endif // TEST_GROUPREP -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) if (strncasecmp("SELECT @@global.read_only read_only ",query_no_space, k_select_read_only_len)==0 && query_no_space_length > k_select_read_only_len+5) { pthread_mutex_lock(&GloSQLite3Server->test_readonly_mutex); @@ -1048,7 +1048,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p query_length = formatted_query.size() + 1; pthread_mutex_unlock(&GloSQLite3Server->test_readonly_mutex); } -#endif // TEST_READONLY || TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG const bool replica_status = strncasecmp("SELECT REPLICA STATUS ", query_no_space, k_select_replica_status_len) == 0; const uint64_t addr_offset { @@ -1150,7 +1150,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p bool deprecate_eof = sess->client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot, in_trans, deprecate_eof); delete resultset; -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) if (strncasecmp("SELECT",query_no_space,6)) { if (strstr(query_no_space,(char *)"READONLY_STATUS")) { // the table is writable @@ -1159,7 +1159,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p pthread_mutex_unlock(&GloSQLite3Server->test_readonly_mutex); } } -#endif // TEST_READONLY || TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG if (strncasecmp("SELECT", query_no_space, 6)) { if (strstr(query_no_space, (char*)"REPLICATIONLAG_HOST_STATUS")) { @@ -1469,15 +1469,15 @@ SQLite3_Server::~SQLite3_Server() { delete tables_defs_grouprep; #endif // TEST_GROUPREP -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) drop_tables_defs(tables_defs_readonly); delete tables_defs_readonly; #endif -#ifdef TEST_RDS_BGD - drop_tables_defs(tables_defs_rds_bgd); - delete tables_defs_rds_bgd; -#endif // TEST_RDS_BGD +#if defined(TEST_AURORA) || defined(TEST_RDS_BGD) + drop_tables_defs(tables_defs_aws_bgd); + delete tables_defs_aws_bgd; +#endif // TEST_AURORA || TEST_RDS_BGD }; #ifdef TEST_AURORA @@ -1579,13 +1579,13 @@ SQLite3_Server::SQLite3_Server() { #ifdef TEST_GROUPREP init_grouprep_ifaces_string(s); #endif // TEST_GROUPREP -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) // Read-only simulation listens on all IPs because it can simulate many clusters. if (!s.empty()) s += ";"; s += "0.0.0.0:3306"; pthread_mutex_init(&test_readonly_mutex, NULL); -#endif // TEST_READONLY || TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG // for replication test we listen on all IPs if (!s.empty()) @@ -2041,7 +2041,7 @@ bool SQLite3_Server::init() { check_and_build_standard_tables(sessdb, tables_defs_grouprep); GloAdmin->enable_grouprep_testing(); #endif // TEST_GALERA -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) tables_defs_readonly = new std::vector; insert_into_tables_defs(tables_defs_readonly, (const char *)"READONLY_STATUS", @@ -2050,32 +2050,32 @@ bool SQLite3_Server::init() { #ifdef TEST_READONLY GloAdmin->enable_readonly_testing(); #endif // TEST_READONLY -#endif // TEST_READONLY || TEST_RDS_BGD -#ifdef TEST_RDS_BGD - tables_defs_rds_bgd = new std::vector; - insert_into_tables_defs(tables_defs_rds_bgd, - (const char *)"RDS_BGD_CONTROL", - (const char *)"CREATE TABLE RDS_BGD_CONTROL (" +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD +#if defined(TEST_AURORA) || defined(TEST_RDS_BGD) + tables_defs_aws_bgd = new std::vector; + insert_into_tables_defs(tables_defs_aws_bgd, + (const char *)"AWS_BGD_CONTROL", + (const char *)"CREATE TABLE AWS_BGD_CONTROL (" "backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, " "topology_present INTEGER NOT NULL DEFAULT 0 CHECK (topology_present IN (0,1)), " "error_code INTEGER NOT NULL DEFAULT 0, error_msg TEXT NOT NULL DEFAULT '', " "PRIMARY KEY (backend_ip, backend_port))"); - insert_into_tables_defs(tables_defs_rds_bgd, - (const char *)"RDS_BGD_TOPOLOGY", - (const char *)"CREATE TABLE RDS_BGD_TOPOLOGY (" + insert_into_tables_defs(tables_defs_aws_bgd, + (const char *)"AWS_BGD_TOPOLOGY", + (const char *)"CREATE TABLE AWS_BGD_TOPOLOGY (" "backend_ip TEXT NOT NULL, backend_port INTEGER NOT NULL, row_order INTEGER NOT NULL, " "id TEXT NOT NULL, endpoint TEXT NOT NULL, topology_port INTEGER NOT NULL, " "role TEXT NOT NULL, status TEXT NOT NULL, " "PRIMARY KEY (backend_ip, backend_port, row_order))"); - insert_into_tables_defs(tables_defs_rds_bgd, - (const char *)"RDS_BGD_PROBE_LOG", - (const char *)"CREATE TABLE RDS_BGD_PROBE_LOG (" + insert_into_tables_defs(tables_defs_aws_bgd, + (const char *)"AWS_BGD_PROBE_LOG", + (const char *)"CREATE TABLE AWS_BGD_PROBE_LOG (" "sequence_id INTEGER PRIMARY KEY AUTOINCREMENT, backend_ip TEXT NOT NULL, " "backend_port INTEGER NOT NULL, probe_kind TEXT NOT NULL " "CHECK (probe_kind IN ('table_check','metadata')), encrypted INTEGER NOT NULL " "CHECK (encrypted IN (0,1)))"); - check_and_build_standard_tables(sessdb, tables_defs_rds_bgd); -#endif // TEST_RDS_BGD + check_and_build_standard_tables(sessdb, tables_defs_aws_bgd); +#endif // TEST_AURORA || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG tables_defs_replicationlag = new std::vector; insert_into_tables_defs(tables_defs_replicationlag, @@ -2205,7 +2205,7 @@ void SQLite3_Server::send_MySQL_ERR(MySQL_Protocol *myprot, uint16_t error_code, myds->DSS=STATE_SLEEP; } -#if defined(TEST_READONLY) || defined(TEST_RDS_BGD) +#if defined(TEST_AURORA) || defined(TEST_READONLY) || defined(TEST_RDS_BGD) void SQLite3_Server::load_readonly_table(MySQL_Session *sess) { // this function needs to be called with lock on mutex readonly_mutex already acquired GloAdmin->mysql_servers_wrlock(); @@ -2246,7 +2246,7 @@ int SQLite3_Server::readonly_test_value(char *p) { } return rc; } -#endif // TEST_READONLY || TEST_RDS_BGD +#endif // TEST_AURORA || TEST_READONLY || TEST_RDS_BGD #ifdef TEST_REPLICATIONLAG void SQLite3_Server::load_replicationlag_table(MySQL_Session* sess) { diff --git a/test/tap/tap/Makefile b/test/tap/tap/Makefile index ead866e075..6e090f4ad9 100644 --- a/test/tap/tap/Makefile +++ b/test/tap/tap/Makefile @@ -83,11 +83,14 @@ mcp_client.o: mcp_client.cpp mcp_client.h libcurl$(SHLIB_EXT) cluster_simulator.o: cluster_simulator.cpp cluster_simulator.h $(CXX) -fPIC -c cluster_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) -rds_bgd_simulator.o: rds_bgd_simulator.cpp rds_bgd_simulator.h cluster_simulator.h utils.h +bgd_simulator.o: bgd_simulator.cpp bgd_simulator.h cluster_simulator.h utils.h + $(CXX) -fPIC -c bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) + +rds_bgd_simulator.o: rds_bgd_simulator.cpp rds_bgd_simulator.h bgd_simulator.h $(CXX) -fPIC -c rds_bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) -libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o rds_bgd_simulator.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a - $(AR) rcs libtap_mariadb.a tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o rds_bgd_simulator.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo +libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o bgd_simulator.o rds_bgd_simulator.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a + $(AR) rcs libtap_mariadb.a tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o bgd_simulator.o rds_bgd_simulator.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo libtap_mysql57.a: tap.o command_line.o utils_mysql57.o noise_utils_mysql57.o mcp_client.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a $(AR) rcs libtap_mysql57.a tap.o command_line.o utils_mysql57.o noise_utils_mysql57.o mcp_client.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo diff --git a/test/tap/tap/bgd_simulator.cpp b/test/tap/tap/bgd_simulator.cpp new file mode 100644 index 0000000000..c0f476e706 --- /dev/null +++ b/test/tap/tap/bgd_simulator.cpp @@ -0,0 +1,223 @@ +#include "bgd_simulator.h" + +#include +#include +#include +#include + +#include "tap.h" + +using namespace std; + +namespace { + +const char* probe_kind_string(BGD_Probe_Kind kind) { + return kind == BGD_Probe_Kind::table_check ? "table_check" : "metadata"; +} + +rc_t parse_probe_kind(string value) { + if (value == "table_check") { + return { EXIT_SUCCESS, BGD_Probe_Kind::table_check }; + } + if (value == "metadata") { + return { EXIT_SUCCESS, BGD_Probe_Kind::metadata }; + } + return { EXIT_FAILURE, BGD_Probe_Kind::table_check }; +} + +} // namespace + +int BGD_Simulator::topology_update(vector backends, vector rows) { + if (backends.empty()) { + return EXIT_FAILURE; + } + + vector statements {}; + for (Endpoint& backend : backends) { + string predicate { backend_predicate(backend) }; + statements.push_back("DELETE FROM AWS_BGD_TOPOLOGY WHERE " + predicate); + statements.push_back( + "INSERT OR REPLACE INTO AWS_BGD_CONTROL" + "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + ",1,0,'')"); + + for (size_t row_order = 0; row_order < rows.size(); ++row_order) { + BGD_Topology_Row& row = rows[row_order]; + statements.push_back( + "INSERT INTO AWS_BGD_TOPOLOGY" + "(backend_ip,backend_port,row_order,id,endpoint,topology_port,role,status) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + to_string(row_order) + "," + sql_quote(row.id) + "," + + sql_quote(row.endpoint) + "," + to_string(row.port) + "," + + sql_quote(row.role) + "," + sql_quote(row.status) + ")"); + } + } + + return execute_transaction(statements); +} + +int BGD_Simulator::topology_delete(vector backends) { + if (backends.empty()) { + return EXIT_FAILURE; + } + + vector statements {}; + for (Endpoint& backend : backends) { + statements.push_back( + "DELETE FROM AWS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); + statements.push_back( + "INSERT OR REPLACE INTO AWS_BGD_CONTROL" + "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + ",1,0,'')"); + } + return execute_transaction(statements); +} + +int BGD_Simulator::topology_drop(vector backends) { + return topology_error(backends, 1146, "Table 'mysql.rds_topology' doesn't exist"); +} + +int BGD_Simulator::topology_error(vector backends, int error_code, string error_msg) { + if (backends.empty() || error_code == 0) { + return EXIT_FAILURE; + } + + bool topology_present = error_code != 1146; + vector statements {}; + for (Endpoint& backend : backends) { + if (!topology_present) { + statements.push_back( + "DELETE FROM AWS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); + } + statements.push_back( + "INSERT OR REPLACE INTO AWS_BGD_CONTROL" + "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + (topology_present ? "1" : "0") + "," + to_string(error_code) + "," + + sql_quote(error_msg) + ")"); + } + return execute_transaction(statements); +} + +int BGD_Simulator::cleanup() { + vector statements { + "DELETE FROM READONLY_STATUS", + "DELETE FROM AWS_BGD_TOPOLOGY", + "DELETE FROM AWS_BGD_CONTROL", + "DELETE FROM AWS_BGD_PROBE_LOG", + }; + return execute_transaction(statements); +} + +rc_t BGD_Simulator::probe_log_last_sequence() { + if (connection() == nullptr) { + return { EXIT_FAILURE, 0 }; + } + + auto [rc, rows] = mysql_query_ext_rows( + connection(), "SELECT COALESCE(MAX(sequence_id),0) FROM AWS_BGD_PROBE_LOG"); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { + return { EXIT_FAILURE, 0 }; + } + + return { + EXIT_SUCCESS, + static_cast(strtoull(rows.front().front().c_str(), nullptr, 10)) + }; +} + +rc_t> BGD_Simulator::probe_log_since(uint64_t sequence_id) { + if (connection() == nullptr) { + return { EXIT_FAILURE, {} }; + } + + string query { + "SELECT sequence_id,backend_ip,backend_port,probe_kind,encrypted " + "FROM AWS_BGD_PROBE_LOG WHERE sequence_id>" + to_string(sequence_id) + + " ORDER BY sequence_id" + }; + auto [rc, rows] = mysql_query_ext_rows(connection(), query); + if (rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + + vector logs {}; + for (mysql_res_row& row : rows) { + if (row.size() != 5) { + return { EXIT_FAILURE, {} }; + } + auto [kind_rc, kind] = parse_probe_kind(row[3]); + if (kind_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + logs.push_back({ + static_cast(strtoull(row[0].c_str(), nullptr, 10)), + { row[1], atoi(row[2].c_str()) }, + kind, + atoi(row[4].c_str()) != 0, + }); + } + + return { EXIT_SUCCESS, move(logs) }; +} + +rc_t BGD_Simulator::wait_for_probe_log( + uint64_t sequence_id, + Endpoint backend, + BGD_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted) +{ + uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + auto [rc, logs] = probe_log_since(sequence_id); + if (rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + for (BGD_Probe_Log& log : logs) { + if (log.backend.host == backend.host && log.backend.port == backend.port && + log.probe_kind == probe_kind && + (encrypted < 0 || log.encrypted == (encrypted != 0))) { + return { EXIT_SUCCESS, log }; + } + } + usleep(50000); + } while (monotonic_time() < deadline); + + auto [rc, logs] = probe_log_since(sequence_id); + if (rc == EXIT_SUCCESS) { + for (BGD_Probe_Log& log : logs) { + diag( + "Observed BGD probe sequence=%llu backend=%s:%d kind=%s encrypted=%d", + static_cast(log.sequence_id), + log.backend.host.c_str(), log.backend.port, + probe_kind_string(log.probe_kind), log.encrypted ? 1 : 0); + } + } + diag( + "Timed out waiting for BGD probe backend=%s:%d kind=%s encrypted=%d", + backend.host.c_str(), backend.port, probe_kind_string(probe_kind), encrypted); + return { ETIMEDOUT, {} }; +} + +int BGD_Simulator::execute_transaction(vector& statements) { + if (execute("START TRANSACTION") != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + for (string& statement : statements) { + if (execute(statement) != EXIT_SUCCESS) { + (void)execute("ROLLBACK"); + return EXIT_FAILURE; + } + } + if (execute("COMMIT") != EXIT_SUCCESS) { + (void)execute("ROLLBACK"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +string BGD_Simulator::backend_predicate(Endpoint backend) { + return "backend_ip=" + sql_quote(backend.host) + + " AND backend_port=" + to_string(backend.port); +} diff --git a/test/tap/tap/bgd_simulator.h b/test/tap/tap/bgd_simulator.h new file mode 100644 index 0000000000..a98013319d --- /dev/null +++ b/test/tap/tap/bgd_simulator.h @@ -0,0 +1,60 @@ +#ifndef TAP_BGD_SIMULATOR_H +#define TAP_BGD_SIMULATOR_H + +#include +#include +#include + +#include "cluster_simulator.h" +#include "utils.h" + +using namespace std; + +/** One row returned by the simulated mysql.rds_topology table. */ +struct BGD_Topology_Row { + string id; + string endpoint; + int port; + string role; + string status; +}; + +/** Identifies a topology query recorded by the shared AWS BGD simulator. */ +enum class BGD_Probe_Kind { + table_check, + metadata, +}; + +/** One topology query observed by the shared AWS BGD simulator. */ +struct BGD_Probe_Log { + uint64_t sequence_id; + Endpoint backend; + BGD_Probe_Kind probe_kind; + bool encrypted; +}; + +/** Controls shared AWS BGD topology responses and reads their probe log. */ +class BGD_Simulator : public Cluster_Simulator { +public: + int topology_update(vector backends, vector rows); + int topology_delete(vector backends); + int topology_drop(vector backends); + int topology_error(vector backends, int error_code, string error_msg); + int cleanup(); + + rc_t probe_log_last_sequence(); + rc_t> probe_log_since(uint64_t sequence_id); + rc_t wait_for_probe_log( + uint64_t sequence_id, + Endpoint backend, + BGD_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted = -1 + ); + +protected: + static string backend_predicate(Endpoint backend); + int execute_transaction(vector& statements); +}; + +#endif // TAP_BGD_SIMULATOR_H diff --git a/test/tap/tap/rds_bgd_simulator.cpp b/test/tap/tap/rds_bgd_simulator.cpp index 5b53f0aad4..673aa50790 100644 --- a/test/tap/tap/rds_bgd_simulator.cpp +++ b/test/tap/tap/rds_bgd_simulator.cpp @@ -1,28 +1,7 @@ #include "rds_bgd_simulator.h" -#include -#include -#include -#include - -#include "tap.h" - using namespace std; -const char* probe_kind_string(RDS_BGD_Probe_Kind kind) { - return kind == RDS_BGD_Probe_Kind::table_check ? "table_check" : "metadata"; -} - -rc_t parse_probe_kind(string value) { - if (value == "table_check") { - return { EXIT_SUCCESS, RDS_BGD_Probe_Kind::table_check }; - } - if (value == "metadata") { - return { EXIT_SUCCESS, RDS_BGD_Probe_Kind::metadata }; - } - return { EXIT_FAILURE, RDS_BGD_Probe_Kind::table_check }; -} - Endpoint RDS_BGD_Host::endpoint() { return { ip, port }; } @@ -54,7 +33,7 @@ vector RDS_BGD_Cluster::get_endpoints() { return endpoints; } -vector RDS_BGD_Cluster::get_topology(string status) { +vector RDS_BGD_Cluster::get_topology(string status) { return { { blue_writer.hostname, blue_writer.hostname, blue_writer.port, "BLUE_GREEN_DEPLOYMENT_SOURCE", status }, @@ -62,200 +41,3 @@ vector RDS_BGD_Cluster::get_topology(string status) { "BLUE_GREEN_DEPLOYMENT_TARGET", status }, }; } - -int RDS_BGD_Simulator::topology_update(vector backends, vector rows) { - if (backends.empty()) { - return EXIT_FAILURE; - } - - vector statements {}; - for (Endpoint& backend : backends) { - string predicate { backend_predicate(backend) }; - statements.push_back("DELETE FROM RDS_BGD_TOPOLOGY WHERE " + predicate); - statements.push_back( - "INSERT OR REPLACE INTO RDS_BGD_CONTROL" - "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + - sql_quote(backend.host) + "," + to_string(backend.port) + ",1,0,'')"); - - for (size_t row_order = 0; row_order < rows.size(); ++row_order) { - RDS_BGD_Topology_Row& row = rows[row_order]; - statements.push_back( - "INSERT INTO RDS_BGD_TOPOLOGY" - "(backend_ip,backend_port,row_order,id,endpoint,topology_port,role,status) VALUES (" + - sql_quote(backend.host) + "," + to_string(backend.port) + "," + - to_string(row_order) + "," + sql_quote(row.id) + "," + - sql_quote(row.endpoint) + "," + to_string(row.port) + "," + - sql_quote(row.role) + "," + sql_quote(row.status) + ")"); - } - } - - return execute_transaction(statements); -} - -int RDS_BGD_Simulator::topology_delete(vector backends) { - if (backends.empty()) { - return EXIT_FAILURE; - } - - vector statements {}; - for (Endpoint& backend : backends) { - statements.push_back( - "DELETE FROM RDS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); - statements.push_back( - "INSERT OR REPLACE INTO RDS_BGD_CONTROL" - "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + - sql_quote(backend.host) + "," + to_string(backend.port) + ",1,0,'')"); - } - return execute_transaction(statements); -} - -int RDS_BGD_Simulator::topology_drop(vector backends) { - return topology_error(backends, 1146, "Table 'mysql.rds_topology' doesn't exist"); -} - -int RDS_BGD_Simulator::topology_error(vector backends, int error_code, string error_msg) { - if (backends.empty() || error_code == 0) { - return EXIT_FAILURE; - } - - bool topology_present = error_code != 1146; - vector statements {}; - for (Endpoint& backend : backends) { - if (!topology_present) { - statements.push_back( - "DELETE FROM RDS_BGD_TOPOLOGY WHERE " + backend_predicate(backend)); - } - statements.push_back( - "INSERT OR REPLACE INTO RDS_BGD_CONTROL" - "(backend_ip,backend_port,topology_present,error_code,error_msg) VALUES (" + - sql_quote(backend.host) + "," + to_string(backend.port) + "," + - (topology_present ? "1" : "0") + "," + to_string(error_code) + "," + - sql_quote(error_msg) + ")"); - } - return execute_transaction(statements); -} - -int RDS_BGD_Simulator::cleanup() { - vector statements { - "DELETE FROM READONLY_STATUS", - "DELETE FROM RDS_BGD_TOPOLOGY", - "DELETE FROM RDS_BGD_CONTROL", - "DELETE FROM RDS_BGD_PROBE_LOG", - }; - return execute_transaction(statements); -} - -rc_t RDS_BGD_Simulator::probe_log_last_sequence() { - if (connection() == nullptr) { - return { EXIT_FAILURE, 0 }; - } - - auto [rc, rows] = mysql_query_ext_rows( - connection(), "SELECT COALESCE(MAX(sequence_id),0) FROM RDS_BGD_PROBE_LOG"); - if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { - return { EXIT_FAILURE, 0 }; - } - - return { - EXIT_SUCCESS, - static_cast(strtoull(rows.front().front().c_str(), nullptr, 10)) - }; -} - -rc_t> RDS_BGD_Simulator::probe_log_since( - uint64_t sequence_id) -{ - if (connection() == nullptr) { - return { EXIT_FAILURE, {} }; - } - - string query { - "SELECT sequence_id,backend_ip,backend_port,probe_kind,encrypted " - "FROM RDS_BGD_PROBE_LOG WHERE sequence_id>" + to_string(sequence_id) + - " ORDER BY sequence_id" - }; - auto [rc, rows] = mysql_query_ext_rows(connection(), query); - if (rc != EXIT_SUCCESS) { - return { EXIT_FAILURE, {} }; - } - - vector logs {}; - for (mysql_res_row& row : rows) { - if (row.size() != 5) { - return { EXIT_FAILURE, {} }; - } - auto [kind_rc, kind] = parse_probe_kind(row[3]); - if (kind_rc != EXIT_SUCCESS) { - return { EXIT_FAILURE, {} }; - } - logs.push_back({ - static_cast(strtoull(row[0].c_str(), nullptr, 10)), - { row[1], atoi(row[2].c_str()) }, - kind, - atoi(row[4].c_str()) != 0, - }); - } - - return { EXIT_SUCCESS, move(logs) }; -} - -rc_t RDS_BGD_Simulator::wait_for_probe_log( - uint64_t sequence_id, - Endpoint backend, - RDS_BGD_Probe_Kind probe_kind, - uint32_t timeout_ms, - int encrypted) -{ - uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; - do { - auto [rc, logs] = probe_log_since(sequence_id); - if (rc != EXIT_SUCCESS) { - return { EXIT_FAILURE, {} }; - } - for (RDS_BGD_Probe_Log& log : logs) { - if (log.backend.host == backend.host && log.backend.port == backend.port && - log.probe_kind == probe_kind && - (encrypted < 0 || log.encrypted == (encrypted != 0))) { - return { EXIT_SUCCESS, log }; - } - } - usleep(50000); - } while (monotonic_time() < deadline); - - auto [rc, logs] = probe_log_since(sequence_id); - if (rc == EXIT_SUCCESS) { - for (RDS_BGD_Probe_Log& log : logs) { - diag( - "Observed BGD probe sequence=%llu backend=%s:%d kind=%s encrypted=%d", - static_cast(log.sequence_id), - log.backend.host.c_str(), log.backend.port, - probe_kind_string(log.probe_kind), log.encrypted ? 1 : 0); - } - } - diag( - "Timed out waiting for BGD probe backend=%s:%d kind=%s encrypted=%d", - backend.host.c_str(), backend.port, probe_kind_string(probe_kind), encrypted); - return { ETIMEDOUT, {} }; -} - -int RDS_BGD_Simulator::execute_transaction(vector& statements) { - if (execute("START TRANSACTION") != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - for (string& statement : statements) { - if (execute(statement) != EXIT_SUCCESS) { - (void)execute("ROLLBACK"); - return EXIT_FAILURE; - } - } - if (execute("COMMIT") != EXIT_SUCCESS) { - (void)execute("ROLLBACK"); - return EXIT_FAILURE; - } - return EXIT_SUCCESS; -} - -string RDS_BGD_Simulator::backend_predicate(Endpoint backend) { - return "backend_ip=" + sql_quote(backend.host) + - " AND backend_port=" + to_string(backend.port); -} diff --git a/test/tap/tap/rds_bgd_simulator.h b/test/tap/tap/rds_bgd_simulator.h index 2827d39435..a112d760a3 100644 --- a/test/tap/tap/rds_bgd_simulator.h +++ b/test/tap/tap/rds_bgd_simulator.h @@ -1,258 +1,36 @@ #ifndef TAP_RDS_BGD_SIMULATOR_H #define TAP_RDS_BGD_SIMULATOR_H -#include #include #include -#include "cluster_simulator.h" -#include "utils.h" +#include "bgd_simulator.h" using namespace std; -/** - * @brief Represents one row returned by the simulated `mysql.rds_topology` table. - */ -struct RDS_BGD_Topology_Row { - string id; ///< RDS topology node identifier. - string endpoint; ///< RDS hostname exposed by the topology row. - int port; ///< MySQL port exposed by the topology row. - string role; ///< Blue/green deployment role reported by RDS. - string status; ///< Blue/green deployment status reported by RDS. -}; - -/** - * @brief Describes one RDS BGD host and its fixed simulator address. - */ +/** Describes one RDS BGD host and its fixed simulator address. */ struct RDS_BGD_Host { - string hostname; ///< AWS-style RDS hostname configured in ProxySQL. - string ip; ///< Fixed loopback address used by the simulator. - int port; ///< MySQL listener port shared by the hostname and IP. + string hostname; + string ip; + int port; - /** - * @brief Returns the IP/port endpoint used for topology simulation. - * - * @return Simulator endpoint containing this host's IP address and port. - */ Endpoint endpoint(); - - /** - * @brief Returns the hostname/port endpoint used for read-only simulation. - * - * @return Simulator endpoint containing this host's RDS hostname and port. - */ Endpoint host_endpoint(); }; -/** - * @brief Holds the blue and green hosts participating in one simulated RDS BGD cluster. - * - * @details TAP tests populate the cluster with the deployment topology required by each - * scenario. Helper methods derive writer endpoints and AWS topology rows from - * the configured hosts. - */ +/** Holds the blue and green hosts in one simulated RDS BGD deployment. */ class RDS_BGD_Cluster { public: - RDS_BGD_Host blue_writer; ///< Source writer configured in ProxySQL. - RDS_BGD_Host green_writer; ///< Target writer discovered from the topology. - vector blue_readers; ///< Source readers configured in ProxySQL. - vector green_readers; ///< Target readers discovered from the topology. + RDS_BGD_Host blue_writer; + RDS_BGD_Host green_writer; + vector blue_readers; + vector green_readers; - /** - * @brief Returns both writer IP/port endpoints for topology simulation. - * - * @return Blue and green writer endpoints keyed by simulator IP address. - */ vector get_writers(); - - /** - * @brief Returns the blue writer and configured blue readers. - * - * @return Blue deployment endpoints keyed by simulator IP address. - */ vector get_blue_endpoints(); - - /** - * @brief Returns the green writer and configured green readers. - * - * @return Green deployment endpoints keyed by simulator IP address. - */ vector get_green_endpoints(); - - /** - * @brief Returns every simulator IP/port endpoint in this cluster. - * - * @details Includes both writers and all configured blue and green readers. - * Tests use this list when resetting or publishing topology for a complete - * simulated deployment. - * - * @return Writer and reader endpoints keyed by simulator IP address. - */ vector get_endpoints(); - - /** - * @brief Builds the topology rows published by the simulated writers. - * - * @details Creates one source row for the blue writer and one target row for the green - * writer. The supplied deployment status is applied to both rows. - * - * @param status RDS blue/green deployment status to publish. - * - * @return Source and target rows for the simulated topology table. - */ - vector get_topology(string status); -}; - -/** - * @brief Identifies the RDS BGD monitor query recorded in the simulator probe log. - */ -enum class RDS_BGD_Probe_Kind { - table_check, ///< Query checking whether `mysql.rds_topology` exists. - metadata, ///< Query fetching rows from `mysql.rds_topology`. -}; - -/** - * @brief Describes one RDS BGD monitor query observed by the simulator. - */ -struct RDS_BGD_Probe_Log { - uint64_t sequence_id; ///< Monotonically increasing probe-log sequence. - Endpoint backend; ///< Accepted backend IP address and port. - RDS_BGD_Probe_Kind probe_kind; ///< Type of topology query observed. - bool encrypted; ///< Whether the monitor connection used TLS. -}; - -/** - * @brief Controls RDS BGD topology responses and inspects monitor probes from TAP tests. - * - * @details Publishes per-backend topology rows, empty results, missing tables, or MySQL - * errors. It also reads the ordered probe log generated by the SQLite3-server - * simulator and reuses `Cluster_Simulator` for shared control operations. - */ -class RDS_BGD_Simulator : public Cluster_Simulator { -public: - /** - * @brief Replaces the simulated topology returned by each backend. - * - * @details Deletes existing topology rows before inserting the supplied rows. The - * topology table is marked present, configured errors are cleared, and the - * complete update is applied atomically across all supplied backends. - * - * @param backends Backend IP/port endpoints that must return the topology. - * @param rows Topology rows to publish on each backend. - * - * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. - */ - int topology_update(vector backends, vector rows); - - /** - * @brief Configures each backend to return an empty topology result. - * - * @details Deletes all topology rows associated with the supplied backends while keeping - * the topology table present and clearing any configured metadata error. - * - * @param backends Backend IP/port endpoints that must return an empty result. - * - * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. - */ - int topology_delete(vector backends); - - /** - * @brief Configures each backend to report that the topology table does not exist. - * - * @param backends Backend IP/port endpoints that must return MySQL error 1146. - * - * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. - */ - int topology_drop(vector backends); - - /** - * @brief Configures a MySQL error for topology queries on each backend. - * - * @details Stores the nonzero error code and message returned by subsequent metadata - * probes. Error 1146 marks the topology table absent and removes its existing - * rows; other error codes leave the table marked present. - * - * @param backends Backend IP/port endpoints that must return the error. - * @param error_code Nonzero MySQL error code to return. - * @param error_msg MySQL error message to return. - * - * @return EXIT_SUCCESS when every backend is updated; EXIT_FAILURE otherwise. - */ - int topology_error(vector backends, int error_code, string error_msg); - - /** - * @brief Removes all read-only, topology-control, topology-row, and probe state. - * - * @return EXIT_SUCCESS when the simulator state is empty; EXIT_FAILURE otherwise. - */ - int cleanup(); - - /** - * @brief Reads the latest sequence from the RDS BGD probe log. - * - * @return EXIT_SUCCESS and the latest sequence, or zero when the log is empty; - * EXIT_FAILURE and zero when the query fails. - */ - rc_t probe_log_last_sequence(); - - /** - * @brief Returns probe-log records newer than a sequence. - * - * @details Selects records with `sequence_id` strictly greater than the supplied value - * and preserves database sequence order in the returned vector. - * - * @param sequence_id Last probe-log sequence already observed by the TAP test. - * - * @return EXIT_SUCCESS and the matching records; EXIT_FAILURE and an empty vector - * when the query or record parsing fails. - */ - rc_t> probe_log_since(uint64_t sequence_id); - - /** - * @brief Waits for a matching RDS BGD probe-log record. - * - * @details Matches records newer than `sequence_id` by backend and probe kind. TLS state - * is matched when `encrypted` is zero or one; `-1` accepts either state. Observed - * probes are emitted through TAP diagnostics when the wait expires. - * - * @param sequence_id Last probe-log sequence observed before the expected probe. - * @param backend Backend IP/port endpoint expected to receive the probe. - * @param probe_kind Type of topology query expected. - * @param timeout_ms Maximum time to wait in milliseconds. - * @param encrypted Expected TLS state, or -1 to accept either state. - * - * @return EXIT_SUCCESS and the matching record; ETIMEDOUT and an empty record when - * the deadline expires; EXIT_FAILURE and an empty record when log retrieval fails. - */ - rc_t wait_for_probe_log( - uint64_t sequence_id, - Endpoint backend, - RDS_BGD_Probe_Kind probe_kind, - uint32_t timeout_ms, - int encrypted = -1 - ); - -private: - /** - * @brief Builds the SQL predicate identifying one simulated backend. - * - * @param backend Backend IP/port endpoint to match. - * - * @return SQL predicate matching the backend control-table key. - */ - static string backend_predicate(Endpoint backend); - - /** - * @brief Executes simulator control statements in one transaction. - * - * @details Executes the supplied statements in order and commits only after every - * statement succeeds. A statement or commit failure triggers a rollback. - * - * @param statements SQL statements to execute atomically. - * - * @return EXIT_SUCCESS when the transaction commits; EXIT_FAILURE otherwise. - */ - int execute_transaction(vector& statements); + vector get_topology(string status); }; #endif // TAP_RDS_BGD_SIMULATOR_H diff --git a/test/tap/tap/rds_bgd_tap.h b/test/tap/tap/rds_bgd_tap.h index 619450c7af..16bb5091f7 100644 --- a/test/tap/tap/rds_bgd_tap.h +++ b/test/tap/tap/rds_bgd_tap.h @@ -90,8 +90,8 @@ struct BGD_Hostgroups { int green_reader; }; -inline vector bgd_topology_with_readers(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +inline vector bgd_topology_with_readers(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); for (RDS_BGD_Host& host : cluster.blue_readers) { rows.push_back({ host.hostname, host.hostname, host.port, "BLUE_GREEN_DEPLOYMENT_SOURCE", status }); } @@ -101,7 +101,7 @@ inline vector bgd_topology_with_readers(RDS_BGD_Cluster& c return rows; } -inline int bgd_set_writer_read_only_0(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +inline int bgd_set_writer_read_only_0(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (sim.read_only_update(cluster.blue_writer.host_endpoint(), false) != EXIT_SUCCESS) { diag("Error: failed to set read_only=0 for the simulated blue writer"); return EXIT_FAILURE; @@ -114,12 +114,12 @@ inline int bgd_set_writer_read_only_0(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& c return EXIT_SUCCESS; } -inline int bgd_set_host_read_only_0(RDS_BGD_Simulator& sim, RDS_BGD_Host& host) { +inline int bgd_set_host_read_only_0(BGD_Simulator& sim, RDS_BGD_Host& host) { int rc = sim.read_only_update(host.host_endpoint(), false); return rc; } -inline int bgd_set_host_read_only_1(RDS_BGD_Simulator& sim, RDS_BGD_Host& host) { +inline int bgd_set_host_read_only_1(BGD_Simulator& sim, RDS_BGD_Host& host) { int rc = sim.read_only_update(host.host_endpoint(), true); return rc; } @@ -310,7 +310,7 @@ inline rc_t bgd_backend_ip_echo(MYSQL* proxy) { } inline rc_t bgd_probe_count_since( - RDS_BGD_Simulator& sim, uint64_t sequence, Endpoint backend, RDS_BGD_Probe_Kind kind) + BGD_Simulator& sim, uint64_t sequence, Endpoint backend, BGD_Probe_Kind kind) { auto [rc, logs] = sim.probe_log_since(sequence); if (rc != EXIT_SUCCESS) { @@ -319,7 +319,7 @@ inline rc_t bgd_probe_count_since( } uint64_t count = 0; - for (const RDS_BGD_Probe_Log& log : logs) { + for (const BGD_Probe_Log& log : logs) { bool backend_matches = log.backend.host == backend.host && log.backend.port == backend.port; @@ -417,19 +417,19 @@ inline int bgd_wait_for_server_placement( return rc; } -inline rc_t bgd_wait_for_probe_from_backends( - RDS_BGD_Simulator& sim, uint64_t sequence, vector backends, - RDS_BGD_Probe_Kind kind, uint32_t timeout_ms, int encrypted = -1) +inline rc_t bgd_wait_for_probe_from_backends( + BGD_Simulator& sim, uint64_t sequence, vector backends, + BGD_Probe_Kind kind, uint32_t timeout_ms, int encrypted = -1) { uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; do { auto [rc, logs] = sim.probe_log_since(sequence); if (rc != EXIT_SUCCESS) { - rc_t result { EXIT_FAILURE, {} }; + rc_t result { EXIT_FAILURE, {} }; return result; } - for (const RDS_BGD_Probe_Log& log : logs) { + for (const BGD_Probe_Log& log : logs) { for (const Endpoint& backend : backends) { bool backend_matches = log.backend.host == backend.host && @@ -439,7 +439,7 @@ inline rc_t bgd_wait_for_probe_from_backends( encrypted < 0 || log.encrypted == (encrypted != 0); if (backend_matches && kind_matches && encryption_matches) { - rc_t result { EXIT_SUCCESS, log }; + rc_t result { EXIT_SUCCESS, log }; return result; } } @@ -448,7 +448,7 @@ inline rc_t bgd_wait_for_probe_from_backends( usleep(50000); } while (monotonic_time() < deadline); - rc_t result { ETIMEDOUT, {} }; + rc_t result { ETIMEDOUT, {} }; return result; } @@ -459,10 +459,10 @@ inline rc_t bgd_wait_for_probe_from_backends( * after the given sequence. */ inline int bgd_expect_no_table_check( - RDS_BGD_Simulator& sim, uint64_t sequence, vector backends, uint32_t timeout_ms) + BGD_Simulator& sim, uint64_t sequence, vector backends, uint32_t timeout_ms) { auto [probe_rc, probe] = bgd_wait_for_probe_from_backends( - sim, sequence, backends, RDS_BGD_Probe_Kind::table_check, timeout_ms + sim, sequence, backends, BGD_Probe_Kind::table_check, timeout_ms ); if (probe_rc == ETIMEDOUT) { @@ -478,12 +478,12 @@ inline int bgd_expect_no_table_check( * endpoint after the given sequence. */ inline int bgd_expect_no_metadata_probe( - RDS_BGD_Simulator& sim, uint64_t sequence, Endpoint backend, uint32_t timeout_ms) + BGD_Simulator& sim, uint64_t sequence, Endpoint backend, uint32_t timeout_ms) { vector backends { backend }; auto [probe_rc, probe] = bgd_wait_for_probe_from_backends( - sim, sequence, backends, RDS_BGD_Probe_Kind::metadata, timeout_ms + sim, sequence, backends, BGD_Probe_Kind::metadata, timeout_ms ); if (probe_rc == ETIMEDOUT) { @@ -499,10 +499,10 @@ inline int bgd_expect_no_metadata_probe( * endpoint after the given sequence. */ inline int bgd_expect_no_metadata_probe_from_backends( - RDS_BGD_Simulator& sim, uint64_t sequence, vector backends, uint32_t timeout_ms) + BGD_Simulator& sim, uint64_t sequence, vector backends, uint32_t timeout_ms) { auto [probe_rc, probe] = bgd_wait_for_probe_from_backends( - sim, sequence, backends, RDS_BGD_Probe_Kind::metadata, timeout_ms + sim, sequence, backends, BGD_Probe_Kind::metadata, timeout_ms ); if (probe_rc == ETIMEDOUT) { diff --git a/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp b/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp index f7ac46e125..b9ed27422c 100644 --- a/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp +++ b/test/tap/tests/test_rds_bgd_automatic_discovery-t.cpp @@ -36,7 +36,7 @@ struct TestState { uint64_t absent_available_sequence { 0 }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -58,7 +58,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -141,7 +141,7 @@ bool runtime_bgd_row_count_matches(MYSQL* admin, int writer_hostgroup, int expec * - Verify one auto-generated runtime row with NULL green hostgroups. * - Verify automatic discovery does not create a persistent BGD row. */ -int test_topology_before_writer(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_topology_before_writer(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.topology_first; BGD_Hostgroups& hg = state.topology_first_hg; @@ -151,7 +151,7 @@ int test_topology_before_writer(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.topology_first_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish topology-first AVAILABLE topology"); @@ -189,7 +189,7 @@ int test_topology_before_writer(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Verify no runtime or persistent BGD row is created. * - Publish AVAILABLE topology and verify automatic row creation. */ -int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_topology_absent_then_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.absent_first; BGD_Hostgroups& hg = state.absent_first_hg; @@ -220,7 +220,7 @@ int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, Te } auto [absent_probe_rc, absent_probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (absent_probe_rc != EXIT_SUCCESS) { diag("Error: automatic discovery did not issue the absent-table metadata probe"); return EXIT_FAILURE; @@ -236,7 +236,7 @@ int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, Te return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.absent_first_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 820"); @@ -250,7 +250,7 @@ int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, Te } auto [green_probe_rc, green_probe] = - sim.wait_for_probe_log(available_seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(available_seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (green_probe_rc != EXIT_SUCCESS) { diag("Error: wHG 820 did not probe the AVAILABLE green writer"); return EXIT_FAILURE; @@ -269,12 +269,12 @@ int test_topology_absent_then_available(MYSQL* admin, RDS_BGD_Simulator& sim, Te * - Verify runtime contains one auto-generated BGD row. * - Verify the automatic row remains absent from persistent configuration. */ -int test_repeated_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_repeated_discovery(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.absent_first; BGD_Hostgroups& hg = state.absent_first_hg; auto [probe_rc, probe] = sim.wait_for_probe_log( - state.absent_available_sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + state.absent_available_sequence, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: automatic wHG 820 did not continue green-writer metadata polling"); return EXIT_FAILURE; @@ -291,7 +291,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp b/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp index 14329cd04f..2b5fff8841 100644 --- a/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp +++ b/test/tap/tests/test_rds_bgd_concurrent_isolation-t.cpp @@ -36,7 +36,7 @@ struct TestState { BGD_Hostgroups cluster_3_hg { 1430, 1431, 1432, 1433 }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -58,7 +58,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -76,8 +76,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -95,7 +95,7 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -116,13 +116,13 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) } int configure_available( - MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, int green_use_ssl) + MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, int green_use_ssl) { if (configure_read_only_values(sim, cluster) != EXIT_SUCCESS) { return EXIT_FAILURE; } - vector topology = topology_with_reader_pair(cluster, "AVAILABLE"); + vector topology = topology_with_reader_pair(cluster, "AVAILABLE"); if (sim.topology_update(cluster.get_endpoints(), topology) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -215,7 +215,7 @@ int replace_cluster_1_green_membership(MYSQL* admin, TestState& state) { * - Use plaintext green metadata for clusters 1 and 3 and TLS for cluster 2. * - Verify that each BGD row reaches AVAILABLE through its own green writer. */ -int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_three_workers_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { auto [seq_rc, seq] = sim.probe_log_last_sequence(); if (seq_rc != EXIT_SUCCESS) { diag("Error: failed to read the probe sequence before starting three BGD workers"); @@ -241,7 +241,7 @@ int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState } auto [cluster_1_probe_rc, cluster_1_probe] = sim.wait_for_probe_log( - seq, state.cluster_1.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + seq, state.cluster_1.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (cluster_1_probe_rc != EXIT_SUCCESS) { diag("Error: BGD wHG 1410 did not probe its plaintext green writer"); @@ -250,7 +250,7 @@ int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState ok(true, "BGD wHG 1410 reports AVAILABLE from its own plaintext green writer"); auto [cluster_2_probe_rc, cluster_2_probe] = sim.wait_for_probe_log( - seq, state.cluster_2.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, state.cluster_2.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (cluster_2_probe_rc != EXIT_SUCCESS) { diag("Error: BGD wHG 1420 did not probe its TLS green writer"); @@ -259,7 +259,7 @@ int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState ok(true, "BGD wHG 1420 reports AVAILABLE from its own TLS green writer"); auto [cluster_3_probe_rc, cluster_3_probe] = sim.wait_for_probe_log( - seq, state.cluster_3.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + seq, state.cluster_3.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (cluster_3_probe_rc != EXIT_SUCCESS) { diag("Error: BGD wHG 1430 did not probe its plaintext green writer"); @@ -278,8 +278,8 @@ int test_three_workers_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState * - After each change, verify that the other two statuses and blue-writer * placements remain unchanged. */ -int test_independent_phase_changes(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { - vector cluster_1_topology = +int test_independent_phase_changes(MYSQL* admin, BGD_Simulator& sim, TestState& state) { + vector cluster_1_topology = topology_with_reader_pair(state.cluster_1, "SWITCHOVER_IN_PROGRESS"); if (sim.topology_update(state.cluster_1.get_endpoints(), cluster_1_topology) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS for BGD wHG 1410"); @@ -308,7 +308,7 @@ int test_independent_phase_changes(MYSQL* admin, RDS_BGD_Simulator& sim, TestSta ok(cluster_2_available && cluster_3_available, "advancing wHG 1410 leaves wHG 1420 and wHG 1430 in AVAILABLE with unchanged blue placement"); - vector cluster_2_topology = + vector cluster_2_topology = topology_with_reader_pair(state.cluster_2, "SWITCHOVER_IN_POST_PROCESSING"); if (sim.topology_update(state.cluster_2.get_endpoints(), cluster_2_topology) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING for BGD wHG 1420"); @@ -338,7 +338,7 @@ int test_independent_phase_changes(MYSQL* admin, RDS_BGD_Simulator& sim, TestSta ok(cluster_1_in_progress && cluster_3_still_available, "advancing wHG 1420 preserves wHG 1410 progress and wHG 1430 availability"); - vector cluster_3_topology = + vector cluster_3_topology = topology_with_reader_pair(state.cluster_3, "SWITCHOVER_INITIATED"); if (sim.topology_update(state.cluster_3.get_endpoints(), cluster_3_topology) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_INITIATED for BGD wHG 1430"); @@ -380,7 +380,7 @@ int test_independent_phase_changes(MYSQL* admin, RDS_BGD_Simulator& sim, TestSta * - Verify that cluster 1 stops probing its removed target while clusters 2 * and 3 keep their status, placement, metadata target, and TLS value. */ -int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_independent_config_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { if (configure_read_only_values(sim, state.cluster_1_b) != EXIT_SUCCESS) { diag("Error: failed to configure simulated read_only values for cluster 1 deployment B"); return EXIT_FAILURE; @@ -398,7 +398,7 @@ int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt return EXIT_FAILURE; } - vector topology = + vector topology = topology_with_reader_pair(state.cluster_1_b, "SWITCHOVER_IN_PROGRESS"); if (sim.topology_update(state.cluster_1_b.get_endpoints(), topology) != EXIT_SUCCESS) { diag("Error: failed to publish deployment B topology for BGD wHG 1410"); @@ -406,7 +406,7 @@ int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt } auto [cluster_1_probe_rc, cluster_1_probe] = sim.wait_for_probe_log( - seq, state.cluster_1_b.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, state.cluster_1_b.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (cluster_1_probe_rc != EXIT_SUCCESS) { diag("Error: refreshed BGD wHG 1410 did not probe its TLS deployment B green writer"); @@ -423,7 +423,7 @@ int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt auto [cluster_2_probe_rc, cluster_2_probe] = sim.wait_for_probe_log( cluster_1_probe.sequence_id, state.cluster_2.green_writer.endpoint(), - RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (cluster_2_probe_rc != EXIT_SUCCESS) { diag("Error: BGD wHG 1420 did not continue probing its TLS green writer"); @@ -432,7 +432,7 @@ int test_independent_config_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt auto [cluster_3_probe_rc, cluster_3_probe] = sim.wait_for_probe_log( cluster_1_probe.sequence_id, state.cluster_3.green_writer.endpoint(), - RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (cluster_3_probe_rc != EXIT_SUCCESS) { diag("Error: BGD wHG 1430 did not continue probing its plaintext green writer"); @@ -457,7 +457,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp b/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp index 0d375e13a1..c45fce02c6 100644 --- a/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp +++ b/test/tap/tests/test_rds_bgd_config_refresh_after_writer_completion-t.cpp @@ -32,7 +32,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -54,7 +54,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -72,8 +72,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_readers(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_readers(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); for (RDS_BGD_Host& host : cluster.blue_readers) { rows.push_back({ host.hostname, @@ -95,8 +95,8 @@ vector topology_with_readers(RDS_BGD_Cluster& cluster, str return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -106,7 +106,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -134,7 +134,7 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) * - Publish target-only SWITCHOVER_COMPLETED. * - Verify BGD status READER_SWITCHOVER_IN_PROGRESS. */ -int test_reader_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_reader_switchover_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -144,7 +144,7 @@ int test_reader_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, Tes return EXIT_FAILURE; } - vector progress = topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + vector progress = topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); int topology_rc = sim.topology_update(state.topology_endpoints, progress); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for wHG 1360"); @@ -168,7 +168,7 @@ int test_reader_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, Tes return EXIT_FAILURE; } - vector completed = target_only_completed(cluster); + vector completed = target_only_completed(cluster); int completed_rc = sim.topology_update(state.topology_endpoints, completed); if (completed_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG 1360"); @@ -194,7 +194,7 @@ int test_reader_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, Tes * - Verify that blue-writer metadata follows the table check. * - Verify BGD status returns to READER_SWITCHOVER_IN_PROGRESS. */ -int test_config_refresh_after_writer_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_config_refresh_after_writer_completion(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -218,14 +218,14 @@ int test_config_refresh_after_writer_completion(MYSQL* admin, RDS_BGD_Simulator& } auto [table_rc, table] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (table_rc != EXIT_SUCCESS) { diag("Error: post-completion refresh did not start with a blue-writer table check for wHG 1360"); return EXIT_FAILURE; } auto [blue_rc, blue] = sim.wait_for_probe_log( - table.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + table.sequence_id, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (blue_rc != EXIT_SUCCESS) { diag("Error: post-completion refresh did not probe blue-writer metadata for wHG 1360"); @@ -251,7 +251,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp b/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp index 2d01d467b4..231401788a 100644 --- a/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp +++ b/test/tap/tests/test_rds_bgd_configuration_persistence-t.cpp @@ -42,7 +42,7 @@ struct TestState { vector admin_owned_endpoints { admin_owned.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -64,7 +64,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -227,7 +227,7 @@ rc_t> green_server_snapshot(MYSQL* admin, const string& ta * - Disable automatic discovery and load explicit hostgroups 890-893. * - Verify explicit values replace the automatic row and persist. */ -int test_automatic_to_explicit(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_automatic_to_explicit(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.conversion; BGD_Hostgroups& hg = state.conversion_hg; @@ -237,7 +237,7 @@ int test_automatic_to_explicit(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.conversion_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 890"); @@ -372,7 +372,7 @@ int test_persistent_row_validation(MYSQL* admin, TestState& state) { * - Execute SAVE MYSQL SERVERS FROM RUNTIME. * - Verify SAVE restores wHG 920 and skips auto-generated wHG 930. */ -int test_save_from_runtime(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_save_from_runtime(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& explicit_cluster = state.explicit_save; RDS_BGD_Cluster& automatic_cluster = state.automatic_save; BGD_Hostgroups& explicit_hg = state.explicit_save_hg; @@ -390,14 +390,14 @@ int test_save_from_runtime(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat return EXIT_FAILURE; } - vector explicit_topology = bgd_topology_with_readers(explicit_cluster, "AVAILABLE"); + vector explicit_topology = bgd_topology_with_readers(explicit_cluster, "AVAILABLE"); int explicit_topology_rc = sim.topology_update(state.explicit_save_endpoints, explicit_topology); if (explicit_topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 920"); return EXIT_FAILURE; } - vector automatic_topology = bgd_topology_with_readers(automatic_cluster, "AVAILABLE"); + vector automatic_topology = bgd_topology_with_readers(automatic_cluster, "AVAILABLE"); int automatic_topology_rc = sim.topology_update(state.automatic_save_endpoints, automatic_topology); if (automatic_topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 930"); @@ -491,7 +491,7 @@ int test_save_from_runtime(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat * - Enable automatic discovery and publish AVAILABLE topology. * - Verify the BGD row and green-server status remain unchanged. */ -int test_admin_server_status_preserved(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_admin_server_status_preserved(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.admin_owned; BGD_Hostgroups& hg = state.admin_owned_hg; @@ -571,7 +571,7 @@ int test_admin_server_status_preserved(MYSQL* admin, RDS_BGD_Simulator& sim, Tes return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.admin_owned_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology beside administrator-owned wHG 1310"); @@ -579,7 +579,7 @@ int test_admin_server_status_preserved(MYSQL* admin, RDS_BGD_Simulator& sim, Tes } auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: automatic discovery did not probe beside administrator-owned wHG 1310"); return EXIT_FAILURE; @@ -609,7 +609,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp b/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp index 07c7327bf9..86a4f435b5 100644 --- a/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp +++ b/test/tap/tests/test_rds_bgd_disable_during_switchover-t.cpp @@ -28,7 +28,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -50,7 +50,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -76,7 +76,7 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { * - Configure `mysql_servers` and `mysql_aws_rds_bgd_hostgroups`. * - Verify that the runtime BGD row reaches `AVAILABLE`. */ -int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_bgd_status_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -88,7 +88,7 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s } // Publish AVAILABLE topology. - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology"); @@ -123,12 +123,12 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Verify `WRITER_SWITCHOVER_IN_PROGRESS`. * - Verify that the blue writer moves to the blue reader hostgroup. */ -int test_writer_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_writer_switchover_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; // Publish SWITCHOVER_IN_PROGRESS and wait for the runtime BGD status. - vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); @@ -191,7 +191,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp b/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp index 2ddc6ee15f..f99db300c9 100644 --- a/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp +++ b/test/tap/tests/test_rds_bgd_explicit_startup-t.cpp @@ -33,7 +33,7 @@ struct TestState { vector servers_first_endpoints { servers_first.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -55,7 +55,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -167,7 +167,7 @@ bool runtime_membership_matches(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Host * - Verify no table-check probe starts. * - Load all servers and verify AVAILABLE with explicit runtime membership. */ -int test_bgd_row_before_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_bgd_row_before_servers(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.row_first; BGD_Hostgroups& hg = state.row_first_hg; @@ -177,7 +177,7 @@ int test_bgd_row_before_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.row_first_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for hostgroups 840-843"); @@ -224,7 +224,7 @@ int test_bgd_row_before_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& } auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: loading the blue writer did not start the wHG 840 table check"); return EXIT_FAILURE; @@ -249,7 +249,7 @@ int test_bgd_row_before_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Verify no table-check probe starts. * - Load the explicit row and verify AVAILABLE with explicit membership. */ -int test_servers_before_bgd_row(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_servers_before_bgd_row(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.servers_first; BGD_Hostgroups& hg = state.servers_first_hg; @@ -259,7 +259,7 @@ int test_servers_before_bgd_row(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.servers_first_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for hostgroups 850-853"); @@ -306,7 +306,7 @@ int test_servers_before_bgd_row(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& } auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: loading wHG 850 did not start the blue table check"); return EXIT_FAILURE; @@ -328,7 +328,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp b/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp index 08f1e2bfc4..cf0ab931cb 100644 --- a/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp +++ b/test/tap/tests/test_rds_bgd_green_membership_ordering-t.cpp @@ -36,7 +36,7 @@ struct TestState { vector after_worker_start_endpoints { after_worker_start.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -58,7 +58,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -175,7 +175,7 @@ bool runtime_green_membership_matches(MYSQL* admin, RDS_BGD_Cluster& cluster, BG * - Publish AVAILABLE topology after all membership exists. * - Verify runtime hostgroups 862 and 863 contain the configured green set. */ -int test_green_before_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_green_before_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.before_available; BGD_Hostgroups& hg = state.before_available_hg; @@ -209,7 +209,7 @@ int test_green_before_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.before_available_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 860"); @@ -235,7 +235,7 @@ int test_green_before_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Verify runtime_mysql_servers contains the complete green membership after * the worker observes the configuration change. */ -int test_green_after_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_green_after_discovery(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.after_discovery; BGD_Hostgroups& hg = state.after_discovery_hg; @@ -245,7 +245,7 @@ int test_green_after_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.after_discovery_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 870"); @@ -289,7 +289,7 @@ int test_green_after_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& } auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: wHG 870 did not probe the green writer after membership load"); return EXIT_FAILURE; @@ -307,7 +307,7 @@ int test_green_after_discovery(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Load green membership into hostgroups 882 and 883. * - Publish AVAILABLE and verify complete runtime green membership. */ -int test_green_after_worker_start(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_green_after_worker_start(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.after_worker_start; BGD_Hostgroups& hg = state.after_worker_start_hg; @@ -348,7 +348,7 @@ int test_green_after_worker_start(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat } auto [start_rc, start_probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (start_rc != EXIT_SUCCESS) { diag("Error: wHG 880 did not start the blue table-check probe"); return EXIT_FAILURE; @@ -360,7 +360,7 @@ int test_green_after_worker_start(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat return EXIT_FAILURE; } - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.after_worker_start_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 880"); @@ -383,7 +383,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp b/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp index 181915b473..0e63511f11 100644 --- a/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp +++ b/test/tap/tests/test_rds_bgd_green_pool_cleanup-t.cpp @@ -52,7 +52,7 @@ struct TestState { } }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -74,7 +74,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -92,8 +92,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_readers(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_readers(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); for (RDS_BGD_Host& host : cluster.blue_readers) { rows.push_back({ host.hostname, @@ -115,8 +115,8 @@ vector topology_with_readers(RDS_BGD_Cluster& cluster, str return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -126,7 +126,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -250,7 +250,7 @@ bool all_pools_nonzero(vector& pools) { return true; } -int configure_status_matrix(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int configure_status_matrix(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -336,8 +336,8 @@ int configure_status_matrix(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& si return EXIT_SUCCESS; } -int publish_topology(RDS_BGD_Simulator& sim, TestState& state, string status) { - vector topology = topology_with_readers(state.cluster, status); +int publish_topology(BGD_Simulator& sim, TestState& state, string status) { + vector topology = topology_with_readers(state.cluster, status); int rc = sim.topology_update(state.topology_endpoints, topology); return rc; @@ -362,7 +362,7 @@ bool snapshots_match(MYSQL* admin, TestState& state) { * - Publish AVAILABLE, SWITCHOVER_IN_PROGRESS, then AVAILABLE. * - Verify rollback preserves every green pool and exact configured row. */ -int test_rollback_preserves_green_pools(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_rollback_preserves_green_pools(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { int config_rc = configure_status_matrix(cl, admin, sim, state); if (config_rc != EXIT_SUCCESS) { return EXIT_FAILURE; @@ -432,7 +432,7 @@ int test_rollback_preserves_green_pools(CommandLine& cl, MYSQL* admin, RDS_BGD_S * - Require every green pool to remain nonzero immediately before cleanup. * - Delete topology rows and verify that ONLINE and SHUNNED pools drain. */ -int test_successful_cleanup_drains_non_offline(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_successful_cleanup_drains_non_offline(MYSQL* admin, BGD_Simulator& sim, TestState& state) { int post_rc = publish_topology(sim, state, "SWITCHOVER_IN_POST_PROCESSING"); if (post_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology for wHG 1300"); @@ -446,7 +446,7 @@ int test_successful_cleanup_drains_non_offline(MYSQL* admin, RDS_BGD_Simulator& return EXIT_FAILURE; } - vector completed = target_only_completed(state.cluster); + vector completed = target_only_completed(state.cluster); int completed_rc = sim.topology_update(state.topology_endpoints, completed); if (completed_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG 1300"); @@ -517,7 +517,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp b/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp index 6e212f11b2..8ad356da90 100644 --- a/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp +++ b/test/tap/tests/test_rds_bgd_late_entry_completed-t.cpp @@ -30,7 +30,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -52,7 +52,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -70,8 +70,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -81,7 +81,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -105,9 +105,9 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) return EXIT_SUCCESS; } -int wait_for_blue_writer(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { +int wait_for_blue_writer(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { auto [probe_rc, probe] = - sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); return probe_rc; } @@ -169,7 +169,7 @@ int wait_for_green_pool_drain(MYSQL* admin, BGD_Hostgroups& hg) { * the blue writer is not demoted. * - Establish green writer/reader pools for terminal cleanup. */ -int test_first_completed(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_first_completed(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -185,7 +185,7 @@ int test_first_completed(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, return EXIT_FAILURE; } - vector topology = target_only_completed(cluster); + vector topology = target_only_completed(cluster); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology"); @@ -280,7 +280,7 @@ int test_first_completed(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, * - Verify eligible green writer/reader pools are drained. * - Verify blue-writer and blue-reader placement remains available. */ -int test_completed_empty_topology(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_completed_empty_topology(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -318,7 +318,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp b/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp index 970378605e..aaf550f1c8 100644 --- a/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp +++ b/test/tap/tests/test_rds_bgd_late_entry_writer_phases-t.cpp @@ -41,7 +41,7 @@ struct TestState { vector post_endpoints { post_cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -63,7 +63,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -81,8 +81,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -100,7 +100,7 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -124,16 +124,16 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) return EXIT_SUCCESS; } -int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { - vector topology = topology_with_reader_pair(cluster, status); +int publish_topology(BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); int rc = sim.topology_update(endpoints, topology); return rc; } -int wait_for_green_writer(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { +int wait_for_green_writer(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { auto [probe_rc, probe] = - sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); return probe_rc; } @@ -266,7 +266,7 @@ int enable_bgd_worker(MYSQL* admin, BGD_Hostgroups& hg) { * - Change simulated writer/reader read_only values. * - Verify read_only monitoring is suppressed for deployment members. */ -int test_first_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_first_initiated(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.initiated_cluster; BGD_Hostgroups& hg = state.initiated_hg; @@ -374,7 +374,7 @@ int test_first_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) * - Change the simulated blue-reader read_only value. * - Verify read_only monitoring remains suppressed after demotion. */ -int test_first_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_first_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.progress_cluster; BGD_Hostgroups& hg = state.progress_hg; @@ -468,7 +468,7 @@ int test_first_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat * - Verify the old blue pool drains and new connections route to green. * - Verify mapped blue readers remain under read_only suppression. */ -int test_first_post_processing(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_first_post_processing(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.post_cluster; BGD_Hostgroups& hg = state.post_hg; @@ -586,7 +586,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_probe_tls-t.cpp b/test/tap/tests/test_rds_bgd_probe_tls-t.cpp index f6c616306b..b3898cf409 100644 --- a/test/tap/tests/test_rds_bgd_probe_tls-t.cpp +++ b/test/tap/tests/test_rds_bgd_probe_tls-t.cpp @@ -42,12 +42,12 @@ struct TestState { }; struct ProbeChain { - RDS_BGD_Probe_Log table; - RDS_BGD_Probe_Log blue; - RDS_BGD_Probe_Log green; + BGD_Probe_Log table; + BGD_Probe_Log blue; + BGD_Probe_Log green; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -69,7 +69,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { vector attribute_queries { "DELETE FROM mysql_hostgroup_attributes", "LOAD MYSQL SERVERS TO RUNTIME", @@ -96,23 +96,23 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int wait_for_probe_chain(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster, +int wait_for_probe_chain(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster, int blue_use_ssl, int green_use_ssl, ProbeChain& chain) { auto [table_rc, table] = - sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, blue_use_ssl); + sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, blue_use_ssl); if (table_rc != EXIT_SUCCESS) { return EXIT_FAILURE; } auto [blue_rc, blue] = - sim.wait_for_probe_log(table.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, blue_use_ssl); + sim.wait_for_probe_log(table.sequence_id, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, blue_use_ssl); if (blue_rc != EXIT_SUCCESS) { return EXIT_FAILURE; } auto [green_rc, green] = - sim.wait_for_probe_log(blue.sequence_id, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, green_use_ssl); + sim.wait_for_probe_log(blue.sequence_id, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, green_use_ssl); if (green_rc != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -125,9 +125,9 @@ bool probe_chain_ordered(ProbeChain& chain) { bool ordered = chain.table.sequence_id < chain.blue.sequence_id && chain.blue.sequence_id < chain.green.sequence_id && - chain.table.probe_kind == RDS_BGD_Probe_Kind::table_check && - chain.blue.probe_kind == RDS_BGD_Probe_Kind::metadata && - chain.green.probe_kind == RDS_BGD_Probe_Kind::metadata; + chain.table.probe_kind == BGD_Probe_Kind::table_check && + chain.blue.probe_kind == BGD_Probe_Kind::metadata && + chain.green.probe_kind == BGD_Probe_Kind::metadata; return ordered; } @@ -154,7 +154,7 @@ bool runtime_server_tls_matches(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, * - Verify table-check and blue metadata use the writer with TLS. * - Verify the mapped green writer metadata probe also uses TLS. */ -int test_automatic_writer_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_automatic_writer_tls(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.automatic; BGD_Hostgroups& hg = state.automatic_hg; @@ -198,7 +198,7 @@ int test_automatic_writer_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s return EXIT_FAILURE; } - vector topology = cluster.get_topology("AVAILABLE"); + vector topology = cluster.get_topology("AVAILABLE"); int topology_rc = sim.topology_update(state.automatic_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish automatic AVAILABLE topology"); @@ -237,7 +237,7 @@ int test_automatic_writer_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Publish AVAILABLE topology for the exact deployment. * - Verify the exact TARGET and TLS value are used in probe order. */ -int test_explicit_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_explicit_target_tls(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.explicit_target; RDS_BGD_Cluster& distractor = state.distractor; BGD_Hostgroups& hg = state.explicit_target_hg; @@ -285,7 +285,7 @@ int test_explicit_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& st vector topology_endpoints = state.explicit_target_endpoints; vector distractor_endpoints = distractor.get_endpoints(); topology_endpoints.insert(topology_endpoints.end(), distractor_endpoints.begin(), distractor_endpoints.end()); - vector topology = cluster.get_topology("AVAILABLE"); + vector topology = cluster.get_topology("AVAILABLE"); int topology_rc = sim.topology_update(topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish explicit AVAILABLE topology"); @@ -331,7 +331,7 @@ int test_explicit_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& st * - Publish AVAILABLE topology. * - Verify the created TARGET runtime row and metadata probe use TLS. */ -int test_created_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_created_target_tls(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.created_target; BGD_Hostgroups& hg = state.created_target_hg; @@ -380,7 +380,7 @@ int test_created_target_tls(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& sta return EXIT_FAILURE; } - vector topology = cluster.get_topology("AVAILABLE"); + vector topology = cluster.get_topology("AVAILABLE"); int topology_rc = sim.topology_update(state.created_target_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish created-TARGET AVAILABLE topology"); @@ -423,7 +423,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_reader_policy-t.cpp b/test/tap/tests/test_rds_bgd_reader_policy-t.cpp index b274b4e9c5..3e3ef9aec8 100644 --- a/test/tap/tests/test_rds_bgd_reader_policy-t.cpp +++ b/test/tap/tests/test_rds_bgd_reader_policy-t.cpp @@ -36,7 +36,7 @@ struct TestState { vector fallback_endpoints { fallback.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -58,7 +58,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -76,8 +76,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pairs(RDS_BGD_Cluster& cluster, string status, size_t pairs) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pairs(RDS_BGD_Cluster& cluster, string status, size_t pairs) { + vector rows = cluster.get_topology(status); for (size_t i = 0; i < pairs; ++i) { rows.push_back({ cluster.blue_readers[i].hostname, @@ -97,7 +97,7 @@ vector topology_with_reader_pairs(RDS_BGD_Cluster& cluster return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -118,7 +118,7 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) } int configure_bgd( - MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, size_t green_reader_count) + MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, size_t green_reader_count) { int read_only_rc = configure_read_only_values(sim, cluster); if (read_only_rc != EXIT_SUCCESS) { @@ -144,9 +144,9 @@ int configure_bgd( } int publish_post_processing( - RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, size_t pairs) + BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, size_t pairs) { - vector topology = + vector topology = topology_with_reader_pairs(cluster, "SWITCHOVER_IN_POST_PROCESSING", pairs); int rc = sim.topology_update(endpoints, topology); @@ -229,7 +229,7 @@ rc_t connect_and_echo(CommandLine& cl) { * - Route a client through hostgroup 1281 and verify that it reaches the * mapped green reader instead of the unmapped blue reader. */ -int test_matched_unmatched_readers(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_matched_unmatched_readers(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.matched; BGD_Hostgroups& hg = state.matched_hg; @@ -303,7 +303,7 @@ int test_matched_unmatched_readers(CommandLine& cl, MYSQL* admin, RDS_BGD_Simula * - Publish SWITCHOVER_IN_POST_PROCESSING topology without reader pairs. * - Verify that the blue writer is not added to reader hostgroup 1291. */ -int test_offline_blue_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_offline_blue_servers(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.fallback; BGD_Hostgroups& hg = state.fallback_hg; @@ -350,7 +350,7 @@ int test_offline_blue_servers(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Connect through hostgroup 1291 and verify routing reaches the green writer * IP pinned for the blue writer hostname. */ -int test_writer_reader_fallback(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_writer_reader_fallback(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.fallback; BGD_Hostgroups& hg = state.fallback_hg; @@ -394,7 +394,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp b/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp index 3d10a06c50..e3f0185ed7 100644 --- a/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp +++ b/test/tap/tests/test_rds_bgd_reader_switchover_cleanup-t.cpp @@ -30,7 +30,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -52,7 +52,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -70,8 +70,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -89,8 +89,8 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows { +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows { { cluster.green_writer.hostname, cluster.green_writer.hostname, @@ -145,11 +145,11 @@ bool green_rows_online(MYSQL* admin, RDS_BGD_Cluster& cluster, BGD_Hostgroups& h return rows_online; } -int advance_to_post_processing(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int advance_to_post_processing(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; - vector available = topology_with_reader_pair(cluster, "AVAILABLE"); + vector available = topology_with_reader_pair(cluster, "AVAILABLE"); int available_rc = sim.topology_update(state.topology_endpoints, available); if (available_rc != EXIT_SUCCESS) { return EXIT_FAILURE; @@ -160,7 +160,7 @@ int advance_to_post_processing(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector in_progress = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); + vector in_progress = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); int progress_rc = sim.topology_update(state.topology_endpoints, in_progress); if (progress_rc != EXIT_SUCCESS) { return EXIT_FAILURE; @@ -171,7 +171,7 @@ int advance_to_post_processing(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - vector post_processing = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + vector post_processing = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); int post_rc = sim.topology_update(state.topology_endpoints, post_processing); if (post_rc != EXIT_SUCCESS) { return EXIT_FAILURE; @@ -193,7 +193,7 @@ int advance_to_post_processing(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Publish target-only SWITCHOVER_COMPLETED twice. * - Verify READER_SWITCHOVER_IN_PROGRESS and retained green rows. */ -int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -259,7 +259,7 @@ int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Si return EXIT_FAILURE; } - vector completed = target_only_completed(cluster); + vector completed = target_only_completed(cluster); int completed_rc = sim.topology_update(state.topology_endpoints, completed); if (completed_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology"); @@ -283,7 +283,7 @@ int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Si return EXIT_FAILURE; } - vector repeated_completed = target_only_completed(cluster); + vector repeated_completed = target_only_completed(cluster); int repeat_rc = sim.topology_update(state.topology_endpoints, repeated_completed); if (repeat_rc != EXIT_SUCCESS) { diag("Error: failed to repeat target-only SWITCHOVER_COMPLETED topology"); @@ -291,7 +291,7 @@ int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Si } auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: BGD did not observe repeated target-only SWITCHOVER_COMPLETED topology"); return EXIT_FAILURE; @@ -316,7 +316,7 @@ int test_reader_switchover_in_progress(CommandLine& cl, MYSQL* admin, RDS_BGD_Si * - Verify metadata probing returns from the green pin to the blue writer. * - Verify green pools drain while configured green rows remain ONLINE. */ -int test_reader_switchover_cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_reader_switchover_cleanup(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -352,14 +352,14 @@ int test_reader_switchover_cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestSta ok(unmatched_reader_online, "reader cleanup restores the unmatched blue reader in hostgroup 981"); auto [green_probe_rc, green_probe] = - sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (green_probe_rc != EXIT_SUCCESS) { diag("Error: reader cleanup did not observe empty topology through the green pin"); return EXIT_FAILURE; } auto [blue_probe_rc, blue_probe] = - sim.wait_for_probe_log(green_probe.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(green_probe.sequence_id, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (blue_probe_rc != EXIT_SUCCESS) { diag("Error: metadata probing did not return to the blue writer after reader cleanup"); return EXIT_FAILURE; @@ -387,7 +387,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp b/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp index dbfa042c38..589edd1291 100644 --- a/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp +++ b/test/tap/tests/test_rds_bgd_remove_during_switchover-t.cpp @@ -29,7 +29,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -51,7 +51,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -86,7 +86,7 @@ int wait_for_bgd_row_absent(MYSQL* admin, BGD_Hostgroups& hg) { * - Configure `mysql_servers` and `mysql_aws_rds_bgd_hostgroups`. * - Verify that the runtime BGD row reaches `AVAILABLE`. */ -int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_bgd_status_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -98,7 +98,7 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s } // Publish AVAILABLE topology. - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology"); @@ -133,12 +133,12 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Verify `WRITER_SWITCHOVER_IN_PROGRESS`. * - Verify that the blue writer moves to the blue reader hostgroup. */ -int test_writer_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_writer_switchover_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; // Publish SWITCHOVER_IN_PROGRESS and wait for the runtime BGD status. - vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); @@ -214,7 +214,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp b/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp index b1852dedf8..a066cf5ea6 100644 --- a/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp +++ b/test/tap/tests/test_rds_bgd_repeated_deployment-t.cpp @@ -36,7 +36,7 @@ struct TestState { } }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -58,7 +58,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -76,8 +76,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -95,8 +95,8 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -106,7 +106,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -196,9 +196,9 @@ int replace_green_membership( } int publish_writer_lifecycle( - MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state, RDS_BGD_Cluster& cluster, string deployment) + MYSQL* admin, BGD_Simulator& sim, TestState& state, RDS_BGD_Cluster& cluster, string deployment) { - vector initiated = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); + vector initiated = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); if (sim.topology_update(cluster.get_endpoints(), initiated) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_INITIATED topology for deployment %s", deployment.c_str()); return EXIT_FAILURE; @@ -209,7 +209,7 @@ int publish_writer_lifecycle( return EXIT_FAILURE; } - vector progress = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); + vector progress = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); if (sim.topology_update(cluster.get_endpoints(), progress) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for deployment %s", deployment.c_str()); return EXIT_FAILURE; @@ -220,7 +220,7 @@ int publish_writer_lifecycle( return EXIT_FAILURE; } - vector post = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + vector post = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); if (sim.topology_update(cluster.get_endpoints(), post) != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology for deployment %s", deployment.c_str()); return EXIT_FAILURE; @@ -235,9 +235,9 @@ int publish_writer_lifecycle( return EXIT_SUCCESS; } -int publish_reader_cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state, RDS_BGD_Cluster& cluster, +int publish_reader_cleanup(MYSQL* admin, BGD_Simulator& sim, TestState& state, RDS_BGD_Cluster& cluster, string deployment) { - vector completed = target_only_completed(cluster); + vector completed = target_only_completed(cluster); if (sim.topology_update(cluster.get_endpoints(), completed) != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED for deployment %s", deployment.c_str()); return EXIT_FAILURE; @@ -275,7 +275,7 @@ int publish_reader_cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat * - Complete writer and reader switchover, then publish empty topology. * - Verify NONE and baseline blue-writer placement. */ -int test_deployment_a(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_deployment_a(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& deployment = state.deployment_a; BGD_Hostgroups& hg = state.hostgroups; @@ -285,7 +285,7 @@ int test_deployment_a(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { return EXIT_FAILURE; } - vector available = topology_with_reader_pair(deployment, "AVAILABLE"); + vector available = topology_with_reader_pair(deployment, "AVAILABLE"); int topology_rc = sim.topology_update(deployment.get_endpoints(), available); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for deployment A"); @@ -332,7 +332,7 @@ int test_deployment_a(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { * - Complete deployment B and verify that routing uses deployment B without * recreating deployment A rows. */ -int test_deployment_b(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_deployment_b(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& deployment_a = state.deployment_a; RDS_BGD_Cluster& deployment_b = state.deployment_b; BGD_Hostgroups& hg = state.hostgroups; @@ -355,7 +355,7 @@ int test_deployment_b(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, Tes return EXIT_FAILURE; } - vector available = topology_with_reader_pair(deployment_b, "AVAILABLE"); + vector available = topology_with_reader_pair(deployment_b, "AVAILABLE"); int topology_rc = sim.topology_update(deployment_b.get_endpoints(), available); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for deployment B"); @@ -372,7 +372,7 @@ int test_deployment_b(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, Tes ok(membership_matches, "runtime_mysql_servers contains only TLS-enabled deployment B green rows"); auto [probe_rc, probe] = sim.wait_for_probe_log( - seq, deployment_b.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, deployment_b.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: deployment B green writer did not receive a TLS metadata probe"); @@ -416,7 +416,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_rollback-t.cpp b/test/tap/tests/test_rds_bgd_rollback-t.cpp index 1e1bbe270f..0c0ec1e4c6 100644 --- a/test/tap/tests/test_rds_bgd_rollback-t.cpp +++ b/test/tap/tests/test_rds_bgd_rollback-t.cpp @@ -42,7 +42,7 @@ struct TestState { vector progress_endpoints { progress_cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -64,7 +64,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -82,8 +82,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -101,7 +101,7 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -121,16 +121,16 @@ int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) return EXIT_SUCCESS; } -int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { - vector topology = topology_with_reader_pair(cluster, status); +int publish_topology(BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); int rc = sim.topology_update(endpoints, topology); return rc; } -int wait_for_green_writer(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { +int wait_for_green_writer(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { auto [probe_rc, probe] = - sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); return probe_rc; } @@ -263,7 +263,7 @@ bool green_rows_match(MYSQL* admin, BGD_Hostgroups& hg, RDS_BGD_Cluster& cluster * - Verify blue-writer placement and normal read_only processing are restored. * - Repeat AVAILABLE and verify the monitor-created green writer remains. */ -int test_initiated_rollback(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_initiated_rollback(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.initiated_cluster; BGD_Hostgroups& hg = state.initiated_hg; @@ -407,7 +407,7 @@ int test_initiated_rollback(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& sta * - Verify explicit green rows and pools remain unchanged. * - Repeat AVAILABLE and verify rollback remains stable. */ -int test_in_progress_rollback(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_in_progress_rollback(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.progress_cluster; BGD_Hostgroups& hg = state.progress_hg; @@ -600,7 +600,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_smoke-t.cpp b/test/tap/tests/test_rds_bgd_smoke-t.cpp index 8b72573e6b..f87f101274 100644 --- a/test/tap/tests/test_rds_bgd_smoke-t.cpp +++ b/test/tap/tests/test_rds_bgd_smoke-t.cpp @@ -28,7 +28,7 @@ struct TestState { uint64_t probe_sequence { 0 }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -50,7 +50,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -109,7 +109,7 @@ int configure_explicit_bgd(MYSQL* admin, TestState& state) { * - Record the probe sequence before publishing topology. * - Publish AVAILABLE topology to the blue and green writer endpoints. */ -int publish_available_topology(RDS_BGD_Simulator& sim, TestState& state) { +int publish_available_topology(BGD_Simulator& sim, TestState& state) { int writer_rc = bgd_set_writer_read_only_0(sim, state.cluster); if (writer_rc != EXIT_SUCCESS) { diag("Error: failed to set read_only=0 for the simulated writers"); @@ -123,7 +123,7 @@ int publish_available_topology(RDS_BGD_Simulator& sim, TestState& state) { } state.probe_sequence = seq; - vector topology = state.cluster.get_topology("AVAILABLE"); + vector topology = state.cluster.get_topology("AVAILABLE"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology"); @@ -164,10 +164,10 @@ int configure_bgd_available(MYSQL* admin, TestState& state) { * - Wait for a metadata probe after the topology publication sequence. * - Require the probe on the green writer IP without TLS. */ -int test_plaintext_green_writer_probe(RDS_BGD_Simulator& sim, TestState& state) { +int test_plaintext_green_writer_probe(BGD_Simulator& sim, TestState& state) { auto [probe_rc, probe] = sim.wait_for_probe_log( state.probe_sequence, state.cluster.green_writer.endpoint(), - RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: green writer did not receive a plaintext metadata probe"); @@ -183,7 +183,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp b/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp index 1c938b306f..1f0f6c67d1 100644 --- a/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp +++ b/test/tap/tests/test_rds_bgd_topology_empty_absent-t.cpp @@ -44,7 +44,7 @@ struct TestState { vector absent_reader_endpoints { absent_reader.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -66,7 +66,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -84,8 +84,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -103,8 +103,8 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -114,7 +114,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -153,7 +153,7 @@ bool runtime_server_online(MYSQL* admin, int hostgroup, RDS_BGD_Host& host) { return online; } -int configure_bgd(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { +int configure_bgd(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { int read_only_rc = configure_read_only_values(sim, cluster); if (read_only_rc != EXIT_SUCCESS) { diag("Error: failed to configure simulated read_only values for wHG %d", hg.blue_writer); @@ -173,14 +173,14 @@ int configure_bgd(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster return EXIT_SUCCESS; } -int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { - vector topology = topology_with_reader_pair(cluster, status); +int publish_topology(BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); int rc = sim.topology_update(endpoints, topology); return rc; } -int enter_writer_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, +int enter_writer_switchover(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, vector endpoints) { int config_rc = configure_bgd(admin, sim, cluster, hg); @@ -222,7 +222,7 @@ int enter_writer_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluste return EXIT_SUCCESS; } -int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, +int enter_reader_switchover(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, vector endpoints) { int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, endpoints); @@ -249,7 +249,7 @@ int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluste return EXIT_FAILURE; } - vector completed = target_only_completed(cluster); + vector completed = target_only_completed(cluster); int completed_rc = sim.topology_update(endpoints, completed); if (completed_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG %d", hg.blue_writer); @@ -286,7 +286,7 @@ int disable_bgd(MYSQL* admin, BGD_Hostgroups& hg) { * - Verify BGD status NONE, restored blue-writer placement, and metadata * telemetry from the pinned green writer. */ -int test_empty_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_empty_before_completion(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.empty_before; BGD_Hostgroups& hg = state.empty_before_hg; @@ -324,7 +324,7 @@ int test_empty_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState ok(true, "empty topology restores the blue writer and sets BGD status for wHG 1100 to NONE"); auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: empty topology for wHG 1100 was not observed through green-writer metadata"); return EXIT_FAILURE; @@ -348,7 +348,7 @@ int test_empty_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState * - Verify BGD status NONE, restored blue-writer placement, and a new * blue-writer table-check probe. */ -int test_absent_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_absent_before_completion(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.absent_before; BGD_Hostgroups& hg = state.absent_before_hg; @@ -386,7 +386,7 @@ int test_absent_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat ok(true, "absent topology restores the blue writer and sets BGD status for wHG 1110 to NONE"); auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: absent topology for wHG 1110 did not return to blue-writer table checks"); return EXIT_FAILURE; @@ -410,7 +410,7 @@ int test_absent_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat * - Verify BGD status NONE, restored blue-reader routing, retained green * rows, and metadata telemetry from the pinned green writer. */ -int test_empty_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_empty_during_reader_switchover(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.empty_reader; BGD_Hostgroups& hg = state.empty_reader_hg; @@ -445,7 +445,7 @@ int test_empty_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, Te "empty topology completes reader cleanup for wHG 1120 and retains configured green rows"); auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: empty topology for wHG 1120 was not observed through green-writer metadata"); return EXIT_FAILURE; @@ -469,7 +469,7 @@ int test_empty_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, Te * - Verify BGD status NONE, restored blue-reader routing, retained green * rows, and a new blue-writer table-check probe. */ -int test_absent_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_absent_during_reader_switchover(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.absent_reader; BGD_Hostgroups& hg = state.absent_reader_hg; @@ -504,7 +504,7 @@ int test_absent_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, T "absent topology completes reader cleanup for wHG 1130 and retains configured green rows"); auto [probe_rc, probe] = - sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { diag("Error: absent topology for wHG 1130 did not return to blue-writer table checks"); return EXIT_FAILURE; @@ -519,7 +519,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_topology_errors-t.cpp b/test/tap/tests/test_rds_bgd_topology_errors-t.cpp index 06bc414ffe..cfecaec568 100644 --- a/test/tap/tests/test_rds_bgd_topology_errors-t.cpp +++ b/test/tap/tests/test_rds_bgd_topology_errors-t.cpp @@ -39,7 +39,7 @@ struct TestState { vector generic_error_endpoints { generic_error.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -61,7 +61,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -79,8 +79,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -98,8 +98,8 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -vector target_only_completed(RDS_BGD_Cluster& cluster) { - vector rows {{ +vector target_only_completed(RDS_BGD_Cluster& cluster) { + vector rows {{ cluster.green_writer.hostname, cluster.green_writer.hostname, cluster.green_writer.port, @@ -109,7 +109,7 @@ vector target_only_completed(RDS_BGD_Cluster& cluster) { return rows; } -int configure_read_only_values(RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { +int configure_read_only_values(BGD_Simulator& sim, RDS_BGD_Cluster& cluster) { if (bgd_set_host_read_only_0(sim, cluster.blue_writer) != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -148,7 +148,7 @@ bool runtime_server_online(MYSQL* admin, int hostgroup, RDS_BGD_Host& host) { return online; } -int configure_bgd(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { +int configure_bgd(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg) { int read_only_rc = configure_read_only_values(sim, cluster); if (read_only_rc != EXIT_SUCCESS) { diag("Error: failed to configure simulated read_only values for wHG %d", hg.blue_writer); @@ -168,14 +168,14 @@ int configure_bgd(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster return EXIT_SUCCESS; } -int publish_topology(RDS_BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { - vector topology = topology_with_reader_pair(cluster, status); +int publish_topology(BGD_Simulator& sim, vector endpoints, RDS_BGD_Cluster& cluster, string status) { + vector topology = topology_with_reader_pair(cluster, status); int rc = sim.topology_update(endpoints, topology); return rc; } -int enter_writer_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, +int enter_writer_switchover(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, vector endpoints) { int config_rc = configure_bgd(admin, sim, cluster, hg); @@ -217,7 +217,7 @@ int enter_writer_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluste return EXIT_SUCCESS; } -int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluster& cluster, +int enter_reader_switchover(MYSQL* admin, BGD_Simulator& sim, RDS_BGD_Cluster& cluster, BGD_Hostgroups& hg, vector endpoints) { int progress_rc = enter_writer_switchover(admin, sim, cluster, hg, endpoints); @@ -244,7 +244,7 @@ int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluste return EXIT_FAILURE; } - vector completed = target_only_completed(cluster); + vector completed = target_only_completed(cluster); int completed_rc = sim.topology_update(endpoints, completed); if (completed_rc != EXIT_SUCCESS) { diag("Error: failed to publish target-only SWITCHOVER_COMPLETED topology for wHG %d", hg.blue_writer); @@ -260,8 +260,8 @@ int enter_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, RDS_BGD_Cluste return EXIT_SUCCESS; } -int wait_for_metadata_error(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster, - int error_number, string error_message, RDS_BGD_Probe_Log& probe) +int wait_for_metadata_error(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster, + int error_number, string error_message, BGD_Probe_Log& probe) { vector green_endpoint { cluster.green_writer.endpoint() }; int error_rc = sim.topology_error(green_endpoint, error_number, error_message); @@ -270,7 +270,7 @@ int wait_for_metadata_error(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_C } auto [probe_rc, metadata_probe] = - sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); if (probe_rc != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -279,7 +279,7 @@ int wait_for_metadata_error(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_C return EXIT_SUCCESS; } -int wait_for_blue_table_check(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { +int wait_for_blue_table_check(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { vector blue_endpoint { cluster.blue_writer.endpoint() }; int drop_rc = sim.topology_drop(blue_endpoint); if (drop_rc != EXIT_SUCCESS) { @@ -287,7 +287,7 @@ int wait_for_blue_table_check(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD } auto [probe_rc, probe] = - sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.blue_writer.endpoint(), BGD_Probe_Kind::table_check, kProbeTimeoutMs, 0); return probe_rc; } @@ -299,7 +299,7 @@ int wait_for_blue_table_check(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD * - Verify BGD status NONE, restored blue-writer placement, and a subsequent * blue-writer table check. */ -int test_error_1146_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_error_1146_before_completion(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.before_completion; BGD_Hostgroups& hg = state.before_completion_hg; @@ -315,7 +315,7 @@ int test_error_1146_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, Test return EXIT_FAILURE; } - RDS_BGD_Probe_Log metadata {}; + BGD_Probe_Log metadata {}; int metadata_rc = wait_for_metadata_error(sim, seq, cluster, 1146, "Table 'mysql.rds_topology' doesn't exist", metadata); if (metadata_rc != EXIT_SUCCESS) { @@ -356,7 +356,7 @@ int test_error_1146_before_completion(MYSQL* admin, RDS_BGD_Simulator& sim, Test * - Verify BGD status NONE, restored blue-reader routing, retained green rows, * and a subsequent blue-writer table check. */ -int test_error_1146_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_error_1146_during_reader_switchover(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.reader_switchover; BGD_Hostgroups& hg = state.reader_switchover_hg; @@ -372,7 +372,7 @@ int test_error_1146_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& si return EXIT_FAILURE; } - RDS_BGD_Probe_Log metadata {}; + BGD_Probe_Log metadata {}; int metadata_rc = wait_for_metadata_error(sim, seq, cluster, 1146, "Table 'mysql.rds_topology' doesn't exist", metadata); if (metadata_rc != EXIT_SUCCESS) { @@ -410,7 +410,7 @@ int test_error_1146_during_reader_switchover(MYSQL* admin, RDS_BGD_Simulator& si * - Verify that table checking does not restart and that the in-progress * status and blue-writer demotion remain unchanged. */ -int test_generic_metadata_error(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_generic_metadata_error(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.generic_error; BGD_Hostgroups& hg = state.generic_error_hg; @@ -426,7 +426,7 @@ int test_generic_metadata_error(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& return EXIT_FAILURE; } - RDS_BGD_Probe_Log metadata {}; + BGD_Probe_Log metadata {}; int metadata_rc = wait_for_metadata_error(sim, seq, cluster, 1105, "simulated generic metadata failure", metadata); if (metadata_rc != EXIT_SUCCESS) { diag("Error: wHG 1160 did not observe the generic metadata error on the green writer"); @@ -462,7 +462,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp b/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp index 1643200102..020c3b7869 100644 --- a/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp +++ b/test/tap/tests/test_rds_bgd_worker_config_refresh-t.cpp @@ -39,7 +39,7 @@ struct TestState { bool topology_discovery_interval_saved { false }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -75,7 +75,7 @@ int restore_topology_discovery_interval(MYSQL* admin, TestState& state) { return rc; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int cleanup(MYSQL* admin, BGD_Simulator& sim, TestState& state) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -144,7 +144,7 @@ int wait_for_server_status(MYSQL* admin, int hostgroup, RDS_BGD_Host& host, stri * - Change blue-writer `weight` and `comment`. * - Verify that the active worker does not restart with a table check. */ -int test_irrelevant_server_fields(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_irrelevant_server_fields(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -156,7 +156,7 @@ int test_irrelevant_server_fields(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat } // Publish AVAILABLE topology. - vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); + vector topology = bgd_topology_with_readers(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology"); @@ -197,7 +197,7 @@ int test_irrelevant_server_fields(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat } auto [probe_rc, probe] = sim.wait_for_probe_log( - probe_seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + probe_seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: green writer did not receive the AVAILABLE metadata probe"); @@ -239,7 +239,7 @@ int test_irrelevant_server_fields(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat * - Verify that the next green-writer metadata probe uses TLS. * - Verify that discovery does not restart. */ -int test_tls_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_tls_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -268,7 +268,7 @@ int test_tls_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { // Wait for the active worker to use TLS without starting a table check. auto [probe_rc, probe] = sim.wait_for_probe_log( - seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: green-writer metadata probe did not use TLS after refresh"); @@ -292,7 +292,7 @@ int test_tls_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { * - Delete the first green reader from hostgroup 1373. * - Verify that discovery does not restart. */ -int test_green_membership_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_green_membership_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -342,7 +342,7 @@ int test_green_membership_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestStat * - Return the green writer to `ONLINE`. * - Verify that TLS metadata probing resumes without a table-check restart. */ -int test_server_eligibility_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_server_eligibility_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -377,7 +377,7 @@ int test_server_eligibility_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt // Wait for the worker to apply the refreshed server list and return to its eligible blue writer. auto [blue_probe_rc, blue_probe] = sim.wait_for_probe_log( - refresh_seq, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 + refresh_seq, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0 ); if (blue_probe_rc != EXIT_SUCCESS) { diag("Error: OFFLINE_SOFT refresh did not return metadata probing to the blue writer"); @@ -423,7 +423,7 @@ int test_server_eligibility_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt } auto [probe_rc, probe] = sim.wait_for_probe_log( - online_seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + online_seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: ONLINE green writer did not resume TLS metadata probes"); @@ -449,7 +449,7 @@ int test_server_eligibility_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestSt * - Verify that the next metadata probe occurs between 500 and 1500 milliseconds. * - Verify that the configuration refresh does not restart with a table check. */ -int test_check_interval_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_check_interval_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -497,14 +497,14 @@ int test_check_interval_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& // Consume the immediate refresh probe and the first blue probe after the worker reaches NONE. auto [first_rc, first_probe] = - sim.wait_for_probe_log(baseline, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, -1); + sim.wait_for_probe_log(baseline, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, -1); if (first_rc != EXIT_SUCCESS) { diag("Error: failed to observe the first blue metadata probe after the check_interval_ms refresh"); return EXIT_FAILURE; } auto [settled_rc, settled_probe] = - sim.wait_for_probe_log(first_probe.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, -1); + sim.wait_for_probe_log(first_probe.sequence_id, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, -1); if (settled_rc != EXIT_SUCCESS) { diag("Error: failed to observe the settled blue metadata probe after the check_interval_ms refresh"); return EXIT_FAILURE; @@ -513,7 +513,7 @@ int test_check_interval_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& // Measure the steady-state interval between consecutive blue metadata probes. unsigned long long interval_start = monotonic_time(); auto [next_rc, next_probe] = sim.wait_for_probe_log( - settled_probe.sequence_id, cluster.blue_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kMaximumProbeIntervalMs, -1 + settled_probe.sequence_id, cluster.blue_writer.endpoint(), BGD_Probe_Kind::metadata, kMaximumProbeIntervalMs, -1 ); if (next_rc != EXIT_SUCCESS) { diag("Error: metadata probing did not occur within 1.5 times check_interval_ms"); @@ -541,7 +541,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp b/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp index 6bd3d7ac64..84214fb22c 100644 --- a/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp +++ b/test/tap/tests/test_rds_bgd_worker_hostgroup_refresh-t.cpp @@ -39,7 +39,7 @@ struct TestState { vector topology_endpoints { cluster.get_endpoints() }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -61,7 +61,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -79,11 +79,11 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_as_writer(RDS_BGD_Cluster& cluster) { +vector topology_with_reader_as_writer(RDS_BGD_Cluster& cluster) { RDS_BGD_Host& blue_writer = cluster.blue_readers[0]; RDS_BGD_Host& green_writer = cluster.green_readers[0]; - vector rows { + vector rows { { blue_writer.hostname, blue_writer.hostname, blue_writer.port, "BLUE_GREEN_DEPLOYMENT_SOURCE", "SWITCHOVER_IN_PROGRESS" }, { green_writer.hostname, green_writer.hostname, green_writer.port, @@ -103,7 +103,7 @@ vector topology_with_reader_as_writer(RDS_BGD_Cluster& clu * - Verify writer placement in hostgroup 1384. * - Verify metadata probes use TLS from green hostgroups 1385 and 1386. */ -int test_hostgroup_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_hostgroup_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; BGD_Hostgroups& refreshed_hg = state.refreshed_hostgroups; @@ -116,7 +116,7 @@ int test_hostgroup_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat } // Publish SWITCHOVER_IN_PROGRESS topology. - vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); + vector topology = bgd_topology_with_readers(cluster, "SWITCHOVER_IN_PROGRESS"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); @@ -207,7 +207,7 @@ int test_hostgroup_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat // Require TLS from the green writer row in refreshed green writer hostgroup 1385. auto [probe_rc, probe] = sim.wait_for_probe_log( - seq, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: metadata probe did not use TLS from green writer hostgroup 1385"); @@ -230,7 +230,7 @@ int test_hostgroup_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& stat * - Verify that the newly mapped writer moves to hostgroup 1384. * - Verify that metadata probing uses TLS from the new green writer target. */ -int test_mapped_writer_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_mapped_writer_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.refreshed_hostgroups; RDS_BGD_Host& previous_writer = cluster.blue_writer; @@ -238,7 +238,7 @@ int test_mapped_writer_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& RDS_BGD_Host& mapped_target = cluster.green_readers[0]; // Publish topology that maps the first blue reader to the first green reader. - vector topology = topology_with_reader_as_writer(cluster); + vector topology = topology_with_reader_as_writer(cluster); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology for the new mapped writer"); @@ -282,7 +282,7 @@ int test_mapped_writer_refresh(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& } auto [probe_rc, probe] = sim.wait_for_probe_log( - seq, mapped_target.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 + seq, mapped_target.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1 ); if (probe_rc != EXIT_SUCCESS) { diag("Error: metadata probing did not use TLS from green writer hostgroup 1385"); @@ -328,7 +328,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); diff --git a/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp b/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp index 9bf8b8ee94..333580707c 100644 --- a/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp +++ b/test/tap/tests/test_rds_bgd_writer_switchover-t.cpp @@ -35,7 +35,7 @@ struct TestState { int64_t reader_log_baseline { -1 }; }; -int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); return EXIT_FAILURE; @@ -57,7 +57,7 @@ int setup(CommandLine& cl, MYSQL*& admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { +int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = bgd_admin_cleanup(admin); if (admin_rc != EXIT_SUCCESS) { diag("Error: failed to clean ProxySQL BGD test state"); @@ -75,8 +75,8 @@ int cleanup(MYSQL* admin, RDS_BGD_Simulator& sim) { return EXIT_SUCCESS; } -vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { - vector rows = cluster.get_topology(status); +vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, string status) { + vector rows = cluster.get_topology(status); rows.push_back({ cluster.blue_readers[0].hostname, cluster.blue_readers[0].hostname, @@ -94,9 +94,9 @@ vector topology_with_reader_pair(RDS_BGD_Cluster& cluster, return rows; } -int wait_for_green_observation(RDS_BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { +int wait_for_green_observation(BGD_Simulator& sim, uint64_t sequence, RDS_BGD_Cluster& cluster) { auto [probe_rc, probe] = - sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), RDS_BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); + sim.wait_for_probe_log(sequence, cluster.green_writer.endpoint(), BGD_Probe_Kind::metadata, kProbeTimeoutMs, 0); return probe_rc; } @@ -211,7 +211,7 @@ int create_blue_writer_pool(CommandLine& cl, MYSQL* admin, TestState& state) { * - Configure mysql_servers and mysql_aws_rds_bgd_hostgroups. * - Verify BGD status AVAILABLE. */ -int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_bgd_status_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -239,7 +239,7 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s return EXIT_FAILURE; } - vector topology = topology_with_reader_pair(cluster, "AVAILABLE"); + vector topology = topology_with_reader_pair(cluster, "AVAILABLE"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish AVAILABLE topology for wHG 970"); @@ -272,7 +272,7 @@ int test_bgd_status_available(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Change simulated blue writer/reader read_only values. * - Verify BGD suppresses their normal placement changes. */ -int test_switchover_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_switchover_initiated(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -282,7 +282,7 @@ int test_switchover_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s return EXIT_FAILURE; } - vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); + vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_INITIATED topology"); @@ -318,7 +318,7 @@ int test_switchover_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s return EXIT_FAILURE; } - vector repeat_topology = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); + vector repeat_topology = topology_with_reader_pair(cluster, "SWITCHOVER_INITIATED"); int repeat_rc = sim.topology_update(state.topology_endpoints, repeat_topology); if (repeat_rc != EXIT_SUCCESS) { diag("Error: failed to repeat SWITCHOVER_INITIATED topology"); @@ -362,11 +362,11 @@ int test_switchover_initiated(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& s * - Verify the blue writer moves from hostgroup 970 to 971. * - Verify the mapped blue reader remains suppressed in hostgroup 971. */ -int test_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_switchover_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; - vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); + vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_PROGRESS"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS topology"); @@ -412,7 +412,7 @@ int test_switchover_in_progress(MYSQL* admin, RDS_BGD_Simulator& sim, TestState& * - Verify writer restoration, blue-pool drain, and green backend routing. * - Repeat POST_PROCESSING and verify the post-cutover pool is not drained. */ -int test_switchover_post_processing(CommandLine& cl, MYSQL* admin, RDS_BGD_Simulator& sim, TestState& state) { +int test_switchover_post_processing(CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { RDS_BGD_Cluster& cluster = state.cluster; BGD_Hostgroups& hg = state.hostgroups; @@ -422,7 +422,7 @@ int test_switchover_post_processing(CommandLine& cl, MYSQL* admin, RDS_BGD_Simul return EXIT_FAILURE; } - vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + vector topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); int topology_rc = sim.topology_update(state.topology_endpoints, topology); if (topology_rc != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING topology"); @@ -484,7 +484,7 @@ int test_switchover_post_processing(CommandLine& cl, MYSQL* admin, RDS_BGD_Simul return EXIT_FAILURE; } - vector repeat_topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); + vector repeat_topology = topology_with_reader_pair(cluster, "SWITCHOVER_IN_POST_PROCESSING"); int repeat_rc = sim.topology_update(state.topology_endpoints, repeat_topology); if (repeat_rc != EXIT_SUCCESS) { diag("Error: failed to repeat SWITCHOVER_IN_POST_PROCESSING topology"); @@ -512,7 +512,7 @@ int main() { CommandLine cl {}; MYSQL* admin = nullptr; - RDS_BGD_Simulator sim {}; + BGD_Simulator sim {}; if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); From 982baeb362453530bf42427fccad4e5a2a4dae3a Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 23:46:16 +0000 Subject: [PATCH 17/34] feat: add Aurora replica simulator service --- include/MySQL_Monitor.hpp | 28 ++- include/SQLite3_Server.h | 11 - lib/MySQL_Monitor.cpp | 29 +-- lib/ProxySQL_Admin.cpp | 2 +- src/SQLite3_Server.cpp | 424 +++++++++++++------------------------- 5 files changed, 170 insertions(+), 324 deletions(-) diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index ad599493a5..38c379d825 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -62,6 +62,26 @@ struct cmp_str { #define AWS_ENDPOINT_SUFFIX_STRING "rds.amazonaws.com" #define QUERY_AWS_RDS_TOPOLOGY_DISCOVERY "SELECT * FROM mysql.rds_topology" #define QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK "SELECT 1 FROM information_schema.TABLES WHERE TABLE_SCHEMA='mysql' AND TABLE_NAME='rds_topology'" +#define QUERY_AWS_AURORA_REPLICA_HOST_STATUS \ + "SELECT SERVER_ID," \ + "IF(" \ + "SESSION_ID = 'MASTER_SESSION_ID' AND " \ + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " \ + "'probably_former_MASTER_SESSION_ID', SESSION_ID" \ + ") SESSION_ID, " \ + "LAST_UPDATE_TIMESTAMP, " \ + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " \ + "CPU " \ + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" \ + " ( " \ + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" \ + " OR SESSION_ID = 'MASTER_SESSION_ID'" \ + " ) " \ + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" \ + " ORDER BY SERVER_ID" +#define QUERY_AWS_AURORA_BGD_REPLICA_HOST_STATUS \ + "SELECT SERVER_ID, SESSION_ID, LAST_UPDATE_TIMESTAMP, IS_CURRENT " \ + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS ORDER BY SERVER_ID" /* @@ -77,14 +97,6 @@ A single AWS_Aurora_monitor_node will have a AWS_Aurora_status_entry per check. */ -#ifdef TEST_AURORA - -#define TEST_AURORA_MONITOR_BASE_QUERY \ - "SELECT SERVER_ID, SESSION_ID, LAST_UPDATE_TIMESTAMP, REPLICA_LAG_IN_MILLISECONDS, CPU"\ - " FROM REPLICA_HOST_STATUS ORDER BY SERVER_ID " - -#endif - class AWS_Aurora_replica_host_status_entry { public: char * server_id = nullptr; diff --git a/include/SQLite3_Server.h b/include/SQLite3_Server.h index 20b5f30ab5..eaf3cd017a 100644 --- a/include/SQLite3_Server.h +++ b/include/SQLite3_Server.h @@ -70,20 +70,9 @@ class SQLite3_Server { public: SQLite3DB *sessdb; #ifdef TEST_AURORA - unsigned int cur_aurora_writer[3]; - unsigned int num_aurora_servers[3]; - unsigned int max_num_aurora_servers; - pthread_mutex_t aurora_mutex; - /** - * @brief Handles queries to table 'REPLICA_HOST_STATUS'. - * @details This function needs to be called with lock on mutex aurora_mutex already acquired. - * @param sess The session which request is to be handled. - */ - void populate_aws_aurora_table(MySQL_Session *sess, uint32_t whg); void init_aurora_ifaces_string(std::string& s); #endif // TEST_AURORA #ifdef TEST_GALERA - //unsigned int cur_aurora_writer[3]; unsigned int num_galera_servers[3]; unsigned int max_num_galera_servers; pthread_mutex_t galera_mutex; diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index ff3c31389b..a76bb3b2f5 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6290,12 +6290,6 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { mmsd->t1=monotonic_time(); mmsd->interr=0; // reset the value -#ifdef TEST_AURORA - { - string query { TEST_AURORA_MONITOR_BASE_QUERY + std::to_string(wHG) }; - mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, query.c_str()); - } -#else // for reference we list the old queries. // original implementation: // mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, "SELECT SERVER_ID, SESSION_ID, LAST_UPDATE_TIMESTAMP, IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, CPU FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE (REPLICA_LAG_IN_MILLISECONDS > 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000) OR SESSION_ID = 'MASTER_SESSION_ID' ORDER BY SERVER_ID"); @@ -6306,27 +6300,8 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { // with: // "REPLICA_LAG_IN_MILLISECONDS >= 0" // mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, "SELECT SERVER_ID, SESSION_ID, LAST_UPDATE_TIMESTAMP, IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, CPU FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE (REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000) OR SESSION_ID = 'MASTER_SESSION_ID' ORDER BY SERVER_ID"); - { - const char * query = - "SELECT SERVER_ID," - "IF(" - "SESSION_ID = 'MASTER_SESSION_ID' AND " - "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " - "'probably_former_MASTER_SESSION_ID', SESSION_ID" - ") SESSION_ID, " // it seems that during a failover, the old writer can keep MASTER_SESSION_ID because not updated - "LAST_UPDATE_TIMESTAMP, " - "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " - "CPU " - "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" - " ( " - "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" // lag between 0 and 10 minutes - " OR SESSION_ID = 'MASTER_SESSION_ID'" // or server with MASTER_SESSION_ID - " ) " - "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" // ignore decommissioned or renamed nodes, see https://github.com/sysown/proxysql/issues/3484 - " ORDER BY SERVER_ID"; - mmsd->async_exit_status = mysql_query_start(&mmsd->interr, mmsd->mysql, query); - } -#endif // TEST_AURORA + mmsd->async_exit_status = mysql_query_start( + &mmsd->interr, mmsd->mysql, QUERY_AWS_AURORA_REPLICA_HOST_STATUS); while (mmsd->async_exit_status) { mmsd->async_exit_status=wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); #ifdef DEBUG diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 9c54baf9cd..0a8dcdae95 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -9225,7 +9225,7 @@ void ProxySQL_Admin::enable_galera_testing() { void ProxySQL_Admin::enable_aurora_testing_populate_mysql_servers() { sqlite3_stmt *statement=NULL; - unsigned int num_aurora_servers = GloSQLite3Server->num_aurora_servers[0]; + constexpr unsigned int num_aurora_servers = 4; admindb->execute("DELETE FROM mysql_servers WHERE hostgroup_id BETWEEN 1271 AND 1276"); char *query=(char *)"INSERT INTO mysql_servers (hostgroup_id,hostname,use_ssl,comment) VALUES (?1, ?2, ?3, ?4)"; auto [rc1, statement_unique] = admindb->prepare_v2(query); diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index 093b284045..c435577530 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -47,6 +47,20 @@ static int random_replication_lag_seconds() { } #endif +#ifdef TEST_AURORA +static std::string simulator_sql_quote(const std::string& value) { + std::string quoted { "'" }; + for (char c : value) { + quoted += c; + if (c == '\'') { + quoted += '\''; + } + } + quoted += '\''; + return quoted; +} +#endif + #ifdef TEST_REPLICATIONLAG static void ensure_replicationlag_table_loaded(SQLite3_Server* server, MySQL_Session* sess) { if (server->replicationlag_map_size() == 0) { @@ -917,7 +931,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } delete control_result; - if (run_query && aws_bgd_table_check) { + if (run_query && aws_bgd_table_check) { const char* topology_sql = topology_present ? "SELECT 1" : "SELECT 1 WHERE 0"; static constexpr size_t topology_sql_len = sizeof("SELECT 1") - 1; @@ -952,39 +966,117 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } #endif // TEST_AURORA || TEST_RDS_BGD - #ifdef TEST_AURORA - if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { - pthread_mutex_lock(&GloSQLite3Server->aurora_mutex); - - if (strcasestr(query_no_space, TEST_AURORA_MONITOR_BASE_QUERY)) { - string s_whg { query_no_space + (sizeof(TEST_AURORA_MONITOR_BASE_QUERY) - 1) }; - uint32_t whg = atoi(s_whg.c_str()); - - GloSQLite3Server->populate_aws_aurora_table(sess, whg); - vector hgs_info { get_hgs_info(GloAdmin->admindb) }; +#ifdef TEST_AURORA + const bool aws_aurora_replica_query = + strcasecmp(query_no_space, QUERY_AWS_AURORA_REPLICA_HOST_STATUS) == 0; + const bool aws_aurora_bgd_replica_query = + strcasecmp(query_no_space, QUERY_AWS_AURORA_BGD_REPLICA_HOST_STATUS) == 0; + if (aws_aurora_replica_query || aws_aurora_bgd_replica_query) { + if (sess->client_myds->proxy_addr.addr == NULL || + sess->client_myds->proxy_addr.port <= 0) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, + "AWS Aurora simulator could not identify the accepted backend address"); + run_query=false; + } else { + SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; + const std::string backend_ip { sess->client_myds->proxy_addr.addr }; + const int backend_port = sess->client_myds->proxy_addr.port; + const std::string predicate { + "backend_ip=" + simulator_sql_quote(backend_ip) + + " AND backend_port=" + std::to_string(backend_port) + }; + char *control_error=nullptr; + int control_cols=0; + int control_affected_rows=0; + SQLite3_result *control_result=nullptr; + const std::string control_query { + "SELECT replica_set_id,replica_table_present,error_code,error_msg " + "FROM AWS_AURORA_REPLICA_CONTROL WHERE " + predicate + }; + sqlite_sess->sessdb->execute_statement( + control_query.c_str(), &control_error, &control_cols, + &control_affected_rows, &control_result); + + std::string replica_set_id {}; + bool replica_table_present=false; + unsigned int configured_error=0; + std::string configured_error_msg {}; + if (control_error == nullptr && control_result && + control_result->rows_count == 1) { + SQLite3_row *row=control_result->rows.front(); + replica_set_id=row->fields[0] ? row->fields[0] : ""; + replica_table_present=atoi(row->fields[1]) != 0; + configured_error=static_cast(atoi(row->fields[2])); + configured_error_msg=row->fields[3] ? row->fields[3] : ""; + } + delete control_result; - const auto match_writer = [&whg](const aurora_hg_info_t& hg_info) { - return std::get(hg_info) == whg; - }; - const auto hg_info_it = std::find_if(hgs_info.begin(), hgs_info.end(), match_writer); - string select_query { - "SELECT SERVER_ID,SESSION_ID,LAST_UPDATE_TIMESTAMP,REPLICA_LAG_IN_MILLISECONDS,CPU" - " FROM REPLICA_HOST_STATUS " - }; + const std::string log_query { + "INSERT INTO AWS_AURORA_REPLICA_PROBE_LOG" + "(backend_ip,backend_port,probe_kind,replica_set_id,encrypted) VALUES (" + + simulator_sql_quote(backend_ip) + "," + + std::to_string(backend_port) + "," + + simulator_sql_quote( + aws_aurora_replica_query ? "ordinary" : "bgd_membership") + "," + + (replica_set_id.empty() ? "NULL" : simulator_sql_quote(replica_set_id)) + + "," + (sess->client_myds->encrypted ? "1" : "0") + ")" + }; - if (hg_info_it == hgs_info.end()) { - select_query += " LIMIT 0"; + if (control_error != nullptr) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, control_error); + free(control_error); + run_query=false; + } else if (!sqlite_sess->sessdb->execute(log_query.c_str())) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, + "AWS Aurora simulator failed to record the replica probe"); + run_query=false; + } else if (!replica_table_present) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1146, + "Table 'information_schema.REPLICA_HOST_STATUS' doesn't exist"); + run_query=false; + } else if (configured_error != 0) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, + static_cast(configured_error), + configured_error_msg.c_str()); + run_query=false; + } else { + const std::string set_literal { simulator_sql_quote(replica_set_id) }; + std::string select_query {}; + if (aws_aurora_replica_query) { + select_query = + "SELECT SERVER_ID," + "CASE WHEN SESSION_ID='MASTER_SESSION_ID' AND SERVER_ID<>(" + "SELECT SERVER_ID FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + + set_literal + " AND SESSION_ID='MASTER_SESSION_ID' " + "ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1) " + "THEN 'probably_former_MASTER_SESSION_ID' ELSE SESSION_ID END AS SESSION_ID," + "LAST_UPDATE_TIMESTAMP," + "CASE WHEN SESSION_ID='MASTER_SESSION_ID' THEN 0 " + "ELSE REPLICA_LAG_IN_MILLISECONDS END AS REPLICA_LAG_IN_MILLISECONDS," + "CPU FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + set_literal + + " AND ((REPLICA_LAG_IN_MILLISECONDS>=0 AND " + "REPLICA_LAG_IN_MILLISECONDS<=600000) OR " + "SESSION_ID='MASTER_SESSION_ID')" + " AND LAST_UPDATE_TIMESTAMP>datetime('now','-180 seconds')" + " ORDER BY SERVER_ID"; } else { - const string& domain_name { std::get(*hg_info_it) }; - select_query += " WHERE DOMAIN_NAME='" + domain_name + "' ORDER BY SERVER_ID"; + select_query = + "SELECT SERVER_ID,SESSION_ID,LAST_UPDATE_TIMESTAMP,IS_CURRENT " + "FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + set_literal + + " ORDER BY SERVER_ID"; } - free(query); - query = static_cast(malloc(select_query.length() + 1)); - memcpy(query, select_query.c_str(), select_query.length()); - query[select_query.length()] = '\0'; + l_free(query_length,query); + query=l_strdup(select_query.c_str()); + query_length=select_query.length()+1; } } + } #endif // TEST_AURORA #ifdef TEST_GALERA if (strstr(query_no_space,(char *)"HOST_STATUS_GALERA")) { @@ -1096,19 +1188,8 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } sqlite_sess->sessdb->execute_statement(query, &error , &cols , &affected_rows , &resultset); -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) +#if defined(TEST_GALERA) || defined(TEST_GROUPREP) if (strncasecmp("SELECT",query_no_space,6)==0) { -#ifdef TEST_AURORA - if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { - pthread_mutex_unlock(&GloSQLite3Server->aurora_mutex); -#ifdef TEST_AURORA_RANDOM - if (rand() % 100 == 0) { - // randomly add some latency on 1% of the traffic - sleep(2); - } -#endif - } -#endif // TEST_AURORA #ifdef TEST_GALERA if (strstr(query_no_space,(char *)"HOST_STATUS_GALERA")) { pthread_mutex_unlock(&GloSQLite3Server->galera_mutex); @@ -1141,7 +1222,7 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } } -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP +#endif // TEST_GALERA || TEST_GROUPREP sqlite3 *db = sqlite_sess->sessdb->get_db(); bool in_trans = false; if ((*proxy_sqlite3_get_autocommit)(db)==0) { @@ -1484,14 +1565,8 @@ SQLite3_Server::~SQLite3_Server() { void SQLite3_Server::init_aurora_ifaces_string(std::string& s) { if(!s.empty()) s += ";"; - pthread_mutex_init(&aurora_mutex,NULL); - unsigned int nas = time(NULL); - nas = nas % 3; // range - nas += 4; // min - max_num_aurora_servers = 10; // hypothetical maximum number of nodes + constexpr unsigned int max_num_aurora_servers = 10; for (unsigned int j=1; j<4; j++) { - cur_aurora_writer[j-1] = 0; - num_aurora_servers[j-1] = nas; for (unsigned int i=11; i(cpu_i) / 100; - - return cpu; -} - -string get_curtime_str() { - time_t __timer; - char lut[30]; - struct tm __tm_info; - time(&__timer); - localtime_r(&__timer, &__tm_info); - strftime(lut, 25, "%Y-%m-%d %H:%M:%S", &__tm_info); - string s = string(lut); - return s; -} - -void bind_query_params( - SQLite3DB* db, - sqlite3_stmt* stmt, - const string& server_id, - const string& domain, - const string& session_id, - float cpu, - const string& lut, - int32_t lag_ms -) { - int rc = 0; - - rc=(*proxy_sqlite3_bind_text)(stmt, 1, server_id.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_text)(stmt, 2, domain.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_text)(stmt, 3, session_id.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_double)(stmt, 4, cpu); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_text)(stmt, 5, lut.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_double)(stmt, 6, lag_ms); ASSERT_SQLITE_OK(rc, db); - SAFE_SQLITE3_STEP2(stmt); - rc=(*proxy_sqlite3_clear_bindings)(stmt); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_reset)(stmt); ASSERT_SQLITE_OK(rc, db); -} - -/** - * @brief Extracts SERVER_ID from the supplied hostname using DOMAIN_NAME. - * @param hostname The server hostname (SERVER_ID + DOMAIN_NAME)). - * @param domain_name The server DOMAIN_NAME as in 'mysql_aws_aurora_hostgroups' - * @return Either the SERVER_ID in the supplied hostname or empty if DOMAIN_NAME failed to match. - */ -string get_server_id(const string& hostname, const string& domain_name) { - string::size_type pos = hostname.find(domain_name); - - if (pos == string::npos) { - return {}; - } else { - return hostname.substr(0, pos); - } -} - -void SQLite3_Server::populate_aws_aurora_table(MySQL_Session *sess, uint32_t whg) { - int rc = 0; - sqlite3_stmt* stmt = NULL; - const char query[] { "INSERT INTO REPLICA_HOST_STATUS VALUES (?1, ?2, ?3, ?4, ?5, ?6)" }; - - auto [stmt_rc, stmt_unique] = sessdb->prepare_v2(query); - rc = stmt_rc; - stmt = stmt_unique.get(); - ASSERT_SQLITE_OK(rc, sessdb); - -#ifndef TEST_AURORA_RANDOM - SQLite3_result* host_status = NULL; - - { - char* error = NULL; - int cols = 0; - int affected_rows = 0; - - string query { - "SELECT SERVER_ID,DOMAIN_NAME,SESSION_ID,LAST_UPDATE_TIMESTAMP,REPLICA_LAG_IN_MILLISECONDS" - " FROM REPLICA_HOST_STATUS" - }; - sessdb->execute_statement(query.c_str(), &error, &cols, &affected_rows, &host_status); - } - - // If empty, we fill the map with sensible defaults for performing manual testing. - if (host_status->rows.empty()) { - vector hgs_info { get_hgs_info(GloAdmin->admindb) }; - SQLite3_result* resultset = nullptr; - - { - char* error = nullptr; - int cols = 0; - int affected_rows = 0; - - GloAdmin->admindb->execute_statement( - "SELECT hostname, hostgroup_id FROM mysql_servers WHERE hostgroup_id BETWEEN 1270 AND 1300" - " GROUP BY HOSTNAME", - &error, &cols, &affected_rows, &resultset - ); - } - - sessdb->execute("DELETE FROM REPLICA_HOST_STATUS"); - vector proc_srvs {}; - - for (const aurora_hg_info_t& hg_info : hgs_info) { - const auto match_writer = [&hg_info](const SQLite3_row* row) { - return atoi(row->fields[1]) == std::get(hg_info); - }; - const auto mysrv_it = std::find_if(resultset->rows.begin(), resultset->rows.end(), match_writer); - bool writer_set = false; - - for (const SQLite3_row* r : resultset->rows) { - const string srv_hostname { r->fields[0] }; - const uint32_t srv_hg_id = atoi(r->fields[1]); - const string& aurora_domain { std::get(hg_info) }; - - if ( - srv_hostname.find(aurora_domain) == string::npos || - std::find(proc_srvs.begin(), proc_srvs.end(), srv_hostname) != proc_srvs.end() - ) { - continue; - } - - const string server_id { - get_server_id(srv_hostname, std::get(hg_info)) - }; - - string session_id {}; - - if ( - (mysrv_it == resultset->rows.end() && writer_set == false) || - (srv_hg_id == std::get(hg_info) && writer_set == false) - ) { - session_id = "MASTER_SESSION_ID"; - writer_set = true; - } else { - session_id = "TESTID-" + server_id + aurora_domain + "-R"; - } - - const float cpu = get_rand_cpu(); - const string lut { get_curtime_str() }; - const int lag_ms = 0; - - bind_query_params(sessdb, stmt, server_id, aurora_domain, session_id, cpu, lut, lag_ms); - proc_srvs.push_back(srv_hostname); - } - } - - delete resultset; - } else { - // We just re-generate deterministic 'SESSION_IDS', preserving 'MASTER_SESSION_ID' values: - // 'SESSION_IDS' are preserved, 'MASTER_SESSION_ID' or others. - for (SQLite3_row* row : host_status->rows) { - const char* server_id = row->fields[0]; - const char* domain_name = row->fields[1]; - - const char update_query_t[] { - "UPDATE REPLICA_HOST_STATUS SET SESSION_ID='%s',CPU=%f,LAST_UPDATE_TIMESTAMP='%s'" - " WHERE SERVER_ID='%s' AND DOMAIN_NAME='%s' AND SESSION_ID!='MASTER_SESSION_ID'" - }; - - const string session_id { "TESTID-" + string { server_id } + domain_name + "-R" }; - const float cpu = get_rand_cpu(); - const string lut { get_curtime_str() }; - - const string update_query { - cstr_format(update_query_t, session_id.c_str(), cpu, lut.c_str(), server_id, domain_name).str - }; - - sessdb->execute(update_query.c_str()); - } - } - - delete host_status; -#else - sessdb->execute("DELETE FROM REPLICA_HOST_STATUS"); - - string lut { get_curtime_str() }; - string myip = string(sess->client_myds->proxy_addr.addr); - string clu_id_s = myip.substr(6,1); - unsigned int cluster_id = atoi(clu_id_s.c_str()); - cluster_id--; - - if (rand() % 20000 == 0) { - // simulate a failover - cur_aurora_writer[cluster_id] = rand() % num_aurora_servers[cluster_id]; - proxy_info("Simulating a failover for AWS Aurora cluster %d , HGs (%d:%d)\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2); - } - if (rand() % 1000 == 0) { - if (num_aurora_servers[cluster_id] < max_num_aurora_servers) { - num_aurora_servers[cluster_id]++; - proxy_info("Simulating the add of a new server for AWS Aurora Cluster %d , HGs (%d:%d). Now adding server num %d\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2, num_aurora_servers[cluster_id]); - } - } - if (rand() % 1000 == 0) { - if (num_aurora_servers[cluster_id] > 1) { - if (cur_aurora_writer[cluster_id] != (num_aurora_servers[cluster_id] - 1) ) { - num_aurora_servers[cluster_id]--; - proxy_info("Simulating the deletion of a server from AWS Aurora Cluster %d , HGs (%d:%d). Removing server num %d\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2, num_aurora_servers[cluster_id]+1); - } - } - } - for (unsigned int i=0; i Date: Thu, 13 Aug 2026 23:46:25 +0000 Subject: [PATCH 18/34] test: add Aurora BGD simulator controls --- test/tap/tap/Makefile | 7 +- test/tap/tap/aurora_bgd_simulator.cpp | 40 +++++ test/tap/tap/aurora_bgd_simulator.h | 52 +++++++ test/tap/tap/bgd_simulator.cpp | 206 ++++++++++++++++++++++++++ test/tap/tap/bgd_simulator.h | 45 +++++- 5 files changed, 347 insertions(+), 3 deletions(-) create mode 100644 test/tap/tap/aurora_bgd_simulator.cpp create mode 100644 test/tap/tap/aurora_bgd_simulator.h diff --git a/test/tap/tap/Makefile b/test/tap/tap/Makefile index 6e090f4ad9..4ceb4b1ca5 100644 --- a/test/tap/tap/Makefile +++ b/test/tap/tap/Makefile @@ -86,11 +86,14 @@ cluster_simulator.o: cluster_simulator.cpp cluster_simulator.h bgd_simulator.o: bgd_simulator.cpp bgd_simulator.h cluster_simulator.h utils.h $(CXX) -fPIC -c bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) +aurora_bgd_simulator.o: aurora_bgd_simulator.cpp aurora_bgd_simulator.h bgd_simulator.h + $(CXX) -fPIC -c aurora_bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) + rds_bgd_simulator.o: rds_bgd_simulator.cpp rds_bgd_simulator.h bgd_simulator.h $(CXX) -fPIC -c rds_bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) -libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o bgd_simulator.o rds_bgd_simulator.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a - $(AR) rcs libtap_mariadb.a tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o bgd_simulator.o rds_bgd_simulator.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo +libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o bgd_simulator.o aurora_bgd_simulator.o rds_bgd_simulator.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a + $(AR) rcs libtap_mariadb.a tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o bgd_simulator.o aurora_bgd_simulator.o rds_bgd_simulator.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo libtap_mysql57.a: tap.o command_line.o utils_mysql57.o noise_utils_mysql57.o mcp_client.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a $(AR) rcs libtap_mysql57.a tap.o command_line.o utils_mysql57.o noise_utils_mysql57.o mcp_client.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo diff --git a/test/tap/tap/aurora_bgd_simulator.cpp b/test/tap/tap/aurora_bgd_simulator.cpp new file mode 100644 index 0000000000..1fb37f78c6 --- /dev/null +++ b/test/tap/tap/aurora_bgd_simulator.cpp @@ -0,0 +1,40 @@ +#include "aurora_bgd_simulator.h" + +using namespace std; + +Endpoint Aurora_BGD_Endpoint::backend() { + return { ip, port }; +} + +Endpoint Aurora_BGD_Endpoint::host_endpoint() { + return { hostname, port }; +} + +Aurora_Replica_Row Aurora_BGD_Member::replica_row() { + return { + server_id, + session_id, + cpu, + last_update_timestamp, + replica_lag_in_milliseconds, + is_current, + }; +} + +vector Aurora_BGD_Membership_Set::replica_rows() { + vector rows {}; + rows.reserve(members.size()); + for (Aurora_BGD_Member& member : members) { + rows.push_back(member.replica_row()); + } + return rows; +} + +vector Aurora_BGD_Membership_Set::backends() { + vector endpoints {}; + endpoints.reserve(serving_endpoints.size()); + for (Aurora_BGD_Endpoint& endpoint : serving_endpoints) { + endpoints.push_back(endpoint.backend()); + } + return endpoints; +} diff --git a/test/tap/tap/aurora_bgd_simulator.h b/test/tap/tap/aurora_bgd_simulator.h new file mode 100644 index 0000000000..6df8f01db2 --- /dev/null +++ b/test/tap/tap/aurora_bgd_simulator.h @@ -0,0 +1,52 @@ +#ifndef TAP_AURORA_BGD_SIMULATOR_H +#define TAP_AURORA_BGD_SIMULATOR_H + +#include +#include + +#include "bgd_simulator.h" + +using namespace std; + +/** An AWS-style Aurora hostname and its fixed simulator backend address. */ +struct Aurora_BGD_Endpoint { + string hostname; + string ip; + int port; + + Endpoint backend(); + Endpoint host_endpoint(); +}; + +/** One Aurora cluster member published through REPLICA_HOST_STATUS. */ +struct Aurora_BGD_Member { + string server_id; + string session_id; + Aurora_BGD_Endpoint endpoint; + double cpu { 0 }; + string last_update_timestamp; + double replica_lag_in_milliseconds { 0 }; + bool is_current { true }; + + Aurora_Replica_Row replica_row(); +}; + +/** A membership snapshot and every backend allowed to return it. */ +struct Aurora_BGD_Membership_Set { + string replica_set_id; + vector members; + vector serving_endpoints; + + vector replica_rows(); + vector backends(); +}; + +/** Stable identity used to describe an AWS member rename during switchover. */ +struct Aurora_BGD_Rename { + string production_server_id; + string target_server_id; + string session_id; + string cached_target_ip; +}; + +#endif // TAP_AURORA_BGD_SIMULATOR_H diff --git a/test/tap/tap/bgd_simulator.cpp b/test/tap/tap/bgd_simulator.cpp index c0f476e706..8c0641fa24 100644 --- a/test/tap/tap/bgd_simulator.cpp +++ b/test/tap/tap/bgd_simulator.cpp @@ -25,6 +25,20 @@ rc_t parse_probe_kind(string value) { return { EXIT_FAILURE, BGD_Probe_Kind::table_check }; } +const char* replica_probe_kind_string(Aurora_Replica_Probe_Kind kind) { + return kind == Aurora_Replica_Probe_Kind::ordinary ? "ordinary" : "bgd_membership"; +} + +rc_t parse_replica_probe_kind(string value) { + if (value == "ordinary") { + return { EXIT_SUCCESS, Aurora_Replica_Probe_Kind::ordinary }; + } + if (value == "bgd_membership") { + return { EXIT_SUCCESS, Aurora_Replica_Probe_Kind::bgd_membership }; + } + return { EXIT_FAILURE, Aurora_Replica_Probe_Kind::ordinary }; +} + } // namespace int BGD_Simulator::topology_update(vector backends, vector rows) { @@ -99,6 +113,84 @@ int BGD_Simulator::topology_error(vector backends, int error_code, str return execute_transaction(statements); } +int BGD_Simulator::replica_update( + string replica_set_id, + vector rows, + vector backends) +{ + if (replica_set_id.empty() || backends.empty()) { + return EXIT_FAILURE; + } + + vector statements { + "DELETE FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + + sql_quote(replica_set_id), + "DELETE FROM AWS_AURORA_REPLICA_CONTROL WHERE replica_set_id=" + + sql_quote(replica_set_id), + }; + for (Aurora_Replica_Row& row : rows) { + statements.push_back( + "INSERT INTO REPLICA_HOST_STATUS" + "(REPLICA_SET_ID,SERVER_ID,SESSION_ID,CPU,LAST_UPDATE_TIMESTAMP," + "REPLICA_LAG_IN_MILLISECONDS,IS_CURRENT) VALUES (" + + sql_quote(replica_set_id) + "," + sql_quote(row.server_id) + "," + + sql_quote(row.session_id) + "," + to_string(row.cpu) + "," + + sql_quote(row.last_update_timestamp) + "," + + to_string(row.replica_lag_in_milliseconds) + "," + + (row.is_current ? "1" : "0") + ")"); + } + for (Endpoint& backend : backends) { + statements.push_back( + "INSERT OR REPLACE INTO AWS_AURORA_REPLICA_CONTROL" + "(backend_ip,backend_port,replica_set_id,replica_table_present,error_code,error_msg) " + "VALUES (" + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + sql_quote(replica_set_id) + ",1,0,'')"); + } + + return execute_transaction(statements); +} + +int BGD_Simulator::replica_delete(string replica_set_id) { + if (replica_set_id.empty()) { + return EXIT_FAILURE; + } + vector statements { + "DELETE FROM AWS_AURORA_REPLICA_CONTROL WHERE replica_set_id=" + + sql_quote(replica_set_id), + "DELETE FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + + sql_quote(replica_set_id), + }; + return execute_transaction(statements); +} + +int BGD_Simulator::replica_drop(vector backends) { + return replica_error( + backends, 1146, + "Table 'information_schema.REPLICA_HOST_STATUS' doesn't exist"); +} + +int BGD_Simulator::replica_error( + vector backends, int error_code, string error_msg) +{ + if (backends.empty() || error_code == 0) { + return EXIT_FAILURE; + } + + const bool table_present = error_code != 1146; + vector statements {}; + for (Endpoint& backend : backends) { + const string predicate { backend_predicate(backend) }; + statements.push_back( + "INSERT OR REPLACE INTO AWS_AURORA_REPLICA_CONTROL" + "(backend_ip,backend_port,replica_set_id,replica_table_present,error_code,error_msg) " + "VALUES (" + sql_quote(backend.host) + "," + to_string(backend.port) + "," + + "COALESCE((SELECT replica_set_id FROM AWS_AURORA_REPLICA_CONTROL WHERE " + + predicate + "),'')," + (table_present ? "1" : "0") + "," + + to_string(error_code) + "," + sql_quote(error_msg) + ")"); + } + return execute_transaction(statements); +} + int BGD_Simulator::cleanup() { vector statements { "DELETE FROM READONLY_STATUS", @@ -106,6 +198,24 @@ int BGD_Simulator::cleanup() { "DELETE FROM AWS_BGD_CONTROL", "DELETE FROM AWS_BGD_PROBE_LOG", }; + + if (connection() == nullptr) { + return EXIT_FAILURE; + } + auto [rc, rows] = mysql_query_ext_rows( + connection(), + "SELECT name FROM sqlite_master WHERE type='table' AND name IN (" + "'AWS_AURORA_REPLICA_CONTROL','AWS_AURORA_REPLICA_PROBE_LOG'," + "'REPLICA_HOST_STATUS')"); + if (rc != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + for (mysql_res_row& row : rows) { + if (row.size() != 1) { + return EXIT_FAILURE; + } + statements.push_back("DELETE FROM " + row.front()); + } return execute_transaction(statements); } @@ -200,6 +310,102 @@ rc_t BGD_Simulator::wait_for_probe_log( return { ETIMEDOUT, {} }; } +rc_t BGD_Simulator::replica_probe_log_last_sequence() { + if (connection() == nullptr) { + return { EXIT_FAILURE, 0 }; + } + auto [rc, rows] = mysql_query_ext_rows( + connection(), + "SELECT COALESCE(MAX(sequence_id),0) FROM AWS_AURORA_REPLICA_PROBE_LOG"); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { + return { EXIT_FAILURE, 0 }; + } + return { + EXIT_SUCCESS, + static_cast(strtoull(rows.front().front().c_str(), nullptr, 10)) + }; +} + +rc_t> BGD_Simulator::replica_probe_log_since( + uint64_t sequence_id) +{ + if (connection() == nullptr) { + return { EXIT_FAILURE, {} }; + } + string query { + "SELECT sequence_id,backend_ip,backend_port,probe_kind," + "COALESCE(replica_set_id,''),encrypted " + "FROM AWS_AURORA_REPLICA_PROBE_LOG WHERE sequence_id>" + + to_string(sequence_id) + " ORDER BY sequence_id" + }; + auto [rc, rows] = mysql_query_ext_rows(connection(), query); + if (rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + + vector logs {}; + for (mysql_res_row& row : rows) { + if (row.size() != 6) { + return { EXIT_FAILURE, {} }; + } + auto [kind_rc, probe_kind] = parse_replica_probe_kind(row[3]); + if (kind_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + logs.push_back({ + static_cast(strtoull(row[0].c_str(), nullptr, 10)), + { row[1], atoi(row[2].c_str()) }, + probe_kind, + row[4], + atoi(row[5].c_str()) != 0, + }); + } + return { EXIT_SUCCESS, move(logs) }; +} + +rc_t BGD_Simulator::wait_for_replica_probe_log( + uint64_t sequence_id, + Endpoint backend, + Aurora_Replica_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted, + string replica_set_id) +{ + uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + auto [rc, logs] = replica_probe_log_since(sequence_id); + if (rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, {} }; + } + for (Aurora_Replica_Probe_Log& log : logs) { + if (log.backend.host == backend.host && log.backend.port == backend.port && + log.probe_kind == probe_kind && + (replica_set_id.empty() || log.replica_set_id == replica_set_id) && + (encrypted < 0 || log.encrypted == (encrypted != 0))) { + return { EXIT_SUCCESS, log }; + } + } + usleep(50000); + } while (monotonic_time() < deadline); + + auto [rc, logs] = replica_probe_log_since(sequence_id); + if (rc == EXIT_SUCCESS) { + for (Aurora_Replica_Probe_Log& log : logs) { + diag( + "Observed Aurora replica probe sequence=%llu backend=%s:%d kind=%s set=%s encrypted=%d", + static_cast(log.sequence_id), + log.backend.host.c_str(), log.backend.port, + replica_probe_kind_string(log.probe_kind), + log.replica_set_id.c_str(), log.encrypted ? 1 : 0); + } + } + diag( + "Timed out waiting for Aurora replica probe backend=%s:%d kind=%s set=%s encrypted=%d", + backend.host.c_str(), backend.port, replica_probe_kind_string(probe_kind), + replica_set_id.c_str(), encrypted); + return { ETIMEDOUT, {} }; +} + int BGD_Simulator::execute_transaction(vector& statements) { if (execute("START TRANSACTION") != EXIT_SUCCESS) { return EXIT_FAILURE; diff --git a/test/tap/tap/bgd_simulator.h b/test/tap/tap/bgd_simulator.h index a98013319d..30650da764 100644 --- a/test/tap/tap/bgd_simulator.h +++ b/test/tap/tap/bgd_simulator.h @@ -33,13 +33,46 @@ struct BGD_Probe_Log { bool encrypted; }; -/** Controls shared AWS BGD topology responses and reads their probe log. */ +/** One row returned by the simulated Aurora replica-status service. */ +struct Aurora_Replica_Row { + string server_id; + string session_id; + double cpu; + string last_update_timestamp; + double replica_lag_in_milliseconds; + bool is_current; +}; + +/** Identifies which Aurora replica-status query reached the simulator. */ +enum class Aurora_Replica_Probe_Kind { + ordinary, + bgd_membership, +}; + +/** One Aurora replica-status query observed by the shared AWS simulator. */ +struct Aurora_Replica_Probe_Log { + uint64_t sequence_id; + Endpoint backend; + Aurora_Replica_Probe_Kind probe_kind; + string replica_set_id; + bool encrypted; +}; + +/** Controls shared AWS BGD topology and Aurora replica simulator responses. */ class BGD_Simulator : public Cluster_Simulator { public: int topology_update(vector backends, vector rows); int topology_delete(vector backends); int topology_drop(vector backends); int topology_error(vector backends, int error_code, string error_msg); + int replica_update( + string replica_set_id, + vector rows, + vector backends + ); + int replica_delete(string replica_set_id); + int replica_drop(vector backends); + int replica_error(vector backends, int error_code, string error_msg); int cleanup(); rc_t probe_log_last_sequence(); @@ -51,6 +84,16 @@ class BGD_Simulator : public Cluster_Simulator { uint32_t timeout_ms, int encrypted = -1 ); + rc_t replica_probe_log_last_sequence(); + rc_t> replica_probe_log_since(uint64_t sequence_id); + rc_t wait_for_replica_probe_log( + uint64_t sequence_id, + Endpoint backend, + Aurora_Replica_Probe_Kind probe_kind, + uint32_t timeout_ms, + int encrypted = -1, + string replica_set_id = "" + ); protected: static string backend_predicate(Endpoint backend); From 1c3702c821a270103a165e6509e9d4ee4323f6af Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 13 Aug 2026 23:46:35 +0000 Subject: [PATCH 19/34] test: migrate Aurora scenarios to replica sets --- .../cluster_simulator/cluster_simulator.cpp | 6 +- .../cluster_simulator/lib/aurora_utils.cpp | 207 +++++++++++++----- .../deps/cluster_simulator/lib/aurora_utils.h | 6 +- test/tap/groups/cluster_sim_aurora/add-hosts | 28 +++ 4 files changed, 184 insertions(+), 63 deletions(-) diff --git a/test/deps/cluster_simulator/cluster_simulator.cpp b/test/deps/cluster_simulator/cluster_simulator.cpp index 2ee974de1a..cced5822a4 100644 --- a/test/deps/cluster_simulator/cluster_simulator.cpp +++ b/test/deps/cluster_simulator/cluster_simulator.cpp @@ -1798,10 +1798,8 @@ std::pair simulate_aws_aurora_cluster_state( const auto& aurora_new_state_to_set { aurora_update_cluster_state(aurora_init_servers_state, aurora_new_servers_state) }; - // cleanup=1 deletes REPLICA_HOST_STATUS rows whose SERVER_ID is not in the new - // state, so payloads can simulate a server being removed from the topology - // (e.g. for autopurge_missing_checks coverage). Backwards-compatible: existing - // payloads keep the full server list in new_state, so nothing extra is purged. + // Replace the published replica sets and their complete backend mappings so + // payloads can simulate members being removed (for example, autopurge coverage). prep_aurora_final_state_err = prepare_aurora_cluster_state(proxysql_sqlite, aurora_new_servers_state, 1); diff --git a/test/deps/cluster_simulator/lib/aurora_utils.cpp b/test/deps/cluster_simulator/lib/aurora_utils.cpp index 920acd30d9..68ea1d4c86 100644 --- a/test/deps/cluster_simulator/lib/aurora_utils.cpp +++ b/test/deps/cluster_simulator/lib/aurora_utils.cpp @@ -1,10 +1,14 @@ #include "aurora_utils.h" #include +#include +#include #include +#include #include #include #include +#include // NOTE: Only needed during testing #include @@ -354,89 +358,176 @@ std::pair prepare_mysql_aurora_hostgroups( return { EXIT_SUCCESS, "" }; } -const char t_aurora_server_state_insert[] { - "INSERT OR REPLACE INTO REPLICA_HOST_STATUS(" - " SERVER_ID," - " DOMAIN_NAME," - " SESSION_ID," - " CPU," - " LAST_UPDATE_TIMESTAMP," - " REPLICA_LAG_IN_MILLISECONDS" - ") VALUES (" - "'%s', '%s', '%s', %d, '%s', %d" - ")" -}; +namespace { -std::pair prepare_aurora_cluster_state( - MYSQL* proxysql_sqlite, - const vector& servers, - uint32_t cleanup -) { - int query_error = 0; +string aurora_sql_quote(const string& value) { + string quoted { "'" }; + for (char c : value) { + quoted += c; + if (c == '\'') { + quoted += '\''; + } + } + quoted += '\''; + return quoted; +} - if (cleanup) { - string srv_ids {}; - string domain_names {}; +std::pair load_aurora_backend_addresses( + std::unordered_map& addresses) +{ + const char* host_file_path = getenv("CLUSTER_SIM_HOST_FILE"); + if (host_file_path == nullptr || *host_file_path == '\0') { + return { EXIT_FAILURE, "CLUSTER_SIM_HOST_FILE is not configured" }; + } - for (const auto& server : servers) { - srv_ids += "'" + std::get(server) + "'"; - domain_names += "'" + std::get(server) + "'"; + std::ifstream host_file { host_file_path }; + if (!host_file.is_open()) { + return { + EXIT_FAILURE, + "Unable to open CLUSTER_SIM_HOST_FILE '" + string { host_file_path } + "'" + }; + } - if (&server != &servers.back()) { - srv_ids += ","; - domain_names += ","; - } + string line {}; + uint64_t line_number = 0; + while (std::getline(host_file, line)) { + ++line_number; + std::istringstream fields { line }; + string hostname {}; + string ip {}; + if (!(fields >> hostname) || hostname.front() == '#') { + continue; } - - string cleanup_query {}; - - if (cleanup == 1) { - cleanup_query = "DELETE FROM REPLICA_HOST_STATUS WHERE SERVER_ID NOT IN (" + - srv_ids + ") OR DOMAIN_NAME NOT IN (" + domain_names + ")"; - } else { - cleanup_query = "DELETE FROM REPLICA_HOST_STATUS"; + if (!(fields >> ip)) { + return { + EXIT_FAILURE, + "Missing IP in CLUSTER_SIM_HOST_FILE at line " + std::to_string(line_number) + }; } - query_error = mysql_query(proxysql_sqlite, cleanup_query.c_str()); - if (query_error) { - return create_query_error(proxysql_sqlite, cleanup_query, __FILE__, __LINE__); + auto existing = addresses.find(hostname); + if (existing != addresses.end() && existing->second != ip) { + return { + EXIT_FAILURE, + "Conflicting CLUSTER_SIM_HOST_FILE mappings for '" + hostname + "'" + }; } + addresses[hostname] = ip; } - usleep(1000 * 1000); + return { EXIT_SUCCESS, "" }; +} + +} // namespace - // NOTE: We adquire a 'write lock' so there are no dirty reads on ProxySQL side - // while we write the new values. - query_error = mysql_query(proxysql_sqlite, "BEGIN IMMEDIATE"); +std::pair prepare_aurora_cluster_state( + MYSQL* proxysql_sqlite, + const vector& servers, + uint32_t cleanup +) { + std::unordered_map backend_addresses {}; + auto [host_file_rc, host_file_error] = + load_aurora_backend_addresses(backend_addresses); + if (host_file_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, host_file_error }; + } + + std::map> replica_sets {}; + for (const aurora_server_state_t& server : servers) { + replica_sets[std::get(server)].push_back(server); + } + + int query_error = mysql_query(proxysql_sqlite, "BEGIN IMMEDIATE"); if (query_error) { return create_query_error(proxysql_sqlite, "BEGIN IMMEDIATE", __FILE__, __LINE__); } - for (const auto& server : servers) { - string server_insert_query {}; + const auto execute_or_rollback = [proxysql_sqlite](const string& query) { + if (mysql_query(proxysql_sqlite, query.c_str()) == 0) { + return std::pair { EXIT_SUCCESS, "" }; + } + auto error = create_query_error(proxysql_sqlite, query, __FILE__, __LINE__); + (void)mysql_query(proxysql_sqlite, "ROLLBACK"); + return error; + }; - string_format( - t_aurora_server_state_insert, - server_insert_query, - std::get(server).c_str(), - std::get(server).c_str(), - std::get(server).c_str(), - 0, - "", - std::get(server) - ); + if (cleanup) { + auto [control_rc, control_error] = + execute_or_rollback("DELETE FROM AWS_AURORA_REPLICA_CONTROL"); + if (control_rc != EXIT_SUCCESS) return { control_rc, control_error }; + auto [rows_rc, rows_error] = execute_or_rollback("DELETE FROM REPLICA_HOST_STATUS"); + if (rows_rc != EXIT_SUCCESS) return { rows_rc, rows_error }; + } else { + for (const auto& replica_set : replica_sets) { + const string set_literal { aurora_sql_quote(replica_set.first) }; + auto [control_rc, control_error] = execute_or_rollback( + "DELETE FROM AWS_AURORA_REPLICA_CONTROL WHERE replica_set_id=" + set_literal); + if (control_rc != EXIT_SUCCESS) return { control_rc, control_error }; + auto [rows_rc, rows_error] = execute_or_rollback( + "DELETE FROM REPLICA_HOST_STATUS WHERE REPLICA_SET_ID=" + set_literal); + if (rows_rc != EXIT_SUCCESS) return { rows_rc, rows_error }; + } + } - query_error = mysql_query(proxysql_sqlite, server_insert_query.c_str()); - if (query_error) { - return create_query_error(proxysql_sqlite, server_insert_query, __FILE__, __LINE__); + const string timestamp { get_fmt_time() }; + for (const auto& replica_set : replica_sets) { + const string& replica_set_id = replica_set.first; + std::map, bool> mapped_backends {}; + for (const aurora_server_state_t& server : replica_set.second) { + const string& server_id = std::get(server); + string session_id = std::get(server); + if (session_id.empty()) { + session_id = "TESTID-" + server_id + replica_set_id + "-R"; + } + const string hostname { server_id + replica_set_id }; + auto address = backend_addresses.find(hostname); + if (address == backend_addresses.end()) { + (void)mysql_query(proxysql_sqlite, "ROLLBACK"); + return { + EXIT_FAILURE, + "Missing CLUSTER_SIM_HOST_FILE mapping for Aurora member '" + + hostname + "'" + }; + } + + const string row_query { + "INSERT INTO REPLICA_HOST_STATUS" + "(REPLICA_SET_ID,SERVER_ID,SESSION_ID,CPU,LAST_UPDATE_TIMESTAMP," + "REPLICA_LAG_IN_MILLISECONDS,IS_CURRENT) VALUES (" + + aurora_sql_quote(replica_set_id) + "," + aurora_sql_quote(server_id) + + "," + aurora_sql_quote(session_id) + ",0," + + aurora_sql_quote(timestamp) + "," + + std::to_string(std::get(server)) + + ",1)" + }; + auto [row_rc, row_error] = execute_or_rollback(row_query); + if (row_rc != EXIT_SUCCESS) return { row_rc, row_error }; + + mapped_backends[{ address->second, 3306 }] = true; + } + + for (const auto& backend : mapped_backends) { + const string control_query { + "INSERT OR REPLACE INTO AWS_AURORA_REPLICA_CONTROL" + "(backend_ip,backend_port,replica_set_id,replica_table_present,error_code,error_msg) " + "VALUES (" + aurora_sql_quote(backend.first.first) + "," + + std::to_string(backend.first.second) + "," + aurora_sql_quote(replica_set_id) + + ",1,0,'')" + }; + auto [control_rc, control_error] = execute_or_rollback(control_query); + if (control_rc != EXIT_SUCCESS) return { control_rc, control_error }; } } query_error = mysql_query(proxysql_sqlite, "COMMIT"); if (query_error) { + (void)mysql_query(proxysql_sqlite, "ROLLBACK"); return create_query_error(proxysql_sqlite, "COMMIT", __FILE__, __LINE__); } + // Allow an already scheduled monitor probe to observe the committed snapshot. + usleep(1000 * 1000); + return { EXIT_SUCCESS, "" }; } diff --git a/test/deps/cluster_simulator/lib/aurora_utils.h b/test/deps/cluster_simulator/lib/aurora_utils.h index 1c75495abf..f8ba8dd2e3 100644 --- a/test/deps/cluster_simulator/lib/aurora_utils.h +++ b/test/deps/cluster_simulator/lib/aurora_utils.h @@ -61,6 +61,11 @@ std::pair prepare_mysql_aurora_hostgroups( const std::vector& hostgroups_configs ); +/** + * Publishes ordinary Aurora JSON state through backend-address replica sets. + * DOMAIN_NAME identifies the set, and CLUSTER_SIM_HOST_FILE resolves every + * SERVER_ID + DOMAIN_NAME member hostname to its simulated backend address. + */ std::pair prepare_aurora_cluster_state( MYSQL* proxysql_sqlite, const std::vector& servers, @@ -94,4 +99,3 @@ std::vector aurora_update_cluster_state( ); #endif - diff --git a/test/tap/groups/cluster_sim_aurora/add-hosts b/test/tap/groups/cluster_sim_aurora/add-hosts index 77d02bd058..94b8d811c7 100644 --- a/test/tap/groups/cluster_sim_aurora/add-hosts +++ b/test/tap/groups/cluster_sim_aurora/add-hosts @@ -28,6 +28,11 @@ host.2.18.cluster2.aws.test 127.0.2.18 host.2.19.cluster2.aws.test 127.0.2.19 host.2.20.cluster2.aws.test 127.0.2.20 +# Cluster 2 members reused with the first test domain. +host.2.11.aws-test.com 127.0.2.11 +host.2.12.aws-test.com 127.0.2.12 +host.2.13.aws-test.com 127.0.2.13 + # Cluster 3 host.3.11.aws.3.test.com 127.0.3.11 host.3.12.aws.3.test.com 127.0.3.12 @@ -39,3 +44,26 @@ host.3.17.aws.3.test.com 127.0.3.17 host.3.18.aws.3.test.com 127.0.3.18 host.3.19.aws.3.test.com 127.0.3.19 host.3.20.aws.3.test.com 127.0.3.20 + +# Aurora blue/green deployment A: production members and target resources. +aurora-a-writer.a1.us-east-1.rds.amazonaws.com 127.0.11.11 +aurora-a-reader-1.a1.us-east-1.rds.amazonaws.com 127.0.11.12 +aurora-a-reader-2.a1.us-east-1.rds.amazonaws.com 127.0.11.13 +aurora-a-green.cluster-a1.us-east-1.rds.amazonaws.com 127.0.11.20 +aurora-a-green-writer.a1.us-east-1.rds.amazonaws.com 127.0.11.21 +aurora-a-green-reader-1.a1.us-east-1.rds.amazonaws.com 127.0.11.22 +aurora-a-green-reader-2.a1.us-east-1.rds.amazonaws.com 127.0.11.23 + +# Canonical names observed after target-member rename retain target addresses. +aurora-a-canonical-writer.a1.us-east-1.rds.amazonaws.com 127.0.11.21 +aurora-a-canonical-reader-1.a1.us-east-1.rds.amazonaws.com 127.0.11.22 +aurora-a-canonical-reader-2.a1.us-east-1.rds.amazonaws.com 127.0.11.23 + +# Aurora blue/green deployment B supports repeated and concurrent scenarios. +aurora-b-writer.b1.us-east-1.rds.amazonaws.com 127.0.12.11 +aurora-b-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.12 +aurora-b-green.cluster-b1.us-east-1.rds.amazonaws.com 127.0.12.20 +aurora-b-green-writer.b1.us-east-1.rds.amazonaws.com 127.0.12.21 +aurora-b-green-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.22 +aurora-b-canonical-writer.b1.us-east-1.rds.amazonaws.com 127.0.12.21 +aurora-b-canonical-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.22 From 5f6ed806b1b19dc0e615748a2e7f9969d5e27d80 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Fri, 14 Aug 2026 00:10:25 +0000 Subject: [PATCH 20/34] fix: harden Aurora simulator publication --- src/SQLite3_Server.cpp | 40 +- .../cluster_simulator/cluster_simulator.cpp | 2 +- .../cluster_simulator/lib/aurora_utils.cpp | 88 ++- test/tap/tap/Makefile | 6 +- test/tap/tests/Makefile | 6 - test/tap/tests/aurora.cpp | 507 ------------------ 6 files changed, 123 insertions(+), 526 deletions(-) delete mode 100644 test/tap/tests/aurora.cpp diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index c435577530..c88ef45595 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -427,6 +427,9 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p int cols; int affected_rows; bool run_query=true; +#ifdef TEST_AURORA + bool aws_aurora_replica_transaction=false; +#endif SQLite3_result *resultset=NULL; char *strA=NULL; char *strB=NULL; @@ -980,6 +983,18 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p run_query=false; } else { SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; + sqlite3 *db = sqlite_sess->sessdb->get_db(); + if ((*proxy_sqlite3_get_autocommit)(db)==1) { + if (!sqlite_sess->sessdb->execute("BEGIN IMMEDIATE")) { + GloSQLite3Server->send_MySQL_ERR( + &sess->client_myds->myprot, 1105, + "AWS Aurora simulator failed to start the replica probe transaction"); + run_query=false; + } else { + aws_aurora_replica_transaction=true; + } + } + const std::string backend_ip { sess->client_myds->proxy_addr.addr }; const int backend_port = sess->client_myds->proxy_addr.port; const std::string predicate { @@ -994,15 +1009,17 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p "SELECT replica_set_id,replica_table_present,error_code,error_msg " "FROM AWS_AURORA_REPLICA_CONTROL WHERE " + predicate }; - sqlite_sess->sessdb->execute_statement( - control_query.c_str(), &control_error, &control_cols, - &control_affected_rows, &control_result); + if (run_query) { + sqlite_sess->sessdb->execute_statement( + control_query.c_str(), &control_error, &control_cols, + &control_affected_rows, &control_result); + } std::string replica_set_id {}; bool replica_table_present=false; unsigned int configured_error=0; std::string configured_error_msg {}; - if (control_error == nullptr && control_result && + if (run_query && control_error == nullptr && control_result && control_result->rows_count == 1) { SQLite3_row *row=control_result->rows.front(); replica_set_id=row->fields[0] ? row->fields[0] : ""; @@ -1023,7 +1040,9 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p "," + (sess->client_myds->encrypted ? "1" : "0") + ")" }; - if (control_error != nullptr) { + if (!run_query) { + // The transaction-start error was already sent above. + } else if (control_error != nullptr) { GloSQLite3Server->send_MySQL_ERR( &sess->client_myds->myprot, 1105, control_error); free(control_error); @@ -1175,6 +1194,12 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } #endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY || TEST_REPLICATIONLAG || TEST_RDS_BGD if (!run_query) { +#ifdef TEST_AURORA + if (aws_aurora_replica_transaction) { + SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; + sqlite_sess->sessdb->execute("COMMIT"); + } +#endif l_free(pkt->size-sizeof(mysql_hdr),query_no_space); l_free(query_length,query); return; @@ -1188,6 +1213,11 @@ void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *p } } sqlite_sess->sessdb->execute_statement(query, &error , &cols , &affected_rows , &resultset); +#ifdef TEST_AURORA + if (aws_aurora_replica_transaction) { + sqlite_sess->sessdb->execute("COMMIT"); + } +#endif #if defined(TEST_GALERA) || defined(TEST_GROUPREP) if (strncasecmp("SELECT",query_no_space,6)==0) { #ifdef TEST_GALERA diff --git a/test/deps/cluster_simulator/cluster_simulator.cpp b/test/deps/cluster_simulator/cluster_simulator.cpp index cced5822a4..530742fc67 100644 --- a/test/deps/cluster_simulator/cluster_simulator.cpp +++ b/test/deps/cluster_simulator/cluster_simulator.cpp @@ -1719,7 +1719,7 @@ std::pair simulate_aws_aurora_cluster_state( // change the values for the cluster state as a last step prep_aurora_init_state_err = prepare_aurora_cluster_state( - proxysql_sqlite, aurora_init_servers_state, hostgroup_attributes.empty() ? 1 : 2 + proxysql_sqlite, aurora_init_servers_state, 2 ); if (prep_aurora_init_state_err.first) { diff --git a/test/deps/cluster_simulator/lib/aurora_utils.cpp b/test/deps/cluster_simulator/lib/aurora_utils.cpp index 68ea1d4c86..fa63104373 100644 --- a/test/deps/cluster_simulator/lib/aurora_utils.cpp +++ b/test/deps/cluster_simulator/lib/aurora_utils.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -372,6 +373,30 @@ string aurora_sql_quote(const string& value) { return quoted; } +string aurora_utc_timestamp() { + time_t now = time(nullptr); + struct tm utc_time {}; + gmtime_r(&now, &utc_time); + char timestamp[20] {}; + strftime(timestamp, sizeof(timestamp), "%Y-%m-%d %H:%M:%S", &utc_time); + return timestamp; +} + +std::pair aurora_scalar_uint64(MYSQL* connection, const string& query) { + if (mysql_query(connection, query.c_str()) != 0) { + return { EXIT_FAILURE, 0 }; + } + MYSQL_RES* result = mysql_store_result(connection); + if (result == nullptr) { + return { EXIT_FAILURE, 0 }; + } + MYSQL_ROW row = mysql_fetch_row(result); + const bool valid = row != nullptr && row[0] != nullptr; + const uint64_t value = valid ? strtoull(row[0], nullptr, 10) : 0; + mysql_free_result(result); + return { valid ? EXIT_SUCCESS : EXIT_FAILURE, value }; +} + std::pair load_aurora_backend_addresses( std::unordered_map& addresses) { @@ -451,12 +476,44 @@ std::pair prepare_aurora_cluster_state( return error; }; - if (cleanup) { + string replica_set_list {}; + for (const auto& replica_set : replica_sets) { + if (!replica_set_list.empty()) replica_set_list += ","; + replica_set_list += aurora_sql_quote(replica_set.first); + } + + auto [checkpoint_rc, probe_checkpoint] = aurora_scalar_uint64( + proxysql_sqlite, + "SELECT COALESCE(MAX(sequence_id),0) FROM AWS_AURORA_REPLICA_PROBE_LOG"); + if (checkpoint_rc != EXIT_SUCCESS) { + (void)mysql_query(proxysql_sqlite, "ROLLBACK"); + return { EXIT_FAILURE, "Unable to read the Aurora replica probe checkpoint" }; + } + + if (cleanup > 1) { auto [control_rc, control_error] = execute_or_rollback("DELETE FROM AWS_AURORA_REPLICA_CONTROL"); if (control_rc != EXIT_SUCCESS) return { control_rc, control_error }; auto [rows_rc, rows_error] = execute_or_rollback("DELETE FROM REPLICA_HOST_STATUS"); if (rows_rc != EXIT_SUCCESS) return { rows_rc, rows_error }; + } else if (cleanup == 1) { + const string delete_controls { + replica_set_list.empty() + ? "DELETE FROM AWS_AURORA_REPLICA_CONTROL" + : "DELETE FROM AWS_AURORA_REPLICA_CONTROL WHERE replica_set_id NOT IN (" + + replica_set_list + ")" + }; + auto [control_rc, control_error] = execute_or_rollback(delete_controls); + if (control_rc != EXIT_SUCCESS) return { control_rc, control_error }; + if (!replica_set_list.empty()) { + auto [reset_rc, reset_error] = execute_or_rollback( + "UPDATE AWS_AURORA_REPLICA_CONTROL SET replica_table_present=1," + "error_code=0,error_msg='' WHERE replica_set_id IN (" + + replica_set_list + ")"); + if (reset_rc != EXIT_SUCCESS) return { reset_rc, reset_error }; + } + auto [rows_rc, rows_error] = execute_or_rollback("DELETE FROM REPLICA_HOST_STATUS"); + if (rows_rc != EXIT_SUCCESS) return { rows_rc, rows_error }; } else { for (const auto& replica_set : replica_sets) { const string set_literal { aurora_sql_quote(replica_set.first) }; @@ -469,7 +526,7 @@ std::pair prepare_aurora_cluster_state( } } - const string timestamp { get_fmt_time() }; + const string timestamp { aurora_utc_timestamp() }; for (const auto& replica_set : replica_sets) { const string& replica_set_id = replica_set.first; std::map, bool> mapped_backends {}; @@ -525,8 +582,31 @@ std::pair prepare_aurora_cluster_state( return create_query_error(proxysql_sqlite, "COMMIT", __FILE__, __LINE__); } - // Allow an already scheduled monitor probe to observe the committed snapshot. - usleep(1000 * 1000); + if (cleanup == 1 && !replica_sets.empty()) { + const uint64_t deadline = monotonic_time() + 10000000; + const string observed_sets_query { + "SELECT COUNT(DISTINCT replica_set_id) FROM AWS_AURORA_REPLICA_PROBE_LOG " + "WHERE sequence_id>" + std::to_string(probe_checkpoint) + + " AND probe_kind='ordinary' AND replica_set_id IN (" + + replica_set_list + ")" + }; + do { + auto [observed_rc, observed_sets] = + aurora_scalar_uint64(proxysql_sqlite, observed_sets_query); + if (observed_rc != EXIT_SUCCESS) { + return { EXIT_FAILURE, "Unable to read the Aurora replica probe log" }; + } + if (observed_sets == replica_sets.size()) { + return { EXIT_SUCCESS, "" }; + } + usleep(50000); + } while (monotonic_time() < deadline); + + return { + EXIT_FAILURE, + "Timed out waiting for every Aurora replica set to be probed" + }; + } return { EXIT_SUCCESS, "" }; } diff --git a/test/tap/tap/Makefile b/test/tap/tap/Makefile index 4ceb4b1ca5..568de7a136 100644 --- a/test/tap/tap/Makefile +++ b/test/tap/tap/Makefile @@ -83,13 +83,13 @@ mcp_client.o: mcp_client.cpp mcp_client.h libcurl$(SHLIB_EXT) cluster_simulator.o: cluster_simulator.cpp cluster_simulator.h $(CXX) -fPIC -c cluster_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) -bgd_simulator.o: bgd_simulator.cpp bgd_simulator.h cluster_simulator.h utils.h +bgd_simulator.o: bgd_simulator.cpp bgd_simulator.h cluster_simulator.h utils.h tap.h command_line.h $(CXX) -fPIC -c bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) -aurora_bgd_simulator.o: aurora_bgd_simulator.cpp aurora_bgd_simulator.h bgd_simulator.h +aurora_bgd_simulator.o: aurora_bgd_simulator.cpp aurora_bgd_simulator.h bgd_simulator.h cluster_simulator.h utils.h command_line.h $(CXX) -fPIC -c aurora_bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) -rds_bgd_simulator.o: rds_bgd_simulator.cpp rds_bgd_simulator.h bgd_simulator.h +rds_bgd_simulator.o: rds_bgd_simulator.cpp rds_bgd_simulator.h bgd_simulator.h cluster_simulator.h utils.h command_line.h $(CXX) -fPIC -c rds_bgd_simulator.cpp $(IDIRS) -I$(MARIADB_IDIR) $(OPT) libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils_mariadb.o mcp_client.o cluster_simulator.o bgd_simulator.o aurora_bgd_simulator.o rds_bgd_simulator.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a diff --git a/test/tap/tests/Makefile b/test/tap/tests/Makefile index fedf75dea6..48912cd41c 100644 --- a/test/tap/tests/Makefile +++ b/test/tap/tests/Makefile @@ -227,9 +227,6 @@ tests-php: $(patsubst %,php-%,$(wildcard *-t.php)) tests-py: $(patsubst %,py-%,$(wildcard *-t.py)) tests-sh: $(patsubst %,sh-%,$(wildcard *-t.sh)) testgalera: galera_1_timeout_count galera_2_timeout_no_count -testaurora: aurora - - ### test deps targets #build_test_deps: $(TAP_LDIR)/libtap$(SHLIB_EXT) $(TEST_MARIADB_LDIR)/libmariadbclient.a $(TEST_MYSQL_LDIR)/libmysqlclient.a @@ -275,9 +272,6 @@ galera_2_timeout_no_count: galera_2_timeout_no_count.cpp $(TAP_LDIR)/libtap$(SHL generate_set_session_csv: generate_set_session_csv.cpp $(CXX) $< $(OPT) -o $@ -aurora: aurora.cpp $(TAP_LDIR)/libtap$(SHLIB_EXT) - $(CXX) -DTEST_AURORA $< ../tap/SQLite3_Server.cpp $(IDIRS) $(LDIRS) $(OPT) $(OBJ) $(MYLIBSJEMALLOC) $(MYLIBS) $(STATIC_LIBS) -o $@ - test_tokenizer-t: test_tokenizer-t.cpp $(TAP_LDIR)/libtap$(SHLIB_EXT) $(CXX) $< $(IDIRS) $(LDIRS) $(OPT) $(MYLIBS) -o $@ diff --git a/test/tap/tests/aurora.cpp b/test/tap/tests/aurora.cpp deleted file mode 100644 index 97a7b0dd0d..0000000000 --- a/test/tap/tests/aurora.cpp +++ /dev/null @@ -1,507 +0,0 @@ -#include // std::cout -#include // std::sort -#include // std::vector -#include "re2/re2.h" -#include "re2/regexp.h" -#include "proxysql.h" -#include "cpp.h" - -#include "MySQL_Logger.hpp" -#include "MySQL_Data_Stream.h" -#include "query_processor.h" -#include "SQLite3_Server.h" -#include "proxysql_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "tap.h" - -#define SELECT_VERSION_COMMENT "select @@version_comment limit 1" -#define SELECT_VERSION_COMMENT_LEN 32 -#define SELECT_DB_USER "select DATABASE(), USER() limit 1" -#define SELECT_DB_USER_LEN 33 -#define SELECT_CHARSET_VARIOUS "select @@character_set_client, @@character_set_connection, @@character_set_server, @@character_set_database limit 1" -#define SELECT_CHARSET_VARIOUS_LEN 115 - -#define READ_ONLY_OFF "\x01\x00\x00\x01\x02\x23\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x0d\x56\x61\x72\x69\x61\x62\x6c\x65\x5f\x6e\x61\x6d\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x1b\x00\x00\x03\x03\x64\x65\x66\x00\x00\x00\x05\x56\x61\x6c\x75\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x05\x00\x00\x04\xfe\x00\x00\x02\x00\x0e\x00\x00\x05\x09\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x03\x4f\x46\x46\x05\x00\x00\x06\xfe\x00\x00\x02\x00" -#define READ_ONLY_ON "\x01\x00\x00\x01\x02\x23\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x0d\x56\x61\x72\x69\x61\x62\x6c\x65\x5f\x6e\x61\x6d\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x1b\x00\x00\x03\x03\x64\x65\x66\x00\x00\x00\x05\x56\x61\x6c\x75\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x05\x00\x00\x04\xfe\x00\x00\x02\x00\x0d\x00\x00\x05\x09\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x02\x4f\x4e\x05\x00\x00\x06\xfe\x00\x00\x02\x00" - -extern SQLite3_Server *GloSQLite3Server; - -static int random_replication_lag_seconds() { - static thread_local std::random_device random_source; - static thread_local std::uniform_int_distribution distribution(10, 39); - return distribution(random_source); -} - -void SQLite3_Server::init_aurora_ifaces_string(std::string& s) { - if(!s.empty()) - s += ";"; - pthread_mutex_init(&aurora_mutex,NULL); - unsigned int nas = time(NULL); - nas = nas % 3; // range - nas += 4; // min - max_num_aurora_servers = 10; // hypothetical maximum number of nodes - for (unsigned int j=1; j<4; j++) { - cur_aurora_writer[j-1] = 0; - num_aurora_servers[j-1] = nas; - for (unsigned int i=11; iexecute("DELETE FROM REPLICA_HOST_STATUS"); - sqlite3_stmt *statement=NULL; - //sqlite3 *mydb3=sessdb->get_db(); - int rc; - char *query=(char *)"INSERT INTO REPLICA_HOST_STATUS VALUES (?1, ?2, ?3, ?4, ?5)"; - //rc=sqlite3_prepare_v2(mydb3, query, -1, &statement, 0); - auto prepared_statement = sessdb->prepare_v2(query); - rc = prepared_statement.first; - stmt_unique_ptr statement_unique = std::move(prepared_statement.second); - statement = statement_unique.get(); - ASSERT_SQLITE_OK(rc, sessdb); - time_t __timer; - char lut[30]; - struct tm __tm_info; - time(&__timer); - localtime_r(&__timer, &__tm_info); - strftime(lut, 25, "%Y-%m-%d %H:%M:%S", &__tm_info); - string myip = string(sess->client_myds->proxy_addr.addr); - string clu_id_s = myip.substr(6,1); - unsigned int cluster_id = atoi(clu_id_s.c_str()); - cluster_id--; - //if (rand() % 200 == 0) { - if (rand() % 20000 == 0) { - // simulate a failover - cur_aurora_writer[cluster_id] = rand() % num_aurora_servers[cluster_id]; - proxy_info("Simulating a failover for AWS Aurora cluster %d , HGs (%d:%d)\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2); - } - if (rand() % 1000 == 0) { - if (num_aurora_servers[cluster_id] < max_num_aurora_servers) { - num_aurora_servers[cluster_id]++; - proxy_info("Simulating the add of a new server for AWS Aurora Cluster %d , HGs (%d:%d). Now adding server num %d\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2, num_aurora_servers[cluster_id]); - } - } - if (rand() % 1000 == 0) { - if (num_aurora_servers[cluster_id] > 1) { - if (cur_aurora_writer[cluster_id] != (num_aurora_servers[cluster_id] - 1) ) { - num_aurora_servers[cluster_id]--; - proxy_info("Simulating the deletion of a server from AWS Aurora Cluster %d , HGs (%d:%d). Removing server num %d\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2, num_aurora_servers[cluster_id]+1); - } - } - } - for (unsigned int i=0; isize-sizeof(mysql_hdr); - query=(char *)l_alloc(query_length); - memcpy(query,(char *)pkt->ptr+sizeof(mysql_hdr)+1,query_length-1); - query[query_length-1]=0; - - if (sess->client_myds->proxy_addr.addr == NULL) { - struct sockaddr addr; - socklen_t addr_len=sizeof(struct sockaddr); - memset(&addr,0,addr_len); - int rc; - rc=getsockname(sess->client_myds->fd, &addr, &addr_len); - if (rc==0) { - char buf[512]; - switch (addr.sa_family) { - case AF_INET: { - struct sockaddr_in *ipv4 = (struct sockaddr_in *)&addr; - inet_ntop(addr.sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); - sess->client_myds->proxy_addr.addr = strdup(buf); - } - break; - case AF_INET6: { - struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)&addr; - inet_ntop(addr.sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); - sess->client_myds->proxy_addr.addr = strdup(buf); - } - break; - default: - sess->client_myds->proxy_addr.addr = strdup("unknown"); - break; - } - } else { - sess->client_myds->proxy_addr.addr = strdup("unknown"); - } - } - - char *query_no_space=(char *)l_alloc(query_length); - memcpy(query_no_space,query,query_length); - - unsigned int query_no_space_length=remove_spaces(query_no_space); - - // fix bug #925 - while (query_no_space[query_no_space_length-1]==';' || query_no_space[query_no_space_length-1]==' ') { - query_no_space_length--; - query_no_space[query_no_space_length]=0; - } - - // fix bug #1047 - if ( -/* - (!strncasecmp("BEGIN", query_no_space, strlen("BEGIN"))) - || - (!strncasecmp("START TRANSACTION", query_no_space, strlen("START TRANSACTION"))) - || - (!strncasecmp("COMMIT", query_no_space, strlen("COMMIT"))) - || - (!strncasecmp("ROLLBACK", query_no_space, strlen("ROLLBACK"))) - || -*/ - (!strncasecmp("SET character_set_results", query_no_space, strlen("SET character_set_results"))) - || - (!strncasecmp("SET SQL_AUTO_IS_NULL", query_no_space, strlen("SET SQL_AUTO_IS_NULL"))) - || - (!strncasecmp("SET NAMES", query_no_space, strlen("SET NAMES"))) - || - (!strncasecmp("SET AUTOCOMMIT", query_no_space, strlen("SET AUTOCOMMIT"))) - || - (!strncasecmp("/*!40100 SET @@SQL_MODE='' */", query_no_space, strlen("/*!40100 SET @@SQL_MODE='' */"))) - || - (!strncasecmp("/*!40103 SET TIME_ZONE=", query_no_space, strlen("/*!40103 SET TIME_ZONE="))) - || - (!strncasecmp("/*!80000 SET SESSION", query_no_space, strlen("/*!80000 SET SESSION"))) - || - (!strncasecmp("SET SESSION", query_no_space, strlen("SET SESSION"))) - || - (!strncasecmp("SET wait_timeout", query_no_space, strlen("SET wait_timeout"))) - ) { - SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; - sqlite3 *db = sqlite_sess->sessdb->get_db(); - uint16_t status=2; // autocommit - if (sqlite3_get_autocommit(db)==0) { - status = 3; // autocommit + transaction - } - GloSQLite3Server->send_MySQL_OK(&sess->client_myds->myprot, NULL, 0, status); - run_query=false; - goto __run_query; - } - - if (query_no_space_length==17) { - if (!strncasecmp((char *)"START TRANSACTION", query_no_space, query_no_space_length)) { - l_free(query_length,query); - query = l_strdup((char *)"BEGIN IMMEDIATE"); - query_length=strlen(query)+1; - goto __run_query; - } - } - - if (query_no_space_length==5) { - if (!strncasecmp((char *)"BEGIN", query_no_space, query_no_space_length)) { - l_free(query_length,query); - query = l_strdup((char *)"BEGIN IMMEDIATE"); - query_length=strlen(query)+1; - goto __run_query; - } - } - - if (query_no_space_length==SELECT_VERSION_COMMENT_LEN) { - if (!strncasecmp(SELECT_VERSION_COMMENT, query_no_space, query_no_space_length)) { - l_free(query_length,query); - const char* proxy_addr = sess->client_myds->proxy_addr.addr; - const std::string query_text = - std::string("SELECT '(ProxySQL Automated Test Server) - ") - + (proxy_addr ? proxy_addr : "") + "'"; - query = l_strdup(query_text.c_str()); - if (!query) { - l_free(pkt->size-sizeof(mysql_hdr), query_no_space); - return; - } - query_length = query_text.size() + 1; - goto __run_query; - } - } - - if (query_no_space_length==SELECT_DB_USER_LEN) { - if (!strncasecmp(SELECT_DB_USER, query_no_space, query_no_space_length)) { - l_free(query_length,query); - const char* username = sess->client_myds->myconn->userinfo->username; - const std::string query_text = - std::string("SELECT \"admin\" AS 'DATABASE()', \"") - + (username ? username : "") + "\" AS 'USER()'"; - query = l_strdup(query_text.c_str()); - if (!query) { - l_free(pkt->size-sizeof(mysql_hdr), query_no_space); - return; - } - query_length = query_text.size() + 1; - goto __run_query; - } - } - - if (query_no_space_length==SELECT_CHARSET_VARIOUS_LEN) { - if (!strncasecmp(SELECT_CHARSET_VARIOUS, query_no_space, query_no_space_length)) { - l_free(query_length,query); - char *query1=(char *)"select 'utf8' as '@@character_set_client', 'utf8' as '@@character_set_connection', 'utf8' as '@@character_set_server', 'utf8' as '@@character_set_database' limit 1"; - query=l_strdup(query1); - query_length=strlen(query1)+1; - goto __run_query; - } - } - - if (!strncasecmp("SELECT @@version", query_no_space, strlen("SELECT @@version"))) { - l_free(query_length,query); - char *q=(char *)"SELECT '%s' AS '@@version'"; - query_length=strlen(q)+20; - query=(char *)l_alloc(query_length); - snprintf(query, query_length, q, PROXYSQL_VERSION); - goto __run_query; - } - - if (!strncasecmp("SELECT version()", query_no_space, strlen("SELECT version()"))) { - l_free(query_length,query); - char *q=(char *)"SELECT '%s' AS 'version()'"; - query_length=strlen(q)+20; - query=(char *)l_alloc(query_length); - snprintf(query, query_length, q, PROXYSQL_VERSION); - goto __run_query; - } - - if (strncasecmp("SHOW ", query_no_space, 5)) { - goto __end_show_commands; // in the next block there are only SHOW commands - } - - if (query_no_space_length==strlen("SHOW TABLES") && !strncasecmp("SHOW TABLES",query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT name AS tables FROM sqlite_master WHERE type='table' AND name NOT IN ('sqlite_sequence') ORDER BY name"); - query_length=strlen(query)+1; - goto __run_query; - } - - if ((query_no_space_length>17) && (!strncasecmp("SHOW TABLES FROM ", query_no_space, 17))) { - strA=query_no_space+17; - strAl=strlen(strA); - strB=(char *)"SELECT name AS tables FROM %s.sqlite_master WHERE type='table' AND name NOT IN ('sqlite_sequence') ORDER BY name"; - strBl=strlen(strB); - int l=strBl+strAl-2; - char *b=(char *)l_alloc(l+1); - snprintf(b,l+1,strB,strA); - b[l]=0; - l_free(query_length,query); - query=b; - query_length=l+1; - goto __run_query; - } - - if ((query_no_space_length>17) && (!strncasecmp("SHOW TABLES LIKE ", query_no_space, 17))) { - strA=query_no_space+17; - strAl=strlen(strA); - strB=(char *)"SELECT name AS tables FROM sqlite_master WHERE type='table' AND name LIKE '%s'"; - strBl=strlen(strB); - char *tn=NULL; // tablename - tn=(char *)malloc(strlen(strA)); - unsigned int i=0, j=0; - while (i=3 && tbh[0]=='`' && tbh[tbh_len-1]=='`') { // tablename is quoted - size_t db_len = tbh_len - 2; - const std::string unquoted_table(tbh + 1, db_len); - free(tbh); - tbh = l_strdup(unquoted_table.c_str()); - if (!tbh) { - free(dbh); - l_free(query_length, query); - return; - } - } - const std::string table_query = - std::string("SELECT name AS 'table' , REPLACE(REPLACE(sql,' , ', X'2C0A20202020'),") - + "'CREATE TABLE " + tbh + " (','CREATE TABLE " + tbh - + " ('||X'0A20202020') AS 'Create Table' FROM " + dbh - + ".sqlite_master WHERE type='table' AND name='" + tbh + "'"; - char *buff = l_strdup(table_query.c_str()); - if (!buff) { - l_free(0, tbh); - l_free(0, dbh); - l_free(query_length, query); - return; - } - l_free(0, tbh); - l_free(0, dbh); - l_free(query_length,query); - query=buff; - query_length=table_query.size()+1; - goto __run_query; - } - - if ( - (query_no_space_length==strlen("SHOW DATABASES") && !strncasecmp("SHOW DATABASES",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SHOW SCHEMAS") && !strncasecmp("SHOW SCHEMAS",query_no_space, query_no_space_length)) - ) { - l_free(query_length,query); - query=l_strdup("PRAGMA DATABASE_LIST"); - query_length=strlen(query)+1; - goto __run_query; - } - -__end_show_commands: - - if (query_no_space_length==strlen("SELECT DATABASE()") && !strncasecmp("SELECT DATABASE()",query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT \"main\" AS 'DATABASE()'"); - query_length=strlen(query)+1; - goto __run_query; - } - - // see issue #1022 - if (query_no_space_length==strlen("SELECT DATABASE() AS name") && !strncasecmp("SELECT DATABASE() AS name",query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT \"main\" AS 'DATABASE()'"); - query_length=strlen(query)+1; - goto __run_query; - } - - if (sess->session_type == PROXYSQL_SESSION_SQLITE) { // no admin - if ( - (strncasecmp("PRAGMA",query_no_space,6)==0) - || - (strncasecmp("ATTACH",query_no_space,6)==0) - ) { - proxy_error("[WARNING]: Commands executed from stats interface in Admin Module: \"%s\"\n", query_no_space); - GloSQLite3Server->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Command not allowed"); - run_query=false; - } - } - -__run_query: - if (run_query) { - if (strncasecmp("SELECT",query_no_space,6)==0) { - if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { - pthread_mutex_lock(&GloSQLite3Server->aurora_mutex); - GloSQLite3Server->populate_aws_aurora_table(sess); - } - if (strstr(query_no_space,(char *)"Seconds_Behind_Master")) { - l_free(0, query); - const std::string formatted_query = cstr_format( - "SELECT %d as Seconds_Behind_Master", random_replication_lag_seconds() - ).str; - query = l_strdup(formatted_query.c_str()); - } - } - SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; - sqlite_sess->sessdb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - if (strncasecmp("SELECT",query_no_space,6)==0) { - if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { - pthread_mutex_unlock(&GloSQLite3Server->aurora_mutex); - if (rand() % 100 == 0) { - // randomly add some latency on 1% of the traffic - sleep(2); - } - } - if (strstr(query_no_space,(char *)"Seconds_Behind_Master")) { - if (rand() % 10 == 0) { - // randomly add some latency on 10% of the traffic - sleep(2); - } - } - } - sqlite3 *db = sqlite_sess->sessdb->get_db(); - bool in_trans = false; - if (sqlite3_get_autocommit(db)==0) { - in_trans = true; - } - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot, in_trans); - delete resultset; - } - l_free(pkt->size-sizeof(mysql_hdr),query_no_space); // it is always freed here - l_free(query_length,query); -} From 25716bb041a7a2a6641d6874ea80900cd9a106ff Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Fri, 14 Aug 2026 06:31:18 +0000 Subject: [PATCH 21/34] docs: clarify Aurora BGD status publisher contract --- include/MySQL_HostGroups_Manager.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index a340e6fc10..2dbc0e199c 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -1127,8 +1127,9 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { /** * @brief Publish the node-local Aurora BGD state for one runtime row. * - * Invalid status strings and writer hostgroups not present at runtime are - * ignored. Configuration reloads do not write this column. + * The caller owns validation of the state vocabulary. Writer hostgroups not + * present at runtime are ignored. Configuration reloads do not write this + * column. */ void update_aws_aurora_bgd_status(int writer_hostgroup, const std::string& bgd_status); /** From 49c1c2995f58c4c3381b39bcd5d0e9e98c5841ec Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Fri, 14 Aug 2026 08:57:33 +0000 Subject: [PATCH 22/34] refactor: clarify Aurora simulator publication modes --- ...ra-bgd-cluster-simulator-testing-design.md | 6 +- .../cluster_simulator/cluster_simulator.cpp | 15 ++-- .../cluster_simulator/lib/aurora_utils.cpp | 89 ++++--------------- .../deps/cluster_simulator/lib/aurora_utils.h | 13 +-- test/tap/groups/cluster_sim_aurora/add-hosts | 2 +- 5 files changed, 36 insertions(+), 89 deletions(-) diff --git a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md index 95e01ddba6..5acaa5f887 100644 --- a/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md +++ b/docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md @@ -205,13 +205,17 @@ Every intercepted production `REPLICA_HOST_STATUS` query records: sequence_id INTEGER PRIMARY KEY AUTOINCREMENT backend_ip TEXT NOT NULL backend_port INTEGER NOT NULL +probe_kind TEXT NOT NULL + CHECK (probe_kind IN ('ordinary', 'bgd_membership')) replica_set_id TEXT NULL encrypted INTEGER NOT NULL CHECK (encrypted IN (0, 1)) ``` The accepted address is logged even when no control mapping exists or the query -returns an error. `replica_set_id` is NULL when the backend has no mapping. +returns an error. `probe_kind` distinguishes the ordinary production probe from +the Aurora BGD target-membership probe. `replica_set_id` is NULL when the +backend has no mapping. ### 4.4 Query handling diff --git a/test/deps/cluster_simulator/cluster_simulator.cpp b/test/deps/cluster_simulator/cluster_simulator.cpp index 530742fc67..dc23952a0a 100644 --- a/test/deps/cluster_simulator/cluster_simulator.cpp +++ b/test/deps/cluster_simulator/cluster_simulator.cpp @@ -1719,7 +1719,9 @@ std::pair simulate_aws_aurora_cluster_state( // change the values for the cluster state as a last step prep_aurora_init_state_err = prepare_aurora_cluster_state( - proxysql_sqlite, aurora_init_servers_state, 2 + proxysql_sqlite, + aurora_init_servers_state, + aurora_publication_mode::reset_scenario ); if (prep_aurora_init_state_err.first) { @@ -1794,14 +1796,13 @@ std::pair simulate_aws_aurora_cluster_state( aurora_servers_state_diff(aurora_init_servers_state, aurora_new_servers_state) }; - // set the new servers state - const auto& aurora_new_state_to_set { - aurora_update_cluster_state(aurora_init_servers_state, aurora_new_servers_state) - }; // Replace the published replica sets and their complete backend mappings so // payloads can simulate members being removed (for example, autopurge coverage). - prep_aurora_final_state_err = - prepare_aurora_cluster_state(proxysql_sqlite, aurora_new_servers_state, 1); + prep_aurora_final_state_err = prepare_aurora_cluster_state( + proxysql_sqlite, + aurora_new_servers_state, + aurora_publication_mode::replace_snapshot_retaining_backends + ); if (prep_aurora_final_state_err.first) { result = internal_error(prep_aurora_final_state_err.second, __FILE__, __LINE__); diff --git a/test/deps/cluster_simulator/lib/aurora_utils.cpp b/test/deps/cluster_simulator/lib/aurora_utils.cpp index fa63104373..fb7b1ef9ec 100644 --- a/test/deps/cluster_simulator/lib/aurora_utils.cpp +++ b/test/deps/cluster_simulator/lib/aurora_utils.cpp @@ -448,7 +448,7 @@ std::pair load_aurora_backend_addresses( std::pair prepare_aurora_cluster_state( MYSQL* proxysql_sqlite, const vector& servers, - uint32_t cleanup + aurora_publication_mode mode ) { std::unordered_map backend_addresses {}; auto [host_file_rc, host_file_error] = @@ -482,21 +482,25 @@ std::pair prepare_aurora_cluster_state( replica_set_list += aurora_sql_quote(replica_set.first); } - auto [checkpoint_rc, probe_checkpoint] = aurora_scalar_uint64( - proxysql_sqlite, - "SELECT COALESCE(MAX(sequence_id),0) FROM AWS_AURORA_REPLICA_PROBE_LOG"); - if (checkpoint_rc != EXIT_SUCCESS) { - (void)mysql_query(proxysql_sqlite, "ROLLBACK"); - return { EXIT_FAILURE, "Unable to read the Aurora replica probe checkpoint" }; + uint64_t probe_checkpoint = 0; + if (mode == aurora_publication_mode::replace_snapshot_retaining_backends) { + auto [checkpoint_rc, checkpoint] = aurora_scalar_uint64( + proxysql_sqlite, + "SELECT COALESCE(MAX(sequence_id),0) FROM AWS_AURORA_REPLICA_PROBE_LOG"); + if (checkpoint_rc != EXIT_SUCCESS) { + (void)mysql_query(proxysql_sqlite, "ROLLBACK"); + return { EXIT_FAILURE, "Unable to read the Aurora replica probe checkpoint" }; + } + probe_checkpoint = checkpoint; } - if (cleanup > 1) { + if (mode == aurora_publication_mode::reset_scenario) { auto [control_rc, control_error] = execute_or_rollback("DELETE FROM AWS_AURORA_REPLICA_CONTROL"); if (control_rc != EXIT_SUCCESS) return { control_rc, control_error }; auto [rows_rc, rows_error] = execute_or_rollback("DELETE FROM REPLICA_HOST_STATUS"); if (rows_rc != EXIT_SUCCESS) return { rows_rc, rows_error }; - } else if (cleanup == 1) { + } else if (mode == aurora_publication_mode::replace_snapshot_retaining_backends) { const string delete_controls { replica_set_list.empty() ? "DELETE FROM AWS_AURORA_REPLICA_CONTROL" @@ -582,7 +586,8 @@ std::pair prepare_aurora_cluster_state( return create_query_error(proxysql_sqlite, "COMMIT", __FILE__, __LINE__); } - if (cleanup == 1 && !replica_sets.empty()) { + if (mode == aurora_publication_mode::replace_snapshot_retaining_backends && + !replica_sets.empty()) { const uint64_t deadline = monotonic_time() + 10000000; const string observed_sets_query { "SELECT COUNT(DISTINCT replica_set_id) FROM AWS_AURORA_REPLICA_PROBE_LOG " @@ -719,67 +724,3 @@ cluster_state_changes aurora_servers_state_diff( return result; } - -aurora_server_state_t aurora_update_state( - const aurora_server_state_t& st1, - const aurora_server_state_t& st2 -) { - aurora_server_state_t result {}; - - // SERVER_ID and DOMAIN_NAME **can't** be changed, because the are part of the server 'id'. Only the other - // fields are allowed to change, otherwise, the verification step should have failed. - - const string st1_session_id { std::get(st1) }; - const string st2_session_id { std::get(st2) }; - - int32_t st1_read_only { std::get(st1) }; - int32_t st2_read_only { std::get(st2) }; - - // Since empty 'SESSION_IDs' have no meaning, we ignore them for updated states - if (st2_session_id != "" && st1_session_id != st2_session_id) { - std::get<2>(result) = st2_session_id; - } - - if (st2_read_only != -1 && st1_read_only != st2_read_only) { - std::get<3>(result) = st2_read_only; - } - - return result; -} - -vector aurora_update_cluster_state( - const vector& servers_state_p, - const vector& servers_state_n -) { - vector result {}; - - vector s_servers_state_p { sort_aurora_server_state(servers_state_p) }; - vector s_servers_state_n { sort_aurora_server_state(servers_state_n) }; - - // find the differences - for (const auto& server_state_n : s_servers_state_n) { - for (const auto& server_state_p : s_servers_state_p) { - const string n_server_state_id { - std::get(server_state_n) + ":" + - std::get(server_state_n) - }; - const string p_server_state_id { - std::get(server_state_p) + ":" + - std::get(server_state_p) - }; - - bool diff_server_status = - ( n_server_state_id == p_server_state_id ) && - ( server_state_n != server_state_p ); - - if (diff_server_status) { - const aurora_server_state_t server_state_update { - aurora_update_state(server_state_p, server_state_n) - }; - result.push_back(server_state_update); - } - } - } - - return result; -} diff --git a/test/deps/cluster_simulator/lib/aurora_utils.h b/test/deps/cluster_simulator/lib/aurora_utils.h index f8ba8dd2e3..5006d50ccd 100644 --- a/test/deps/cluster_simulator/lib/aurora_utils.h +++ b/test/deps/cluster_simulator/lib/aurora_utils.h @@ -50,6 +50,12 @@ enum class aurora_state_id { new_state = 1 }; +enum class aurora_publication_mode { + replace_sets, + replace_snapshot_retaining_backends, + reset_scenario +}; + std::pair extract_aurora_servers_state( const aurora_state_id& state_id, const json& aurora_test_def, @@ -69,7 +75,7 @@ std::pair prepare_mysql_aurora_hostgroups( std::pair prepare_aurora_cluster_state( MYSQL* proxysql_sqlite, const std::vector& servers, - uint32_t cleanup = 0 + aurora_publication_mode mode = aurora_publication_mode::replace_sets ); /** @@ -93,9 +99,4 @@ cluster_state_changes aurora_servers_state_diff( const std::vector& servers_state_n ); -std::vector aurora_update_cluster_state( - const std::vector& servers_state_p, - const std::vector& servers_state_n -); - #endif diff --git a/test/tap/groups/cluster_sim_aurora/add-hosts b/test/tap/groups/cluster_sim_aurora/add-hosts index 94b8d811c7..128b632715 100644 --- a/test/tap/groups/cluster_sim_aurora/add-hosts +++ b/test/tap/groups/cluster_sim_aurora/add-hosts @@ -2,7 +2,7 @@ # Format: " " per line; '#' comments allowed. # These are injected into the ProxySQL container's /etc/hosts via Docker # --add-host by test/infra/control/start-proxysql-isolated.bash when -# CLUSTER_SIM_HOST_FILE points at this file (see test/tap/groups/aurora-sim/env.sh). +# CLUSTER_SIM_HOST_FILE points at this file (see test/tap/groups/cluster_sim_aurora/env.sh). # Cluster 1 host.1.11.aws-test.com 127.0.1.11 From e437b3b1ac3f42f00f9604001c55fb21f10b4a81 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sat, 15 Aug 2026 17:51:17 +0000 Subject: [PATCH 23/34] feat: add Aurora BGD discovery and membership tracking --- include/MySQL_Monitor.hpp | 94 ++++ lib/MySQL_Monitor.cpp | 522 ++++++++++++++++++ test/tap/groups/cluster_sim_aurora/add-hosts | 5 + test/tap/groups/groups.json | 1 + test/tap/tap/aurora_bgd_tap.h | 290 ++++++++++ .../tap/tests/test_aurora_bgd_discovery-t.cpp | 278 ++++++++++ 6 files changed, 1190 insertions(+) create mode 100644 test/tap/tap/aurora_bgd_tap.h create mode 100644 test/tap/tests/test_aurora_bgd_discovery-t.cpp diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 38c379d825..163d05106f 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -564,6 +564,85 @@ struct AWS_RDS_BGD_Probe_Host { int use_ssl = 0; }; +/** + * @brief Aurora blue/green deployment phase published by the Aurora worker. + */ +enum class AWS_Aurora_BGD_Status { + NONE = 0, + AVAILABLE, + SWITCHOVER_INITIATED, + SWITCHOVER_IN_PROGRESS, + SWITCHOVER_IN_POST_PROCESSING, + SWITCHOVER_COMPLETED, +}; + +/** + * @brief Stable identity of one Aurora blue/green deployment. + */ +struct AWS_Aurora_BGD_Fingerprint { + std::string target_id; + std::string target_endpoint; + int target_port = 0; + + bool empty() const { + return target_id.empty() || target_endpoint.empty() || target_port <= 0; + } + + bool operator==(const AWS_Aurora_BGD_Fingerprint& rhs) const { + return target_id == rhs.target_id + && target_endpoint == rhs.target_endpoint + && target_port == rhs.target_port; + } +}; + +/** + * @brief One production or target Aurora member retained by the BGD worker. + */ +struct AWS_Aurora_BGD_Member { + std::string server_id; + std::string normalized_server_id; + std::string session_id; + std::string hostname; + std::string production_hostname; + std::string target_ip; + int port = 0; + int use_ssl = 0; + bool is_writer = false; + bool traffic_pin_applied = false; +}; + +/** + * @brief State carried by one existing per-writer Aurora monitor worker. + */ +struct AWS_Aurora_BGD_State { + unsigned int writer_hg = 0; + unsigned int reader_hg = 0; + int green_writer_hg = -1; + int green_reader_hg = -1; + unsigned int check_interval_ms = 0; + unsigned int check_timeout_ms = 0; + int target_use_ssl = 0; + std::string domain_name; + + AWS_Aurora_BGD_Status status = AWS_Aurora_BGD_Status::NONE; + RDS_BGD_Topology_Monitor_State topology_state = TOPOLOGY_TABLE_CHECK; + AWS_Aurora_BGD_Fingerprint fingerprint; + std::vector production_probe_hosts; + std::vector production_members; + std::vector target_members; + + bool production_snapshot_frozen = false; + bool production_probe_suspended = false; + bool target_snapshot_complete = false; + + bool has_complete_target_snapshot() const { + return target_snapshot_complete; + } +}; + +// Maps an Aurora switchover status enum to its runtime string. +const char* aws_aurora_bgd_status_str(AWS_Aurora_BGD_Status status); + /** * @brief Switchover state carried by RDS BGD worker thread. * @@ -723,6 +802,21 @@ class MySQL_Monitor { void * monitor_group_replication_2(); void * monitor_galera(); void * monitor_aws_aurora(); + /** + * @brief Refresh the Aurora BGD worker's last complete production snapshot. + * + * @details Invalid or incomplete observations retain the previous snapshot. + */ + void aws_aurora_bgd_refresh_production_snapshot( + AWS_Aurora_BGD_State& st, const AWS_Aurora_status_entry& result); + /** + * @brief Run the topology and target-membership probes owned by an Aurora worker. + * + * @details Discovery is serialized with the ordinary Aurora probe. This method + * validates topology before publishing status and replaces target membership + * only with a complete, unambiguous, fully resolved snapshot. + */ + void aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st); /** * @brief AWS RDS BGD monitor thread entry point. * diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index a76bb3b2f5..53ec72e02a 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6057,6 +6057,504 @@ static void shuffle_hosts(host_def_t *array, size_t n) { } } +const char* aws_aurora_bgd_status_str(AWS_Aurora_BGD_Status status) { + switch (status) { + case AWS_Aurora_BGD_Status::NONE: + return "NONE"; + case AWS_Aurora_BGD_Status::AVAILABLE: + return BGD_STATUS_AVAILABLE; + case AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED: + return BGD_STATUS_INITIATED; + case AWS_Aurora_BGD_Status::SWITCHOVER_IN_PROGRESS: + return BGD_STATUS_IN_PROGRESS; + case AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING: + return BGD_STATUS_POST_PROC; + case AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED: + return BGD_STATUS_COMPLETED; + } + return "NONE"; +} + +namespace { + +struct AWS_Aurora_BGD_Query_Result { + unique_ptr mmsd; + int rc = 1; + unsigned int mysql_error = 0; +}; + +struct AWS_Aurora_BGD_Topology_Observation { + bool valid = false; + bool completed = false; + AWS_Aurora_BGD_Status status = AWS_Aurora_BGD_Status::NONE; + AWS_Aurora_BGD_Fingerprint fingerprint; +}; + +static std::string aws_aurora_bgd_normalize_server_id(const std::string& server_id) { + const size_t green_pos = server_id.find("-green-"); + return green_pos == std::string::npos ? server_id : server_id.substr(0, green_pos); +} + +static std::string aws_aurora_bgd_member_hostname( + const std::string& server_id, const std::string& domain_name +) { + if (domain_name.empty()) { + return server_id; + } + return domain_name.front() == '.' + ? server_id + domain_name + : server_id + "." + domain_name; +} + +static bool aws_aurora_bgd_status_from_raw( + const std::string& raw_status, AWS_Aurora_BGD_Status& status +) { + if (strcasecmp(raw_status.c_str(), BGD_STATUS_AVAILABLE) == 0) { + status = AWS_Aurora_BGD_Status::AVAILABLE; + } else if (strcasecmp(raw_status.c_str(), BGD_STATUS_INITIATED) == 0) { + status = AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED; + } else if (strcasecmp(raw_status.c_str(), BGD_STATUS_IN_PROGRESS) == 0) { + status = AWS_Aurora_BGD_Status::SWITCHOVER_IN_PROGRESS; + } else if (strcasecmp(raw_status.c_str(), BGD_STATUS_POST_PROC) == 0) { + status = AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING; + } else if (strcasecmp(raw_status.c_str(), BGD_STATUS_COMPLETED) == 0) { + status = AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED; + } else { + return false; + } + return true; +} + +static bool aws_aurora_bgd_required_topology_fields(const AWS_RDS_Topology_Node& node) { + return !node.id.empty() && !node.endpoint.empty() && node.port > 0 + && !node.role.empty() && !node.status.empty(); +} + +static AWS_Aurora_BGD_Topology_Observation aws_aurora_bgd_validate_topology( + const AWS_RDS_Topology_Result& topology +) { + AWS_Aurora_BGD_Topology_Observation observation; + if (!topology.blue_green || topology.nodes.empty()) { + return observation; + } + + const AWS_RDS_Topology_Node* source = nullptr; + const AWS_RDS_Topology_Node* target = nullptr; + for (const AWS_RDS_Topology_Node& node : topology.nodes) { + if (!aws_aurora_bgd_required_topology_fields(node)) { + return observation; + } + if (strcasecmp(node.role.c_str(), BGD_ROLE_SOURCE) == 0) { + if (source != nullptr) { + return observation; + } + source = &node; + } else if (strcasecmp(node.role.c_str(), BGD_ROLE_TARGET) == 0) { + if (target != nullptr) { + return observation; + } + target = &node; + } else { + return observation; + } + } + + if (target == nullptr || !aws_aurora_bgd_status_from_raw(target->status, observation.status)) { + return observation; + } + + if (observation.status == AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED) { + if (topology.nodes.size() != 1 || source != nullptr) { + return AWS_Aurora_BGD_Topology_Observation {}; + } + observation.completed = true; + } else { + if (topology.nodes.size() != 2 || source == nullptr + || strcasecmp(source->status.c_str(), target->status.c_str()) != 0) { + return AWS_Aurora_BGD_Topology_Observation {}; + } + } + + observation.fingerprint = { + target->id, + target->endpoint, + target->port, + }; + observation.valid = !observation.fingerprint.empty(); + return observation; +} + +static bool aws_aurora_bgd_select_reachable_host( + const std::vector& hosts, + unsigned int writer_hg, + AWS_RDS_BGD_Probe_Host& selected +) { + if (hosts.empty()) { + return false; + } + + const size_t first = static_cast(rand()) % hosts.size(); + for (size_t offset = 0; offset < hosts.size(); ++offset) { + const AWS_RDS_BGD_Probe_Host& host = hosts[(first + offset) % hosts.size()]; + if (GloMyMon->server_responds_to_ping( + const_cast(host.hostname.c_str()), host.port)) { + selected = host; + return true; + } + MyHGM->p_update_mysql_error_counter( + p_mysql_error_type::proxysql, writer_hg, + const_cast(host.hostname.c_str()), host.port, + ER_PROXYSQL_AWS_NO_PINGABLE_SRV + ); + } + return false; +} + +static AWS_Aurora_BGD_Query_Result aws_aurora_bgd_query( + const AWS_RDS_BGD_Probe_Host& host, + unsigned int writer_hg, + unsigned int timeout_ms, + MySQL_Monitor_State_Data_Task_Type task_type, + const char* query +) { + AWS_Aurora_BGD_Query_Result out; + out.mmsd.reset(new MySQL_Monitor_State_Data( + task_type, const_cast(host.hostname.c_str()), host.port, host.use_ssl, writer_hg)); + MySQL_Monitor_State_Data* mmsd = out.mmsd.get(); + mmsd->writer_hostgroup = writer_hg; + mmsd->aws_aurora_check_timeout_ms = timeout_ms; + mmsd->mysql = GloMyMon->My_Conn_Pool->get_connection(mmsd->hostname, mmsd->port, mmsd); + + bool new_connection = false; + if (mmsd->mysql == nullptr) { + new_connection = true; + if (!mmsd->create_new_connection()) { + out.mysql_error = mmsd->mysql ? mysql_errno(mmsd->mysql) : 0; + return out; + } + GloMyMon->My_Conn_Pool->conn_register(mmsd); + } + + std::atomic_bool worker_stop {false}; + out.rc = GloMyMon->aws_rds_bgd_async_query(mmsd, query, worker_stop); + out.mysql_error = mmsd->mysql ? mysql_errno(mmsd->mysql) : 0; + + if (out.rc != 0) { + GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); + } else if (new_connection) { + if (mmsd->set_wait_timeout()) { + GloMyMon->My_Conn_Pool->put_connection(mmsd->hostname, mmsd); + } else { + GloMyMon->My_Conn_Pool->destroy_mysql_connection(mmsd); + } + } else { + GloMyMon->My_Conn_Pool->put_connection(mmsd->hostname, mmsd); + } + + return out; +} + +static void aws_aurora_bgd_set_status( + AWS_Aurora_BGD_State& st, AWS_Aurora_BGD_Status status +) { + if (st.status == status) { + return; + } + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: switchover status '%s' -> '%s'\n", + st.writer_hg, st.reader_hg, + aws_aurora_bgd_status_str(st.status), aws_aurora_bgd_status_str(status)); + st.status = status; + MyHGM->update_aws_aurora_bgd_status(st.writer_hg, aws_aurora_bgd_status_str(status)); +} + +static bool aws_aurora_bgd_same_production_snapshot( + const std::vector& lhs, + const std::vector& rhs +) { + if (lhs.size() != rhs.size()) { + return false; + } + for (const AWS_Aurora_BGD_Member& member : lhs) { + auto found = std::find_if(rhs.begin(), rhs.end(), [&](const AWS_Aurora_BGD_Member& candidate) { + return candidate.normalized_server_id == member.normalized_server_id + && candidate.is_writer == member.is_writer; + }); + if (found == rhs.end()) { + return false; + } + } + return true; +} + +static bool aws_aurora_bgd_parse_target_membership( + AWS_Aurora_BGD_State& st, MYSQL_RES* result, + std::vector& snapshot +) { + if (result == nullptr || st.production_members.empty()) { + return false; + } + + const unsigned int num_fields = mysql_num_fields(result); + MYSQL_FIELD* fields = mysql_fetch_fields(result); + int server_id_idx = -1; + int session_id_idx = -1; + int is_current_idx = -1; + for (unsigned int i = 0; i < num_fields; ++i) { + if (fields[i].name == nullptr) { + continue; + } + if (strcasecmp(fields[i].name, "SERVER_ID") == 0) { + server_id_idx = static_cast(i); + } else if (strcasecmp(fields[i].name, "SESSION_ID") == 0) { + session_id_idx = static_cast(i); + } else if (strcasecmp(fields[i].name, "IS_CURRENT") == 0) { + is_current_idx = static_cast(i); + } + } + if (server_id_idx < 0 || session_id_idx < 0 || is_current_idx < 0) { + return false; + } + + std::unordered_map production_by_id; + for (const AWS_Aurora_BGD_Member& member : st.production_members) { + if (!production_by_id.emplace(member.normalized_server_id, &member).second) { + return false; + } + } + + std::unordered_map previous_reader_session_by_id; + std::unordered_map previous_reader_id_by_session; + std::unordered_map previous_ip_by_id; + for (const AWS_Aurora_BGD_Member& member : st.target_members) { + previous_ip_by_id[member.normalized_server_id] = member.target_ip; + if (!member.is_writer) { + previous_reader_session_by_id[member.normalized_server_id] = member.session_id; + previous_reader_id_by_session[member.session_id] = member.normalized_server_id; + } + } + + std::unordered_set target_ids; + std::unordered_set reader_sessions; + unsigned int writers = 0; + MYSQL_ROW row; + while ((row = mysql_fetch_row(result))) { + if (row[is_current_idx] == nullptr) { + continue; + } + const bool is_current = atoi(row[is_current_idx]) != 0 + || strcasecmp(row[is_current_idx], "true") == 0; + if (!is_current) { + continue; + } + if (row[server_id_idx] == nullptr || row[server_id_idx][0] == '\0' + || row[session_id_idx] == nullptr || row[session_id_idx][0] == '\0') { + return false; + } + + AWS_Aurora_BGD_Member member; + member.server_id = row[server_id_idx]; + member.normalized_server_id = aws_aurora_bgd_normalize_server_id(member.server_id); + member.session_id = row[session_id_idx]; + member.is_writer = strcasecmp(member.session_id.c_str(), "MASTER_SESSION_ID") == 0; + if (member.normalized_server_id.empty() || !target_ids.insert(member.normalized_server_id).second) { + return false; + } + + auto production = production_by_id.find(member.normalized_server_id); + if (production == production_by_id.end() + || production->second->is_writer != member.is_writer) { + return false; + } + member.production_hostname = production->second->hostname; + member.hostname = aws_aurora_bgd_member_hostname(member.server_id, st.domain_name); + member.port = st.fingerprint.target_port; + member.use_ssl = st.target_use_ssl; + + if (member.is_writer) { + writers++; + } else { + if (!reader_sessions.insert(member.session_id).second) { + return false; + } + auto old_session = previous_reader_session_by_id.find(member.normalized_server_id); + if (old_session != previous_reader_session_by_id.end() + && old_session->second != member.session_id) { + return false; + } + auto old_id = previous_reader_id_by_session.find(member.session_id); + if (old_id != previous_reader_id_by_session.end() + && old_id->second != member.normalized_server_id) { + return false; + } + } + + size_t ip_count = 0; + std::string resolved_ip = MySQL_Monitor::dns_lookup(member.hostname, false, &ip_count); + if (resolved_ip.empty()) { + const int ai_family = mysql_resolution_family_to_ai_family(mysql_thread___resolution_family); + std::vector ips = dns_resolve(member.hostname, ai_family); + if (!ips.empty()) { + resolved_ip = ips.front(); + } + } + if (resolved_ip.empty()) { + return false; + } + auto previous_ip = previous_ip_by_id.find(member.normalized_server_id); + member.target_ip = previous_ip != previous_ip_by_id.end() && !previous_ip->second.empty() + ? previous_ip->second : resolved_ip; + snapshot.push_back(std::move(member)); + } + + return writers == 1 && snapshot.size() == st.production_members.size(); +} + +static std::vector aws_aurora_bgd_target_probe_hosts( + const AWS_Aurora_BGD_State& st +) { + std::vector hosts; + if (st.has_complete_target_snapshot()) { + hosts.reserve(st.target_members.size()); + for (const AWS_Aurora_BGD_Member& member : st.target_members) { + hosts.push_back({member.hostname, member.port, member.use_ssl}); + } + } else if (!st.fingerprint.empty()) { + hosts.push_back({ + st.fingerprint.target_endpoint, + st.fingerprint.target_port, + st.target_use_ssl, + }); + } + return hosts; +} + +} // namespace + +void MySQL_Monitor::aws_aurora_bgd_refresh_production_snapshot( + AWS_Aurora_BGD_State& st, const AWS_Aurora_status_entry& result +) { + if (st.production_snapshot_frozen || result.error != nullptr + || result.host_statuses == nullptr || result.host_statuses->empty()) { + return; + } + + std::vector snapshot; + std::unordered_set member_ids; + unsigned int writers = 0; + for (const AWS_Aurora_replica_host_status_entry* row : *result.host_statuses) { + if (row == nullptr || row->server_id == nullptr || row->server_id[0] == '\0' + || row->session_id == nullptr) { + return; + } + AWS_Aurora_BGD_Member member; + member.server_id = row->server_id; + member.normalized_server_id = aws_aurora_bgd_normalize_server_id(member.server_id); + member.session_id = row->session_id; + member.hostname = aws_aurora_bgd_member_hostname(member.server_id, st.domain_name); + member.production_hostname = member.hostname; + member.port = st.production_probe_hosts.empty() ? 0 : st.production_probe_hosts.front().port; + member.use_ssl = st.production_probe_hosts.empty() ? 0 : st.production_probe_hosts.front().use_ssl; + member.is_writer = strcasecmp(member.session_id.c_str(), "MASTER_SESSION_ID") == 0; + if (member.normalized_server_id.empty() || !member_ids.insert(member.normalized_server_id).second) { + return; + } + writers += member.is_writer ? 1 : 0; + snapshot.push_back(std::move(member)); + } + if (writers != 1) { + return; + } + + if (!aws_aurora_bgd_same_production_snapshot(st.production_members, snapshot)) { + st.target_snapshot_complete = false; + } + st.production_members = std::move(snapshot); +} + +void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) { + const bool configured_green_hgs = st.green_writer_hg >= 0 && st.green_reader_hg >= 0; + const bool discovery_admitted = configured_green_hgs + || st.status != AWS_Aurora_BGD_Status::NONE + || mysql_thread___aws_blue_green_deployment_auto_discovery != 0; + if (!discovery_admitted) { + return; + } + + std::vector topology_hosts = st.has_complete_target_snapshot() + ? aws_aurora_bgd_target_probe_hosts(st) + : st.production_probe_hosts; + AWS_RDS_BGD_Probe_Host topology_host; + if (!aws_aurora_bgd_select_reachable_host(topology_hosts, st.writer_hg, topology_host)) { + proxy_error("No node is pingable for AWS Aurora BGD topology checks with writer HG %u\n", st.writer_hg); + return; + } + + if (st.topology_state == TOPOLOGY_TABLE_CHECK) { + AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query( + topology_host, st.writer_hg, st.check_timeout_ms, + MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK); + if (query.rc == 0 && query.mmsd->result + && mysql_num_rows(query.mmsd->result) > 0) { + st.topology_state = TOPOLOGY_METADATA_FETCH; + } + } else { + AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query( + topology_host, st.writer_hg, st.check_timeout_ms, + MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY); + if (query.rc != 0) { + if (query.mysql_error == 1146) { + st.topology_state = TOPOLOGY_TABLE_CHECK; + } + } else if (query.mmsd->result && mysql_num_rows(query.mmsd->result) > 0) { + AWS_RDS_Topology_Result topology = parse_aws_rds_topology(query.mmsd->result); + AWS_Aurora_BGD_Topology_Observation observation = + aws_aurora_bgd_validate_topology(topology); + if (observation.valid && !observation.completed) { + const bool same_deployment = st.fingerprint.empty() + || st.fingerprint == observation.fingerprint; + const bool rollback_transition = + static_cast(observation.status) < static_cast(st.status); + if (same_deployment && !rollback_transition) { + st.fingerprint = observation.fingerprint; + st.target_use_ssl = topology_host.use_ssl; + if (observation.status >= AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED) { + st.production_snapshot_frozen = true; + } + aws_aurora_bgd_set_status(st, observation.status); + } + } + } + } + + if (st.status == AWS_Aurora_BGD_Status::NONE || st.fingerprint.empty()) { + return; + } + + std::vector membership_hosts = aws_aurora_bgd_target_probe_hosts(st); + AWS_RDS_BGD_Probe_Host membership_host; + if (!aws_aurora_bgd_select_reachable_host(membership_hosts, st.writer_hg, membership_host)) { + proxy_error("No node is pingable for AWS Aurora BGD membership checks with writer HG %u\n", st.writer_hg); + return; + } + + AWS_Aurora_BGD_Query_Result membership = aws_aurora_bgd_query( + membership_host, st.writer_hg, st.check_timeout_ms, + MON_AWS_AURORA, QUERY_AWS_AURORA_BGD_REPLICA_HOST_STATUS); + if (membership.rc != 0) { + return; + } + + std::vector snapshot; + if (aws_aurora_bgd_parse_target_membership(st, membership.mmsd->result, snapshot)) { + st.target_members = std::move(snapshot); + st.target_snapshot_complete = true; + proxy_debug(PROXY_DEBUG_MONITOR, 7, + "AWS Aurora BGD [wHG=%u rHG=%u]: retained complete target membership with %zu members\n", + st.writer_hg, st.reader_hg, st.target_members.size()); + } +} + void * monitor_AWS_Aurora_thread_HG(void *arg) { unsigned int wHG = *(unsigned int *)arg; unsigned int rHG = 0; @@ -6071,6 +6569,8 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { unsigned int autopurge_missing_checks = 0; std::string domain_name; std::map autopurge_counter; + AWS_Aurora_BGD_State bgd_state; + bgd_state.writer_hg = wHG; set_thread_name("MonitorAuroraHG", GloVars.set_thread_name); proxy_info("Started Monitor thread for AWS Aurora writer HG %u\n", wHG); @@ -6118,6 +6618,12 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { if (domain_name.empty() && r->fields[12]) { domain_name = r->fields[12]; } + if (r->fields[13] && r->fields[13][0]) { + bgd_state.green_writer_hg = atoi(r->fields[13]); + } + if (r->fields[14] && r->fields[14][0]) { + bgd_state.green_reader_hg = atoi(r->fields[14]); + } } } host_def_t *hpa = (host_def_t *)malloc(sizeof(host_def_t)*num_hosts); @@ -6127,9 +6633,21 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { hpa[cur_host_idx].host = strdup(r->fields[2]); hpa[cur_host_idx].port = atoi(r->fields[3]); hpa[cur_host_idx].use_ssl = atoi(r->fields[4]); + bgd_state.production_probe_hosts.push_back({ + r->fields[2], + atoi(r->fields[3]), + atoi(r->fields[4]), + }); cur_host_idx++; } } + bgd_state.reader_hg = rHG; + bgd_state.check_interval_ms = check_interval_ms; + bgd_state.check_timeout_ms = check_timeout_ms; + bgd_state.domain_name = domain_name; + if (!bgd_state.production_probe_hosts.empty()) { + bgd_state.target_use_ssl = bgd_state.production_probe_hosts.front().use_ssl; + } // NOTE: 'cur_host_idx' should never be higher than 'num_hosts' otherwise later an invalid memory access // can table place later when accessing 'hpa[cur_host_idx]'. if (cur_host_idx >= num_hosts) { @@ -6388,6 +6906,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { mysql_free_result(mmsd->result); mmsd->result=NULL; } + GloMyMon->aws_aurora_bgd_refresh_production_snapshot(bgd_state, *ase); if (lasts_ase[ase_idx]) { AWS_Aurora_status_entry * l_ase = lasts_ase[ase_idx]; @@ -6477,6 +6996,9 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { } } } + if (GloMyMon->shutdown == false && exit_now == false) { + GloMyMon->aws_aurora_bgd_run_discovery_cycle(bgd_state); + } } __exit_monitor_AWS_Aurora_thread_HG_now: if (mmsd) { diff --git a/test/tap/groups/cluster_sim_aurora/add-hosts b/test/tap/groups/cluster_sim_aurora/add-hosts index 128b632715..cb6b8a8746 100644 --- a/test/tap/groups/cluster_sim_aurora/add-hosts +++ b/test/tap/groups/cluster_sim_aurora/add-hosts @@ -53,6 +53,9 @@ aurora-a-green.cluster-a1.us-east-1.rds.amazonaws.com 127.0.11.20 aurora-a-green-writer.a1.us-east-1.rds.amazonaws.com 127.0.11.21 aurora-a-green-reader-1.a1.us-east-1.rds.amazonaws.com 127.0.11.22 aurora-a-green-reader-2.a1.us-east-1.rds.amazonaws.com 127.0.11.23 +aurora-a-writer-green-k7m2.a1.us-east-1.rds.amazonaws.com 127.0.11.21 +aurora-a-reader-1-green-p4q8.a1.us-east-1.rds.amazonaws.com 127.0.11.22 +aurora-a-reader-2-green-v9n3.a1.us-east-1.rds.amazonaws.com 127.0.11.23 # Canonical names observed after target-member rename retain target addresses. aurora-a-canonical-writer.a1.us-east-1.rds.amazonaws.com 127.0.11.21 @@ -65,5 +68,7 @@ aurora-b-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.12 aurora-b-green.cluster-b1.us-east-1.rds.amazonaws.com 127.0.12.20 aurora-b-green-writer.b1.us-east-1.rds.amazonaws.com 127.0.12.21 aurora-b-green-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.22 +aurora-b-writer-green-h2s6.b1.us-east-1.rds.amazonaws.com 127.0.12.21 +aurora-b-reader-1-green-r8t4.b1.us-east-1.rds.amazonaws.com 127.0.12.22 aurora-b-canonical-writer.b1.us-east-1.rds.amazonaws.com 127.0.12.21 aurora-b-canonical-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.22 diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 1c53e22d9b..631ac58cec 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -363,6 +363,7 @@ "test_client_limit_error-t" : [ "todo-g1" ], "test_cluster1-t" : [ "legacy-g5","mariadb10-galera-g5","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3","mysql84-g5","mysql84-gr-g5","mysql90-g5","mysql95-g5" ], "test_cluster_sim_aurora-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_discovery-t" : [ "cluster_sim_aurora-g1" ], "test_cluster_sim_galera-t" : [ "cluster_sim_galera-g1" ], "test_cluster_sim_group_repl-t" : [ "cluster_sim_group_repl-g1" ], "test_cluster_sim_read_only-t" : [ "cluster_sim_read_only-g1" ], diff --git a/test/tap/tap/aurora_bgd_tap.h b/test/tap/tap/aurora_bgd_tap.h new file mode 100644 index 0000000000..09e12ff4b3 --- /dev/null +++ b/test/tap/tap/aurora_bgd_tap.h @@ -0,0 +1,290 @@ +#ifndef TAP_TESTS_AURORA_BGD_TAP_H +#define TAP_TESTS_AURORA_BGD_TAP_H + +#include +#include +#include +#include +#include +#include + +#include "aurora_bgd_simulator.h" +#include "tap.h" + +using namespace std; + +struct Aurora_BGD_Test_Deployment { + string name; + string domain_name; + string blue_replica_set; + string target_replica_set; + Aurora_BGD_Endpoint target_cluster_endpoint; + Aurora_BGD_Membership_Set production; + Aurora_BGD_Membership_Set target; + string source_topology_id; + string target_topology_id; +}; + +inline Aurora_BGD_Member aurora_bgd_member( + string server_id, string session_id, Aurora_BGD_Endpoint endpoint, bool current = true +) { + Aurora_BGD_Member member; + member.server_id = move(server_id); + member.session_id = move(session_id); + member.endpoint = move(endpoint); + member.last_update_timestamp = "2099-01-01 00:00:00"; + member.is_current = current; + return member; +} + +inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_a() { + Aurora_BGD_Test_Deployment deployment; + deployment.name = "Aurora BGD deployment A"; + deployment.domain_name = ".a1.us-east-1.rds.amazonaws.com"; + deployment.blue_replica_set = "aurora-bgd-blue-a"; + deployment.target_replica_set = "aurora-bgd-target-a"; + deployment.source_topology_id = "aurora-bgd-source-a"; + deployment.target_topology_id = "aurora-bgd-target-a"; + deployment.target_cluster_endpoint = { + "aurora-a-green.cluster-a1.us-east-1.rds.amazonaws.com", "127.0.11.20", 3306 + }; + deployment.production = { + deployment.blue_replica_set, + { + aurora_bgd_member("aurora-a-writer", "MASTER_SESSION_ID", + {"aurora-a-writer.a1.us-east-1.rds.amazonaws.com", "127.0.11.11", 3306}), + aurora_bgd_member("aurora-a-reader-1", "reader-a-1", + {"aurora-a-reader-1.a1.us-east-1.rds.amazonaws.com", "127.0.11.12", 3306}), + aurora_bgd_member("aurora-a-reader-2", "reader-a-2", + {"aurora-a-reader-2.a1.us-east-1.rds.amazonaws.com", "127.0.11.13", 3306}), + }, + {} + }; + for (Aurora_BGD_Member& member : deployment.production.members) { + deployment.production.serving_endpoints.push_back(member.endpoint); + } + deployment.target = { + deployment.target_replica_set, + { + aurora_bgd_member("aurora-a-writer-green-k7m2", "MASTER_SESSION_ID", + {"aurora-a-writer-green-k7m2.a1.us-east-1.rds.amazonaws.com", "127.0.11.21", 3306}), + aurora_bgd_member("aurora-a-reader-1-green-p4q8", "reader-a-1", + {"aurora-a-reader-1-green-p4q8.a1.us-east-1.rds.amazonaws.com", "127.0.11.22", 3306}), + aurora_bgd_member("aurora-a-reader-2-green-v9n3", "reader-a-2", + {"aurora-a-reader-2-green-v9n3.a1.us-east-1.rds.amazonaws.com", "127.0.11.23", 3306}), + }, + {deployment.target_cluster_endpoint} + }; + for (Aurora_BGD_Member& member : deployment.target.members) { + deployment.target.serving_endpoints.push_back(member.endpoint); + } + return deployment; +} + +inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_b_writer_only() { + Aurora_BGD_Test_Deployment deployment; + deployment.name = "Aurora BGD deployment B writer-only"; + deployment.domain_name = ".b1.us-east-1.rds.amazonaws.com"; + deployment.blue_replica_set = "aurora-bgd-blue-b"; + deployment.target_replica_set = "aurora-bgd-target-b"; + deployment.source_topology_id = "aurora-bgd-source-b"; + deployment.target_topology_id = "aurora-bgd-target-b"; + deployment.target_cluster_endpoint = { + "aurora-b-green.cluster-b1.us-east-1.rds.amazonaws.com", "127.0.12.20", 3306 + }; + deployment.production = { + deployment.blue_replica_set, + { + aurora_bgd_member("aurora-b-writer", "MASTER_SESSION_ID", + {"aurora-b-writer.b1.us-east-1.rds.amazonaws.com", "127.0.12.11", 3306}), + }, + {} + }; + deployment.production.serving_endpoints.push_back(deployment.production.members.front().endpoint); + deployment.target = { + deployment.target_replica_set, + { + aurora_bgd_member("aurora-b-writer-green-h2s6", "MASTER_SESSION_ID", + {"aurora-b-writer-green-h2s6.b1.us-east-1.rds.amazonaws.com", "127.0.12.21", 3306}), + }, + {deployment.target_cluster_endpoint} + }; + deployment.target.serving_endpoints.push_back(deployment.target.members.front().endpoint); + return deployment; +} + +inline string aurora_bgd_sql_quote(const string& value) { + string quoted {"'"}; + for (char c : value) { + quoted += c; + if (c == '\'') { + quoted += '\''; + } + } + quoted += '\''; + return quoted; +} + +inline int aurora_bgd_execute_all(MYSQL* admin, const vector& queries) { + for (const string& query : queries) { + if (mysql_query(admin, query.c_str()) != 0) { + diag("Aurora BGD admin query failed: %s; query: %s", mysql_error(admin), query.c_str()); + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; +} + +inline int aurora_bgd_admin_cleanup(MYSQL* admin) { + return aurora_bgd_execute_all(admin, { + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + "DELETE FROM mysql_aws_aurora_hostgroups", + "DELETE FROM mysql_servers", + "LOAD MYSQL SERVERS TO RUNTIME", + }); +} + +inline int aurora_bgd_admin_setup( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, + int writer_hg, int reader_hg, int green_writer_hg, int green_reader_hg, + bool auto_discovery +) { + vector queries { + "SET mysql-monitor_username='aurora1'", + "SET mysql-monitor_password='pass1'", + "SET mysql-monitor_enabled='true'", + "SET mysql-aws_blue_green_deployment_auto_discovery='" + + string(auto_discovery ? "true" : "false") + "'", + }; + + string green_columns = green_writer_hg >= 0 + ? to_string(green_writer_hg) + "," + to_string(green_reader_hg) + : "NULL,NULL"; + queries.push_back( + "INSERT INTO mysql_aws_aurora_hostgroups(" + "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup," + "active,aurora_port,domain_name,max_lag_ms,check_interval_ms,check_timeout_ms," + "writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks," + "autopurge_missing_checks,comment) VALUES (" + + to_string(writer_hg) + "," + to_string(reader_hg) + "," + green_columns + + ",1,3306," + aurora_bgd_sql_quote(deployment.domain_name) + + ",200,100,800,0,1,30,30,1,0," + aurora_bgd_sql_quote(deployment.name) + ")"); + + for (Aurora_BGD_Member& member : deployment.production.members) { + const int hostgroup = member.session_id == "MASTER_SESSION_ID" ? writer_hg : reader_hg; + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + + to_string(hostgroup) + "," + aurora_bgd_sql_quote(member.endpoint.hostname) + + ",3306,'ONLINE',0,'Aurora BGD production member')"); + } + queries.push_back("LOAD MYSQL VARIABLES TO RUNTIME"); + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +inline vector aurora_bgd_available_topology( + Aurora_BGD_Test_Deployment& deployment +) { + return { + { + deployment.source_topology_id, + deployment.production.members.front().endpoint.hostname, + 3306, + "BLUE_GREEN_DEPLOYMENT_SOURCE", + "AVAILABLE", + }, + { + deployment.target_topology_id, + deployment.target_cluster_endpoint.hostname, + 3306, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "AVAILABLE", + }, + }; +} + +inline vector aurora_bgd_topology_backends(Aurora_BGD_Test_Deployment& deployment) { + vector backends = deployment.production.backends(); + vector target_backends = deployment.target.backends(); + backends.insert(backends.end(), target_backends.begin(), target_backends.end()); + return backends; +} + +inline int aurora_bgd_publish( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment +) { + if (sim.replica_update( + deployment.production.replica_set_id, + deployment.production.replica_rows(), + deployment.production.backends()) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + if (sim.replica_update( + deployment.target.replica_set_id, + deployment.target.replica_rows(), + deployment.target.backends()) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_available_topology(deployment)); +} + +inline int aurora_bgd_wait_for_status( + MYSQL* admin, int writer_hg, const string& status, uint32_t timeout_seconds +) { + string query = + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hg) + " AND bgd_status=" + aurora_bgd_sql_quote(status); + return wait_for_cond(admin, query, timeout_seconds); +} + +inline rc_t aurora_bgd_wait_for_topology_probe( + BGD_Simulator& sim, uint64_t sequence, const vector& backends, + BGD_Probe_Kind kind, uint32_t timeout_ms +) { + const uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + auto [rc, logs] = sim.probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return {EXIT_FAILURE, {}}; + } + for (const BGD_Probe_Log& log : logs) { + for (const Endpoint& backend : backends) { + if (log.backend.host == backend.host && log.backend.port == backend.port + && log.probe_kind == kind) { + return {EXIT_SUCCESS, log}; + } + } + } + usleep(50000); + } while (monotonic_time() < deadline); + return {ETIMEDOUT, {}}; +} + +inline rc_t aurora_bgd_wait_for_replica_probe( + BGD_Simulator& sim, uint64_t sequence, const vector& backends, + Aurora_Replica_Probe_Kind kind, uint32_t timeout_ms, const string& replica_set = "" +) { + const uint64_t deadline = monotonic_time() + static_cast(timeout_ms) * 1000; + do { + auto [rc, logs] = sim.replica_probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return {EXIT_FAILURE, {}}; + } + for (const Aurora_Replica_Probe_Log& log : logs) { + for (const Endpoint& backend : backends) { + if (log.backend.host == backend.host && log.backend.port == backend.port + && log.probe_kind == kind + && (replica_set.empty() || log.replica_set_id == replica_set)) { + return {EXIT_SUCCESS, log}; + } + } + } + usleep(50000); + } while (monotonic_time() < deadline); + return {ETIMEDOUT, {}}; +} + +#endif // TAP_TESTS_AURORA_BGD_TAP_H diff --git a/test/tap/tests/test_aurora_bgd_discovery-t.cpp b/test/tap/tests/test_aurora_bgd_discovery-t.cpp new file mode 100644 index 0000000000..6462d1c141 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_discovery-t.cpp @@ -0,0 +1,278 @@ +/** + * @file test_aurora_bgd_discovery-t.cpp + * @brief Aurora BGD AVAILABLE discovery, three probes, and fail-closed mapping. + */ + +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const uint32_t kProbeTimeoutMs = 5000; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + char simulator_username[] = "aurora1"; + char simulator_password[] = "pass1"; + if (sim.connect(cl.host, 3306, simulator_username, simulator_password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the shared AWS simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to clear prior Aurora BGD state"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int admin_rc = aurora_bgd_admin_cleanup(admin); + int simulator_rc = sim.cleanup(); + if (admin) { + mysql_close(admin); + } + return admin_rc == EXIT_SUCCESS && simulator_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +bool runtime_status_is(MYSQL* admin, int writer_hg, const string& status) { + auto [rc, rows] = mysql_query_ext_rows( + admin, + "SELECT bgd_status FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hg)); + return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 + && rows.front().front() == status; +} + +bool runtime_production_unchanged(MYSQL* admin, int writer_hg, int reader_hg, size_t members) { + auto [rc, rows] = mysql_query_ext_rows( + admin, + "SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id IN (" + + to_string(writer_hg) + "," + to_string(reader_hg) + ")"); + return rc == EXIT_SUCCESS && rows.size() == members; +} + +bool retain_complete_target_after_membership( + BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, + const vector& rows +) { + auto [seq_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (seq_rc != EXIT_SUCCESS || sim.replica_update( + deployment.target_replica_set, rows, deployment.target.backends()) != EXIT_SUCCESS) { + return false; + } + + vector target_members; + for (Aurora_BGD_Member& member : deployment.target.members) { + target_members.push_back(member.endpoint.backend()); + } + auto [first_rc, first_probe] = aurora_bgd_wait_for_replica_probe( + sim, sequence, target_members, Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, deployment.target_replica_set); + if (first_rc != EXIT_SUCCESS) { + return false; + } + auto [next_rc, next_probe] = aurora_bgd_wait_for_replica_probe( + sim, first_probe.sequence_id, target_members, + Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, deployment.target_replica_set); + return next_rc == EXIT_SUCCESS; +} + +bool invalid_topology_retains_available( + MYSQL* admin, + BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, + const vector& rows, + int writer_hg +) { + auto [seq_rc, sequence] = sim.probe_log_last_sequence(); + vector target_members; + for (Aurora_BGD_Member& member : deployment.target.members) { + target_members.push_back(member.endpoint.backend()); + } + if (seq_rc != EXIT_SUCCESS || sim.topology_update( + aurora_bgd_topology_backends(deployment), rows) != EXIT_SUCCESS) { + return false; + } + auto [probe_rc, probe] = aurora_bgd_wait_for_topology_probe( + sim, sequence, target_members, BGD_Probe_Kind::metadata, kProbeTimeoutMs); + return probe_rc == EXIT_SUCCESS && runtime_status_is(admin, writer_hg, "AVAILABLE"); +} + +int main() { + plan(18); + + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + const int writer_hg = 1510; + const int reader_hg = 1511; + Aurora_BGD_Test_Deployment deployment = aurora_bgd_deployment_a(); + if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, deployment, writer_hg, reader_hg, 1512, 1513, false) + != EXIT_SUCCESS) { + diag("Error: failed to publish or configure deployment A"); + cleanup(admin, sim); + return exit_status(); + } + + ok(aurora_bgd_wait_for_status(admin, writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "valid SOURCE/TARGET topology publishes AVAILABLE"); + + { + auto [rc, logs] = sim.replica_probe_log_since(0); + bool ordinary_seen = false; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary + && log.replica_set_id == deployment.blue_replica_set) { + ordinary_seen = true; + } + } + ok(rc == EXIT_SUCCESS && ordinary_seen, + "the existing worker continues ordinary Aurora probing in AVAILABLE"); + } + + vector blue_backends = deployment.production.backends(); + { + auto [rc, probe] = aurora_bgd_wait_for_topology_probe( + sim, 0, blue_backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); + ok(rc == EXIT_SUCCESS, "topology discovery starts on a reachable production member"); + } + { + auto [rc, probe] = sim.wait_for_replica_probe_log( + 0, deployment.target_cluster_endpoint.backend(), + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, + 0, deployment.target_replica_set); + ok(rc == EXIT_SUCCESS, "TARGET cluster endpoint bootstraps target membership"); + } + + vector target_backends; + for (Aurora_BGD_Member& member : deployment.target.members) { + target_backends.push_back(member.endpoint.backend()); + } + { + auto [rc, probe] = aurora_bgd_wait_for_topology_probe( + sim, 0, target_backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); + ok(rc == EXIT_SUCCESS, "complete membership moves topology probes to target members"); + } + { + auto [rc, probe] = aurora_bgd_wait_for_replica_probe( + sim, 0, target_backends, Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, deployment.target_replica_set); + ok(rc == EXIT_SUCCESS, "complete membership moves membership probes to target members"); + } + + vector complete_rows = deployment.target.replica_rows(); + vector incomplete_rows {complete_rows[0], complete_rows[1]}; + ok(retain_complete_target_after_membership(sim, deployment, incomplete_rows), + "an incomplete target result retains the previous complete target selection"); + + vector duplicate_writer_rows = complete_rows; + duplicate_writer_rows[1].session_id = "MASTER_SESSION_ID"; + ok(retain_complete_target_after_membership(sim, deployment, duplicate_writer_rows), + "multiple current target writers retain the previous complete snapshot"); + + vector unresolved_rows = complete_rows; + unresolved_rows[2].server_id = "aurora-a-reader-2-green-missing"; + ok(retain_complete_target_after_membership(sim, deployment, unresolved_rows), + "an unresolved target member retains the previous complete snapshot"); + + vector mismatched_session_rows = complete_rows; + mismatched_session_rows[1].session_id = "reader-a-1-replaced"; + ok(retain_complete_target_after_membership(sim, deployment, mismatched_session_rows), + "a reader SESSION_ID mismatch retains the previous complete snapshot"); + + vector stale_extra_rows = complete_rows; + Aurora_Replica_Row stale_writer = complete_rows.front(); + stale_writer.server_id = "unrelated-stale-writer"; + stale_writer.is_current = false; + stale_extra_rows.push_back(stale_writer); + ok(retain_complete_target_after_membership(sim, deployment, stale_extra_rows), + "IS_CURRENT=0 rows are excluded from the complete target snapshot"); + + if (sim.replica_update( + deployment.target_replica_set, complete_rows, deployment.target.backends()) != EXIT_SUCCESS) { + diag("Error: failed to restore complete target membership"); + cleanup(admin, sim); + return exit_status(); + } + + vector valid_topology = aurora_bgd_available_topology(deployment); + vector missing_identity = valid_topology; + missing_identity[1].id.clear(); + ok(invalid_topology_retains_available(admin, sim, deployment, missing_identity, writer_hg), + "missing TARGET identity does not replace AVAILABLE"); + + vector source_only {valid_topology.front()}; + ok(invalid_topology_retains_available(admin, sim, deployment, source_only, writer_hg), + "SOURCE-only topology does not replace AVAILABLE"); + + vector mismatched_status = valid_topology; + mismatched_status[0].status = "SWITCHOVER_INITIATED"; + ok(invalid_topology_retains_available(admin, sim, deployment, mismatched_status, writer_hg), + "mismatched SOURCE/TARGET statuses do not replace AVAILABLE"); + + vector unknown_status = valid_topology; + unknown_status[0].status = "UNSUPPORTED_STATUS"; + unknown_status[1].status = "UNSUPPORTED_STATUS"; + ok(invalid_topology_retains_available(admin, sim, deployment, unknown_status, writer_hg), + "unsupported topology status is not copied into runtime state"); + + ok(runtime_production_unchanged(admin, writer_hg, reader_hg, deployment.production.members.size()), + "AVAILABLE discovery performs no production routing action"); + + if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to reset state before writer-only deployment"); + cleanup(admin, sim); + return exit_status(); + } + + Aurora_BGD_Test_Deployment writer_only = aurora_bgd_deployment_b_writer_only(); + if (aurora_bgd_publish(sim, writer_only) != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, writer_only, 1520, 1521, -1, -1, true) + != EXIT_SUCCESS) { + diag("Error: failed to publish or configure writer-only deployment"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status(admin, 1520, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "auto-discovery admits a row without configured green hostgroups"); + { + vector writer_target { + writer_only.target.members.front().endpoint.backend() + }; + auto [rc, probe] = aurora_bgd_wait_for_replica_probe( + sim, 0, writer_target, Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, writer_only.target_replica_set); + ok(rc == EXIT_SUCCESS, "a writer-only production cluster accepts a writer-only target snapshot"); + } + + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD discovery state"); + return EXIT_FAILURE; + } + return exit_status(); +} From d5f486a7a6569b8a9a6daa80c85ff84a115c5fdf Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sat, 15 Aug 2026 18:30:39 +0000 Subject: [PATCH 24/34] feat: handle Aurora BGD active switchover --- include/MySQL_Monitor.hpp | 9 + lib/MySQL_Monitor.cpp | 101 +++- test/tap/groups/groups.json | 1 + test/tap/tap/aurora_bgd_tap.h | 22 +- test/tap/tests/test_aurora_bgd_active-t.cpp | 480 ++++++++++++++++++++ 5 files changed, 604 insertions(+), 9 deletions(-) create mode 100644 test/tap/tests/test_aurora_bgd_active-t.cpp diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 163d05106f..b4fb0798f5 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -809,6 +809,15 @@ class MySQL_Monitor { */ void aws_aurora_bgd_refresh_production_snapshot( AWS_Aurora_BGD_State& st, const AWS_Aurora_status_entry& result); + /** + * @brief Apply the routing actions for an accepted active Aurora BGD state. + * + * @details INITIATED only suspends production probing. IN_PROGRESS demotes + * the snapshotted writer on entry. POST_PROCESSING applies each complete + * member mapping once and restores canonical writer placement. + */ + void aws_aurora_bgd_apply_active_actions( + AWS_Aurora_BGD_State& st, bool status_changed); /** * @brief Run the topology and target-membership probes owned by an Aurora worker. * diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 53ec72e02a..094066ef64 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6326,8 +6326,10 @@ static bool aws_aurora_bgd_parse_target_membership( std::unordered_map previous_reader_session_by_id; std::unordered_map previous_reader_id_by_session; std::unordered_map previous_ip_by_id; + std::unordered_map previous_action_by_id; for (const AWS_Aurora_BGD_Member& member : st.target_members) { previous_ip_by_id[member.normalized_server_id] = member.target_ip; + previous_action_by_id[member.normalized_server_id] = member.traffic_pin_applied; if (!member.is_writer) { previous_reader_session_by_id[member.normalized_server_id] = member.session_id; previous_reader_id_by_session[member.session_id] = member.normalized_server_id; @@ -6404,6 +6406,9 @@ static bool aws_aurora_bgd_parse_target_membership( auto previous_ip = previous_ip_by_id.find(member.normalized_server_id); member.target_ip = previous_ip != previous_ip_by_id.end() && !previous_ip->second.empty() ? previous_ip->second : resolved_ip; + auto previous_action = previous_action_by_id.find(member.normalized_server_id); + member.traffic_pin_applied = previous_action != previous_action_by_id.end() + && previous_action->second; snapshot.push_back(std::move(member)); } @@ -6472,6 +6477,71 @@ void MySQL_Monitor::aws_aurora_bgd_refresh_production_snapshot( st.production_members = std::move(snapshot); } +void MySQL_Monitor::aws_aurora_bgd_apply_active_actions( + AWS_Aurora_BGD_State& st, bool status_changed +) { + if (st.status < AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED + || st.status > AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING) { + return; + } + + st.production_snapshot_frozen = true; + st.production_probe_suspended = true; + + if (st.status == AWS_Aurora_BGD_Status::SWITCHOVER_IN_PROGRESS + && status_changed) { + auto writer = std::find_if( + st.production_members.begin(), st.production_members.end(), + [](const AWS_Aurora_BGD_Member& member) { return member.is_writer; }); + if (writer == st.production_members.end()) { + proxy_error( + "AWS Aurora BGD [wHG=%u rHG=%u]: cannot demote writer without a complete production snapshot\n", + st.writer_hg, st.reader_hg); + return; + } + MyHGM->update_aws_aurora_set_reader( + st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str())); + return; + } + + if (st.status != AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING + || !st.has_complete_target_snapshot()) { + return; + } + + bool applied_member_action = false; + bool writer_pin_applied = false; + for (AWS_Aurora_BGD_Member& member : st.target_members) { + if (!member.traffic_pin_applied) { + dns_cache->pin(member.production_hostname, member.target_ip); + MyHGM->wrlock(); + MyHGM->drain_server_connections(member.production_hostname.c_str(), member.port); + MyHGM->wrunlock(); + My_Conn_Pool->purge_connections(member.production_hostname.c_str(), member.port); + member.traffic_pin_applied = true; + applied_member_action = true; + + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: repointed production '%s' to target IP %s\n", + st.writer_hg, st.reader_hg, member.production_hostname.c_str(), + member.target_ip.c_str()); + } + writer_pin_applied |= member.is_writer && member.traffic_pin_applied; + } + + if (!writer_pin_applied || (!status_changed && !applied_member_action)) { + return; + } + + auto writer = std::find_if( + st.production_members.begin(), st.production_members.end(), + [](const AWS_Aurora_BGD_Member& member) { return member.is_writer; }); + if (writer != st.production_members.end()) { + MyHGM->update_aws_aurora_set_writer( + st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str())); + } +} + void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) { const bool configured_green_hgs = st.green_writer_hg >= 0 && st.green_reader_hg >= 0; const bool discovery_admitted = configured_green_hgs @@ -6516,12 +6586,11 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) const bool rollback_transition = static_cast(observation.status) < static_cast(st.status); if (same_deployment && !rollback_transition) { + const bool status_changed = st.status != observation.status; st.fingerprint = observation.fingerprint; st.target_use_ssl = topology_host.use_ssl; - if (observation.status >= AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED) { - st.production_snapshot_frozen = true; - } aws_aurora_bgd_set_status(st, observation.status); + aws_aurora_bgd_apply_active_actions(st, status_changed); } } } @@ -6552,6 +6621,7 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) proxy_debug(PROXY_DEBUG_MONITOR, 7, "AWS Aurora BGD [wHG=%u rHG=%u]: retained complete target membership with %zu members\n", st.writer_hg, st.reader_hg, st.target_members.size()); + aws_aurora_bgd_apply_active_actions(st, false); } } @@ -6713,6 +6783,17 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { //proxy_info("Looping Monitor thread for AWS Aurora writer HG %u\n", wHG); continue; } + if (bgd_state.production_probe_suspended) { + if (mmsd) { + delete mmsd; + mmsd = NULL; + } + GloMyMon->aws_aurora_bgd_run_discovery_cycle(bgd_state); + const unsigned int interval_ms = bgd_state.production_probe_suspended + ? 100 : check_interval_ms; + next_loop_at = t1 + static_cast(interval_ms) * 1000; + continue; + } //proxy_info("Running check AWS Aurora writer HG %u\n", wHG); found_pingable_host = false; @@ -6759,7 +6840,10 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { if (found_pingable_host == false) { proxy_error("No node is pingable for AWS Aurora cluster with writer HG %u\n", wHG); - next_loop_at = t1 + check_interval_ms * 1000; + GloMyMon->aws_aurora_bgd_run_discovery_cycle(bgd_state); + const unsigned int interval_ms = bgd_state.production_probe_suspended + ? 100 : check_interval_ms; + next_loop_at = t1 + static_cast(interval_ms) * 1000; continue; } #ifdef TEST_AURORA @@ -6996,9 +7080,12 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { } } } - if (GloMyMon->shutdown == false && exit_now == false) { - GloMyMon->aws_aurora_bgd_run_discovery_cycle(bgd_state); - } + if (GloMyMon->shutdown == false && exit_now == false) { + GloMyMon->aws_aurora_bgd_run_discovery_cycle(bgd_state); + const unsigned int interval_ms = bgd_state.production_probe_suspended + ? 100 : check_interval_ms; + next_loop_at = t1 + static_cast(interval_ms) * 1000; + } } __exit_monitor_AWS_Aurora_thread_HG_now: if (mmsd) { diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 631ac58cec..b06d7f0ad2 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -364,6 +364,7 @@ "test_cluster1-t" : [ "legacy-g5","mariadb10-galera-g5","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3","mysql84-g5","mysql84-gr-g5","mysql90-g5","mysql95-g5" ], "test_cluster_sim_aurora-t" : [ "cluster_sim_aurora-g1" ], "test_aurora_bgd_discovery-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_active-t" : [ "cluster_sim_aurora-g1" ], "test_cluster_sim_galera-t" : [ "cluster_sim_galera-g1" ], "test_cluster_sim_group_repl-t" : [ "cluster_sim_group_repl-g1" ], "test_cluster_sim_read_only-t" : [ "cluster_sim_read_only-g1" ], diff --git a/test/tap/tap/aurora_bgd_tap.h b/test/tap/tap/aurora_bgd_tap.h index 09e12ff4b3..cfc7aa78fd 100644 --- a/test/tap/tap/aurora_bgd_tap.h +++ b/test/tap/tap/aurora_bgd_tap.h @@ -141,14 +141,17 @@ inline int aurora_bgd_admin_cleanup(MYSQL* admin) { "LOAD MYSQL VARIABLES TO RUNTIME", "DELETE FROM mysql_aws_aurora_hostgroups", "DELETE FROM mysql_servers", + "UPDATE mysql_users SET default_hostgroup=0 WHERE username='testuser'", "LOAD MYSQL SERVERS TO RUNTIME", + "LOAD MYSQL USERS TO RUNTIME", }); } inline int aurora_bgd_admin_setup( MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, int writer_hg, int reader_hg, int green_writer_hg, int green_reader_hg, - bool auto_discovery + bool auto_discovery, int check_interval_ms = 100, + bool writer_is_also_reader = false ) { vector queries { "SET mysql-monitor_username='aurora1'", @@ -156,6 +159,8 @@ inline int aurora_bgd_admin_setup( "SET mysql-monitor_enabled='true'", "SET mysql-aws_blue_green_deployment_auto_discovery='" + string(auto_discovery ? "true" : "false") + "'", + "UPDATE mysql_users SET default_hostgroup=" + to_string(writer_hg) + + " WHERE username='testuser'", }; string green_columns = green_writer_hg >= 0 @@ -169,7 +174,9 @@ inline int aurora_bgd_admin_setup( "autopurge_missing_checks,comment) VALUES (" + to_string(writer_hg) + "," + to_string(reader_hg) + "," + green_columns + ",1,3306," + aurora_bgd_sql_quote(deployment.domain_name) + - ",200,100,800,0,1,30,30,1,0," + aurora_bgd_sql_quote(deployment.name) + ")"); + ",200," + to_string(check_interval_ms) + ",800," + + to_string(writer_is_also_reader ? 1 : 0) + + ",1,30,30,1,0," + aurora_bgd_sql_quote(deployment.name) + ")"); for (Aurora_BGD_Member& member : deployment.production.members) { const int hostgroup = member.session_id == "MASTER_SESSION_ID" ? writer_hg : reader_hg; @@ -180,6 +187,7 @@ inline int aurora_bgd_admin_setup( } queries.push_back("LOAD MYSQL VARIABLES TO RUNTIME"); queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + queries.push_back("LOAD MYSQL USERS TO RUNTIME"); return aurora_bgd_execute_all(admin, queries); } @@ -204,6 +212,16 @@ inline vector aurora_bgd_available_topology( }; } +inline vector aurora_bgd_topology( + Aurora_BGD_Test_Deployment& deployment, const string& status +) { + vector rows = aurora_bgd_available_topology(deployment); + for (BGD_Topology_Row& row : rows) { + row.status = status; + } + return rows; +} + inline vector aurora_bgd_topology_backends(Aurora_BGD_Test_Deployment& deployment) { vector backends = deployment.production.backends(); vector target_backends = deployment.target.backends(); diff --git a/test/tap/tests/test_aurora_bgd_active-t.cpp b/test/tap/tests/test_aurora_bgd_active-t.cpp new file mode 100644 index 0000000000..cbcc256222 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_active-t.cpp @@ -0,0 +1,480 @@ +/** + * @file test_aurora_bgd_active-t.cpp + * @brief Aurora BGD active-state probe suspension and routing behavior. + */ + +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const uint32_t kProbeTimeoutMs = 5000; +const char kOrdinaryAuroraQuery[] = + "SELECT SERVER_ID," + "IF(" + "SESSION_ID = 'MASTER_SESSION_ID' AND " + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " + "'probably_former_MASTER_SESSION_ID', SESSION_ID" + ") SESSION_ID, " + "LAST_UPDATE_TIMESTAMP, " + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " + "CPU " + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" + " ( " + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" + " OR SESSION_ID = 'MASTER_SESSION_ID'" + " ) " + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" + " ORDER BY SERVER_ID"; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + char simulator_username[] = "aurora1"; + char simulator_password[] = "pass1"; + if (sim.connect(cl.host, 3306, simulator_username, simulator_password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the shared AWS simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to clear prior Aurora BGD state"); + return EXIT_FAILURE; + } + if (aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " + "VALUES ('testuser','testuser',1,0,1)", + "LOAD MYSQL USERS TO RUNTIME", + }) != EXIT_SUCCESS) { + diag("Error: failed to configure the routing test user"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int admin_rc = aurora_bgd_admin_cleanup(admin); + int user_rc = admin == nullptr ? EXIT_FAILURE : aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); + int simulator_rc = sim.cleanup(); + if (admin) { + mysql_close(admin); + } + return admin_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS && simulator_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +bool runtime_server_count( + MYSQL* admin, int hostgroup, const string& hostname, int expected, + const string& status = "" +) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(hostgroup) + " AND hostname=" + aurora_bgd_sql_quote(hostname); + if (!status.empty()) { + query += " AND status=" + aurora_bgd_sql_quote(status); + } + auto [rc, rows] = mysql_query_ext_rows(admin, query); + return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 + && rows.front().front() == to_string(expected); +} + +int wait_for_writer_placement( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, + bool demoted +) { + string query = + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "0" : "1") + ") AND " + "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "1" : "0") + ")"; + return wait_for_cond(admin, query, kWaitSeconds); +} + +int publish_status( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, + const string& status +) { + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)); +} + +bool fast_membership_without_ordinary( + BGD_Simulator& sim, uint64_t sequence, const string& target_replica_set, + uint32_t observation_ms, uint64_t minimum_membership_probes +) { + usleep(observation_ms * 1000); + auto [rc, logs] = sim.replica_probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return false; + } + uint64_t membership_probes = 0; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { + return false; + } + if (log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership + && log.replica_set_id == target_replica_set) { + membership_probes++; + } + } + return membership_probes >= minimum_membership_probes; +} + +int add_member_routes( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, + const vector& route_hgs +) { + vector queries; + for (size_t i = 0; i < deployment.production.members.size(); ++i) { + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(route_hgs[i]) + "," + + aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD member route')"); + } + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + return aurora_bgd_execute_all(admin, { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +bool route_to_expected_backend( + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected_backend +) { + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS) { + return false; + } + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return false; + } + auto [rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; + if (rc != EXIT_SUCCESS) { + diag("Backend routing query failed with MySQL error %d: %s", rc, mysql_error(client)); + } + mysql_close(client); + if (rc != EXIT_SUCCESS) { + return false; + } + + auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); + if (logs_rc != EXIT_SUCCESS) { + return false; + } + const Aurora_Replica_Probe_Log* routed_probe = nullptr; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { + routed_probe = &log; + } + } + if (routed_probe == nullptr + || routed_probe->backend.host != expected_backend.host + || routed_probe->backend.port != expected_backend.port) { + diag( + "Ordinary Aurora query reached %s:%d; expected %s:%d", + routed_probe ? routed_probe->backend.host.c_str() : "", + routed_probe ? routed_probe->backend.port : 0, + expected_backend.host.c_str(), expected_backend.port); + return false; + } + return true; +} + +bool route_members_to_expected_ips( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, + const vector& route_hgs, bool target +) { + for (size_t i = 0; i < route_hgs.size(); ++i) { + if (set_default_hostgroup(admin, route_hgs[i]) != EXIT_SUCCESS) { + return false; + } + Endpoint expected_backend = target + ? deployment.target.members[i].endpoint.backend() + : deployment.production.members[i].endpoint.backend(); + if (!route_to_expected_backend(cl, sim, expected_backend)) { + return false; + } + } + return true; +} + +int64_t member_route_pool_count(MYSQL* admin, const vector& route_hgs) { + string hostgroups; + for (int hostgroup : route_hgs) { + if (!hostgroups.empty()) { + hostgroups += ","; + } + hostgroups += to_string(hostgroup); + } + auto [rc, rows] = mysql_query_ext_rows( + admin, + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool " + "WHERE hostgroup IN (" + hostgroups + ")"); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { + return -1; + } + return strtoll(rows.front().front().c_str(), nullptr, 10); +} + +int wait_for_member_route_pool_count( + MYSQL* admin, const vector& route_hgs, const string& comparison +) { + string hostgroups; + for (int hostgroup : route_hgs) { + if (!hostgroups.empty()) { + hostgroups += ","; + } + hostgroups += to_string(hostgroup); + } + return wait_for_cond( + admin, + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0)" + comparison + + " FROM stats_mysql_connection_pool WHERE hostgroup IN (" + hostgroups + ")", + kWaitSeconds); +} + +int main() { + plan(20); + + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + const int writer_hg = 1530; + const int reader_hg = 1531; + const vector route_hgs {1534, 1535, 1536}; + Aurora_BGD_Test_Deployment deployment = aurora_bgd_deployment_a(); + if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, writer_hg, reader_hg, 1532, 1533, false, 1000, false) + != EXIT_SUCCESS + || add_member_routes(admin, deployment, route_hgs) != EXIT_SUCCESS) { + diag("Error: failed to configure the active switchover scenario"); + cleanup(admin, sim); + return exit_status(); + } + + ok(aurora_bgd_wait_for_status(admin, writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "active scenario starts from a complete AVAILABLE snapshot"); + + if (publish_status(sim, deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, writer_hg, "SWITCHOVER_INITIATED", kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: worker did not enter SWITCHOVER_INITIATED"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "INITIATED is published"); + ok(wait_for_writer_placement( + admin, writer_hg, reader_hg, + deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + "INITIATED does not change writer placement"); + + auto [active_seq_rc, active_sequence] = sim.replica_probe_log_last_sequence(); + ok(active_seq_rc == EXIT_SUCCESS && fast_membership_without_ordinary( + sim, active_sequence, deployment.target_replica_set, 650, 3), + "INITIATED uses fast membership probes and suspends the ordinary Aurora query"); + + vector competing_source = deployment.production.replica_rows(); + competing_source[0].session_id = "source-observed-reader"; + competing_source[1].session_id = "MASTER_SESSION_ID"; + auto [source_change_rc, source_change_sequence] = sim.replica_probe_log_last_sequence(); + if (source_change_rc != EXIT_SUCCESS || sim.replica_update( + deployment.blue_replica_set, competing_source, deployment.production.backends()) + != EXIT_SUCCESS) { + diag("Error: failed to publish the competing source role observation"); + cleanup(admin, sim); + return exit_status(); + } + ok(fast_membership_without_ordinary( + sim, source_change_sequence, deployment.target_replica_set, 350, 2), + "changed source roles cannot compete while production probing is suspended"); + + if (publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, writer_hg, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: worker did not enter SWITCHOVER_IN_PROGRESS"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "IN_PROGRESS is published"); + ok(wait_for_writer_placement( + admin, writer_hg, reader_hg, + deployment.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, + "IN_PROGRESS demotes the snapshotted production writer"); + ok(runtime_server_count( + admin, writer_hg, deployment.production.members[1].endpoint.hostname, 0) + && runtime_server_count( + admin, reader_hg, deployment.production.members[1].endpoint.hostname, 1, "ONLINE"), + "the competing source observation does not promote a reader"); + + auto [repeat_progress_rc, repeat_progress_sequence] = sim.replica_probe_log_last_sequence(); + if (repeat_progress_rc != EXIT_SUCCESS + || publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS) { + diag("Error: failed to repeat SWITCHOVER_IN_PROGRESS"); + cleanup(admin, sim); + return exit_status(); + } + ok(fast_membership_without_ordinary( + sim, repeat_progress_sequence, deployment.target_replica_set, 350, 2) + && wait_for_writer_placement( + admin, writer_hg, reader_hg, + deployment.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, + "repeated IN_PROGRESS retains the one demotion and active probe policy"); + + ok(route_members_to_expected_ips(cl, admin, sim, deployment, route_hgs, false) + && member_route_pool_count(admin, route_hgs) + >= static_cast(route_hgs.size()), + "all member routes use source IPs and hold pre-cutover pools"); + + if (publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, writer_hg, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: worker did not enter SWITCHOVER_IN_POST_PROCESSING"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "POST_PROCESSING is published without a target-writability gate"); + ok(wait_for_writer_placement( + admin, writer_hg, reader_hg, + deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + "POST_PROCESSING restores the canonical writer placement"); + ok(runtime_server_count( + admin, reader_hg, deployment.production.members[1].endpoint.hostname, 1, "ONLINE") + && runtime_server_count( + admin, reader_hg, deployment.production.members[2].endpoint.hostname, 1, "ONLINE"), + "POST_PROCESSING leaves canonical readers ONLINE and eligible"); + ok(wait_for_member_route_pool_count(admin, route_hgs, "=0") == EXIT_SUCCESS, + "POST_PROCESSING retires the pre-cutover member pools"); + ok(route_members_to_expected_ips(cl, admin, sim, deployment, route_hgs, true), + "POST_PROCESSING pins every production hostname to its cached target IP"); + + const int64_t target_pool_count = member_route_pool_count(admin, route_hgs); + auto [repeat_post_rc, repeat_post_sequence] = sim.probe_log_last_sequence(); + if (target_pool_count < static_cast(route_hgs.size()) + || repeat_post_rc != EXIT_SUCCESS + || publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS) { + diag("Error: failed to prepare repeated POST_PROCESSING"); + cleanup(admin, sim); + return exit_status(); + } + vector target_backends = deployment.target.backends(); + auto [repeat_probe_rc, repeat_probe] = aurora_bgd_wait_for_topology_probe( + sim, repeat_post_sequence, target_backends, BGD_Probe_Kind::metadata, + kProbeTimeoutMs); + ok(repeat_probe_rc == EXIT_SUCCESS + && member_route_pool_count(admin, route_hgs) >= target_pool_count, + "repeated POST_PROCESSING does not replay completed member retirement"); + + auto [post_seq_rc, post_sequence] = sim.replica_probe_log_last_sequence(); + ok(post_seq_rc == EXIT_SUCCESS && fast_membership_without_ordinary( + sim, post_sequence, deployment.target_replica_set, 350, 2), + "POST_PROCESSING keeps fast BGD probes without ordinary Aurora queries"); + + if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to reset before the incomplete-snapshot scenario"); + cleanup(admin, sim); + return exit_status(); + } + + const int gated_writer_hg = 1540; + const int gated_reader_hg = 1541; + const int gated_route_hg = 1544; + Aurora_BGD_Test_Deployment gated = aurora_bgd_deployment_b_writer_only(); + vector invalid_target = gated.target.replica_rows(); + invalid_target.front().server_id = "unpaired-target-writer"; + if (sim.replica_update( + gated.blue_replica_set, gated.production.replica_rows(), gated.production.backends()) + != EXIT_SUCCESS + || sim.replica_update( + gated.target_replica_set, invalid_target, gated.target.backends()) + != EXIT_SUCCESS + || sim.topology_update( + aurora_bgd_topology_backends(gated), aurora_bgd_available_topology(gated)) + != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, gated, gated_writer_hg, gated_reader_hg, 1542, 1543, false, 1000, false) + != EXIT_SUCCESS + || add_member_routes(admin, gated, {gated_route_hg}) != EXIT_SUCCESS) { + diag("Error: failed to configure incomplete target membership"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, gated_writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "an incomplete target snapshot can publish AVAILABLE without routing"); + if (publish_status(sim, gated, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, gated_writer_hg, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: incomplete scenario did not publish POST_PROCESSING"); + cleanup(admin, sim); + return exit_status(); + } + const int gated_default_rc = set_default_hostgroup(admin, gated_route_hg); + ok(gated_default_rc == EXIT_SUCCESS && route_to_expected_backend( + cl, sim, gated.production.members.front().endpoint.backend()), + "POST_PROCESSING leaves routing unchanged without a complete target snapshot"); + + auto [gated_refresh_seq_rc, gated_refresh_sequence] = sim.replica_probe_log_last_sequence(); + if (gated_refresh_seq_rc != EXIT_SUCCESS || sim.replica_update( + gated.target_replica_set, gated.target.replica_rows(), gated.target.backends()) + != EXIT_SUCCESS) { + diag("Error: failed to restore complete target membership"); + cleanup(admin, sim); + return exit_status(); + } + auto [gated_probe_rc, gated_probe] = aurora_bgd_wait_for_replica_probe( + sim, gated_refresh_sequence, gated.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, gated.target_replica_set); + bool gated_target_routing = false; + if (gated_probe_rc == EXIT_SUCCESS + && wait_for_member_route_pool_count(admin, {gated_route_hg}, "=0") == EXIT_SUCCESS + && set_default_hostgroup(admin, gated_route_hg) == EXIT_SUCCESS) { + gated_target_routing = route_to_expected_backend( + cl, sim, gated.target.members.front().endpoint.backend()); + } + ok(gated_target_routing, + "repeated POST_PROCESSING applies routing after membership becomes complete"); + + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD active-state test data"); + return EXIT_FAILURE; + } + return exit_status(); +} From 931df1e4d43d8194e61732da1a1d6288add460b3 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sat, 15 Aug 2026 18:58:41 +0000 Subject: [PATCH 25/34] feat: complete Aurora BGD switchover --- include/MySQL_Monitor.hpp | 14 + lib/MySQL_Monitor.cpp | 144 ++++- test/tap/groups/groups.json | 1 + test/tap/tap/aurora_bgd_tap.h | 14 + .../tests/test_aurora_bgd_completion-t.cpp | 549 ++++++++++++++++++ 5 files changed, 702 insertions(+), 20 deletions(-) create mode 100644 test/tap/tests/test_aurora_bgd_completion-t.cpp diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index b4fb0798f5..6727c6db0d 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -818,6 +818,20 @@ class MySQL_Monitor { */ void aws_aurora_bgd_apply_active_actions( AWS_Aurora_BGD_State& st, bool status_changed); + /** + * @brief Run immediate effect-driven cleanup for TARGET completion. + * + * @details Reconciles writer placement, removes mapped traffic pins, drains + * eligible configured green pools, resumes production probing, and enters + * the completed latch while retaining the deployment fingerprint. + */ + void aws_aurora_bgd_apply_completion( + AWS_Aurora_BGD_State& st, + const AWS_Aurora_BGD_Fingerprint& completed_fingerprint); + /** + * @brief Release the completed latch after a successful topology drain. + */ + void aws_aurora_bgd_release_completed_latch(AWS_Aurora_BGD_State& st); /** * @brief Run the topology and target-membership probes owned by an Aurora worker. * diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 094066ef64..1504b1cdf7 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6542,6 +6542,91 @@ void MySQL_Monitor::aws_aurora_bgd_apply_active_actions( } } +void MySQL_Monitor::aws_aurora_bgd_apply_completion( + AWS_Aurora_BGD_State& st, + const AWS_Aurora_BGD_Fingerprint& completed_fingerprint +) { + if (st.status == AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED + && st.fingerprint == completed_fingerprint) { + return; + } + + auto writer = std::find_if( + st.production_members.begin(), st.production_members.end(), + [](const AWS_Aurora_BGD_Member& member) { return member.is_writer; }); + if (writer != st.production_members.end()) { + MyHGM->update_aws_aurora_set_writer( + st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str())); + } + + for (const AWS_Aurora_BGD_Member& member : st.target_members) { + dns_cache->remove(member.production_hostname); + My_Conn_Pool->purge_connections(member.production_hostname.c_str(), member.port); + } + + struct green_server_t { + int hostgroup; + std::string hostname; + int port; + }; + std::vector green_servers; + MyHGM->wrlock(); + for (int hostgroup : {st.green_writer_hg, st.green_reader_hg}) { + if (hostgroup < 0) { + continue; + } + MyHGC* hgc = MyHGM->MyHGC_find(hostgroup); + if (hgc == nullptr || hgc->mysrvs == nullptr) { + continue; + } + for (unsigned int i = 0; i < hgc->mysrvs->cnt(); ++i) { + MySrvC* server = hgc->mysrvs->idx(i); + if (server->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT + || server->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + continue; + } + green_servers.push_back({hostgroup, server->address, server->port}); + MyHGM->drain_server_connections(server->address, server->port); + } + } + MyHGM->wrunlock(); + + for (const green_server_t& server : green_servers) { + dns_cache->remove(server.hostname); + My_Conn_Pool->purge_connections(server.hostname.c_str(), server.port); + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: connections drained from green HG %d server '%s:%d'\n", + st.writer_hg, st.reader_hg, server.hostgroup, + server.hostname.c_str(), server.port); + } + + st.fingerprint = completed_fingerprint; + st.production_snapshot_frozen = false; + st.production_probe_suspended = false; + st.target_snapshot_complete = false; + st.target_members.clear(); + aws_aurora_bgd_set_status(st, AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED); + + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: switchover cleanup complete; completion latched\n", + st.writer_hg, st.reader_hg); +} + +void MySQL_Monitor::aws_aurora_bgd_release_completed_latch(AWS_Aurora_BGD_State& st) { + if (st.status != AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED) { + return; + } + + st.fingerprint = AWS_Aurora_BGD_Fingerprint {}; + st.target_members.clear(); + st.target_snapshot_complete = false; + aws_aurora_bgd_set_status(st, AWS_Aurora_BGD_Status::NONE); + + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: topology drained; completion latch released\n", + st.writer_hg, st.reader_hg); +} + void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) { const bool configured_green_hgs = st.green_writer_hg >= 0 && st.green_reader_hg >= 0; const bool discovery_admitted = configured_green_hgs @@ -6564,9 +6649,12 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query( topology_host, st.writer_hg, st.check_timeout_ms, MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK); - if (query.rc == 0 && query.mmsd->result - && mysql_num_rows(query.mmsd->result) > 0) { - st.topology_state = TOPOLOGY_METADATA_FETCH; + if (query.rc == 0 && query.mmsd->result) { + if (mysql_num_rows(query.mmsd->result) > 0) { + st.topology_state = TOPOLOGY_METADATA_FETCH; + } else { + aws_aurora_bgd_release_completed_latch(st); + } } } else { AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query( @@ -6576,27 +6664,40 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) if (query.mysql_error == 1146) { st.topology_state = TOPOLOGY_TABLE_CHECK; } - } else if (query.mmsd->result && mysql_num_rows(query.mmsd->result) > 0) { - AWS_RDS_Topology_Result topology = parse_aws_rds_topology(query.mmsd->result); - AWS_Aurora_BGD_Topology_Observation observation = - aws_aurora_bgd_validate_topology(topology); - if (observation.valid && !observation.completed) { - const bool same_deployment = st.fingerprint.empty() - || st.fingerprint == observation.fingerprint; - const bool rollback_transition = - static_cast(observation.status) < static_cast(st.status); - if (same_deployment && !rollback_transition) { - const bool status_changed = st.status != observation.status; - st.fingerprint = observation.fingerprint; - st.target_use_ssl = topology_host.use_ssl; - aws_aurora_bgd_set_status(st, observation.status); - aws_aurora_bgd_apply_active_actions(st, status_changed); + } else if (query.mmsd->result) { + if (mysql_num_rows(query.mmsd->result) == 0) { + aws_aurora_bgd_release_completed_latch(st); + } else { + AWS_RDS_Topology_Result topology = parse_aws_rds_topology(query.mmsd->result); + AWS_Aurora_BGD_Topology_Observation observation = + aws_aurora_bgd_validate_topology(topology); + if (observation.valid && observation.completed) { + const bool same_active_deployment = st.fingerprint.empty() + || st.fingerprint == observation.fingerprint + || st.status == AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED; + if (same_active_deployment) { + aws_aurora_bgd_apply_completion(st, observation.fingerprint); + } + } else if (observation.valid) { + const bool same_deployment = st.fingerprint.empty() + || st.fingerprint == observation.fingerprint; + const bool rollback_transition = + static_cast(observation.status) < static_cast(st.status); + if (same_deployment && !rollback_transition) { + const bool status_changed = st.status != observation.status; + st.fingerprint = observation.fingerprint; + st.target_use_ssl = topology_host.use_ssl; + aws_aurora_bgd_set_status(st, observation.status); + aws_aurora_bgd_apply_active_actions(st, status_changed); + } } } } } - if (st.status == AWS_Aurora_BGD_Status::NONE || st.fingerprint.empty()) { + if (st.status == AWS_Aurora_BGD_Status::NONE + || st.status == AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED + || st.fingerprint.empty()) { return; } @@ -7154,7 +7255,6 @@ void * MySQL_Monitor::monitor_aws_aurora() { pthread_mutex_unlock(&aws_aurora_mutex); if (new_raw_checksum != last_raw_checksum) { proxy_info("Detected new/changed definition for AWS Aurora monitoring\n"); - last_raw_checksum = new_raw_checksum; if (pthreads_array) { // wait all threads to terminate for (unsigned int i=0; i < hgs_num; i++) { @@ -7168,6 +7268,10 @@ void * MySQL_Monitor::monitor_aws_aurora() { } hgs_num = 0; pthread_mutex_lock(&aws_aurora_mutex); + // Configuration can change while the previous workers are being joined. + // Launch replacements from the current snapshot and retain that checksum, + // so the coordinator never waits on a worker created from newer state. + last_raw_checksum = AWS_Aurora_Hosts_resultset->raw_checksum(); // scan all the writer HGs unsigned int num_rows = AWS_Aurora_Hosts_resultset->rows_count; if (num_rows) { diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index b06d7f0ad2..9afa842be7 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -365,6 +365,7 @@ "test_cluster_sim_aurora-t" : [ "cluster_sim_aurora-g1" ], "test_aurora_bgd_discovery-t" : [ "cluster_sim_aurora-g1" ], "test_aurora_bgd_active-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_completion-t" : [ "cluster_sim_aurora-g1" ], "test_cluster_sim_galera-t" : [ "cluster_sim_galera-g1" ], "test_cluster_sim_group_repl-t" : [ "cluster_sim_group_repl-g1" ], "test_cluster_sim_read_only-t" : [ "cluster_sim_read_only-g1" ], diff --git a/test/tap/tap/aurora_bgd_tap.h b/test/tap/tap/aurora_bgd_tap.h index cfc7aa78fd..a38cb4bb5f 100644 --- a/test/tap/tap/aurora_bgd_tap.h +++ b/test/tap/tap/aurora_bgd_tap.h @@ -222,6 +222,20 @@ inline vector aurora_bgd_topology( return rows; } +inline vector aurora_bgd_completed_topology( + Aurora_BGD_Test_Deployment& deployment +) { + return { + { + deployment.target_topology_id, + deployment.target_cluster_endpoint.hostname, + deployment.target_cluster_endpoint.port, + "BLUE_GREEN_DEPLOYMENT_TARGET", + "SWITCHOVER_COMPLETED", + }, + }; +} + inline vector aurora_bgd_topology_backends(Aurora_BGD_Test_Deployment& deployment) { vector backends = deployment.production.backends(); vector target_backends = deployment.target.backends(); diff --git a/test/tap/tests/test_aurora_bgd_completion-t.cpp b/test/tap/tests/test_aurora_bgd_completion-t.cpp new file mode 100644 index 0000000000..fdc22e1cbe --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_completion-t.cpp @@ -0,0 +1,549 @@ +/** + * @file test_aurora_bgd_completion-t.cpp + * @brief Aurora BGD completion cleanup and terminal-latch behavior. + */ + +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const uint32_t kProbeTimeoutMs = 5000; +const char kOrdinaryAuroraQuery[] = + "SELECT SERVER_ID," + "IF(" + "SESSION_ID = 'MASTER_SESSION_ID' AND " + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " + "'probably_former_MASTER_SESSION_ID', SESSION_ID" + ") SESSION_ID, " + "LAST_UPDATE_TIMESTAMP, " + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " + "CPU " + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" + " ( " + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" + " OR SESSION_ID = 'MASTER_SESSION_ID'" + " ) " + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" + " ORDER BY SERVER_ID"; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + char simulator_username[] = "aurora1"; + char simulator_password[] = "pass1"; + if (sim.connect(cl.host, 3306, simulator_username, simulator_password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the shared AWS simulator"); + mysql_close(admin); + admin = nullptr; + return EXIT_FAILURE; + } + if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to clear prior Aurora BGD state"); + return EXIT_FAILURE; + } + return aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " + "VALUES ('testuser','testuser',1,0,1)", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { + return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS + && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int reset_rc = reset_scenario(admin, sim); + int user_rc = aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); + mysql_close(admin); + return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int publish_status( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)); +} + +int publish_completed( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& serving_deployment, + Aurora_BGD_Test_Deployment& completed_deployment +) { + return sim.topology_update( + aurora_bgd_topology_backends(serving_deployment), + aurora_bgd_completed_topology(completed_deployment)); +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + return aurora_bgd_execute_all(admin, { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int add_route( + MYSQL* admin, int hostgroup, const string& hostname, const string& status = "ONLINE" +) { + return aurora_bgd_execute_all(admin, { + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(hostgroup) + "," + aurora_bgd_sql_quote(hostname) + + ",3306," + aurora_bgd_sql_quote(status) + ",'Aurora BGD completion route')", + "LOAD MYSQL SERVERS TO RUNTIME", + }); +} + +int add_member_routes( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, const vector& hostgroups +) { + vector queries; + for (size_t i = 0; i < hostgroups.size(); ++i) { + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(hostgroups[i]) + "," + + aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD completion member route')"); + } + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +int add_green_servers( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, + int green_writer_hg, int green_reader_hg +) { + vector queries; + for (size_t i = 0; i < deployment.target.members.size(); ++i) { + const int hostgroup = i == 0 ? green_writer_hg : green_reader_hg; + const string status = i + 1 == deployment.target.members.size() + ? "OFFLINE_SOFT" : "ONLINE"; + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(hostgroup) + "," + + aurora_bgd_sql_quote(deployment.target.members[i].endpoint.hostname) + + ",3306," + aurora_bgd_sql_quote(status) + + ",'Aurora BGD configured green member')"); + } + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +bool route_to_expected_backend( + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected_backend +) { + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS) { + return false; + } + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return false; + } + auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; + if (query_rc != EXIT_SUCCESS) { + diag("Backend routing query failed with MySQL error %d: %s", + mysql_errno(client), mysql_error(client)); + } + mysql_close(client); + if (query_rc != EXIT_SUCCESS) { + return false; + } + + auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); + if (logs_rc != EXIT_SUCCESS) { + return false; + } + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary + && log.backend.host == expected_backend.host + && log.backend.port == expected_backend.port) { + return true; + } + } + return false; +} + +bool route_members( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, const vector& hostgroups, bool target +) { + for (size_t i = 0; i < hostgroups.size(); ++i) { + if (set_default_hostgroup(admin, hostgroups[i]) != EXIT_SUCCESS) { + return false; + } + const Endpoint expected = target + ? deployment.target.members[i].endpoint.backend() + : deployment.production.members[i].endpoint.backend(); + if (!route_to_expected_backend(cl, sim, expected)) { + return false; + } + } + return true; +} + +int64_t pool_count(MYSQL* admin, int hostgroup) { + auto [rc, rows] = mysql_query_ext_rows( + admin, + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool " + "WHERE hostgroup=" + to_string(hostgroup)); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { + return -1; + } + return strtoll(rows.front().front().c_str(), nullptr, 10); +} + +int wait_for_pool_count(MYSQL* admin, int hostgroup, const string& comparison) { + return wait_for_cond( + admin, + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0)" + comparison + + " FROM stats_mysql_connection_pool WHERE hostgroup=" + to_string(hostgroup), + kWaitSeconds); +} + +bool server_count( + MYSQL* admin, int hostgroup, const string& hostname, int expected, + const string& status = "" +) { + string query = + "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(hostgroup) + " AND hostname=" + aurora_bgd_sql_quote(hostname); + if (!status.empty()) { + query += " AND status=" + aurora_bgd_sql_quote(status); + } + auto [rc, rows] = mysql_query_ext_rows(admin, query); + return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 + && rows.front().front() == to_string(expected); +} + +int wait_for_writer_policy( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, + bool writer_is_also_reader +) { + return wait_for_cond( + admin, + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=1) AND ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (writer_is_also_reader ? "1" : "0") + ")", + kWaitSeconds); +} + +int wait_for_writer_demotion( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname +) { + return wait_for_cond( + admin, + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=0) AND ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=1)", + kWaitSeconds); +} + +bool completion_probe_policy( + BGD_Simulator& sim, uint64_t sequence, + Aurora_BGD_Test_Deployment& deployment, uint32_t observation_ms +) { + usleep(observation_ms * 1000); + auto [rc, logs] = sim.replica_probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return false; + } + bool ordinary_on_production = false; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership) { + return false; + } + if (log.probe_kind != Aurora_Replica_Probe_Kind::ordinary) { + continue; + } + for (const Endpoint& backend : deployment.production.backends()) { + ordinary_on_production |= log.backend.host == backend.host + && log.backend.port == backend.port; + } + } + return ordinary_on_production; +} + +bool wait_for_topology_observation( + BGD_Simulator& sim, uint64_t sequence, Aurora_BGD_Test_Deployment& deployment +) { + auto [rc, probe] = aurora_bgd_wait_for_topology_probe( + sim, sequence, deployment.production.backends(), + BGD_Probe_Kind::metadata, kProbeTimeoutMs); + return rc == EXIT_SUCCESS; +} + +int main() { + plan(28); + + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + // Completion from IN_PROGRESS reconciles the writer without replaying POST_PROCESSING. + Aurora_BGD_Test_Deployment progress = aurora_bgd_deployment_b_writer_only(); + const int progress_writer_hg = 1550; + const int progress_reader_hg = 1551; + if (aurora_bgd_publish(sim, progress) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, progress, progress_writer_hg, progress_reader_hg, + 1552, 1553, false, 300, false) != EXIT_SUCCESS) { + diag("Error: failed to configure completion-from-IN_PROGRESS scenario"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, progress_writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "completion-from-IN_PROGRESS scenario reaches AVAILABLE"); + if (publish_status(sim, progress, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, progress_writer_hg, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to reach IN_PROGRESS"); + cleanup(admin, sim); + return exit_status(); + } + ok(wait_for_writer_demotion( + admin, progress_writer_hg, progress_reader_hg, + progress.production.members.front().endpoint.hostname) == EXIT_SUCCESS, + "IN_PROGRESS demotes the writer before completion"); + if (publish_completed(sim, progress, progress) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, progress_writer_hg, "SWITCHOVER_COMPLETED", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to enter the completed latch from IN_PROGRESS"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "TARGET-only completion publishes SWITCHOVER_COMPLETED"); + ok(wait_for_writer_policy( + admin, progress_writer_hg, progress_reader_hg, + progress.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + "completion restores the demoted writer without replaying POST_PROCESSING"); + auto [progress_replica_seq_rc, progress_replica_sequence] = + sim.replica_probe_log_last_sequence(); + ok(progress_replica_seq_rc == EXIT_SUCCESS && completion_probe_policy( + sim, progress_replica_sequence, progress, 750), + "completion resumes ordinary production probing and stops membership probing"); + auto [repeat_progress_seq_rc, repeat_progress_sequence] = sim.probe_log_last_sequence(); + bool repeated_progress = repeat_progress_seq_rc == EXIT_SUCCESS + && publish_completed(sim, progress, progress) == EXIT_SUCCESS + && wait_for_topology_observation(sim, repeat_progress_sequence, progress); + ok(repeated_progress && wait_for_writer_policy( + admin, progress_writer_hg, progress_reader_hg, + progress.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + "repeated completion is a no-op while latched"); + auto [error_seq_rc, error_sequence] = sim.probe_log_last_sequence(); + bool error_retained = error_seq_rc == EXIT_SUCCESS + && sim.topology_error( + aurora_bgd_topology_backends(progress), 1205, "simulated completion timeout") + == EXIT_SUCCESS + && wait_for_topology_observation(sim, error_sequence, progress); + ok(error_retained && aurora_bgd_wait_for_status( + admin, progress_writer_hg, "SWITCHOVER_COMPLETED", 1) == EXIT_SUCCESS, + "topology query errors retain the completed latch"); + ok(sim.topology_drop(aurora_bgd_topology_backends(progress)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, progress_writer_hg, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "confirmed topology absence releases the completed latch to NONE"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before POST_PROCESSING completion scenario"); + cleanup(admin, sim); + return exit_status(); + } + + // Completion after POST_PROCESSING removes pins and drains configured green pools once. + Aurora_BGD_Test_Deployment post = aurora_bgd_deployment_a(); + const int post_writer_hg = 1560; + const int post_reader_hg = 1561; + const int green_writer_hg = 1562; + const int green_reader_hg = 1563; + const vector route_hgs {1564, 1565, 1566}; + if (aurora_bgd_publish(sim, post) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, post, post_writer_hg, post_reader_hg, + green_writer_hg, green_reader_hg, false, 300, true) != EXIT_SUCCESS + || add_green_servers(admin, post, green_writer_hg, green_reader_hg) != EXIT_SUCCESS + || add_member_routes(admin, post, route_hgs) != EXIT_SUCCESS) { + diag("Error: failed to configure POST_PROCESSING completion scenario"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, post_writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "POST_PROCESSING completion scenario reaches AVAILABLE"); + if (publish_status(sim, post, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, post_writer_hg, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) + != EXIT_SUCCESS + || publish_status(sim, post, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, post_writer_hg, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to advance through POST_PROCESSING"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "active deployment advances through POST_PROCESSING"); + ok(route_members(cl, admin, sim, post, route_hgs, true), + "POST_PROCESSING routes every production member to its target IP"); + const int64_t target_route_pool = pool_count(admin, route_hgs.front()); + bool green_pool_ready = set_default_hostgroup(admin, green_writer_hg) == EXIT_SUCCESS + && route_to_expected_backend(cl, sim, post.target.members.front().endpoint.backend()) + && pool_count(admin, green_writer_hg) >= 1; + ok(target_route_pool >= 1 && green_pool_ready, + "pre-completion target and configured-green pools are established"); + if (publish_completed(sim, post, post) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, post_writer_hg, "SWITCHOVER_COMPLETED", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to complete the POST_PROCESSING scenario"); + cleanup(admin, sim); + return exit_status(); + } + ok(true, "completion after POST_PROCESSING enters the terminal latch"); + ok(wait_for_writer_policy( + admin, post_writer_hg, post_reader_hg, + post.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, + "completion preserves canonical writer_is_also_reader placement"); + ok(pool_count(admin, route_hgs.front()) >= target_route_pool, + "completion does not repeat retirement of post-cutover production pools"); + ok(wait_for_pool_count(admin, green_writer_hg, "=0") == EXIT_SUCCESS, + "completion drains eligible configured-green pools immediately"); + ok(server_count( + admin, green_reader_hg, + post.target.members.back().endpoint.hostname, 1, "OFFLINE_SOFT"), + "completion preserves configured green rows and OFFLINE status"); + const int post_completion_route_hg = 1567; + ok(add_route( + admin, post_completion_route_hg, + post.production.members.front().endpoint.hostname) == EXIT_SUCCESS + && set_default_hostgroup(admin, post_completion_route_hg) == EXIT_SUCCESS + && route_to_expected_backend( + cl, sim, post.production.members.front().endpoint.backend()), + "completion removes the production traffic pin without DNS verification"); + auto [post_replica_seq_rc, post_replica_sequence] = sim.replica_probe_log_last_sequence(); + ok(post_replica_seq_rc == EXIT_SUCCESS && completion_probe_policy( + sim, post_replica_sequence, post, 750), + "the completed latch uses configured cadence and canonical production probes"); + bool recreated_green_pool = set_default_hostgroup(admin, green_writer_hg) == EXIT_SUCCESS + && route_to_expected_backend(cl, sim, post.target.members.front().endpoint.backend()) + && pool_count(admin, green_writer_hg) >= 1; + auto [same_completed_seq_rc, same_completed_sequence] = sim.probe_log_last_sequence(); + bool same_completion_seen = same_completed_seq_rc == EXIT_SUCCESS + && publish_completed(sim, post, post) == EXIT_SUCCESS + && wait_for_topology_observation(sim, same_completed_sequence, post); + ok(recreated_green_pool && same_completion_seen && pool_count(admin, green_writer_hg) >= 1, + "repeated completion does not drain a pool created while latched"); + auto [latched_error_seq_rc, latched_error_sequence] = sim.probe_log_last_sequence(); + bool latched_error_seen = latched_error_seq_rc == EXIT_SUCCESS + && sim.topology_error( + aurora_bgd_topology_backends(post), 1205, "simulated latched timeout") + == EXIT_SUCCESS + && wait_for_topology_observation(sim, latched_error_sequence, post); + ok(latched_error_seen && aurora_bgd_wait_for_status( + admin, post_writer_hg, "SWITCHOVER_COMPLETED", 1) == EXIT_SUCCESS + && pool_count(admin, green_writer_hg) >= 1, + "query errors neither release the latch nor repeat completion cleanup"); + Aurora_BGD_Test_Deployment different = aurora_bgd_deployment_b_writer_only(); + auto [different_seq_rc, different_sequence] = sim.probe_log_last_sequence(); + bool different_seen = different_seq_rc == EXIT_SUCCESS + && publish_completed(sim, post, different) == EXIT_SUCCESS + && wait_for_topology_observation(sim, different_sequence, post); + ok(different_seen && wait_for_pool_count(admin, green_writer_hg, "=0") == EXIT_SUCCESS, + "a different completed deployment fingerprint rearms and runs its cleanup"); + bool second_green_pool = set_default_hostgroup(admin, green_writer_hg) == EXIT_SUCCESS + && route_to_expected_backend(cl, sim, post.target.members.front().endpoint.backend()); + auto [repeat_different_seq_rc, repeat_different_sequence] = sim.probe_log_last_sequence(); + bool repeated_different = repeat_different_seq_rc == EXIT_SUCCESS + && publish_completed(sim, post, different) == EXIT_SUCCESS + && wait_for_topology_observation(sim, repeat_different_sequence, post); + ok(second_green_pool && repeated_different && pool_count(admin, green_writer_hg) >= 1, + "the new fingerprint is retained and its repeated completion is a no-op"); + ok(sim.topology_delete(aurora_bgd_topology_backends(post)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, post_writer_hg, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "successful empty topology releases the rearmed completed latch"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before direct-completion scenario"); + cleanup(admin, sim); + return exit_status(); + } + + // A first observation at completion has no target map and does not replay prior phases. + Aurora_BGD_Test_Deployment direct = aurora_bgd_deployment_b_writer_only(); + if (sim.replica_update( + direct.blue_replica_set, direct.production.replica_rows(), + direct.production.backends()) != EXIT_SUCCESS + || sim.topology_update( + aurora_bgd_topology_backends(direct), + aurora_bgd_completed_topology(direct)) != EXIT_SUCCESS) { + diag("Error: failed to publish direct completion inputs"); + cleanup(admin, sim); + return exit_status(); + } + auto [direct_seq_rc, direct_sequence] = sim.replica_probe_log_last_sequence(); + if (aurora_bgd_admin_setup( + admin, direct, 1580, 1581, -1, -1, true, 300, false) != EXIT_SUCCESS) { + diag("Error: failed to configure direct completion"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, 1580, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS, + "late entry directly at completion enters the terminal latch"); + ok(wait_for_writer_policy( + admin, 1580, 1581, + direct.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + "direct completion leaves canonical writer placement unchanged"); + auto [direct_logs_rc, direct_logs] = sim.replica_probe_log_since(direct_sequence); + bool direct_membership_probe = false; + for (const Aurora_Replica_Probe_Log& log : direct_logs) { + direct_membership_probe |= + log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership; + } + ok(direct_seq_rc == EXIT_SUCCESS && direct_logs_rc == EXIT_SUCCESS + && !direct_membership_probe, + "direct completion does not manufacture target membership or replay active phases"); + ok(sim.topology_delete(aurora_bgd_topology_backends(direct)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1580, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "direct-completion latch rearms only after topology drain"); + + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD completion test data"); + return EXIT_FAILURE; + } + return exit_status(); +} From 561bff5bdb54c65644cdddf518be9d0d7718110f Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sat, 15 Aug 2026 20:05:47 +0000 Subject: [PATCH 26/34] feat: harden Aurora BGD lifecycle handling --- include/MySQL_HostGroups_Manager.h | 4 + include/MySQL_Monitor.hpp | 67 ++- lib/MySQL_HostGroups_Manager.cpp | 30 +- lib/MySQL_Monitor.cpp | 560 +++++++++++------- test/tap/groups/cluster_sim_aurora/add-hosts | 5 + test/tap/groups/groups.json | 3 + test/tap/tap/aurora_bgd_tap.h | 32 + .../tests/test_aurora_bgd_cluster_sync-t.cpp | 306 ++++++++++ .../tap/tests/test_aurora_bgd_lifecycle-t.cpp | 397 +++++++++++++ .../tests/test_aurora_bgd_resilience-t.cpp | 412 +++++++++++++ 10 files changed, 1578 insertions(+), 238 deletions(-) create mode 100644 test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_lifecycle-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_resilience-t.cpp diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 3ba9955883..bd81cffc35 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -1227,6 +1227,10 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { //void update_aws_aurora_set_reader(int _whid, int _rhid, char *_hostname, int _port); bool aws_aurora_replication_lag_action(int _whid, int _rhid, char *server_id, float current_replication_lag_ms, bool enable, bool is_writer, bool verbose=true); void update_aws_aurora_set_writer(int _whid, int _rhid, char *server_id, bool verbose=true); + void update_aws_aurora_set_writer( + int _whid, int _rhid, char *server_id, bool verbose, + const char *domain_name, int aurora_port, int writer_is_also_reader, + int new_reader_weight); void update_aws_aurora_set_reader(int _whid, int _rhid, char *server_id); /** * @brief Updates the resultset and corresponding checksum used by Monitor for AWS Aurora. diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 6727c6db0d..4a2637f61a 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -576,6 +576,40 @@ enum class AWS_Aurora_BGD_Status { SWITCHOVER_COMPLETED, }; +/** + * @brief Column positions in `AWS_Aurora_Hosts_resultset`. + */ +enum AWS_Aurora_Hosts_Column { + AWS_AURORA_WRITER_HOSTGROUP = 0, + AWS_AURORA_READER_HOSTGROUP, + AWS_AURORA_HOSTNAME, + AWS_AURORA_PORT, + AWS_AURORA_USE_SSL, + AWS_AURORA_MAX_LAG_MS, + AWS_AURORA_CHECK_INTERVAL_MS, + AWS_AURORA_CHECK_TIMEOUT_MS, + AWS_AURORA_ADD_LAG_MS, + AWS_AURORA_MIN_LAG_MS, + AWS_AURORA_LAG_NUM_CHECKS, + AWS_AURORA_AUTOPURGE_MISSING_CHECKS, + AWS_AURORA_DOMAIN_NAME, + AWS_AURORA_GREEN_WRITER_HOSTGROUP, + AWS_AURORA_GREEN_READER_HOSTGROUP, + AWS_AURORA_WRITER_IS_ALSO_READER, + AWS_AURORA_NEW_READER_WEIGHT, + AWS_AURORA_HOSTS_COLUMNS, +}; + +/** + * @brief Monitor worker owned by one Aurora writer hostgroup. + */ +struct AWS_Aurora_BGD_Worker { + int writer_hg = 0; + pthread_t thread {}; + std::atomic_bool worker_stop {false}; + std::atomic current_checksum {0}; +}; + /** * @brief Stable identity of one Aurora blue/green deployment. */ @@ -619,8 +653,15 @@ struct AWS_Aurora_BGD_State { unsigned int reader_hg = 0; int green_writer_hg = -1; int green_reader_hg = -1; + unsigned int max_lag_ms = 0; unsigned int check_interval_ms = 0; unsigned int check_timeout_ms = 0; + unsigned int add_lag_ms = 0; + unsigned int min_lag_ms = 0; + unsigned int lag_num_checks = 1; + unsigned int autopurge_missing_checks = 0; + int writer_is_also_reader = 0; + int new_reader_weight = 1; int target_use_ssl = 0; std::string domain_name; @@ -802,6 +843,19 @@ class MySQL_Monitor { void * monitor_group_replication_2(); void * monitor_galera(); void * monitor_aws_aurora(); + /** + * @brief Load the configuration rows for one Aurora worker. + * + * @return true when the rows still match the coordinator-provided checksum. + */ + bool aws_aurora_bgd_load_worker_config( + int writer_hg, uint64_t current_checksum, AWS_Aurora_BGD_State& candidate); + /** + * @brief Refresh only configuration-derived fields of a running Aurora worker. + */ + bool aws_aurora_bgd_refresh_worker_config( + AWS_Aurora_BGD_State& st, uint64_t current_checksum, + unsigned long long& next_loop_at); /** * @brief Refresh the Aurora BGD worker's last complete production snapshot. * @@ -828,6 +882,16 @@ class MySQL_Monitor { void aws_aurora_bgd_apply_completion( AWS_Aurora_BGD_State& st, const AWS_Aurora_BGD_Fingerprint& completed_fingerprint); + /** + * @brief Reverse applied pre-completion effects and enter a safe earlier state. + * + * @details Removes only applied traffic pins, drains affected production + * pools, restores canonical writer placement, resumes ordinary probing, + * and optionally clears the deployment identity. + */ + void aws_aurora_bgd_apply_rollback( + AWS_Aurora_BGD_State& st, AWS_Aurora_BGD_Status next_status, + bool clear_deployment); /** * @brief Release the completed latch after a successful topology drain. */ @@ -839,7 +903,8 @@ class MySQL_Monitor { * validates topology before publishing status and replaces target membership * only with a complete, unambiguous, fully resolved snapshot. */ - void aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st); + void aws_aurora_bgd_run_discovery_cycle( + AWS_Aurora_BGD_State& st, std::atomic_bool& worker_stop); /** * @brief AWS RDS BGD monitor thread entry point. * diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 17ca164329..fd572ef66c 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -7196,6 +7196,15 @@ int MySQL_HostGroups_Manager::remove_server_in_hg(uint32_t hid, const string& ad // FIXME: complete this!! void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid, char *_server_id, bool verbose) { + update_aws_aurora_set_writer( + _whid, _rhid, _server_id, verbose, nullptr, -1, -1, -1); +} + +void MySQL_HostGroups_Manager::update_aws_aurora_set_writer( + int _whid, int _rhid, char *_server_id, bool verbose, + const char *domain_name_override, int aurora_port_override, + int writer_is_also_reader_override, int new_reader_weight_override +) { int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; @@ -7203,7 +7212,7 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid char *q=NULL; char *error=NULL; //q=(char *)"SELECT hostgroup_id FROM mysql_servers JOIN mysql_galera_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup OR hostgroup_id=backup_writer_hostgroup OR hostgroup_id=offline_hostgroup WHERE hostname='%s' AND port=%d AND status<>3"; - q=(char *)"SELECT hostgroup_id FROM mysql_servers JOIN mysql_aws_aurora_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE hostname='%s%s' AND port=%d AND status<>3 AND hostgroup_id IN (%d, %d)"; + q=(char *)"SELECT hostgroup_id FROM mysql_servers WHERE hostname='%s%s' AND port=%d AND status<>3 AND hostgroup_id IN (%d, %d)"; int writer_is_also_reader=0; int new_reader_weight = 1; @@ -7213,7 +7222,14 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid int aurora_port = 3306; char *domain_name = strdup((char *)""); int read_HG=-1; - { + if (domain_name_override != nullptr) { + free(domain_name); + domain_name = strdup(domain_name_override); + aurora_port = aurora_port_override; + writer_is_also_reader = writer_is_also_reader_override; + new_reader_weight = new_reader_weight_override; + read_HG = _rhid; + } else { pthread_mutex_lock(&AWS_Aurora_Info_mutex); std::map::iterator it2; it2 = AWS_Aurora_Info_Map.find(_writer_hostgroup); @@ -7317,10 +7333,10 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid char *q1 = NULL; char *q2 = NULL; char *error=NULL; - q1 = (char *)"SELECT DISTINCT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, mysql_servers.comment FROM mysql_servers JOIN mysql_aws_aurora_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE writer_hostgroup=%d ORDER BY hostgroup_id, hostname, port"; - q2 = (char *)"SELECT DISTINCT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, mysql_servers_incoming.comment FROM mysql_servers_incoming JOIN mysql_aws_aurora_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE writer_hostgroup=%d ORDER BY hostgroup_id, hostname, port"; + q1 = (char *)"SELECT DISTINCT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers WHERE hostgroup_id IN (%d,%d) ORDER BY hostgroup_id, hostname, port"; + q2 = (char *)"SELECT DISTINCT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming WHERE hostgroup_id IN (%d,%d) ORDER BY hostgroup_id, hostname, port"; query = (char *)malloc(strlen(q2)+128); - sprintf(query,q1,_writer_hostgroup); + sprintf(query,q1,_writer_hostgroup,_rhid); mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset_servers); if (error == NULL) { if (resultset_servers) { @@ -7331,7 +7347,7 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid delete resultset_servers; resultset_servers = NULL; } - sprintf(query,q2,_writer_hostgroup); + sprintf(query,q2,_writer_hostgroup,_rhid); mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset_servers); if (error == NULL) { if (resultset_servers) { @@ -7550,7 +7566,7 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_reader(int _whid, int _rhid const char SELECT_AWS_AURORA_SERVERS_FOR_MONITOR[] { "SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl, max_lag_ms, check_interval_ms," " check_timeout_ms, add_lag_ms, min_lag_ms, lag_num_checks, autopurge_missing_checks, domain_name," - " green_writer_hostgroup, green_reader_hostgroup FROM mysql_servers" + " green_writer_hostgroup, green_reader_hostgroup, writer_is_also_reader, new_reader_weight FROM mysql_servers" " JOIN mysql_aws_aurora_hostgroups ON" " hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3)" " GROUP BY writer_hostgroup, hostname, port" diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 1504b1cdf7..d9adc334b5 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6034,29 +6034,6 @@ bool AWS_Aurora_monitor_node::add_entry(AWS_Aurora_status_entry *ase) { } -typedef struct _host_def_t { - char *host; - int port; - int use_ssl; -} host_def_t; - -static void shuffle_hosts(host_def_t *array, size_t n) { - char tmp[sizeof(host_def_t)]; - char *arr = (char *)array; - size_t stride = sizeof(host_def_t) * sizeof(char); - - if (n > 1) { - size_t i; - for (i = 0; i < n - 1 ; ++i) { - size_t rnd = (size_t) fastrand(); - size_t j = i + rnd / (0x7FFF / (n - i) + 1); - memcpy(tmp, arr + j * stride, sizeof(host_def_t)); - memcpy(arr + j * stride, arr + i * stride, sizeof(host_def_t)); - memcpy(arr + i * stride, tmp, sizeof(host_def_t)); - } - } -} - const char* aws_aurora_bgd_status_str(AWS_Aurora_BGD_Status status) { switch (status) { case AWS_Aurora_BGD_Status::NONE: @@ -6215,9 +6192,14 @@ static AWS_Aurora_BGD_Query_Result aws_aurora_bgd_query( unsigned int writer_hg, unsigned int timeout_ms, MySQL_Monitor_State_Data_Task_Type task_type, - const char* query + const char* query, + std::atomic_bool& worker_stop ) { AWS_Aurora_BGD_Query_Result out; + if (worker_stop.load()) { + out.rc = 2; + return out; + } out.mmsd.reset(new MySQL_Monitor_State_Data( task_type, const_cast(host.hostname.c_str()), host.port, host.use_ssl, writer_hg)); MySQL_Monitor_State_Data* mmsd = out.mmsd.get(); @@ -6235,7 +6217,6 @@ static AWS_Aurora_BGD_Query_Result aws_aurora_bgd_query( GloMyMon->My_Conn_Pool->conn_register(mmsd); } - std::atomic_bool worker_stop {false}; out.rc = GloMyMon->aws_rds_bgd_async_query(mmsd, query, worker_stop); out.mysql_error = mmsd->mysql ? mysql_errno(mmsd->mysql) : 0; @@ -6538,7 +6519,9 @@ void MySQL_Monitor::aws_aurora_bgd_apply_active_actions( [](const AWS_Aurora_BGD_Member& member) { return member.is_writer; }); if (writer != st.production_members.end()) { MyHGM->update_aws_aurora_set_writer( - st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str())); + st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str()), true, + st.domain_name.c_str(), writer->port, st.writer_is_also_reader, + st.new_reader_weight); } } @@ -6556,7 +6539,9 @@ void MySQL_Monitor::aws_aurora_bgd_apply_completion( [](const AWS_Aurora_BGD_Member& member) { return member.is_writer; }); if (writer != st.production_members.end()) { MyHGM->update_aws_aurora_set_writer( - st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str())); + st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str()), true, + st.domain_name.c_str(), writer->port, st.writer_is_also_reader, + st.new_reader_weight); } for (const AWS_Aurora_BGD_Member& member : st.target_members) { @@ -6612,6 +6597,51 @@ void MySQL_Monitor::aws_aurora_bgd_apply_completion( st.writer_hg, st.reader_hg); } +void MySQL_Monitor::aws_aurora_bgd_apply_rollback( + AWS_Aurora_BGD_State& st, AWS_Aurora_BGD_Status next_status, + bool clear_deployment +) { + for (AWS_Aurora_BGD_Member& member : st.target_members) { + if (!member.traffic_pin_applied) { + continue; + } + dns_cache->remove(member.production_hostname); + MyHGM->wrlock(); + MyHGM->drain_server_connections(member.production_hostname.c_str(), member.port); + MyHGM->wrunlock(); + My_Conn_Pool->purge_connections(member.production_hostname.c_str(), member.port); + member.traffic_pin_applied = false; + } + + auto writer = std::find_if( + st.production_members.begin(), st.production_members.end(), + [](const AWS_Aurora_BGD_Member& member) { return member.is_writer; }); + if (writer != st.production_members.end()) { + MyHGM->update_aws_aurora_set_writer( + st.writer_hg, st.reader_hg, const_cast(writer->server_id.c_str()), true, + st.domain_name.c_str(), writer->port, st.writer_is_also_reader, + st.new_reader_weight); + } + + st.production_snapshot_frozen = false; + st.production_probe_suspended = false; + st.target_snapshot_complete = false; + st.target_members.clear(); + if (clear_deployment) { + st.fingerprint = AWS_Aurora_BGD_Fingerprint {}; + } + + aws_aurora_bgd_set_status(st, next_status); + if (next_status >= AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED + && next_status <= AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING) { + aws_aurora_bgd_apply_active_actions(st, true); + } + + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: rollback cleanup complete; status '%s'\n", + st.writer_hg, st.reader_hg, aws_aurora_bgd_status_str(st.status)); +} + void MySQL_Monitor::aws_aurora_bgd_release_completed_latch(AWS_Aurora_BGD_State& st) { if (st.status != AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED) { return; @@ -6627,7 +6657,12 @@ void MySQL_Monitor::aws_aurora_bgd_release_completed_latch(AWS_Aurora_BGD_State& st.writer_hg, st.reader_hg); } -void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) { +void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle( + AWS_Aurora_BGD_State& st, std::atomic_bool& worker_stop +) { + if (worker_stop.load()) { + return; + } const bool configured_green_hgs = st.green_writer_hg >= 0 && st.green_reader_hg >= 0; const bool discovery_admitted = configured_green_hgs || st.status != AWS_Aurora_BGD_Status::NONE @@ -6648,10 +6683,17 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) if (st.topology_state == TOPOLOGY_TABLE_CHECK) { AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query( topology_host, st.writer_hg, st.check_timeout_ms, - MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK); + MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK, worker_stop); + if (worker_stop.load()) { + return; + } if (query.rc == 0 && query.mmsd->result) { if (mysql_num_rows(query.mmsd->result) > 0) { st.topology_state = TOPOLOGY_METADATA_FETCH; + } else if (st.status != AWS_Aurora_BGD_Status::NONE + && st.status != AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED) { + aws_aurora_bgd_apply_rollback( + st, AWS_Aurora_BGD_Status::NONE, true); } else { aws_aurora_bgd_release_completed_latch(st); } @@ -6659,14 +6701,23 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) } else { AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query( topology_host, st.writer_hg, st.check_timeout_ms, - MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY); + MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY, worker_stop); + if (worker_stop.load()) { + return; + } if (query.rc != 0) { if (query.mysql_error == 1146) { st.topology_state = TOPOLOGY_TABLE_CHECK; } } else if (query.mmsd->result) { if (mysql_num_rows(query.mmsd->result) == 0) { - aws_aurora_bgd_release_completed_latch(st); + if (st.status != AWS_Aurora_BGD_Status::NONE + && st.status != AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED) { + aws_aurora_bgd_apply_rollback( + st, AWS_Aurora_BGD_Status::NONE, true); + } else { + aws_aurora_bgd_release_completed_latch(st); + } } else { AWS_RDS_Topology_Result topology = parse_aws_rds_topology(query.mmsd->result); AWS_Aurora_BGD_Topology_Observation observation = @@ -6683,7 +6734,10 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) || st.fingerprint == observation.fingerprint; const bool rollback_transition = static_cast(observation.status) < static_cast(st.status); - if (same_deployment && !rollback_transition) { + if (same_deployment && rollback_transition) { + aws_aurora_bgd_apply_rollback( + st, observation.status, false); + } else if (same_deployment) { const bool status_changed = st.status != observation.status; st.fingerprint = observation.fingerprint; st.target_use_ssl = topology_host.use_ssl; @@ -6697,7 +6751,7 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) if (st.status == AWS_Aurora_BGD_Status::NONE || st.status == AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED - || st.fingerprint.empty()) { + || st.fingerprint.empty() || worker_stop.load()) { return; } @@ -6710,7 +6764,7 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) AWS_Aurora_BGD_Query_Result membership = aws_aurora_bgd_query( membership_host, st.writer_hg, st.check_timeout_ms, - MON_AWS_AURORA, QUERY_AWS_AURORA_BGD_REPLICA_HOST_STATUS); + MON_AWS_AURORA, QUERY_AWS_AURORA_BGD_REPLICA_HOST_STATUS, worker_stop); if (membership.rc != 0) { return; } @@ -6726,19 +6780,102 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle(AWS_Aurora_BGD_State& st) } } +bool MySQL_Monitor::aws_aurora_bgd_load_worker_config( + int writer_hg, uint64_t current_checksum, AWS_Aurora_BGD_State& candidate +) { + SQLite3_result cluster_result(AWS_AURORA_HOSTS_COLUMNS); + pthread_mutex_lock(&aws_aurora_mutex); + if (AWS_Aurora_Hosts_resultset) { + for (SQLite3_row* row : AWS_Aurora_Hosts_resultset->rows) { + if (atoi(row->fields[AWS_AURORA_WRITER_HOSTGROUP]) == writer_hg) { + cluster_result.add_row(row); + } + } + } + pthread_mutex_unlock(&aws_aurora_mutex); + + if (cluster_result.rows.empty() || cluster_result.raw_checksum() != current_checksum) { + return false; + } + + candidate.writer_hg = writer_hg; + bool first_row = true; + for (SQLite3_row* row : cluster_result.rows) { + if (first_row) { + candidate.reader_hg = atoi(row->fields[AWS_AURORA_READER_HOSTGROUP]); + candidate.green_writer_hg = row->fields[AWS_AURORA_GREEN_WRITER_HOSTGROUP] + ? atoi(row->fields[AWS_AURORA_GREEN_WRITER_HOSTGROUP]) : -1; + candidate.green_reader_hg = row->fields[AWS_AURORA_GREEN_READER_HOSTGROUP] + ? atoi(row->fields[AWS_AURORA_GREEN_READER_HOSTGROUP]) : -1; + candidate.max_lag_ms = atoi(row->fields[AWS_AURORA_MAX_LAG_MS]); + candidate.check_interval_ms = atoi(row->fields[AWS_AURORA_CHECK_INTERVAL_MS]); + candidate.check_timeout_ms = atoi(row->fields[AWS_AURORA_CHECK_TIMEOUT_MS]); + candidate.add_lag_ms = atoi(row->fields[AWS_AURORA_ADD_LAG_MS]); + candidate.min_lag_ms = atoi(row->fields[AWS_AURORA_MIN_LAG_MS]); + candidate.lag_num_checks = atoi(row->fields[AWS_AURORA_LAG_NUM_CHECKS]); + candidate.autopurge_missing_checks = + atoi(row->fields[AWS_AURORA_AUTOPURGE_MISSING_CHECKS]); + candidate.domain_name = row->fields[AWS_AURORA_DOMAIN_NAME] + ? row->fields[AWS_AURORA_DOMAIN_NAME] : ""; + candidate.writer_is_also_reader = + atoi(row->fields[AWS_AURORA_WRITER_IS_ALSO_READER]); + candidate.new_reader_weight = atoi(row->fields[AWS_AURORA_NEW_READER_WEIGHT]); + first_row = false; + } + + candidate.production_probe_hosts.push_back(AWS_RDS_BGD_Probe_Host { + row->fields[AWS_AURORA_HOSTNAME], + atoi(row->fields[AWS_AURORA_PORT]), + atoi(row->fields[AWS_AURORA_USE_SSL]), + }); + } + + if (candidate.production_probe_hosts.empty()) { + return false; + } + candidate.target_use_ssl = candidate.production_probe_hosts.front().use_ssl; + return true; +} + +bool MySQL_Monitor::aws_aurora_bgd_refresh_worker_config( + AWS_Aurora_BGD_State& st, uint64_t current_checksum, + unsigned long long& next_loop_at +) { + AWS_Aurora_BGD_State candidate; + if (!aws_aurora_bgd_load_worker_config(st.writer_hg, current_checksum, candidate)) { + return false; + } + + st.reader_hg = candidate.reader_hg; + st.green_writer_hg = candidate.green_writer_hg; + st.green_reader_hg = candidate.green_reader_hg; + st.max_lag_ms = candidate.max_lag_ms; + st.check_interval_ms = candidate.check_interval_ms; + st.check_timeout_ms = candidate.check_timeout_ms; + st.add_lag_ms = candidate.add_lag_ms; + st.min_lag_ms = candidate.min_lag_ms; + st.lag_num_checks = candidate.lag_num_checks; + st.autopurge_missing_checks = candidate.autopurge_missing_checks; + st.writer_is_also_reader = candidate.writer_is_also_reader; + st.new_reader_weight = candidate.new_reader_weight; + st.domain_name = candidate.domain_name; + st.production_probe_hosts = std::move(candidate.production_probe_hosts); + if (st.status == AWS_Aurora_BGD_Status::NONE && !st.target_snapshot_complete) { + st.target_use_ssl = candidate.target_use_ssl; + } + + next_loop_at = 0; + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: applied checksum %llu with in-place refresh at %s\n", + st.writer_hg, st.reader_hg, (unsigned long long)current_checksum, + aws_aurora_bgd_status_str(st.status)); + return true; +} + void * monitor_AWS_Aurora_thread_HG(void *arg) { - unsigned int wHG = *(unsigned int *)arg; - unsigned int rHG = 0; - unsigned int num_hosts = 0; + AWS_Aurora_BGD_Worker* worker = static_cast(arg); + unsigned int wHG = worker->writer_hg; unsigned int cur_host_idx = 0; - unsigned int max_lag_ms = 0; - unsigned int check_interval_ms = 0; - unsigned int check_timeout_ms = 0; - unsigned int add_lag_ms = 0; - unsigned int min_lag_ms = 0; - unsigned int lag_num_checks = 1; - unsigned int autopurge_missing_checks = 0; - std::string domain_name; std::map autopurge_counter; AWS_Aurora_BGD_State bgd_state; bgd_state.writer_hg = wHG; @@ -6753,8 +6890,6 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { MySQL_Monitor__thread_MySQL_Thread_Variables_version=GloMTH->get_global_version(); mysql_thr->refresh_variables(); - uint64_t initial_raw_checksum = 0; - // this is a static array of the latest reads unsigned int ase_idx = 0; AWS_Aurora_status_entry *lasts_ase[N_L_ASE]; @@ -6762,78 +6897,13 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { lasts_ase[i] = NULL; } - // initial data load - pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); - initial_raw_checksum = GloMyMon->AWS_Aurora_Hosts_resultset_checksum; - // count the number of hosts - for (std::vector::iterator it = GloMyMon->AWS_Aurora_Hosts_resultset->rows.begin() ; it != GloMyMon->AWS_Aurora_Hosts_resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - if (atoi(r->fields[0]) == (int)wHG) { - num_hosts++; - if (max_lag_ms == 0) { - max_lag_ms = atoi(r->fields[5]); - } - if (check_interval_ms == 0) { - check_interval_ms = atoi(r->fields[6]); - } - if (check_timeout_ms == 0) { - check_timeout_ms = atoi(r->fields[7]); - } - if (rHG == 0) { - rHG = atoi(r->fields[1]); - } - add_lag_ms = atoi(r->fields[8]); - min_lag_ms = atoi(r->fields[9]); - lag_num_checks = atoi(r->fields[10]); - autopurge_missing_checks = atoi(r->fields[11]); - if (domain_name.empty() && r->fields[12]) { - domain_name = r->fields[12]; - } - if (r->fields[13] && r->fields[13][0]) { - bgd_state.green_writer_hg = atoi(r->fields[13]); - } - if (r->fields[14] && r->fields[14][0]) { - bgd_state.green_reader_hg = atoi(r->fields[14]); - } - } - } - host_def_t *hpa = (host_def_t *)malloc(sizeof(host_def_t)*num_hosts); - for (std::vector::iterator it = GloMyMon->AWS_Aurora_Hosts_resultset->rows.begin() ; it != GloMyMon->AWS_Aurora_Hosts_resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - if (atoi(r->fields[0]) == (int)wHG) { - hpa[cur_host_idx].host = strdup(r->fields[2]); - hpa[cur_host_idx].port = atoi(r->fields[3]); - hpa[cur_host_idx].use_ssl = atoi(r->fields[4]); - bgd_state.production_probe_hosts.push_back({ - r->fields[2], - atoi(r->fields[3]), - atoi(r->fields[4]), - }); - cur_host_idx++; - } - } - bgd_state.reader_hg = rHG; - bgd_state.check_interval_ms = check_interval_ms; - bgd_state.check_timeout_ms = check_timeout_ms; - bgd_state.domain_name = domain_name; - if (!bgd_state.production_probe_hosts.empty()) { - bgd_state.target_use_ssl = bgd_state.production_probe_hosts.front().use_ssl; - } - // NOTE: 'cur_host_idx' should never be higher than 'num_hosts' otherwise later an invalid memory access - // can table place later when accessing 'hpa[cur_host_idx]'. - if (cur_host_idx >= num_hosts) { - cur_host_idx = num_hosts - 1; - } - pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); - - bool exit_now = false; unsigned long long t1 = 0; //unsigned long long t2 = 0; unsigned long long next_loop_at = 0; bool crc = false; - uint64_t current_raw_checksum = 0; + uint64_t last_checksum = 0; size_t rnd; bool found_pingable_host = false; bool rc_ping = false; @@ -6842,7 +6912,8 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { t1 = monotonic_time(); unsigned long long start_time=t1; - while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true && exit_now==false) { + while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true + && worker->worker_stop.load()==false) { unsigned int glover; @@ -6864,14 +6935,17 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { next_loop_at=0; } - pthread_mutex_lock(&GloMyMon->aws_aurora_mutex); - current_raw_checksum = GloMyMon->AWS_Aurora_Hosts_resultset_checksum; - pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); - - if (current_raw_checksum != initial_raw_checksum) { - // the content of AWS_Aurora_Hosts_resultset has changed. Exit - exit_now=true; - break; + uint64_t current_checksum = worker->current_checksum.load(); + if (current_checksum != last_checksum) { + if (!GloMyMon->aws_aurora_bgd_refresh_worker_config( + bgd_state, current_checksum, next_loop_at)) { + usleep(50000); + continue; + } + last_checksum = current_checksum; + if (cur_host_idx >= bgd_state.production_probe_hosts.size()) { + cur_host_idx = 0; + } } //fprintf(stderr,"%u : %llu %llu\n", wHG, t1, next_loop_at); if (t1 < next_loop_at) { @@ -6889,45 +6963,38 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { delete mmsd; mmsd = NULL; } - GloMyMon->aws_aurora_bgd_run_discovery_cycle(bgd_state); + GloMyMon->aws_aurora_bgd_run_discovery_cycle( + bgd_state, worker->worker_stop); const unsigned int interval_ms = bgd_state.production_probe_suspended - ? 100 : check_interval_ms; + ? 100 : bgd_state.check_interval_ms; next_loop_at = t1 + static_cast(interval_ms) * 1000; continue; } //proxy_info("Running check AWS Aurora writer HG %u\n", wHG); found_pingable_host = false; - rc_ping = false; - // pick a random host - rnd = (size_t) rand(); - rnd %= num_hosts; - rc_ping = GloMyMon->server_responds_to_ping(hpa[rnd].host, hpa[rnd].port); - //proxy_info("Looping Monitor thread for AWS Aurora writer HG %u\n", wHG); + const size_t num_hosts = bgd_state.production_probe_hosts.size(); + if (num_hosts != 0) { + rnd = static_cast(rand()) % num_hosts; + for (size_t offset = 0; !found_pingable_host && offset < num_hosts; ++offset) { + const size_t host_idx = (rnd + offset) % num_hosts; + const AWS_RDS_BGD_Probe_Host& host = + bgd_state.production_probe_hosts[host_idx]; + rc_ping = GloMyMon->server_responds_to_ping( + const_cast(host.hostname.c_str()), host.port); #ifdef TEST_AURORA_RANDOM - if (rand() % 100 < 30) { - // we randomly fail 30% of the requests - rc_ping = false; - } + if (offset == 0 && rand() % 100 < 30) { + rc_ping = false; + } #endif // TEST_AURORA_RANDOM - if (rc_ping) { - found_pingable_host = true; - cur_host_idx = rnd; - } else { - MyHGM->p_update_mysql_error_counter( - p_mysql_error_type::proxysql, wHG, hpa[rnd].host, hpa[rnd].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV - ); - // the randomly picked host didn't work work - shuffle_hosts(hpa,num_hosts); - for (unsigned int i=0; (found_pingable_host == false && iserver_responds_to_ping(hpa[i].host, hpa[i].port); if (rc_ping) { found_pingable_host = true; - cur_host_idx = i; + cur_host_idx = host_idx; } else { MyHGM->p_update_mysql_error_counter( - p_mysql_error_type::proxysql, wHG, hpa[i].host, hpa[i].port, ER_PROXYSQL_AWS_NO_PINGABLE_SRV - ); + p_mysql_error_type::proxysql, wHG, + const_cast(host.hostname.c_str()), host.port, + ER_PROXYSQL_AWS_NO_PINGABLE_SRV); } } } @@ -6941,25 +7008,31 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { if (found_pingable_host == false) { proxy_error("No node is pingable for AWS Aurora cluster with writer HG %u\n", wHG); - GloMyMon->aws_aurora_bgd_run_discovery_cycle(bgd_state); + GloMyMon->aws_aurora_bgd_run_discovery_cycle( + bgd_state, worker->worker_stop); const unsigned int interval_ms = bgd_state.production_probe_suspended - ? 100 : check_interval_ms; + ? 100 : bgd_state.check_interval_ms; next_loop_at = t1 + static_cast(interval_ms) * 1000; continue; } #ifdef TEST_AURORA if (rand() % 1000 == 0) { // suppress 99.9% of the output, too verbose - proxy_info("Running check for AWS Aurora writer HG %u on %s:%d\n", wHG , hpa[cur_host_idx].host, hpa[cur_host_idx].port); + const AWS_RDS_BGD_Probe_Host& host = bgd_state.production_probe_hosts[cur_host_idx]; + proxy_info("Running check for AWS Aurora writer HG %u on %s:%d\n", wHG, + host.hostname.c_str(), host.port); } #endif // TEST_AURORA if (mmsd) { delete mmsd; mmsd = NULL; } - //mmsd = NULL; - mmsd = new MySQL_Monitor_State_Data(MON_AWS_AURORA, hpa[cur_host_idx].host, hpa[cur_host_idx].port, hpa[cur_host_idx].use_ssl); + const AWS_RDS_BGD_Probe_Host& probe_host = + bgd_state.production_probe_hosts[cur_host_idx]; + mmsd = new MySQL_Monitor_State_Data( + MON_AWS_AURORA, const_cast(probe_host.hostname.c_str()), + probe_host.port, probe_host.use_ssl); mmsd->writer_hostgroup = wHG; - mmsd->aws_aurora_check_timeout_ms = check_timeout_ms; + mmsd->aws_aurora_check_timeout_ms = bgd_state.check_timeout_ms; mmsd->mysql=GloMyMon->My_Conn_Pool->get_connection(mmsd->hostname, mmsd->port, mmsd); //unsigned long long start_time=mysql_thr->curtime; start_time=t1; @@ -7018,7 +7091,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_AWS_HEALTH_CHECK_TIMEOUT); goto __exit_monitor_aws_aurora_HG_thread; } - if (GloMyMon->shutdown==true) { + if (GloMyMon->shutdown==true || worker->worker_stop.load()) { goto __fast_exit_monitor_aws_aurora_HG_thread; // exit immediately } if ((mmsd->async_exit_status & MYSQL_WAIT_TIMEOUT) == 0) { @@ -7039,7 +7112,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, mmsd->hostgroup_id, mmsd->hostname, mmsd->port, ER_PROXYSQL_AWS_HEALTH_CHECK_TIMEOUT); goto __exit_monitor_aws_aurora_HG_thread; } - if (GloMyMon->shutdown==true) { + if (GloMyMon->shutdown==true || worker->worker_stop.load()) { goto __fast_exit_monitor_aws_aurora_HG_thread; // exit immediately } if ((mmsd->async_exit_status & MYSQL_WAIT_TIMEOUT) == 0) { @@ -7053,7 +7126,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { __exit_monitor_aws_aurora_HG_thread: mmsd->t2=monotonic_time(); - next_loop_at = t1 + (check_interval_ms * 1000); + next_loop_at = t1 + (bgd_state.check_interval_ms * 1000); if (mmsd->t2 > t1) { next_loop_at -= (mmsd->t2 - t1); } @@ -7098,12 +7171,19 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { delete l_ase; } lasts_ase[ase_idx] = ase_l; - GloMyMon->evaluate_aws_aurora_results(wHG, rHG, &lasts_ase[0], ase_idx, max_lag_ms, add_lag_ms, min_lag_ms, lag_num_checks); + GloMyMon->evaluate_aws_aurora_results( + wHG, bgd_state.reader_hg, &lasts_ase[0], ase_idx, + bgd_state.max_lag_ms, bgd_state.add_lag_ms, + bgd_state.min_lag_ms, bgd_state.lag_num_checks); // Auto-purge servers that disappear from REPLICA_HOST_STATUS // Only process if autopurge is enabled and query was successful with results - if (autopurge_missing_checks > 0 && mmsd->interr == 0 && ase->host_statuses->size() > 0) { - GloMyMon->aws_aurora_autopurge_servers(wHG, rHG, ase, autopurge_missing_checks, autopurge_counter, domain_name); + if (bgd_state.autopurge_missing_checks > 0 && mmsd->interr == 0 + && ase->host_statuses->size() > 0) { + GloMyMon->aws_aurora_autopurge_servers( + wHG, bgd_state.reader_hg, ase, + bgd_state.autopurge_missing_checks, autopurge_counter, + bgd_state.domain_name); } for (auto h : *(ase_l->host_statuses)) { @@ -7181,14 +7261,19 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { } } } - if (GloMyMon->shutdown == false && exit_now == false) { - GloMyMon->aws_aurora_bgd_run_discovery_cycle(bgd_state); + if (GloMyMon->shutdown == false && worker->worker_stop.load() == false) { + GloMyMon->aws_aurora_bgd_run_discovery_cycle( + bgd_state, worker->worker_stop); const unsigned int interval_ms = bgd_state.production_probe_suspended - ? 100 : check_interval_ms; + ? 100 : bgd_state.check_interval_ms; next_loop_at = t1 + static_cast(interval_ms) * 1000; } } __exit_monitor_AWS_Aurora_thread_HG_now: + if (bgd_state.status != AWS_Aurora_BGD_Status::NONE) { + GloMyMon->aws_aurora_bgd_apply_rollback( + bgd_state, AWS_Aurora_BGD_Status::NONE, true); + } if (mmsd) { delete (mmsd); mmsd = NULL; @@ -7200,7 +7285,6 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { } } - free(hpa); if (mysql_thr) { delete mysql_thr; mysql_thr=NULL; @@ -7217,9 +7301,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { void * MySQL_Monitor::monitor_aws_aurora() { - // initialize the MySQL Thread (note: this is not a real thread, just the structures associated with it) - // Wait for GloMTH to be initialized - if (!wait_for_glo_mth()) return NULL; // quick exit during shutdown/restart + if (!wait_for_glo_mth()) return NULL; unsigned int MySQL_Monitor__thread_MySQL_Thread_Variables_version; MySQL_Thread * mysql_thr = new MySQL_Thread(); mysql_thr->curtime=monotonic_time(); @@ -7227,20 +7309,11 @@ void * MySQL_Monitor::monitor_aws_aurora() { mysql_thr->refresh_variables(); uint64_t last_raw_checksum = 0; - - // ADD here an unordered map , Writer HG => next time at - // when empty, a new map is populated - // when next_loop_at = 0 , the tables is emptied so to be populated again - - unsigned int *hgs_array = NULL; - pthread_t *pthreads_array = NULL; - unsigned int hgs_num = 0; + std::unordered_map> workers; while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true) { - unsigned int glover; - - if (!GloMTH) return NULL; // quick exit during shutdown/restart + if (!GloMTH) break; // if variables has changed, triggers new checks glover=GloMTH->get_global_version(); @@ -7249,69 +7322,96 @@ void * MySQL_Monitor::monitor_aws_aurora() { mysql_thr->refresh_variables(); } - // if list of servers or HG or options has changed, triggers new checks + uint64_t new_raw_checksum = 0; + std::unordered_map cluster_checksums; pthread_mutex_lock(&aws_aurora_mutex); - uint64_t new_raw_checksum = AWS_Aurora_Hosts_resultset->raw_checksum(); + if (AWS_Aurora_Hosts_resultset) { + new_raw_checksum = AWS_Aurora_Hosts_resultset->raw_checksum(); + if (new_raw_checksum != last_raw_checksum) { + std::unordered_map> cluster_results; + for (SQLite3_row* row : AWS_Aurora_Hosts_resultset->rows) { + const int writer_hg = atoi(row->fields[AWS_AURORA_WRITER_HOSTGROUP]); + auto cluster_it = cluster_results.find(writer_hg); + if (cluster_it == cluster_results.end()) { + cluster_it = cluster_results.emplace( + writer_hg, std::unique_ptr( + new SQLite3_result(AWS_AURORA_HOSTS_COLUMNS))).first; + } + cluster_it->second->add_row(row); + } + for (const auto& [writer_hg, result] : cluster_results) { + cluster_checksums[writer_hg] = result->raw_checksum(); + } + } + } pthread_mutex_unlock(&aws_aurora_mutex); + if (new_raw_checksum != last_raw_checksum) { proxy_info("Detected new/changed definition for AWS Aurora monitoring\n"); - if (pthreads_array) { - // wait all threads to terminate - for (unsigned int i=0; i < hgs_num; i++) { - pthread_join(pthreads_array[i], NULL); - proxy_info("Stopped Monitor thread for AWS Aurora writer HG %u\n", hgs_array[i]); - } - free(pthreads_array); - free(hgs_array); - pthreads_array = NULL; - hgs_array = NULL; - } - hgs_num = 0; - pthread_mutex_lock(&aws_aurora_mutex); - // Configuration can change while the previous workers are being joined. - // Launch replacements from the current snapshot and retain that checksum, - // so the coordinator never waits on a worker created from newer state. - last_raw_checksum = AWS_Aurora_Hosts_resultset->raw_checksum(); - // scan all the writer HGs - unsigned int num_rows = AWS_Aurora_Hosts_resultset->rows_count; - if (num_rows) { - unsigned int *tmp_hgs_array = (unsigned int *)malloc(sizeof(unsigned int)*num_rows); - for (std::vector::iterator it = AWS_Aurora_Hosts_resultset->rows.begin() ; it != AWS_Aurora_Hosts_resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - int wHG = atoi(r->fields[0]); - bool found = false; - // very simple search. Far from optimal, but assuming very few HGs it is fast enough - for (unsigned int i=0; i < hgs_num; i++) { - if (tmp_hgs_array[i] == (unsigned int)wHG) { - found = true; - } - } - if (found == false) { - // new wHG found - tmp_hgs_array[hgs_num]=wHG; - hgs_num++; - } + last_raw_checksum = new_raw_checksum; + std::vector stopped_workers; + + for (auto& [writer_hg, worker] : workers) { + auto cluster_it = cluster_checksums.find(writer_hg); + if (cluster_it == cluster_checksums.end()) { + worker->worker_stop.store(true); + stopped_workers.push_back(writer_hg); + proxy_info( + "AWS Aurora BGD [wHG=%d]: stopping worker; owning row is inactive, removed, or has no monitored server\n", + writer_hg); + continue; } - proxy_info("Activating Monitoring of %u AWS Aurora clusters\n", hgs_num); - hgs_array = (unsigned int *)malloc(sizeof(unsigned int)*hgs_num); - pthreads_array = (pthread_t *)malloc(sizeof(pthread_t)*hgs_num); - for (unsigned int i=0; i < hgs_num; i++) { - hgs_array[i] = tmp_hgs_array[i]; - proxy_info("Starting Monitor thread for AWS Aurora writer HG %u\n", hgs_array[i]); - if (pthread_create(&pthreads_array[i], NULL, monitor_AWS_Aurora_thread_HG, &hgs_array[i]) != 0) { - // LCOV_EXCL_START - proxy_error("Thread creation\n"); - assert(0); - // LCOV_EXCL_STOP - } + + uint64_t old_checksum = worker->current_checksum.load(); + if (old_checksum != cluster_it->second) { + worker->current_checksum.store(cluster_it->second); + proxy_info( + "AWS Aurora BGD [wHG=%d]: signaling config refresh, checksum %llu -> %llu\n", + writer_hg, (unsigned long long)old_checksum, + (unsigned long long)cluster_it->second); } - free(tmp_hgs_array); } - pthread_mutex_unlock(&aws_aurora_mutex); + + for (const auto& [writer_hg, checksum] : cluster_checksums) { + if (workers.find(writer_hg) != workers.end()) { + continue; + } + std::unique_ptr worker(new AWS_Aurora_BGD_Worker); + worker->writer_hg = writer_hg; + worker->current_checksum.store(checksum); + AWS_Aurora_BGD_Worker* worker_arg = worker.get(); + workers.emplace(writer_hg, std::move(worker)); + proxy_info("Starting Monitor thread for AWS Aurora writer HG %d\n", writer_hg); + if (pthread_create( + &worker_arg->thread, NULL, monitor_AWS_Aurora_thread_HG, worker_arg) != 0) { + // LCOV_EXCL_START + proxy_error("Thread creation\n"); + assert(0); + // LCOV_EXCL_STOP + } + } + + for (int writer_hg : stopped_workers) { + auto worker_it = workers.find(writer_hg); + if (worker_it == workers.end()) { + continue; + } + pthread_join(worker_it->second->thread, NULL); + proxy_info("Stopped Monitor thread for AWS Aurora writer HG %d\n", writer_hg); + workers.erase(worker_it); + } } usleep(10000); } + for (auto& [writer_hg, worker] : workers) { + worker->worker_stop.store(true); + } + for (auto& [writer_hg, worker] : workers) { + pthread_join(worker->thread, NULL); + proxy_info("Stopped Monitor thread for AWS Aurora writer HG %d\n", writer_hg); + } + workers.clear(); if (mysql_thr) { delete mysql_thr; mysql_thr=NULL; diff --git a/test/tap/groups/cluster_sim_aurora/add-hosts b/test/tap/groups/cluster_sim_aurora/add-hosts index cb6b8a8746..b8853c07d1 100644 --- a/test/tap/groups/cluster_sim_aurora/add-hosts +++ b/test/tap/groups/cluster_sim_aurora/add-hosts @@ -72,3 +72,8 @@ aurora-b-writer-green-h2s6.b1.us-east-1.rds.amazonaws.com 127.0.12.21 aurora-b-reader-1-green-r8t4.b1.us-east-1.rds.amazonaws.com 127.0.12.22 aurora-b-canonical-writer.b1.us-east-1.rds.amazonaws.com 127.0.12.21 aurora-b-canonical-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.22 + +# Aurora blue/green deployment C supports three-cluster isolation scenarios. +aurora-c-writer.c1.us-east-1.rds.amazonaws.com 127.0.13.11 +aurora-c-green.cluster-c1.us-east-1.rds.amazonaws.com 127.0.13.20 +aurora-c-writer-green-m5n9.c1.us-east-1.rds.amazonaws.com 127.0.13.21 diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 9afa842be7..c4de080fa4 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -366,6 +366,9 @@ "test_aurora_bgd_discovery-t" : [ "cluster_sim_aurora-g1" ], "test_aurora_bgd_active-t" : [ "cluster_sim_aurora-g1" ], "test_aurora_bgd_completion-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_cluster_sync-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_resilience-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_lifecycle-t" : [ "cluster_sim_aurora-g1" ], "test_cluster_sim_galera-t" : [ "cluster_sim_galera-g1" ], "test_cluster_sim_group_repl-t" : [ "cluster_sim_group_repl-g1" ], "test_cluster_sim_read_only-t" : [ "cluster_sim_read_only-g1" ], diff --git a/test/tap/tap/aurora_bgd_tap.h b/test/tap/tap/aurora_bgd_tap.h index a38cb4bb5f..c71d8100c1 100644 --- a/test/tap/tap/aurora_bgd_tap.h +++ b/test/tap/tap/aurora_bgd_tap.h @@ -113,6 +113,38 @@ inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_b_writer_only() { return deployment; } +inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_c_writer_only() { + Aurora_BGD_Test_Deployment deployment; + deployment.name = "Aurora BGD deployment C writer-only"; + deployment.domain_name = ".c1.us-east-1.rds.amazonaws.com"; + deployment.blue_replica_set = "aurora-bgd-blue-c"; + deployment.target_replica_set = "aurora-bgd-target-c"; + deployment.source_topology_id = "aurora-bgd-source-c"; + deployment.target_topology_id = "aurora-bgd-target-c"; + deployment.target_cluster_endpoint = { + "aurora-c-green.cluster-c1.us-east-1.rds.amazonaws.com", "127.0.13.20", 3306 + }; + deployment.production = { + deployment.blue_replica_set, + { + aurora_bgd_member("aurora-c-writer", "MASTER_SESSION_ID", + {"aurora-c-writer.c1.us-east-1.rds.amazonaws.com", "127.0.13.11", 3306}), + }, + {} + }; + deployment.production.serving_endpoints.push_back(deployment.production.members.front().endpoint); + deployment.target = { + deployment.target_replica_set, + { + aurora_bgd_member("aurora-c-writer-green-m5n9", "MASTER_SESSION_ID", + {"aurora-c-writer-green-m5n9.c1.us-east-1.rds.amazonaws.com", "127.0.13.21", 3306}), + }, + {deployment.target_cluster_endpoint} + }; + deployment.target.serving_endpoints.push_back(deployment.target.members.front().endpoint); + return deployment; +} + inline string aurora_bgd_sql_quote(const string& value) { string quoted {"'"}; for (char c : value) { diff --git a/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp b/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp new file mode 100644 index 0000000000..743b5e4673 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp @@ -0,0 +1,306 @@ +/** + * @file test_aurora_bgd_cluster_sync-t.cpp + * @brief Aurora BGD configuration sync preserves worker-owned status per node. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; +namespace fs = std::filesystem; + +const uint32_t kWaitSeconds = 10; +const int kReplicaAdminPort = 16062; +const int kReplicaMySQLPort = 16063; +const char kReplicaHost[] = "127.0.0.1"; +const char kSQLiteInterfaces[] = "0.0.0.0:3306;0.0.0.0:3307"; + +struct Replica_Process { + pid_t pid = -1; + string directory; + string config_path; + string stderr_path; +}; + +string config_quote(const string& value) { + string quoted; + for (char c : value) { + if (c == '\\' || c == '"') { + quoted += '\\'; + } + quoted += c; + } + return quoted; +} + +int prepare_replica_config(const CommandLine& cl, Replica_Process& process) { + char directory_template[] = "/tmp/proxysql-aurora-bgd-sync-XXXXXX"; + char* directory = mkdtemp(directory_template); // NOSONAR: mkdtemp creates an owner-only directory. + if (directory == nullptr) { + diag("mkdtemp failed: %s", strerror(errno)); + return EXIT_FAILURE; + } + process.directory = directory; + process.config_path = process.directory + "/proxysql.cnf"; + process.stderr_path = process.directory + "/proxysql.stderr"; + + ofstream config(process.config_path); + if (!config.is_open()) { + diag("failed to create replica config: %s", process.config_path.c_str()); + return EXIT_FAILURE; + } + config + << "datadir=\"" << config_quote(process.directory) << "\"\n" + << "admin_variables={\n" + << " admin_credentials=\"" << config_quote(cl.admin_username) << ":" + << config_quote(cl.admin_password) << ";radmin:radmin\"\n" + << " mysql_ifaces=\"0.0.0.0:" << kReplicaAdminPort << "\"\n" + << " cluster_username=\"radmin\"\n" + << " cluster_password=\"radmin\"\n" + << " cluster_check_interval_ms=200\n" + << " cluster_check_status_frequency=100\n" + << " cluster_admin_variables_diffs_before_sync=0\n" + << " cluster_mysql_servers_diffs_before_sync=1\n" + << " cluster_mysql_servers_save_to_disk=false\n" + << " cluster_mysql_servers_sync_algorithm=3\n" + << "}\n" + << "mysql_variables={\n" + << " interfaces=\"0.0.0.0:" << kReplicaMySQLPort << "\"\n" + << " monitor_username=\"aurora1\"\n" + << " monitor_password=\"pass1\"\n" + << " monitor_connect_timeout=500\n" + << " monitor_ping_interval=10000\n" + << "}\n" + << "proxysql_servers=()\n"; + config.close(); + return config.fail() ? EXIT_FAILURE : EXIT_SUCCESS; +} + +int launch_replica(const CommandLine& cl, Replica_Process& process) { + if (prepare_replica_config(cl, process) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + const string binary = string(cl.workdir) + "../../../src/proxysql"; + process.pid = fork(); + if (process.pid == -1) { + diag("fork failed: %s", strerror(errno)); + return EXIT_FAILURE; + } + if (process.pid == 0) { + int stderr_fd = open(process.stderr_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (stderr_fd >= 0) { + dup2(stderr_fd, STDOUT_FILENO); + dup2(stderr_fd, STDERR_FILENO); + close(stderr_fd); + } + execl( + binary.c_str(), "proxysql", "--sqlite3-server", "-f", "-c", + process.config_path.c_str(), static_cast(nullptr)); + _exit(127); + } + return EXIT_SUCCESS; +} + +void stop_replica(MYSQL*& admin, Replica_Process& process, bool preserve_log) { + if (admin != nullptr) { + mysql_query(admin, "PROXYSQL SHUTDOWN"); + mysql_close(admin); + admin = nullptr; + } + if (process.pid > 0) { + bool exited = false; + for (int i = 0; i < 50; ++i) { + pid_t rc = waitpid(process.pid, nullptr, WNOHANG); + if (rc == process.pid || rc == -1) { + exited = true; + break; + } + usleep(100000); + } + if (!exited) { + kill(process.pid, SIGKILL); + waitpid(process.pid, nullptr, 0); + } + } + if (!preserve_log && !process.directory.empty()) { + fs::remove_all(process.directory); + } else if (preserve_log) { + diag("replica ProxySQL log retained at %s", process.stderr_path.c_str()); + } +} + +Aurora_BGD_Test_Deployment peer_deployment() { + Aurora_BGD_Test_Deployment deployment; + deployment.name = "Aurora BGD peer-local status"; + deployment.domain_name = ".localhost"; + deployment.blue_replica_set = "aurora-bgd-peer-blue"; + deployment.target_replica_set = "aurora-bgd-peer-target"; + deployment.source_topology_id = "aurora-bgd-peer-source"; + deployment.target_topology_id = "aurora-bgd-peer-target"; + deployment.target_cluster_endpoint = { + "aurora-peer-writer-green-sync.localhost", "127.0.0.1", 3307 + }; + deployment.production = { + deployment.blue_replica_set, + {aurora_bgd_member( + "aurora-peer-writer", "MASTER_SESSION_ID", + {"aurora-peer-writer.localhost", "127.0.0.1", 3306})}, + {} + }; + deployment.production.serving_endpoints.push_back( + deployment.production.members.front().endpoint); + deployment.target = { + deployment.target_replica_set, + {aurora_bgd_member( + "aurora-peer-writer-green-sync", "MASTER_SESSION_ID", + {"aurora-peer-writer-green-sync.localhost", "127.0.0.1", 3307})}, + {deployment.target_cluster_endpoint} + }; + deployment.target.serving_endpoints.push_back( + deployment.target.members.front().endpoint); + return deployment; +} + +int configure_sqlite_interfaces(MYSQL* admin, const string& interfaces) { + return aurora_bgd_execute_all(admin, { + "SET sqliteserver-mysql_ifaces=" + aurora_bgd_sql_quote(interfaces), + "LOAD SQLITESERVER VARIABLES TO RUNTIME", + }); +} + +int configure_peer( + MYSQL* admin, BGD_Simulator& simulator, Aurora_BGD_Test_Deployment& deployment, + const string& status +) { + return simulator.cleanup() == EXIT_SUCCESS + && aurora_bgd_publish(simulator, deployment) == EXIT_SUCCESS + && simulator.topology_update( + aurora_bgd_topology_backends(deployment), aurora_bgd_topology(deployment, status)) + == EXIT_SUCCESS + && aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS + && aurora_bgd_admin_setup(admin, deployment, 1800, 1801, 1802, 1803, false) + == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int main() { + plan(5); + CommandLine cl {}; + if (cl.getEnv()) { + diag("failed to load TAP environment"); + return exit_status(); + } + + MYSQL* primary_admin = init_mysql_conn( + cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + MYSQL* replica_admin = nullptr; + Replica_Process replica_process; + string primary_sqlite_interfaces; + BGD_Simulator primary_simulator; + BGD_Simulator replica_simulator; + Aurora_BGD_Test_Deployment deployment = peer_deployment(); + bool primary_simulator_connected = false; + bool replica_simulator_connected = false; + + if (primary_admin == nullptr + || get_variable_value( + primary_admin, "sqliteserver-mysql_ifaces", primary_sqlite_interfaces) != EXIT_SUCCESS + || configure_sqlite_interfaces(primary_admin, kSQLiteInterfaces) != EXIT_SUCCESS + || launch_replica(cl, replica_process) != EXIT_SUCCESS) { + diag("failed to prepare the two ProxySQL nodes"); + goto cleanup; + } + + replica_admin = wait_for_proxysql( + {kReplicaHost, cl.admin_username, cl.admin_password, kReplicaAdminPort}, + kWaitSeconds); + if (replica_admin == nullptr + || configure_sqlite_interfaces(replica_admin, kSQLiteInterfaces) != EXIT_SUCCESS) { + diag("failed to start the replica ProxySQL node"); + goto cleanup; + } + + { + char username[] = "aurora1"; + char password[] = "pass1"; + primary_simulator_connected = primary_simulator.connect( + cl.host, 3306, username, password) == EXIT_SUCCESS; + replica_simulator_connected = replica_simulator.connect( + const_cast(kReplicaHost), 3306, username, password) == EXIT_SUCCESS; + } + if (!primary_simulator_connected || !replica_simulator_connected + || configure_peer(primary_admin, primary_simulator, deployment, "AVAILABLE") + != EXIT_SUCCESS + || configure_peer( + replica_admin, replica_simulator, deployment, "SWITCHOVER_INITIATED") + != EXIT_SUCCESS) { + diag("failed to publish the two node-local Aurora observations"); + goto cleanup; + } + + ok(aurora_bgd_wait_for_status(primary_admin, 1800, "AVAILABLE", kWaitSeconds) + == EXIT_SUCCESS, "primary worker publishes its local AVAILABLE observation"); + ok(aurora_bgd_wait_for_status( + replica_admin, 1800, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, + "replica worker publishes its local SWITCHOVER_INITIATED observation"); + + if (aurora_bgd_execute_all(replica_admin, { + "DELETE FROM proxysql_servers", + "INSERT INTO proxysql_servers(hostname,port,weight,comment) VALUES (" + + aurora_bgd_sql_quote(cl.admin_host) + "," + to_string(cl.admin_port) + + ",0,'Aurora BGD sync primary')", + "LOAD PROXYSQL SERVERS TO RUNTIME", + }) != EXIT_SUCCESS + || aurora_bgd_execute_all(primary_admin, { + "UPDATE mysql_aws_aurora_hostgroups SET comment='peer-config-synced' " + "WHERE writer_hostgroup=1800", + "LOAD MYSQL SERVERS TO RUNTIME", + }) != EXIT_SUCCESS) { + diag("failed to initiate Aurora configuration synchronization"); + goto cleanup; + } + + ok(wait_for_cond( + replica_admin, + "SELECT COUNT(*)=1 FROM mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=1800 AND comment='peer-config-synced'", + kWaitSeconds) == EXIT_SUCCESS, + "Aurora BGD configured fields synchronize to the peer"); + ok(aurora_bgd_wait_for_status(primary_admin, 1800, "AVAILABLE", kWaitSeconds) + == EXIT_SUCCESS, "configuration sync preserves the primary's local status"); + ok(aurora_bgd_wait_for_status( + replica_admin, 1800, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, + "configuration sync preserves the replica's local status"); + +cleanup: + if (primary_simulator_connected) { + primary_simulator.cleanup(); + } + if (replica_simulator_connected) { + replica_simulator.cleanup(); + } + if (replica_admin != nullptr) { + aurora_bgd_admin_cleanup(replica_admin); + } + if (primary_admin != nullptr) { + aurora_bgd_admin_cleanup(primary_admin); + if (!primary_sqlite_interfaces.empty()) { + configure_sqlite_interfaces(primary_admin, primary_sqlite_interfaces); + } + mysql_close(primary_admin); + } + stop_replica(replica_admin, replica_process, tests_failed() != 0); + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_lifecycle-t.cpp b/test/tap/tests/test_aurora_bgd_lifecycle-t.cpp new file mode 100644 index 0000000000..77a2e210d0 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_lifecycle-t.cpp @@ -0,0 +1,397 @@ +/** + * @file test_aurora_bgd_lifecycle-t.cpp + * @brief Aurora BGD reload, worker removal, and concurrent-cluster behavior. + */ + +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const char kOrdinaryAuroraQuery[] = + "SELECT SERVER_ID," + "IF(" + "SESSION_ID = 'MASTER_SESSION_ID' AND " + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " + "'probably_former_MASTER_SESSION_ID', SESSION_ID" + ") SESSION_ID, " + "LAST_UPDATE_TIMESTAMP, " + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " + "CPU " + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" + " ( " + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" + " OR SESSION_ID = 'MASTER_SESSION_ID'" + " ) " + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" + " ORDER BY SERVER_ID"; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + return EXIT_FAILURE; + } + char username[] = "aurora1"; + char password[] = "pass1"; + if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS + || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS + || sim.cleanup() != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " + "VALUES ('testuser','testuser',1,0,1)", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { + return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS + && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int reset_rc = reset_scenario(admin, sim); + int user_rc = aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); + mysql_close(admin); + return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int publish_status( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)); +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + return aurora_bgd_execute_all(admin, { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int add_writer_route( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, int hostgroup +) { + return aurora_bgd_execute_all(admin, { + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(hostgroup) + "," + + aurora_bgd_sql_quote(deployment.production.members.front().endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD lifecycle route')", + "LOAD MYSQL SERVERS TO RUNTIME", + }); +} + +bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected) { + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS) { + return false; + } + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return false; + } + auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; + mysql_close(client); + if (query_rc != EXIT_SUCCESS) { + return false; + } + auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); + if (logs_rc != EXIT_SUCCESS) { + return false; + } + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary + && log.backend.host == expected.host && log.backend.port == expected.port) { + return true; + } + } + return false; +} + +bool route_writer( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, int route_hg, bool target +) { + if (set_default_hostgroup(admin, route_hg) != EXIT_SUCCESS) { + return false; + } + const Endpoint expected = target + ? deployment.target.members.front().endpoint.backend() + : deployment.production.members.front().endpoint.backend(); + return route_to_backend(cl, sim, expected); +} + +bool wait_for_writer_route( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, int route_hg, bool target +) { + if (set_default_hostgroup(admin, route_hg) != EXIT_SUCCESS) { + return false; + } + const Endpoint expected = target + ? deployment.target.members.front().endpoint.backend() + : deployment.production.members.front().endpoint.backend(); + for (uint32_t elapsed_ms = 0; elapsed_ms < kWaitSeconds * 1000; elapsed_ms += 100) { + if (route_to_backend(cl, sim, expected)) { + return true; + } + usleep(100000); + } + return false; +} + +bool writer_placement( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, bool demoted +) { + string query = + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "0" : "1") + ") AND " + "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "1" : "0") + ")"; + return wait_for_cond(admin, query, kWaitSeconds) == EXIT_SUCCESS; +} + +bool wait_for_inactive_none(MYSQL* admin, int writer_hg) { + return wait_for_cond( + admin, + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hg) + " AND active=0 AND bgd_status='NONE'", + kWaitSeconds) == EXIT_SUCCESS; +} + +bool wait_for_runtime_row_absent(MYSQL* admin, int writer_hg) { + return wait_for_cond( + admin, + "SELECT COUNT(*)=0 FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hg), + kWaitSeconds) == EXIT_SUCCESS; +} + +int main() { + plan(28); + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + Aurora_BGD_Test_Deployment reload = aurora_bgd_deployment_a(); + if (aurora_bgd_publish(sim, reload) != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, reload, 1620, 1621, 1622, 1623, false, 300) + != EXIT_SUCCESS + || add_writer_route(admin, reload, 1624) != EXIT_SUCCESS) { + diag("Error: failed to configure the active reload scenario"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status(admin, 1620, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "reload scenario starts from AVAILABLE"); + ok(publish_status(sim, reload, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1620, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS + && route_writer(cl, admin, sim, reload, 1624, true), + "active reload scenario pins writer traffic to the target"); + ok(aurora_bgd_execute_all(admin, {"LOAD MYSQL SERVERS TO RUNTIME"}) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1620, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "an unrelated server reload preserves active FSM status"); + ok(route_writer(cl, admin, sim, reload, 1624, true), + "an unrelated server reload preserves the applied traffic pin"); + ok(aurora_bgd_execute_all(admin, { + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + }) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1620, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "disabling auto-discovery does not abort an admitted deployment"); + ok(route_writer(cl, admin, sim, reload, 1624, true), + "a variable refresh preserves cached membership and target routing"); + ok(aurora_bgd_execute_all(admin, { + "UPDATE mysql_aws_aurora_hostgroups SET green_writer_hostgroup=1626," + "green_reader_hostgroup=1627 WHERE writer_hostgroup=1620", + "LOAD MYSQL SERVERS TO RUNTIME", + }) == EXIT_SUCCESS + && wait_for_cond( + admin, + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=1620 AND green_writer_hostgroup=1626 " + "AND green_reader_hostgroup=1627 " + "AND bgd_status='SWITCHOVER_IN_POST_PROCESSING'", + kWaitSeconds) == EXIT_SUCCESS, + "green hostgroup changes refresh staging references without resetting the FSM"); + ok(route_writer(cl, admin, sim, reload, 1624, true), + "green hostgroup refresh preserves applied pins and cached target IPs"); + ok(aurora_bgd_execute_all(admin, { + "UPDATE mysql_aws_aurora_hostgroups SET active=0 WHERE writer_hostgroup=1620", + "LOAD MYSQL SERVERS TO RUNTIME", + }) == EXIT_SUCCESS && wait_for_inactive_none(admin, 1620), + "deactivating the owning row stops the worker after publishing NONE"); + ok(writer_placement( + admin, 1620, 1621, reload.production.members.front().endpoint.hostname, false), + "worker teardown restores canonical writer placement"); + ok(route_writer(cl, admin, sim, reload, 1624, false), + "worker teardown removes the applied traffic pin"); + ok(wait_for_cond( + admin, + "SELECT ((SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=1620 AND active=0)=1) AND " + "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostname=" + + aurora_bgd_sql_quote(reload.production.members.front().endpoint.hostname) + ")>=1)", + kWaitSeconds) == EXIT_SUCCESS, + "teardown preserves user configuration and server rows"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before terminal reload scenario"); + cleanup(admin, sim); + return exit_status(); + } + Aurora_BGD_Test_Deployment terminal = aurora_bgd_deployment_b_writer_only(); + if (aurora_bgd_publish(sim, terminal) != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, terminal, 1630, 1631, -1, -1, true, 300) + != EXIT_SUCCESS) { + diag("Error: failed to configure the terminal reload scenario"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status(admin, 1630, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "terminal reload scenario starts from AVAILABLE"); + ok(sim.topology_update( + aurora_bgd_topology_backends(terminal), aurora_bgd_completed_topology(terminal)) + == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1630, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS, + "direct completion enters the terminal latch"); + ok(aurora_bgd_execute_all(admin, { + "UPDATE mysql_aws_aurora_hostgroups SET green_writer_hostgroup=1632," + "green_reader_hostgroup=1633 WHERE writer_hostgroup=1630", + "LOAD MYSQL SERVERS TO RUNTIME", + "SET mysql-aws_blue_green_deployment_auto_discovery='false'", + "LOAD MYSQL VARIABLES TO RUNTIME", + }) == EXIT_SUCCESS + && wait_for_cond( + admin, + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=1630 AND green_writer_hostgroup=1632 " + "AND green_reader_hostgroup=1633 AND bgd_status='SWITCHOVER_COMPLETED'", + kWaitSeconds) == EXIT_SUCCESS, + "configuration and variable refresh preserve the terminal latch"); + ok(sim.topology_delete(aurora_bgd_topology_backends(terminal)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1630, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "a successful topology drain rearms a refreshed terminal worker"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before concurrent scenario"); + cleanup(admin, sim); + return exit_status(); + } + Aurora_BGD_Test_Deployment concurrent_a = aurora_bgd_deployment_a(); + Aurora_BGD_Test_Deployment concurrent_b = aurora_bgd_deployment_b_writer_only(); + Aurora_BGD_Test_Deployment concurrent_c = aurora_bgd_deployment_c_writer_only(); + if (aurora_bgd_publish(sim, concurrent_a) != EXIT_SUCCESS + || aurora_bgd_publish(sim, concurrent_b) != EXIT_SUCCESS + || aurora_bgd_publish(sim, concurrent_c) != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, concurrent_a, 1640, 1641, 1642, 1643, false, 300) + != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, concurrent_b, 1650, 1651, -1, -1, true, 300) + != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, concurrent_c, 1660, 1661, -1, -1, true, 300) + != EXIT_SUCCESS + || add_writer_route(admin, concurrent_a, 1644) != EXIT_SUCCESS + || add_writer_route(admin, concurrent_b, 1654) != EXIT_SUCCESS + || add_writer_route(admin, concurrent_c, 1664) != EXIT_SUCCESS) { + diag("Error: failed to configure three concurrent deployments"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status(admin, 1640, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1650, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1660, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "three writer hostgroups discover deployments independently"); + ok(publish_status(sim, concurrent_a, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + && publish_status(sim, concurrent_b, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + && publish_status(sim, concurrent_c, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1640, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1650, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1660, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "three workers enter POST_PROCESSING without sharing FSM state"); + ok(route_writer(cl, admin, sim, concurrent_a, 1644, true), + "deployment A owns its target pin"); + ok(route_writer(cl, admin, sim, concurrent_b, 1654, true), + "deployment B owns its target pin"); + ok(route_writer(cl, admin, sim, concurrent_c, 1664, true), + "deployment C owns its target pin"); + ok(aurora_bgd_execute_all(admin, { + "UPDATE mysql_aws_aurora_hostgroups SET active=0 WHERE writer_hostgroup=1650", + "LOAD MYSQL SERVERS TO RUNTIME", + }) == EXIT_SUCCESS + && wait_for_inactive_none(admin, 1650) + && aurora_bgd_wait_for_status( + admin, 1640, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1660, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "deactivating deployment B cleans only its worker state"); + ok(writer_placement( + admin, 1650, 1651, concurrent_b.production.members.front().endpoint.hostname, false) + && route_writer(cl, admin, sim, concurrent_b, 1654, false), + "deployment B teardown restores only its production routing"); + ok(route_writer(cl, admin, sim, concurrent_a, 1644, true) + && route_writer(cl, admin, sim, concurrent_c, 1664, true), + "deployment B teardown leaves A and C pins intact"); + ok(sim.topology_update( + aurora_bgd_topology_backends(concurrent_a), + aurora_bgd_completed_topology(concurrent_a)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1640, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, 1660, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "completing deployment A leaves deployment C active"); + ok(add_writer_route(admin, concurrent_a, 1645) == EXIT_SUCCESS + && route_writer(cl, admin, sim, concurrent_a, 1645, false) + && route_writer(cl, admin, sim, concurrent_c, 1664, true), + "deployment A cleanup removes only its pin"); + ok(aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=1660", + "LOAD MYSQL SERVERS TO RUNTIME", + }) == EXIT_SUCCESS + && wait_for_runtime_row_absent(admin, 1660) + && writer_placement( + admin, 1660, 1661, + concurrent_c.production.members.front().endpoint.hostname, false) + && wait_for_writer_route(cl, admin, sim, concurrent_c, 1664, false), + "removing deployment C safely restores its production routing"); + ok(aurora_bgd_wait_for_status( + admin, 1640, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS + && route_writer(cl, admin, sim, concurrent_a, 1645, false), + "deployment C removal leaves deployment A terminal state unchanged"); + + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD lifecycle test data"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_resilience-t.cpp b/test/tap/tests/test_aurora_bgd_resilience-t.cpp new file mode 100644 index 0000000000..9d11f19728 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_resilience-t.cpp @@ -0,0 +1,412 @@ +/** + * @file test_aurora_bgd_resilience-t.cpp + * @brief Aurora BGD rollback, error retention, and active-state late entry. + */ + +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const uint32_t kProbeTimeoutMs = 5000; +const char kOrdinaryAuroraQuery[] = + "SELECT SERVER_ID," + "IF(" + "SESSION_ID = 'MASTER_SESSION_ID' AND " + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " + "'probably_former_MASTER_SESSION_ID', SESSION_ID" + ") SESSION_ID, " + "LAST_UPDATE_TIMESTAMP, " + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " + "CPU " + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" + " ( " + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" + " OR SESSION_ID = 'MASTER_SESSION_ID'" + " ) " + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" + " ORDER BY SERVER_ID"; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + return EXIT_FAILURE; + } + char username[] = "aurora1"; + char password[] = "pass1"; + if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS + || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS + || sim.cleanup() != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " + "VALUES ('testuser','testuser',1,0,1)", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { + return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS + && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int reset_rc = reset_scenario(admin, sim); + int user_rc = aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); + mysql_close(admin); + return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int publish_initial( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return sim.replica_update( + deployment.production.replica_set_id, + deployment.production.replica_rows(), deployment.production.backends()) == EXIT_SUCCESS + && sim.replica_update( + deployment.target.replica_set_id, + deployment.target.replica_rows(), deployment.target.backends()) == EXIT_SUCCESS + && sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)) == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int publish_status( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)); +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + return aurora_bgd_execute_all(admin, { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +bool writer_placement( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, bool demoted +) { + string query = + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "0" : "1") + ") AND " + "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "1" : "0") + ")"; + return wait_for_cond(admin, query, kWaitSeconds) == EXIT_SUCCESS; +} + +bool active_probe_policy( + BGD_Simulator& sim, uint64_t sequence, const string& target_replica_set, + uint32_t observation_ms +) { + usleep(observation_ms * 1000); + auto [rc, logs] = sim.replica_probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return false; + } + bool membership = false; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { + return false; + } + membership |= log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership + && log.replica_set_id == target_replica_set; + } + return membership; +} + +bool wait_for_ordinary_probe( + BGD_Simulator& sim, uint64_t sequence, Aurora_BGD_Test_Deployment& deployment +) { + auto [rc, log] = aurora_bgd_wait_for_replica_probe( + sim, sequence, deployment.production.backends(), + Aurora_Replica_Probe_Kind::ordinary, kProbeTimeoutMs); + return rc == EXIT_SUCCESS; +} + +int add_routes( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, + const vector& route_hgs, int green_writer_hg +) { + vector queries; + for (size_t i = 0; i < route_hgs.size(); ++i) { + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(route_hgs[i]) + "," + + aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD rollback route')"); + } + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(green_writer_hg) + "," + + aurora_bgd_sql_quote(deployment.target.members.front().endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD rollback green pool')"); + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected) { + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS) { + return false; + } + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return false; + } + auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; + mysql_close(client); + if (query_rc != EXIT_SUCCESS) { + return false; + } + auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); + if (logs_rc != EXIT_SUCCESS) { + return false; + } + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary + && log.backend.host == expected.host && log.backend.port == expected.port) { + return true; + } + } + return false; +} + +bool route_members( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, const vector& route_hgs, bool target +) { + for (size_t i = 0; i < route_hgs.size(); ++i) { + if (set_default_hostgroup(admin, route_hgs[i]) != EXIT_SUCCESS) { + return false; + } + Endpoint expected = target + ? deployment.target.members[i].endpoint.backend() + : deployment.production.members[i].endpoint.backend(); + if (!route_to_backend(cl, sim, expected)) { + return false; + } + } + return true; +} + +int64_t pool_count(MYSQL* admin, int hostgroup) { + auto [rc, rows] = mysql_query_ext_rows( + admin, "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) " + "FROM stats_mysql_connection_pool WHERE hostgroup=" + to_string(hostgroup)); + return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 + ? strtoll(rows.front().front().c_str(), nullptr, 10) : -1; +} + +int main() { + plan(20); + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + Aurora_BGD_Test_Deployment initiated = aurora_bgd_deployment_b_writer_only(); + const int initiated_writer_hg = 1590; + const int initiated_reader_hg = 1591; + if (publish_initial(sim, initiated, "SWITCHOVER_INITIATED") != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, initiated, initiated_writer_hg, initiated_reader_hg, + 1592, 1593, false, 300, false) != EXIT_SUCCESS) { + diag("Error: failed to configure INITIATED late entry"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, + "late entry at INITIATED publishes the observed state"); + ok(writer_placement( + admin, initiated_writer_hg, initiated_reader_hg, + initiated.production.members.front().endpoint.hostname, false), + "late INITIATED entry leaves writer placement unchanged"); + auto [initiated_seq_rc, initiated_sequence] = sim.replica_probe_log_last_sequence(); + ok(initiated_seq_rc == EXIT_SUCCESS && active_probe_policy( + sim, initiated_sequence, initiated.target_replica_set, 450), + "late INITIATED entry reconstructs fast membership probing"); + auto [topology_error_seq_rc, topology_error_sequence] = sim.probe_log_last_sequence(); + auto topology_error_seen = topology_error_seq_rc == EXIT_SUCCESS + && sim.topology_error( + aurora_bgd_topology_backends(initiated), 1205, "simulated topology timeout") + == EXIT_SUCCESS + && aurora_bgd_wait_for_topology_probe( + sim, topology_error_sequence, initiated.target.backends(), + BGD_Probe_Kind::metadata, kProbeTimeoutMs).first == EXIT_SUCCESS; + ok(topology_error_seen && aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "SWITCHOVER_INITIATED", 1) == EXIT_SUCCESS, + "topology errors retain the active state"); + if (publish_status(sim, initiated, "SWITCHOVER_INITIATED") != EXIT_SUCCESS) { + diag("Error: failed to restore topology after error"); + cleanup(admin, sim); + return exit_status(); + } + auto [membership_error_seq_rc, membership_error_sequence] = + sim.replica_probe_log_last_sequence(); + auto membership_error_seen = membership_error_seq_rc == EXIT_SUCCESS + && sim.replica_error( + initiated.target.backends(), 1205, "simulated membership timeout") == EXIT_SUCCESS + && aurora_bgd_wait_for_replica_probe( + sim, membership_error_sequence, initiated.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs).first == EXIT_SUCCESS; + ok(membership_error_seen && aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "SWITCHOVER_INITIATED", 1) == EXIT_SUCCESS, + "membership errors retain the last complete state"); + if (sim.replica_update( + initiated.target.replica_set_id, initiated.target.replica_rows(), + initiated.target.backends()) != EXIT_SUCCESS + || publish_status(sim, initiated, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to advance to IN_PROGRESS"); + cleanup(admin, sim); + return exit_status(); + } + ok(writer_placement( + admin, initiated_writer_hg, initiated_reader_hg, + initiated.production.members.front().endpoint.hostname, true), + "IN_PROGRESS demotes the reconstructed production writer"); + ok(publish_status(sim, initiated, "SWITCHOVER_INITIATED") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "SWITCHOVER_INITIATED", kWaitSeconds) + == EXIT_SUCCESS + && writer_placement( + admin, initiated_writer_hg, initiated_reader_hg, + initiated.production.members.front().endpoint.hostname, false), + "a backward status rolls back writer placement before entering the earlier state"); + auto [ordinary_seq_rc, ordinary_sequence] = sim.replica_probe_log_last_sequence(); + ok(sim.topology_delete(aurora_bgd_topology_backends(initiated)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "a successful empty topology cancels the active deployment"); + ok(ordinary_seq_rc == EXIT_SUCCESS + && wait_for_ordinary_probe(sim, ordinary_sequence, initiated), + "cancellation resumes ordinary production probing"); + ok(publish_status(sim, initiated, "AVAILABLE") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, initiated_writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "the worker admits a repeated deployment after cancellation"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before IN_PROGRESS late entry"); + cleanup(admin, sim); + return exit_status(); + } + Aurora_BGD_Test_Deployment progress = aurora_bgd_deployment_b_writer_only(); + if (publish_initial(sim, progress, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, progress, 1600, 1601, 1602, 1603, false, 300, false) + != EXIT_SUCCESS) { + diag("Error: failed to configure IN_PROGRESS late entry"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status(admin, 1600, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) + == EXIT_SUCCESS + && writer_placement( + admin, 1600, 1601, progress.production.members.front().endpoint.hostname, true), + "late entry at IN_PROGRESS reconstructs and demotes the writer"); + ok(sim.topology_drop(aurora_bgd_topology_backends(progress)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1600, "NONE", kWaitSeconds) == EXIT_SUCCESS + && writer_placement( + admin, 1600, 1601, progress.production.members.front().endpoint.hostname, false), + "confirmed topology absence rolls back IN_PROGRESS to NONE"); + + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before POST_PROCESSING late entry"); + cleanup(admin, sim); + return exit_status(); + } + Aurora_BGD_Test_Deployment post = aurora_bgd_deployment_a(); + const vector route_hgs {1614, 1615, 1616}; + if (publish_initial(sim, post, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_admin_setup(admin, post, 1610, 1611, 1612, 1613, false, 300, false) + != EXIT_SUCCESS + || add_routes(admin, post, route_hgs, 1612) != EXIT_SUCCESS) { + diag("Error: failed to configure POST_PROCESSING late entry"); + cleanup(admin, sim); + return exit_status(); + } + ok(aurora_bgd_wait_for_status( + admin, 1610, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "late entry at POST_PROCESSING reconstructs the active phase"); + ok(route_members(cl, admin, sim, post, route_hgs, true), + "late POST_PROCESSING entry reconstructs and applies every target pin"); + ok(set_default_hostgroup(admin, 1612) == EXIT_SUCCESS + && route_to_backend(cl, sim, post.target.members.front().endpoint.backend()) + && pool_count(admin, 1612) >= 1, + "a configured green pool is established before rollback"); + auto [post_error_seq_rc, post_error_sequence] = sim.probe_log_last_sequence(); + bool post_error_seen = post_error_seq_rc == EXIT_SUCCESS + && sim.topology_error( + aurora_bgd_topology_backends(post), 1205, "simulated post timeout") == EXIT_SUCCESS + && aurora_bgd_wait_for_topology_probe( + sim, post_error_sequence, post.target.backends(), + BGD_Probe_Kind::metadata, kProbeTimeoutMs).first == EXIT_SUCCESS; + ok(post_error_seen && aurora_bgd_wait_for_status( + admin, 1610, "SWITCHOVER_IN_POST_PROCESSING", 1) == EXIT_SUCCESS + && route_members(cl, admin, sim, post, route_hgs, true), + "topology errors do not roll back applied POST_PROCESSING pins"); + if (publish_status(sim, post, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS) { + diag("Error: failed to restore POST_PROCESSING topology"); + cleanup(admin, sim); + return exit_status(); + } + auto [post_membership_seq_rc, post_membership_sequence] = + sim.replica_probe_log_last_sequence(); + bool post_membership_error = post_membership_seq_rc == EXIT_SUCCESS + && sim.replica_error(post.target.backends(), 1205, "simulated post membership timeout") + == EXIT_SUCCESS + && aurora_bgd_wait_for_replica_probe( + sim, post_membership_sequence, post.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs).first == EXIT_SUCCESS; + ok(post_membership_error + && sim.replica_update( + post.target.replica_set_id, post.target.replica_rows(), post.target.backends()) + == EXIT_SUCCESS + && route_members(cl, admin, sim, post, route_hgs, true), + "membership errors retain the last complete mapped routing"); + ok(publish_status(sim, post, "AVAILABLE") == EXIT_SUCCESS + && aurora_bgd_wait_for_status(admin, 1610, "AVAILABLE", kWaitSeconds) + == EXIT_SUCCESS, + "a backward POST_PROCESSING status completes rollback before AVAILABLE"); + ok(writer_placement( + admin, 1610, 1611, post.production.members.front().endpoint.hostname, false) + && route_members(cl, admin, sim, post, route_hgs, false), + "rollback removes pins and restores canonical writer and member routing"); + ok(pool_count(admin, 1612) >= 1, + "rollback preserves configured green pools instead of draining them"); + + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD resilience test data"); + return EXIT_FAILURE; + } + return exit_status(); +} From 1456064e92fa89d1f2f3d7b06935f36700e990c9 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sun, 16 Aug 2026 08:56:18 +0000 Subject: [PATCH 27/34] test: organize Aurora BGD simulator coverage --- test/tap/groups/groups.json | 6 +- test/tap/tests/test_aurora_bgd_active-t.cpp | 309 +++++++---- .../tests/test_aurora_bgd_cluster_sync-t.cpp | 174 ++++--- .../tests/test_aurora_bgd_completion-t.cpp | 396 ++++++++------ ...test_aurora_bgd_concurrent_isolation-t.cpp | 391 ++++++++++++++ .../tap/tests/test_aurora_bgd_discovery-t.cpp | 254 ++++++--- .../test_aurora_bgd_error_recovery-t.cpp | 489 ++++++++++++++++++ .../tests/test_aurora_bgd_late_entry-t.cpp | 364 +++++++++++++ .../tests/test_aurora_bgd_resilience-t.cpp | 412 --------------- ...=> test_aurora_bgd_worker_lifecycle-t.cpp} | 340 ++++++------ 10 files changed, 2164 insertions(+), 971 deletions(-) create mode 100644 test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_error_recovery-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_late_entry-t.cpp delete mode 100644 test/tap/tests/test_aurora_bgd_resilience-t.cpp rename test/tap/tests/{test_aurora_bgd_lifecycle-t.cpp => test_aurora_bgd_worker_lifecycle-t.cpp} (50%) diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index c4de080fa4..6dd6bc1909 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -367,8 +367,10 @@ "test_aurora_bgd_active-t" : [ "cluster_sim_aurora-g1" ], "test_aurora_bgd_completion-t" : [ "cluster_sim_aurora-g1" ], "test_aurora_bgd_cluster_sync-t" : [ "cluster_sim_aurora-g1" ], - "test_aurora_bgd_resilience-t" : [ "cluster_sim_aurora-g1" ], - "test_aurora_bgd_lifecycle-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_late_entry-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_error_recovery-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_worker_lifecycle-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_concurrent_isolation-t" : [ "cluster_sim_aurora-g1" ], "test_cluster_sim_galera-t" : [ "cluster_sim_galera-g1" ], "test_cluster_sim_group_repl-t" : [ "cluster_sim_group_repl-g1" ], "test_cluster_sim_read_only-t" : [ "cluster_sim_read_only-g1" ], diff --git a/test/tap/tests/test_aurora_bgd_active-t.cpp b/test/tap/tests/test_aurora_bgd_active-t.cpp index cbcc256222..b6a4a7a088 100644 --- a/test/tap/tests/test_aurora_bgd_active-t.cpp +++ b/test/tap/tests/test_aurora_bgd_active-t.cpp @@ -1,6 +1,14 @@ /** * @file test_aurora_bgd_active-t.cpp * @brief Aurora BGD active-state probe suspension and routing behavior. + * + * Steps: + * + * 1. Configure a complete deployment and reach AVAILABLE. + * 2. Enter INITIATED and verify placement suppression and fast target probing. + * 3. Enter IN_PROGRESS and verify one writer demotion with source routing intact. + * 4. Enter POST_PROCESSING and verify restoration, pool retirement, and target pins. + * 5. Complete a previously incomplete target snapshot during POST_PROCESSING. */ #include @@ -34,6 +42,21 @@ const char kOrdinaryAuroraQuery[] = "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" " ORDER BY SERVER_ID"; +struct TestState { + Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_a() }; + int writer_hostgroup { 1530 }; + int reader_hostgroup { 1531 }; + int green_writer_hostgroup { 1532 }; + int green_reader_hostgroup { 1533 }; + vector route_hostgroups { 1534, 1535, 1536 }; + Aurora_BGD_Test_Deployment gated { aurora_bgd_deployment_b_writer_only() }; + int gated_writer_hostgroup { 1540 }; + int gated_reader_hostgroup { 1541 }; + int gated_green_writer_hostgroup { 1542 }; + int gated_green_reader_hostgroup { 1543 }; + int gated_route_hostgroup { 1544 }; +}; + int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); @@ -82,6 +105,11 @@ int cleanup(MYSQL* admin, BGD_Simulator& sim) { ? EXIT_SUCCESS : EXIT_FAILURE; } +int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { + return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS + && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; +} + bool runtime_server_count( MYSQL* admin, int hostgroup, const string& hostname, int expected, const string& status = "" @@ -264,43 +292,45 @@ int wait_for_member_route_pool_count( kWaitSeconds); } -int main() { - plan(20); - - CommandLine cl {}; - MYSQL* admin = nullptr; - BGD_Simulator sim {}; - if (setup(cl, admin, sim) != EXIT_SUCCESS) { - return exit_status(); - } - - const int writer_hg = 1530; - const int reader_hg = 1531; - const vector route_hgs {1534, 1535, 1536}; - Aurora_BGD_Test_Deployment deployment = aurora_bgd_deployment_a(); +/** Configure the active switchover scenario and reach AVAILABLE. */ +int test_bgd_status_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { + Aurora_BGD_Test_Deployment& deployment = state.deployment; if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS || aurora_bgd_admin_setup( - admin, deployment, writer_hg, reader_hg, 1532, 1533, false, 1000, false) - != EXIT_SUCCESS - || add_member_routes(admin, deployment, route_hgs) != EXIT_SUCCESS) { + admin, deployment, state.writer_hostgroup, state.reader_hostgroup, + state.green_writer_hostgroup, state.green_reader_hostgroup, + false, 1000, false) != EXIT_SUCCESS + || add_member_routes(admin, deployment, state.route_hostgroups) != EXIT_SUCCESS) { diag("Error: failed to configure the active switchover scenario"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } - ok(aurora_bgd_wait_for_status(admin, writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + ok(aurora_bgd_wait_for_status( + admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, "active scenario starts from a complete AVAILABLE snapshot"); + return EXIT_SUCCESS; +} +/** + * Enter SWITCHOVER_INITIATED. + * + * - Keep canonical writer placement unchanged. + * - Replace ordinary Aurora probes with fast target-membership probes. + * - Ignore a competing production role observation. + */ +int test_switchover_initiated(MYSQL* admin, BGD_Simulator& sim, TestState& state) { + Aurora_BGD_Test_Deployment& deployment = state.deployment; if (publish_status(sim, deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS || aurora_bgd_wait_for_status( - admin, writer_hg, "SWITCHOVER_INITIATED", kWaitSeconds) != EXIT_SUCCESS) { + admin, state.writer_hostgroup, "SWITCHOVER_INITIATED", kWaitSeconds) + != EXIT_SUCCESS) { diag("Error: worker did not enter SWITCHOVER_INITIATED"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } + ok(true, "INITIATED is published"); ok(wait_for_writer_placement( - admin, writer_hg, reader_hg, + admin, state.writer_hostgroup, state.reader_hostgroup, deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, "INITIATED does not change writer placement"); @@ -317,161 +347,250 @@ int main() { deployment.blue_replica_set, competing_source, deployment.production.backends()) != EXIT_SUCCESS) { diag("Error: failed to publish the competing source role observation"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } ok(fast_membership_without_ordinary( sim, source_change_sequence, deployment.target_replica_set, 350, 2), "changed source roles cannot compete while production probing is suspended"); + return EXIT_SUCCESS; +} +/** + * Enter SWITCHOVER_IN_PROGRESS. + * + * - Demote the snapshotted production writer exactly once. + * - Keep the competing source reader out of the writer hostgroup. + * - Preserve source-backed routing pools. + */ +int test_switchover_in_progress( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) +{ + Aurora_BGD_Test_Deployment& deployment = state.deployment; if (publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS || aurora_bgd_wait_for_status( - admin, writer_hg, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) != EXIT_SUCCESS) { + admin, state.writer_hostgroup, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) + != EXIT_SUCCESS) { diag("Error: worker did not enter SWITCHOVER_IN_PROGRESS"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } + ok(true, "IN_PROGRESS is published"); ok(wait_for_writer_placement( - admin, writer_hg, reader_hg, + admin, state.writer_hostgroup, state.reader_hostgroup, deployment.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, "IN_PROGRESS demotes the snapshotted production writer"); ok(runtime_server_count( - admin, writer_hg, deployment.production.members[1].endpoint.hostname, 0) + admin, state.writer_hostgroup, deployment.production.members[1].endpoint.hostname, 0) && runtime_server_count( - admin, reader_hg, deployment.production.members[1].endpoint.hostname, 1, "ONLINE"), + admin, state.reader_hostgroup, + deployment.production.members[1].endpoint.hostname, 1, "ONLINE"), "the competing source observation does not promote a reader"); auto [repeat_progress_rc, repeat_progress_sequence] = sim.replica_probe_log_last_sequence(); if (repeat_progress_rc != EXIT_SUCCESS || publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS) { diag("Error: failed to repeat SWITCHOVER_IN_PROGRESS"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } ok(fast_membership_without_ordinary( sim, repeat_progress_sequence, deployment.target_replica_set, 350, 2) && wait_for_writer_placement( - admin, writer_hg, reader_hg, + admin, state.writer_hostgroup, state.reader_hostgroup, deployment.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, "repeated IN_PROGRESS retains the one demotion and active probe policy"); - ok(route_members_to_expected_ips(cl, admin, sim, deployment, route_hgs, false) - && member_route_pool_count(admin, route_hgs) - >= static_cast(route_hgs.size()), + ok(route_members_to_expected_ips( + cl, admin, sim, deployment, state.route_hostgroups, false) + && member_route_pool_count(admin, state.route_hostgroups) + >= static_cast(state.route_hostgroups.size()), "all member routes use source IPs and hold pre-cutover pools"); + return EXIT_SUCCESS; +} +/** + * Enter SWITCHOVER_IN_POST_PROCESSING. + * + * - Restore canonical placement and keep readers eligible. + * - Retire pre-cutover pools and pin every production hostname to its target IP. + * - Avoid replaying retirement on repeated POST_PROCESSING. + */ +int test_switchover_post_processing( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) +{ + Aurora_BGD_Test_Deployment& deployment = state.deployment; if (publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS || aurora_bgd_wait_for_status( - admin, writer_hg, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { + admin, state.writer_hostgroup, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) + != EXIT_SUCCESS) { diag("Error: worker did not enter SWITCHOVER_IN_POST_PROCESSING"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } + ok(true, "POST_PROCESSING is published without a target-writability gate"); ok(wait_for_writer_placement( - admin, writer_hg, reader_hg, + admin, state.writer_hostgroup, state.reader_hostgroup, deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, "POST_PROCESSING restores the canonical writer placement"); ok(runtime_server_count( - admin, reader_hg, deployment.production.members[1].endpoint.hostname, 1, "ONLINE") + admin, state.reader_hostgroup, + deployment.production.members[1].endpoint.hostname, 1, "ONLINE") && runtime_server_count( - admin, reader_hg, deployment.production.members[2].endpoint.hostname, 1, "ONLINE"), + admin, state.reader_hostgroup, + deployment.production.members[2].endpoint.hostname, 1, "ONLINE"), "POST_PROCESSING leaves canonical readers ONLINE and eligible"); - ok(wait_for_member_route_pool_count(admin, route_hgs, "=0") == EXIT_SUCCESS, + ok(wait_for_member_route_pool_count( + admin, state.route_hostgroups, "=0") == EXIT_SUCCESS, "POST_PROCESSING retires the pre-cutover member pools"); - ok(route_members_to_expected_ips(cl, admin, sim, deployment, route_hgs, true), + ok(route_members_to_expected_ips( + cl, admin, sim, deployment, state.route_hostgroups, true), "POST_PROCESSING pins every production hostname to its cached target IP"); - const int64_t target_pool_count = member_route_pool_count(admin, route_hgs); + const int64_t target_pool_count = member_route_pool_count(admin, state.route_hostgroups); auto [repeat_post_rc, repeat_post_sequence] = sim.probe_log_last_sequence(); - if (target_pool_count < static_cast(route_hgs.size()) + if (target_pool_count < static_cast(state.route_hostgroups.size()) || repeat_post_rc != EXIT_SUCCESS || publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS) { diag("Error: failed to prepare repeated POST_PROCESSING"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } vector target_backends = deployment.target.backends(); auto [repeat_probe_rc, repeat_probe] = aurora_bgd_wait_for_topology_probe( sim, repeat_post_sequence, target_backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); ok(repeat_probe_rc == EXIT_SUCCESS - && member_route_pool_count(admin, route_hgs) >= target_pool_count, + && member_route_pool_count(admin, state.route_hostgroups) >= target_pool_count, "repeated POST_PROCESSING does not replay completed member retirement"); auto [post_seq_rc, post_sequence] = sim.replica_probe_log_last_sequence(); ok(post_seq_rc == EXIT_SUCCESS && fast_membership_without_ordinary( sim, post_sequence, deployment.target_replica_set, 350, 2), "POST_PROCESSING keeps fast BGD probes without ordinary Aurora queries"); + return EXIT_SUCCESS; +} - if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { +/** + * Complete target membership after entering POST_PROCESSING. + * + * - Leave routing unchanged while the target snapshot is incomplete. + * - Publish complete membership. + * - Apply target routing on the next POST_PROCESSING observation. + */ +int test_post_processing_after_membership_completion( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) +{ + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { diag("Error: failed to reset before the incomplete-snapshot scenario"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } - const int gated_writer_hg = 1540; - const int gated_reader_hg = 1541; - const int gated_route_hg = 1544; - Aurora_BGD_Test_Deployment gated = aurora_bgd_deployment_b_writer_only(); - vector invalid_target = gated.target.replica_rows(); + Aurora_BGD_Test_Deployment& deployment = state.gated; + vector invalid_target = deployment.target.replica_rows(); invalid_target.front().server_id = "unpaired-target-writer"; if (sim.replica_update( - gated.blue_replica_set, gated.production.replica_rows(), gated.production.backends()) - != EXIT_SUCCESS + deployment.blue_replica_set, deployment.production.replica_rows(), + deployment.production.backends()) != EXIT_SUCCESS || sim.replica_update( - gated.target_replica_set, invalid_target, gated.target.backends()) + deployment.target_replica_set, invalid_target, deployment.target.backends()) != EXIT_SUCCESS || sim.topology_update( - aurora_bgd_topology_backends(gated), aurora_bgd_available_topology(gated)) - != EXIT_SUCCESS + aurora_bgd_topology_backends(deployment), + aurora_bgd_available_topology(deployment)) != EXIT_SUCCESS || aurora_bgd_admin_setup( - admin, gated, gated_writer_hg, gated_reader_hg, 1542, 1543, false, 1000, false) - != EXIT_SUCCESS - || add_member_routes(admin, gated, {gated_route_hg}) != EXIT_SUCCESS) { + admin, deployment, state.gated_writer_hostgroup, + state.gated_reader_hostgroup, state.gated_green_writer_hostgroup, + state.gated_green_reader_hostgroup, false, 1000, false) != EXIT_SUCCESS + || add_member_routes( + admin, deployment, { state.gated_route_hostgroup }) != EXIT_SUCCESS) { diag("Error: failed to configure incomplete target membership"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } + ok(aurora_bgd_wait_for_status( - admin, gated_writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + admin, state.gated_writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, "an incomplete target snapshot can publish AVAILABLE without routing"); - if (publish_status(sim, gated, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + if (publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS || aurora_bgd_wait_for_status( - admin, gated_writer_hg, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) - != EXIT_SUCCESS) { + admin, state.gated_writer_hostgroup, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { diag("Error: incomplete scenario did not publish POST_PROCESSING"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } - const int gated_default_rc = set_default_hostgroup(admin, gated_route_hg); - ok(gated_default_rc == EXIT_SUCCESS && route_to_expected_backend( - cl, sim, gated.production.members.front().endpoint.backend()), + + int default_rc = set_default_hostgroup(admin, state.gated_route_hostgroup); + ok(default_rc == EXIT_SUCCESS && route_to_expected_backend( + cl, sim, deployment.production.members.front().endpoint.backend()), "POST_PROCESSING leaves routing unchanged without a complete target snapshot"); - auto [gated_refresh_seq_rc, gated_refresh_sequence] = sim.replica_probe_log_last_sequence(); - if (gated_refresh_seq_rc != EXIT_SUCCESS || sim.replica_update( - gated.target_replica_set, gated.target.replica_rows(), gated.target.backends()) - != EXIT_SUCCESS) { + auto [refresh_seq_rc, refresh_sequence] = sim.replica_probe_log_last_sequence(); + if (refresh_seq_rc != EXIT_SUCCESS || sim.replica_update( + deployment.target_replica_set, deployment.target.replica_rows(), + deployment.target.backends()) != EXIT_SUCCESS) { diag("Error: failed to restore complete target membership"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } - auto [gated_probe_rc, gated_probe] = aurora_bgd_wait_for_replica_probe( - sim, gated_refresh_sequence, gated.target.backends(), + auto [probe_rc, probe] = aurora_bgd_wait_for_replica_probe( + sim, refresh_sequence, deployment.target.backends(), Aurora_Replica_Probe_Kind::bgd_membership, - kProbeTimeoutMs, gated.target_replica_set); - bool gated_target_routing = false; - if (gated_probe_rc == EXIT_SUCCESS - && wait_for_member_route_pool_count(admin, {gated_route_hg}, "=0") == EXIT_SUCCESS - && set_default_hostgroup(admin, gated_route_hg) == EXIT_SUCCESS) { - gated_target_routing = route_to_expected_backend( - cl, sim, gated.target.members.front().endpoint.backend()); - } - ok(gated_target_routing, + kProbeTimeoutMs, deployment.target_replica_set); + bool target_routing = false; + if (probe_rc == EXIT_SUCCESS + && wait_for_member_route_pool_count( + admin, { state.gated_route_hostgroup }, "=0") == EXIT_SUCCESS + && set_default_hostgroup(admin, state.gated_route_hostgroup) == EXIT_SUCCESS) { + target_routing = route_to_expected_backend( + cl, sim, deployment.target.members.front().endpoint.backend()); + } + ok(target_routing, "repeated POST_PROCESSING applies routing after membership becomes complete"); + return EXIT_SUCCESS; +} + +int main() { + plan(20); + + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish a complete deployment. + // ProxySQL: configure Aurora BGD and per-member routing hostgroups. + // Verify: the worker reaches AVAILABLE. + if (test_bgd_status_available(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish INITIATED and a competing production role observation. + // Verify: placement is suppressed and only fast target membership remains active. + if (test_switchover_initiated(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish IN_PROGRESS twice. + // Verify: the snapshotted writer is demoted once and source routes remain active. + if (test_switchover_in_progress(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish POST_PROCESSING twice. + // Verify: canonical placement, target pins, and one-time pool retirement. + if (test_switchover_post_processing(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: enter POST_PROCESSING with incomplete membership, then complete it. + // Verify: routing changes only after a complete target map exists. + if (test_post_processing_after_membership_completion(cl, admin, sim, state) + != EXIT_SUCCESS) { + goto exit_cleanup; + } +exit_cleanup: if (cleanup(admin, sim) != EXIT_SUCCESS) { diag("Error: failed to clean Aurora BGD active-state test data"); return EXIT_FAILURE; diff --git a/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp b/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp index 743b5e4673..961b064368 100644 --- a/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp +++ b/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp @@ -1,6 +1,13 @@ /** * @file test_aurora_bgd_cluster_sync-t.cpp * @brief Aurora BGD configuration sync preserves worker-owned status per node. + * + * Steps: + * + * 1. Start two ProxySQL nodes with independent simulator endpoints. + * 2. Publish AVAILABLE on the primary and INITIATED on the replica. + * 3. Synchronize the configured Aurora hostgroup row from primary to replica. + * 4. Verify configured fields synchronize while runtime status remains node-local. */ #include @@ -34,6 +41,20 @@ struct Replica_Process { string stderr_path; }; +Aurora_BGD_Test_Deployment peer_deployment(); + +struct TestState { + MYSQL* primary_admin { nullptr }; + MYSQL* replica_admin { nullptr }; + Replica_Process replica_process {}; + string primary_sqlite_interfaces; + BGD_Simulator primary_simulator {}; + BGD_Simulator replica_simulator {}; + Aurora_BGD_Test_Deployment deployment { peer_deployment() }; + bool primary_simulator_connected { false }; + bool replica_simulator_connected { false }; +}; + string config_quote(const string& value) { string quoted; for (char c : value) { @@ -195,112 +216,145 @@ int configure_peer( ? EXIT_SUCCESS : EXIT_FAILURE; } -int main() { - plan(5); - CommandLine cl {}; +int setup(CommandLine& cl, TestState& state) { if (cl.getEnv()) { diag("failed to load TAP environment"); - return exit_status(); + return EXIT_FAILURE; } - MYSQL* primary_admin = init_mysql_conn( + state.primary_admin = init_mysql_conn( cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); - MYSQL* replica_admin = nullptr; - Replica_Process replica_process; - string primary_sqlite_interfaces; - BGD_Simulator primary_simulator; - BGD_Simulator replica_simulator; - Aurora_BGD_Test_Deployment deployment = peer_deployment(); - bool primary_simulator_connected = false; - bool replica_simulator_connected = false; - - if (primary_admin == nullptr + if (state.primary_admin == nullptr || get_variable_value( - primary_admin, "sqliteserver-mysql_ifaces", primary_sqlite_interfaces) != EXIT_SUCCESS - || configure_sqlite_interfaces(primary_admin, kSQLiteInterfaces) != EXIT_SUCCESS - || launch_replica(cl, replica_process) != EXIT_SUCCESS) { + state.primary_admin, "sqliteserver-mysql_ifaces", + state.primary_sqlite_interfaces) != EXIT_SUCCESS + || configure_sqlite_interfaces( + state.primary_admin, kSQLiteInterfaces) != EXIT_SUCCESS + || launch_replica(cl, state.replica_process) != EXIT_SUCCESS) { diag("failed to prepare the two ProxySQL nodes"); - goto cleanup; + return EXIT_FAILURE; } - replica_admin = wait_for_proxysql( + state.replica_admin = wait_for_proxysql( {kReplicaHost, cl.admin_username, cl.admin_password, kReplicaAdminPort}, kWaitSeconds); - if (replica_admin == nullptr - || configure_sqlite_interfaces(replica_admin, kSQLiteInterfaces) != EXIT_SUCCESS) { + if (state.replica_admin == nullptr + || configure_sqlite_interfaces( + state.replica_admin, kSQLiteInterfaces) != EXIT_SUCCESS) { diag("failed to start the replica ProxySQL node"); - goto cleanup; + return EXIT_FAILURE; } - { - char username[] = "aurora1"; - char password[] = "pass1"; - primary_simulator_connected = primary_simulator.connect( - cl.host, 3306, username, password) == EXIT_SUCCESS; - replica_simulator_connected = replica_simulator.connect( - const_cast(kReplicaHost), 3306, username, password) == EXIT_SUCCESS; - } - if (!primary_simulator_connected || !replica_simulator_connected - || configure_peer(primary_admin, primary_simulator, deployment, "AVAILABLE") - != EXIT_SUCCESS + char username[] = "aurora1"; + char password[] = "pass1"; + state.primary_simulator_connected = state.primary_simulator.connect( + cl.host, 3306, username, password) == EXIT_SUCCESS; + state.replica_simulator_connected = state.replica_simulator.connect( + const_cast(kReplicaHost), 3306, username, password) == EXIT_SUCCESS; + if (!state.primary_simulator_connected || !state.replica_simulator_connected || configure_peer( - replica_admin, replica_simulator, deployment, "SWITCHOVER_INITIATED") - != EXIT_SUCCESS) { + state.primary_admin, state.primary_simulator, + state.deployment, "AVAILABLE") != EXIT_SUCCESS + || configure_peer( + state.replica_admin, state.replica_simulator, + state.deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS) { diag("failed to publish the two node-local Aurora observations"); - goto cleanup; + return EXIT_FAILURE; } + return EXIT_SUCCESS; +} - ok(aurora_bgd_wait_for_status(primary_admin, 1800, "AVAILABLE", kWaitSeconds) - == EXIT_SUCCESS, "primary worker publishes its local AVAILABLE observation"); +/** + * Synchronize configured Aurora fields without synchronizing worker-owned status. + * + * - Verify each node publishes its local topology observation. + * - Synchronize the primary's configured hostgroup comment to the replica. + * - Verify AVAILABLE and INITIATED remain local to their respective workers. + */ +int test_configuration_sync_preserves_local_status(CommandLine& cl, TestState& state) { ok(aurora_bgd_wait_for_status( - replica_admin, 1800, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, + state.primary_admin, 1800, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "primary worker publishes its local AVAILABLE observation"); + ok(aurora_bgd_wait_for_status( + state.replica_admin, 1800, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, "replica worker publishes its local SWITCHOVER_INITIATED observation"); - if (aurora_bgd_execute_all(replica_admin, { + if (aurora_bgd_execute_all(state.replica_admin, { "DELETE FROM proxysql_servers", "INSERT INTO proxysql_servers(hostname,port,weight,comment) VALUES (" + aurora_bgd_sql_quote(cl.admin_host) + "," + to_string(cl.admin_port) + ",0,'Aurora BGD sync primary')", "LOAD PROXYSQL SERVERS TO RUNTIME", }) != EXIT_SUCCESS - || aurora_bgd_execute_all(primary_admin, { + || aurora_bgd_execute_all(state.primary_admin, { "UPDATE mysql_aws_aurora_hostgroups SET comment='peer-config-synced' " "WHERE writer_hostgroup=1800", "LOAD MYSQL SERVERS TO RUNTIME", }) != EXIT_SUCCESS) { diag("failed to initiate Aurora configuration synchronization"); - goto cleanup; + return EXIT_FAILURE; } ok(wait_for_cond( - replica_admin, + state.replica_admin, "SELECT COUNT(*)=1 FROM mysql_aws_aurora_hostgroups " "WHERE writer_hostgroup=1800 AND comment='peer-config-synced'", kWaitSeconds) == EXIT_SUCCESS, "Aurora BGD configured fields synchronize to the peer"); - ok(aurora_bgd_wait_for_status(primary_admin, 1800, "AVAILABLE", kWaitSeconds) - == EXIT_SUCCESS, "configuration sync preserves the primary's local status"); ok(aurora_bgd_wait_for_status( - replica_admin, 1800, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, + state.primary_admin, 1800, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "configuration sync preserves the primary's local status"); + ok(aurora_bgd_wait_for_status( + state.replica_admin, 1800, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, "configuration sync preserves the replica's local status"); + return EXIT_SUCCESS; +} -cleanup: - if (primary_simulator_connected) { - primary_simulator.cleanup(); +int cleanup(TestState& state) { + if (state.primary_simulator_connected) { + state.primary_simulator.cleanup(); } - if (replica_simulator_connected) { - replica_simulator.cleanup(); + if (state.replica_simulator_connected) { + state.replica_simulator.cleanup(); } - if (replica_admin != nullptr) { - aurora_bgd_admin_cleanup(replica_admin); + if (state.replica_admin != nullptr) { + aurora_bgd_admin_cleanup(state.replica_admin); } - if (primary_admin != nullptr) { - aurora_bgd_admin_cleanup(primary_admin); - if (!primary_sqlite_interfaces.empty()) { - configure_sqlite_interfaces(primary_admin, primary_sqlite_interfaces); + if (state.primary_admin != nullptr) { + aurora_bgd_admin_cleanup(state.primary_admin); + if (!state.primary_sqlite_interfaces.empty()) { + configure_sqlite_interfaces( + state.primary_admin, state.primary_sqlite_interfaces); } - mysql_close(primary_admin); + mysql_close(state.primary_admin); + state.primary_admin = nullptr; + } + stop_replica( + state.replica_admin, state.replica_process, tests_failed() != 0); + return EXIT_SUCCESS; +} + +int main() { + plan(5); + + CommandLine cl {}; + TestState state {}; + + if (setup(cl, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish different Aurora states to two real ProxySQL nodes. + // ProxySQL: synchronize the configured Aurora hostgroup row between them. + // Verify: configured fields synchronize while runtime status remains node-local. + if (test_configuration_sync_preserves_local_status(cl, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(state) != EXIT_SUCCESS) { + diag("failed to clean the Aurora BGD cluster-sync state"); + return EXIT_FAILURE; } - stop_replica(replica_admin, replica_process, tests_failed() != 0); return exit_status(); } diff --git a/test/tap/tests/test_aurora_bgd_completion-t.cpp b/test/tap/tests/test_aurora_bgd_completion-t.cpp index fdc22e1cbe..ce7ba1dfae 100644 --- a/test/tap/tests/test_aurora_bgd_completion-t.cpp +++ b/test/tap/tests/test_aurora_bgd_completion-t.cpp @@ -1,6 +1,13 @@ /** * @file test_aurora_bgd_completion-t.cpp * @brief Aurora BGD completion cleanup and terminal-latch behavior. + * + * Steps: + * + * 1. Complete directly from IN_PROGRESS and reconcile only the writer effect. + * 2. Complete after POST_PROCESSING and clean pins and eligible pools once. + * 3. Rearm completion only for a new fingerprint or confirmed topology drain. + * 4. Start directly at completion and verify no earlier phase is replayed. */ #include @@ -34,6 +41,24 @@ const char kOrdinaryAuroraQuery[] = "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" " ORDER BY SERVER_ID"; +struct TestState { + Aurora_BGD_Test_Deployment progress { aurora_bgd_deployment_b_writer_only() }; + int progress_writer_hostgroup { 1550 }; + int progress_reader_hostgroup { 1551 }; + int progress_green_writer_hostgroup { 1552 }; + int progress_green_reader_hostgroup { 1553 }; + Aurora_BGD_Test_Deployment post { aurora_bgd_deployment_a() }; + int post_writer_hostgroup { 1560 }; + int post_reader_hostgroup { 1561 }; + int green_writer_hostgroup { 1562 }; + int green_reader_hostgroup { 1563 }; + vector route_hostgroups { 1564, 1565, 1566 }; + int post_completion_route_hostgroup { 1567 }; + Aurora_BGD_Test_Deployment direct { aurora_bgd_deployment_b_writer_only() }; + int direct_writer_hostgroup { 1580 }; + int direct_reader_hostgroup { 1581 }; +}; + int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); @@ -300,247 +325,312 @@ bool wait_for_topology_observation( return rc == EXIT_SUCCESS; } -int main() { - plan(28); - - CommandLine cl {}; - MYSQL* admin = nullptr; - BGD_Simulator sim {}; - if (setup(cl, admin, sim) != EXIT_SUCCESS) { - return exit_status(); - } - - // Completion from IN_PROGRESS reconciles the writer without replaying POST_PROCESSING. - Aurora_BGD_Test_Deployment progress = aurora_bgd_deployment_b_writer_only(); - const int progress_writer_hg = 1550; - const int progress_reader_hg = 1551; - if (aurora_bgd_publish(sim, progress) != EXIT_SUCCESS +/** + * Complete a deployment directly from IN_PROGRESS. + * + * - Restore the demoted writer without replaying POST_PROCESSING. + * - Resume ordinary production probing while retaining a terminal latch. + * - Release the latch only after confirmed topology absence. + */ +int test_completion_from_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { + Aurora_BGD_Test_Deployment& deployment = state.progress; + if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS || aurora_bgd_admin_setup( - admin, progress, progress_writer_hg, progress_reader_hg, - 1552, 1553, false, 300, false) != EXIT_SUCCESS) { + admin, deployment, state.progress_writer_hostgroup, + state.progress_reader_hostgroup, state.progress_green_writer_hostgroup, + state.progress_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS) { diag("Error: failed to configure completion-from-IN_PROGRESS scenario"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } + ok(aurora_bgd_wait_for_status( - admin, progress_writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + admin, state.progress_writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, "completion-from-IN_PROGRESS scenario reaches AVAILABLE"); - if (publish_status(sim, progress, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + if (publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS || aurora_bgd_wait_for_status( - admin, progress_writer_hg, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) - != EXIT_SUCCESS) { + admin, state.progress_writer_hostgroup, + "SWITCHOVER_IN_PROGRESS", kWaitSeconds) != EXIT_SUCCESS) { diag("Error: failed to reach IN_PROGRESS"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } ok(wait_for_writer_demotion( - admin, progress_writer_hg, progress_reader_hg, - progress.production.members.front().endpoint.hostname) == EXIT_SUCCESS, + admin, state.progress_writer_hostgroup, state.progress_reader_hostgroup, + deployment.production.members.front().endpoint.hostname) == EXIT_SUCCESS, "IN_PROGRESS demotes the writer before completion"); - if (publish_completed(sim, progress, progress) != EXIT_SUCCESS + + if (publish_completed(sim, deployment, deployment) != EXIT_SUCCESS || aurora_bgd_wait_for_status( - admin, progress_writer_hg, "SWITCHOVER_COMPLETED", kWaitSeconds) - != EXIT_SUCCESS) { + admin, state.progress_writer_hostgroup, + "SWITCHOVER_COMPLETED", kWaitSeconds) != EXIT_SUCCESS) { diag("Error: failed to enter the completed latch from IN_PROGRESS"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } ok(true, "TARGET-only completion publishes SWITCHOVER_COMPLETED"); ok(wait_for_writer_policy( - admin, progress_writer_hg, progress_reader_hg, - progress.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + admin, state.progress_writer_hostgroup, state.progress_reader_hostgroup, + deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, "completion restores the demoted writer without replaying POST_PROCESSING"); - auto [progress_replica_seq_rc, progress_replica_sequence] = - sim.replica_probe_log_last_sequence(); - ok(progress_replica_seq_rc == EXIT_SUCCESS && completion_probe_policy( - sim, progress_replica_sequence, progress, 750), + + auto [replica_seq_rc, replica_sequence] = sim.replica_probe_log_last_sequence(); + ok(replica_seq_rc == EXIT_SUCCESS && completion_probe_policy( + sim, replica_sequence, deployment, 750), "completion resumes ordinary production probing and stops membership probing"); - auto [repeat_progress_seq_rc, repeat_progress_sequence] = sim.probe_log_last_sequence(); - bool repeated_progress = repeat_progress_seq_rc == EXIT_SUCCESS - && publish_completed(sim, progress, progress) == EXIT_SUCCESS - && wait_for_topology_observation(sim, repeat_progress_sequence, progress); - ok(repeated_progress && wait_for_writer_policy( - admin, progress_writer_hg, progress_reader_hg, - progress.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + + auto [repeat_seq_rc, repeat_sequence] = sim.probe_log_last_sequence(); + bool repeated = repeat_seq_rc == EXIT_SUCCESS + && publish_completed(sim, deployment, deployment) == EXIT_SUCCESS + && wait_for_topology_observation(sim, repeat_sequence, deployment); + ok(repeated && wait_for_writer_policy( + admin, state.progress_writer_hostgroup, state.progress_reader_hostgroup, + deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, "repeated completion is a no-op while latched"); + auto [error_seq_rc, error_sequence] = sim.probe_log_last_sequence(); bool error_retained = error_seq_rc == EXIT_SUCCESS && sim.topology_error( - aurora_bgd_topology_backends(progress), 1205, "simulated completion timeout") - == EXIT_SUCCESS - && wait_for_topology_observation(sim, error_sequence, progress); + aurora_bgd_topology_backends(deployment), 1205, + "simulated completion timeout") == EXIT_SUCCESS + && wait_for_topology_observation(sim, error_sequence, deployment); ok(error_retained && aurora_bgd_wait_for_status( - admin, progress_writer_hg, "SWITCHOVER_COMPLETED", 1) == EXIT_SUCCESS, + admin, state.progress_writer_hostgroup, "SWITCHOVER_COMPLETED", 1) + == EXIT_SUCCESS, "topology query errors retain the completed latch"); - ok(sim.topology_drop(aurora_bgd_topology_backends(progress)) == EXIT_SUCCESS + ok(sim.topology_drop(aurora_bgd_topology_backends(deployment)) == EXIT_SUCCESS && aurora_bgd_wait_for_status( - admin, progress_writer_hg, "NONE", kWaitSeconds) == EXIT_SUCCESS, + admin, state.progress_writer_hostgroup, "NONE", kWaitSeconds) == EXIT_SUCCESS, "confirmed topology absence releases the completed latch to NONE"); + return EXIT_SUCCESS; +} +/** + * Complete a deployment after POST_PROCESSING. + * + * - Remove production pins and drain eligible configured-green pools once. + * - Preserve configured rows, public status, and post-cutover production pools. + * - Rearm cleanup only for a different deployment fingerprint or topology drain. + */ +int test_completion_after_post_processing( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) +{ if (reset_scenario(admin, sim) != EXIT_SUCCESS) { diag("Error: failed to reset before POST_PROCESSING completion scenario"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } - // Completion after POST_PROCESSING removes pins and drains configured green pools once. - Aurora_BGD_Test_Deployment post = aurora_bgd_deployment_a(); - const int post_writer_hg = 1560; - const int post_reader_hg = 1561; - const int green_writer_hg = 1562; - const int green_reader_hg = 1563; - const vector route_hgs {1564, 1565, 1566}; - if (aurora_bgd_publish(sim, post) != EXIT_SUCCESS + Aurora_BGD_Test_Deployment& deployment = state.post; + if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS || aurora_bgd_admin_setup( - admin, post, post_writer_hg, post_reader_hg, - green_writer_hg, green_reader_hg, false, 300, true) != EXIT_SUCCESS - || add_green_servers(admin, post, green_writer_hg, green_reader_hg) != EXIT_SUCCESS - || add_member_routes(admin, post, route_hgs) != EXIT_SUCCESS) { + admin, deployment, state.post_writer_hostgroup, state.post_reader_hostgroup, + state.green_writer_hostgroup, state.green_reader_hostgroup, + false, 300, true) != EXIT_SUCCESS + || add_green_servers( + admin, deployment, state.green_writer_hostgroup, + state.green_reader_hostgroup) != EXIT_SUCCESS + || add_member_routes(admin, deployment, state.route_hostgroups) != EXIT_SUCCESS) { diag("Error: failed to configure POST_PROCESSING completion scenario"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } + ok(aurora_bgd_wait_for_status( - admin, post_writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + admin, state.post_writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, "POST_PROCESSING completion scenario reaches AVAILABLE"); - if (publish_status(sim, post, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + if (publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS || aurora_bgd_wait_for_status( - admin, post_writer_hg, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) - != EXIT_SUCCESS - || publish_status(sim, post, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + admin, state.post_writer_hostgroup, + "SWITCHOVER_IN_PROGRESS", kWaitSeconds) != EXIT_SUCCESS + || publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS || aurora_bgd_wait_for_status( - admin, post_writer_hg, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) - != EXIT_SUCCESS) { + admin, state.post_writer_hostgroup, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { diag("Error: failed to advance through POST_PROCESSING"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } ok(true, "active deployment advances through POST_PROCESSING"); - ok(route_members(cl, admin, sim, post, route_hgs, true), + ok(route_members(cl, admin, sim, deployment, state.route_hostgroups, true), "POST_PROCESSING routes every production member to its target IP"); - const int64_t target_route_pool = pool_count(admin, route_hgs.front()); - bool green_pool_ready = set_default_hostgroup(admin, green_writer_hg) == EXIT_SUCCESS - && route_to_expected_backend(cl, sim, post.target.members.front().endpoint.backend()) - && pool_count(admin, green_writer_hg) >= 1; + + const int64_t target_route_pool = pool_count(admin, state.route_hostgroups.front()); + bool green_pool_ready = set_default_hostgroup(admin, state.green_writer_hostgroup) + == EXIT_SUCCESS + && route_to_expected_backend( + cl, sim, deployment.target.members.front().endpoint.backend()) + && pool_count(admin, state.green_writer_hostgroup) >= 1; ok(target_route_pool >= 1 && green_pool_ready, "pre-completion target and configured-green pools are established"); - if (publish_completed(sim, post, post) != EXIT_SUCCESS + + if (publish_completed(sim, deployment, deployment) != EXIT_SUCCESS || aurora_bgd_wait_for_status( - admin, post_writer_hg, "SWITCHOVER_COMPLETED", kWaitSeconds) - != EXIT_SUCCESS) { + admin, state.post_writer_hostgroup, + "SWITCHOVER_COMPLETED", kWaitSeconds) != EXIT_SUCCESS) { diag("Error: failed to complete the POST_PROCESSING scenario"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } ok(true, "completion after POST_PROCESSING enters the terminal latch"); ok(wait_for_writer_policy( - admin, post_writer_hg, post_reader_hg, - post.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, + admin, state.post_writer_hostgroup, state.post_reader_hostgroup, + deployment.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, "completion preserves canonical writer_is_also_reader placement"); - ok(pool_count(admin, route_hgs.front()) >= target_route_pool, + ok(pool_count(admin, state.route_hostgroups.front()) >= target_route_pool, "completion does not repeat retirement of post-cutover production pools"); - ok(wait_for_pool_count(admin, green_writer_hg, "=0") == EXIT_SUCCESS, + ok(wait_for_pool_count( + admin, state.green_writer_hostgroup, "=0") == EXIT_SUCCESS, "completion drains eligible configured-green pools immediately"); ok(server_count( - admin, green_reader_hg, - post.target.members.back().endpoint.hostname, 1, "OFFLINE_SOFT"), + admin, state.green_reader_hostgroup, + deployment.target.members.back().endpoint.hostname, 1, "OFFLINE_SOFT"), "completion preserves configured green rows and OFFLINE status"); - const int post_completion_route_hg = 1567; ok(add_route( - admin, post_completion_route_hg, - post.production.members.front().endpoint.hostname) == EXIT_SUCCESS - && set_default_hostgroup(admin, post_completion_route_hg) == EXIT_SUCCESS + admin, state.post_completion_route_hostgroup, + deployment.production.members.front().endpoint.hostname) == EXIT_SUCCESS + && set_default_hostgroup( + admin, state.post_completion_route_hostgroup) == EXIT_SUCCESS && route_to_expected_backend( - cl, sim, post.production.members.front().endpoint.backend()), + cl, sim, deployment.production.members.front().endpoint.backend()), "completion removes the production traffic pin without DNS verification"); - auto [post_replica_seq_rc, post_replica_sequence] = sim.replica_probe_log_last_sequence(); - ok(post_replica_seq_rc == EXIT_SUCCESS && completion_probe_policy( - sim, post_replica_sequence, post, 750), + + auto [replica_seq_rc, replica_sequence] = sim.replica_probe_log_last_sequence(); + ok(replica_seq_rc == EXIT_SUCCESS && completion_probe_policy( + sim, replica_sequence, deployment, 750), "the completed latch uses configured cadence and canonical production probes"); - bool recreated_green_pool = set_default_hostgroup(admin, green_writer_hg) == EXIT_SUCCESS - && route_to_expected_backend(cl, sim, post.target.members.front().endpoint.backend()) - && pool_count(admin, green_writer_hg) >= 1; - auto [same_completed_seq_rc, same_completed_sequence] = sim.probe_log_last_sequence(); - bool same_completion_seen = same_completed_seq_rc == EXIT_SUCCESS - && publish_completed(sim, post, post) == EXIT_SUCCESS - && wait_for_topology_observation(sim, same_completed_sequence, post); - ok(recreated_green_pool && same_completion_seen && pool_count(admin, green_writer_hg) >= 1, + + bool recreated_green_pool = set_default_hostgroup(admin, state.green_writer_hostgroup) + == EXIT_SUCCESS + && route_to_expected_backend( + cl, sim, deployment.target.members.front().endpoint.backend()) + && pool_count(admin, state.green_writer_hostgroup) >= 1; + auto [same_seq_rc, same_sequence] = sim.probe_log_last_sequence(); + bool same_completion_seen = same_seq_rc == EXIT_SUCCESS + && publish_completed(sim, deployment, deployment) == EXIT_SUCCESS + && wait_for_topology_observation(sim, same_sequence, deployment); + ok(recreated_green_pool && same_completion_seen + && pool_count(admin, state.green_writer_hostgroup) >= 1, "repeated completion does not drain a pool created while latched"); - auto [latched_error_seq_rc, latched_error_sequence] = sim.probe_log_last_sequence(); - bool latched_error_seen = latched_error_seq_rc == EXIT_SUCCESS + + auto [error_seq_rc, error_sequence] = sim.probe_log_last_sequence(); + bool error_seen = error_seq_rc == EXIT_SUCCESS && sim.topology_error( - aurora_bgd_topology_backends(post), 1205, "simulated latched timeout") - == EXIT_SUCCESS - && wait_for_topology_observation(sim, latched_error_sequence, post); - ok(latched_error_seen && aurora_bgd_wait_for_status( - admin, post_writer_hg, "SWITCHOVER_COMPLETED", 1) == EXIT_SUCCESS - && pool_count(admin, green_writer_hg) >= 1, + aurora_bgd_topology_backends(deployment), 1205, + "simulated latched timeout") == EXIT_SUCCESS + && wait_for_topology_observation(sim, error_sequence, deployment); + ok(error_seen && aurora_bgd_wait_for_status( + admin, state.post_writer_hostgroup, "SWITCHOVER_COMPLETED", 1) + == EXIT_SUCCESS + && pool_count(admin, state.green_writer_hostgroup) >= 1, "query errors neither release the latch nor repeat completion cleanup"); + Aurora_BGD_Test_Deployment different = aurora_bgd_deployment_b_writer_only(); auto [different_seq_rc, different_sequence] = sim.probe_log_last_sequence(); bool different_seen = different_seq_rc == EXIT_SUCCESS - && publish_completed(sim, post, different) == EXIT_SUCCESS - && wait_for_topology_observation(sim, different_sequence, post); - ok(different_seen && wait_for_pool_count(admin, green_writer_hg, "=0") == EXIT_SUCCESS, + && publish_completed(sim, deployment, different) == EXIT_SUCCESS + && wait_for_topology_observation(sim, different_sequence, deployment); + ok(different_seen && wait_for_pool_count( + admin, state.green_writer_hostgroup, "=0") == EXIT_SUCCESS, "a different completed deployment fingerprint rearms and runs its cleanup"); - bool second_green_pool = set_default_hostgroup(admin, green_writer_hg) == EXIT_SUCCESS - && route_to_expected_backend(cl, sim, post.target.members.front().endpoint.backend()); + + bool second_green_pool = set_default_hostgroup(admin, state.green_writer_hostgroup) + == EXIT_SUCCESS + && route_to_expected_backend( + cl, sim, deployment.target.members.front().endpoint.backend()); auto [repeat_different_seq_rc, repeat_different_sequence] = sim.probe_log_last_sequence(); bool repeated_different = repeat_different_seq_rc == EXIT_SUCCESS - && publish_completed(sim, post, different) == EXIT_SUCCESS - && wait_for_topology_observation(sim, repeat_different_sequence, post); - ok(second_green_pool && repeated_different && pool_count(admin, green_writer_hg) >= 1, + && publish_completed(sim, deployment, different) == EXIT_SUCCESS + && wait_for_topology_observation(sim, repeat_different_sequence, deployment); + ok(second_green_pool && repeated_different + && pool_count(admin, state.green_writer_hostgroup) >= 1, "the new fingerprint is retained and its repeated completion is a no-op"); - ok(sim.topology_delete(aurora_bgd_topology_backends(post)) == EXIT_SUCCESS + ok(sim.topology_delete(aurora_bgd_topology_backends(deployment)) == EXIT_SUCCESS && aurora_bgd_wait_for_status( - admin, post_writer_hg, "NONE", kWaitSeconds) == EXIT_SUCCESS, + admin, state.post_writer_hostgroup, "NONE", kWaitSeconds) == EXIT_SUCCESS, "successful empty topology releases the rearmed completed latch"); + return EXIT_SUCCESS; +} +/** + * Start a worker from its first SWITCHOVER_COMPLETED observation. + * + * - Enter the terminal latch without a cached target map. + * - Keep canonical writer placement and avoid replaying active phases. + * - Rearm only after a successful topology drain. + */ +int test_first_completed_observation(MYSQL* admin, BGD_Simulator& sim, TestState& state) { if (reset_scenario(admin, sim) != EXIT_SUCCESS) { diag("Error: failed to reset before direct-completion scenario"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } - // A first observation at completion has no target map and does not replay prior phases. - Aurora_BGD_Test_Deployment direct = aurora_bgd_deployment_b_writer_only(); + Aurora_BGD_Test_Deployment& deployment = state.direct; if (sim.replica_update( - direct.blue_replica_set, direct.production.replica_rows(), - direct.production.backends()) != EXIT_SUCCESS + deployment.blue_replica_set, deployment.production.replica_rows(), + deployment.production.backends()) != EXIT_SUCCESS || sim.topology_update( - aurora_bgd_topology_backends(direct), - aurora_bgd_completed_topology(direct)) != EXIT_SUCCESS) { + aurora_bgd_topology_backends(deployment), + aurora_bgd_completed_topology(deployment)) != EXIT_SUCCESS) { diag("Error: failed to publish direct completion inputs"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } - auto [direct_seq_rc, direct_sequence] = sim.replica_probe_log_last_sequence(); + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); if (aurora_bgd_admin_setup( - admin, direct, 1580, 1581, -1, -1, true, 300, false) != EXIT_SUCCESS) { + admin, deployment, state.direct_writer_hostgroup, + state.direct_reader_hostgroup, -1, -1, true, 300, false) != EXIT_SUCCESS) { diag("Error: failed to configure direct completion"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } + ok(aurora_bgd_wait_for_status( - admin, 1580, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS, + admin, state.direct_writer_hostgroup, + "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS, "late entry directly at completion enters the terminal latch"); ok(wait_for_writer_policy( - admin, 1580, 1581, - direct.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, + admin, state.direct_writer_hostgroup, state.direct_reader_hostgroup, + deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, "direct completion leaves canonical writer placement unchanged"); - auto [direct_logs_rc, direct_logs] = sim.replica_probe_log_since(direct_sequence); - bool direct_membership_probe = false; - for (const Aurora_Replica_Probe_Log& log : direct_logs) { - direct_membership_probe |= - log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership; - } - ok(direct_seq_rc == EXIT_SUCCESS && direct_logs_rc == EXIT_SUCCESS - && !direct_membership_probe, + + auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); + bool membership_probe = false; + for (const Aurora_Replica_Probe_Log& log : logs) { + membership_probe |= log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership; + } + ok(sequence_rc == EXIT_SUCCESS && logs_rc == EXIT_SUCCESS && !membership_probe, "direct completion does not manufacture target membership or replay active phases"); - ok(sim.topology_delete(aurora_bgd_topology_backends(direct)) == EXIT_SUCCESS - && aurora_bgd_wait_for_status(admin, 1580, "NONE", kWaitSeconds) == EXIT_SUCCESS, + ok(sim.topology_delete(aurora_bgd_topology_backends(deployment)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, state.direct_writer_hostgroup, "NONE", kWaitSeconds) == EXIT_SUCCESS, "direct-completion latch rearms only after topology drain"); + return EXIT_SUCCESS; +} + +int main() { + plan(28); + + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: advance a deployment to IN_PROGRESS, then publish completion. + // Verify: only the writer effect is reconciled and the completed latch is retained. + if (test_completion_from_in_progress(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: advance through POST_PROCESSING, then publish completion repeatedly. + // Verify: pins and eligible pools are cleaned once per deployment fingerprint. + if (test_completion_after_post_processing(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: make SWITCHOVER_COMPLETED the first observed deployment state. + // Verify: prior phase effects are not manufactured or replayed. + if (test_first_completed_observation(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } +exit_cleanup: if (cleanup(admin, sim) != EXIT_SUCCESS) { diag("Error: failed to clean Aurora BGD completion test data"); return EXIT_FAILURE; diff --git a/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp b/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp new file mode 100644 index 0000000000..03cf0b3576 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp @@ -0,0 +1,391 @@ +/** + * @file test_aurora_bgd_concurrent_isolation-t.cpp + * @brief Isolation across three concurrent Aurora BGD workers. + * + * Steps: + * + * 1. Configure three deployments with independent workers and route hostgroups. + * 2. Enter POST_PROCESSING on all workers and verify each owns its target pin. + * 3. Deactivate one worker without changing the other two. + * 4. Complete and remove different workers without leaking lifecycle effects. + */ + +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const char kOrdinaryAuroraQuery[] = + "SELECT SERVER_ID," + "IF(" + "SESSION_ID = 'MASTER_SESSION_ID' AND " + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " + "'probably_former_MASTER_SESSION_ID', SESSION_ID" + ") SESSION_ID, " + "LAST_UPDATE_TIMESTAMP, " + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " + "CPU " + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" + " ( " + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" + " OR SESSION_ID = 'MASTER_SESSION_ID'" + " ) " + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" + " ORDER BY SERVER_ID"; + +struct TestState { + Aurora_BGD_Test_Deployment deployment_a { aurora_bgd_deployment_a() }; + int writer_hostgroup_a { 1640 }; + int reader_hostgroup_a { 1641 }; + int green_writer_hostgroup_a { 1642 }; + int green_reader_hostgroup_a { 1643 }; + int route_hostgroup_a { 1644 }; + int post_completion_route_hostgroup_a { 1645 }; + Aurora_BGD_Test_Deployment deployment_b { aurora_bgd_deployment_b_writer_only() }; + int writer_hostgroup_b { 1650 }; + int reader_hostgroup_b { 1651 }; + int route_hostgroup_b { 1654 }; + Aurora_BGD_Test_Deployment deployment_c { aurora_bgd_deployment_c_writer_only() }; + int writer_hostgroup_c { 1660 }; + int reader_hostgroup_c { 1661 }; + int route_hostgroup_c { 1664 }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + char username[] = "aurora1"; + char password[] = "pass1"; + if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS + || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS + || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to initialize the shared AWS simulator"); + return EXIT_FAILURE; + } + return aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " + "VALUES ('testuser','testuser',1,0,1)", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int admin_rc = aurora_bgd_admin_cleanup(admin); + int user_rc = aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); + int simulator_rc = sim.cleanup(); + mysql_close(admin); + return admin_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS + && simulator_rc == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int publish_status( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)); +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + return aurora_bgd_execute_all(admin, { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int add_writer_route( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, int hostgroup +) { + return aurora_bgd_execute_all(admin, { + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(hostgroup) + "," + + aurora_bgd_sql_quote(deployment.production.members.front().endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD concurrent route')", + "LOAD MYSQL SERVERS TO RUNTIME", + }); +} + +bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected) { + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS) { + return false; + } + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return false; + } + auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; + mysql_close(client); + if (query_rc != EXIT_SUCCESS) { + return false; + } + auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); + if (logs_rc != EXIT_SUCCESS) { + return false; + } + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary + && log.backend.host == expected.host && log.backend.port == expected.port) { + return true; + } + } + return false; +} + +bool route_writer( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, int route_hg, bool target +) { + if (set_default_hostgroup(admin, route_hg) != EXIT_SUCCESS) { + return false; + } + const Endpoint expected = target + ? deployment.target.members.front().endpoint.backend() + : deployment.production.members.front().endpoint.backend(); + return route_to_backend(cl, sim, expected); +} + +bool wait_for_writer_route( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, int route_hg, bool target +) { + if (set_default_hostgroup(admin, route_hg) != EXIT_SUCCESS) { + return false; + } + const Endpoint expected = target + ? deployment.target.members.front().endpoint.backend() + : deployment.production.members.front().endpoint.backend(); + for (uint32_t elapsed_ms = 0; elapsed_ms < kWaitSeconds * 1000; elapsed_ms += 100) { + if (route_to_backend(cl, sim, expected)) { + return true; + } + usleep(100000); + } + return false; +} + +bool writer_placement( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, bool demoted +) { + string query = + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "0" : "1") + ") AND " + "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "1" : "0") + ")"; + return wait_for_cond(admin, query, kWaitSeconds) == EXIT_SUCCESS; +} + +bool wait_for_inactive_none(MYSQL* admin, int writer_hg) { + return wait_for_cond( + admin, + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hg) + " AND active=0 AND bgd_status='NONE'", + kWaitSeconds) == EXIT_SUCCESS; +} + +bool wait_for_runtime_row_absent(MYSQL* admin, int writer_hg) { + return wait_for_cond( + admin, + "SELECT COUNT(*)=0 FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hg), + kWaitSeconds) == EXIT_SUCCESS; +} + +/** Configure three workers and move each independently to POST_PROCESSING. */ +int test_three_workers_post_processing( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) +{ + if (aurora_bgd_publish(sim, state.deployment_a) != EXIT_SUCCESS + || aurora_bgd_publish(sim, state.deployment_b) != EXIT_SUCCESS + || aurora_bgd_publish(sim, state.deployment_c) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, state.deployment_a, state.writer_hostgroup_a, + state.reader_hostgroup_a, state.green_writer_hostgroup_a, + state.green_reader_hostgroup_a, false, 300) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, state.deployment_b, state.writer_hostgroup_b, + state.reader_hostgroup_b, -1, -1, true, 300) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, state.deployment_c, state.writer_hostgroup_c, + state.reader_hostgroup_c, -1, -1, true, 300) != EXIT_SUCCESS + || add_writer_route( + admin, state.deployment_a, state.route_hostgroup_a) != EXIT_SUCCESS + || add_writer_route( + admin, state.deployment_b, state.route_hostgroup_b) != EXIT_SUCCESS + || add_writer_route( + admin, state.deployment_c, state.route_hostgroup_c) != EXIT_SUCCESS) { + diag("Error: failed to configure three concurrent deployments"); + return EXIT_FAILURE; + } + + ok(aurora_bgd_wait_for_status( + admin, state.writer_hostgroup_a, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, state.writer_hostgroup_b, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, state.writer_hostgroup_c, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "three writer hostgroups discover deployments independently"); + ok(publish_status( + sim, state.deployment_a, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + && publish_status( + sim, state.deployment_b, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + && publish_status( + sim, state.deployment_c, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, state.writer_hostgroup_a, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, state.writer_hostgroup_b, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, state.writer_hostgroup_c, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "three workers enter POST_PROCESSING without sharing FSM state"); + ok(route_writer( + cl, admin, sim, state.deployment_a, state.route_hostgroup_a, true), + "deployment A owns its target pin"); + ok(route_writer( + cl, admin, sim, state.deployment_b, state.route_hostgroup_b, true), + "deployment B owns its target pin"); + ok(route_writer( + cl, admin, sim, state.deployment_c, state.route_hostgroup_c, true), + "deployment C owns its target pin"); + return EXIT_SUCCESS; +} + +/** Deactivate deployment B without changing deployments A or C. */ +int test_deactivate_one_worker( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) +{ + ok(aurora_bgd_execute_all(admin, { + "UPDATE mysql_aws_aurora_hostgroups SET active=0 WHERE writer_hostgroup=" + + to_string(state.writer_hostgroup_b), + "LOAD MYSQL SERVERS TO RUNTIME", + }) == EXIT_SUCCESS + && wait_for_inactive_none(admin, state.writer_hostgroup_b) + && aurora_bgd_wait_for_status( + admin, state.writer_hostgroup_a, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, state.writer_hostgroup_c, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "deactivating deployment B cleans only its worker state"); + ok(writer_placement( + admin, state.writer_hostgroup_b, state.reader_hostgroup_b, + state.deployment_b.production.members.front().endpoint.hostname, false) + && route_writer( + cl, admin, sim, state.deployment_b, state.route_hostgroup_b, false), + "deployment B teardown restores only its production routing"); + ok(route_writer( + cl, admin, sim, state.deployment_a, state.route_hostgroup_a, true) + && route_writer( + cl, admin, sim, state.deployment_c, state.route_hostgroup_c, true), + "deployment B teardown leaves A and C pins intact"); + return EXIT_SUCCESS; +} + +/** Complete deployment A and remove deployment C without cross-worker effects. */ +int test_complete_and_remove_independent_workers( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) +{ + ok(sim.topology_update( + aurora_bgd_topology_backends(state.deployment_a), + aurora_bgd_completed_topology(state.deployment_a)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, state.writer_hostgroup_a, + "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, state.writer_hostgroup_c, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "completing deployment A leaves deployment C active"); + ok(add_writer_route( + admin, state.deployment_a, state.post_completion_route_hostgroup_a) + == EXIT_SUCCESS + && route_writer( + cl, admin, sim, state.deployment_a, + state.post_completion_route_hostgroup_a, false) + && route_writer( + cl, admin, sim, state.deployment_c, state.route_hostgroup_c, true), + "deployment A cleanup removes only its pin"); + ok(aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(state.writer_hostgroup_c), + "LOAD MYSQL SERVERS TO RUNTIME", + }) == EXIT_SUCCESS + && wait_for_runtime_row_absent(admin, state.writer_hostgroup_c) + && writer_placement( + admin, state.writer_hostgroup_c, state.reader_hostgroup_c, + state.deployment_c.production.members.front().endpoint.hostname, false) + && wait_for_writer_route( + cl, admin, sim, state.deployment_c, state.route_hostgroup_c, false), + "removing deployment C safely restores its production routing"); + ok(aurora_bgd_wait_for_status( + admin, state.writer_hostgroup_a, + "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS + && route_writer( + cl, admin, sim, state.deployment_a, + state.post_completion_route_hostgroup_a, false), + "deployment C removal leaves deployment A terminal state unchanged"); + return EXIT_SUCCESS; +} + +int main() { + plan(12); + + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish three independent deployments and POST_PROCESSING states. + // Verify: each worker owns only its target pin and FSM state. + if (test_three_workers_post_processing(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: deactivate deployment B. + // Verify: only deployment B returns to production routing. + if (test_deactivate_one_worker(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator and ProxySQL: complete A and remove C. + // Verify: their cleanup effects remain isolated from each other. + if (test_complete_and_remove_independent_workers(cl, admin, sim, state) + != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD concurrent-isolation test data"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_discovery-t.cpp b/test/tap/tests/test_aurora_bgd_discovery-t.cpp index 6462d1c141..90be2fb6a9 100644 --- a/test/tap/tests/test_aurora_bgd_discovery-t.cpp +++ b/test/tap/tests/test_aurora_bgd_discovery-t.cpp @@ -1,6 +1,14 @@ /** * @file test_aurora_bgd_discovery-t.cpp * @brief Aurora BGD AVAILABLE discovery, three probes, and fail-closed mapping. + * + * Steps: + * + * 1. Publish a complete SOURCE/TARGET deployment and verify AVAILABLE discovery. + * 2. Verify topology and target-membership probes move to the selected target. + * 3. Publish incomplete and inconsistent membership and retain the last complete map. + * 4. Publish invalid topology rows and retain the last valid runtime state. + * 5. Verify automatic discovery accepts a writer-only deployment. */ #include @@ -17,6 +25,17 @@ using namespace std; const uint32_t kWaitSeconds = 5; const uint32_t kProbeTimeoutMs = 5000; +struct TestState { + Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_a() }; + int writer_hostgroup { 1510 }; + int reader_hostgroup { 1511 }; + int green_writer_hostgroup { 1512 }; + int green_reader_hostgroup { 1513 }; + Aurora_BGD_Test_Deployment writer_only { aurora_bgd_deployment_b_writer_only() }; + int writer_only_writer_hostgroup { 1520 }; + int writer_only_reader_hostgroup { 1521 }; +}; + int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); @@ -52,6 +71,11 @@ int cleanup(MYSQL* admin, BGD_Simulator& sim) { ? EXIT_SUCCESS : EXIT_FAILURE; } +int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { + return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS + && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; +} + bool runtime_status_is(MYSQL* admin, int writer_hg, const string& status) { auto [rc, rows] = mysql_query_ext_rows( admin, @@ -118,75 +142,79 @@ bool invalid_topology_retains_available( return probe_rc == EXIT_SUCCESS && runtime_status_is(admin, writer_hg, "AVAILABLE"); } -int main() { - plan(18); - - CommandLine cl {}; - MYSQL* admin = nullptr; - BGD_Simulator sim {}; - if (setup(cl, admin, sim) != EXIT_SUCCESS) { - return exit_status(); - } - - const int writer_hg = 1510; - const int reader_hg = 1511; - Aurora_BGD_Test_Deployment deployment = aurora_bgd_deployment_a(); +/** + * Discover a complete Aurora blue/green deployment. + * + * - Publish complete production and target membership. + * - Verify the worker reaches AVAILABLE without interrupting ordinary Aurora probes. + * - Verify topology discovery, target bootstrap, and steady probes use the expected endpoints. + */ +int test_available_discovery(MYSQL* admin, BGD_Simulator& sim, TestState& state) { + Aurora_BGD_Test_Deployment& deployment = state.deployment; if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS - || aurora_bgd_admin_setup(admin, deployment, writer_hg, reader_hg, 1512, 1513, false) - != EXIT_SUCCESS) { + || aurora_bgd_admin_setup( + admin, deployment, state.writer_hostgroup, state.reader_hostgroup, + state.green_writer_hostgroup, state.green_reader_hostgroup, false) != EXIT_SUCCESS) { diag("Error: failed to publish or configure deployment A"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } - ok(aurora_bgd_wait_for_status(admin, writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + ok(aurora_bgd_wait_for_status( + admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, "valid SOURCE/TARGET topology publishes AVAILABLE"); - { - auto [rc, logs] = sim.replica_probe_log_since(0); - bool ordinary_seen = false; - for (const Aurora_Replica_Probe_Log& log : logs) { - if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary - && log.replica_set_id == deployment.blue_replica_set) { - ordinary_seen = true; - } + auto [logs_rc, logs] = sim.replica_probe_log_since(0); + bool ordinary_seen = false; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary + && log.replica_set_id == deployment.blue_replica_set) { + ordinary_seen = true; } - ok(rc == EXIT_SUCCESS && ordinary_seen, - "the existing worker continues ordinary Aurora probing in AVAILABLE"); } + ok(logs_rc == EXIT_SUCCESS && ordinary_seen, + "the existing worker continues ordinary Aurora probing in AVAILABLE"); vector blue_backends = deployment.production.backends(); - { - auto [rc, probe] = aurora_bgd_wait_for_topology_probe( - sim, 0, blue_backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); - ok(rc == EXIT_SUCCESS, "topology discovery starts on a reachable production member"); - } - { - auto [rc, probe] = sim.wait_for_replica_probe_log( - 0, deployment.target_cluster_endpoint.backend(), - Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, - 0, deployment.target_replica_set); - ok(rc == EXIT_SUCCESS, "TARGET cluster endpoint bootstraps target membership"); - } + auto [topology_rc, topology_probe] = aurora_bgd_wait_for_topology_probe( + sim, 0, blue_backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); + ok(topology_rc == EXIT_SUCCESS, + "topology discovery starts on a reachable production member"); + + auto [bootstrap_rc, bootstrap_probe] = sim.wait_for_replica_probe_log( + 0, deployment.target_cluster_endpoint.backend(), + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, + 0, deployment.target_replica_set); + ok(bootstrap_rc == EXIT_SUCCESS, + "TARGET cluster endpoint bootstraps target membership"); vector target_backends; for (Aurora_BGD_Member& member : deployment.target.members) { target_backends.push_back(member.endpoint.backend()); } - { - auto [rc, probe] = aurora_bgd_wait_for_topology_probe( - sim, 0, target_backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); - ok(rc == EXIT_SUCCESS, "complete membership moves topology probes to target members"); - } - { - auto [rc, probe] = aurora_bgd_wait_for_replica_probe( - sim, 0, target_backends, Aurora_Replica_Probe_Kind::bgd_membership, - kProbeTimeoutMs, deployment.target_replica_set); - ok(rc == EXIT_SUCCESS, "complete membership moves membership probes to target members"); - } + auto [target_topology_rc, target_topology_probe] = aurora_bgd_wait_for_topology_probe( + sim, 0, target_backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); + ok(target_topology_rc == EXIT_SUCCESS, + "complete membership moves topology probes to target members"); + auto [target_membership_rc, target_membership_probe] = aurora_bgd_wait_for_replica_probe( + sim, 0, target_backends, Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, deployment.target_replica_set); + ok(target_membership_rc == EXIT_SUCCESS, + "complete membership moves membership probes to target members"); + return EXIT_SUCCESS; +} + +/** + * Retain the last complete target membership after unusable observations. + * + * - Publish incomplete, ambiguous, unresolved, and mismatched target snapshots. + * - Ignore stale non-current rows. + * - Verify the worker continues probing the last complete target selection. + */ +int test_target_membership_validation(BGD_Simulator& sim, TestState& state) { + Aurora_BGD_Test_Deployment& deployment = state.deployment; vector complete_rows = deployment.target.replica_rows(); - vector incomplete_rows {complete_rows[0], complete_rows[1]}; + vector incomplete_rows { complete_rows[0], complete_rows[1] }; ok(retain_complete_target_after_membership(sim, deployment, incomplete_rows), "an incomplete target result retains the previous complete target selection"); @@ -213,63 +241,129 @@ int main() { ok(retain_complete_target_after_membership(sim, deployment, stale_extra_rows), "IS_CURRENT=0 rows are excluded from the complete target snapshot"); - if (sim.replica_update( - deployment.target_replica_set, complete_rows, deployment.target.backends()) != EXIT_SUCCESS) { + int restore_rc = sim.replica_update( + deployment.target_replica_set, complete_rows, deployment.target.backends()); + if (restore_rc != EXIT_SUCCESS) { diag("Error: failed to restore complete target membership"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } + return EXIT_SUCCESS; +} +/** + * Reject incomplete or unsupported topology observations. + * + * - Publish missing identity, SOURCE-only, mismatched-status, and unknown-status rows. + * - Verify the last AVAILABLE state and production routing remain unchanged. + */ +int test_topology_validation(MYSQL* admin, BGD_Simulator& sim, TestState& state) { + Aurora_BGD_Test_Deployment& deployment = state.deployment; vector valid_topology = aurora_bgd_available_topology(deployment); + vector missing_identity = valid_topology; missing_identity[1].id.clear(); - ok(invalid_topology_retains_available(admin, sim, deployment, missing_identity, writer_hg), + ok(invalid_topology_retains_available( + admin, sim, deployment, missing_identity, state.writer_hostgroup), "missing TARGET identity does not replace AVAILABLE"); - vector source_only {valid_topology.front()}; - ok(invalid_topology_retains_available(admin, sim, deployment, source_only, writer_hg), + vector source_only { valid_topology.front() }; + ok(invalid_topology_retains_available( + admin, sim, deployment, source_only, state.writer_hostgroup), "SOURCE-only topology does not replace AVAILABLE"); vector mismatched_status = valid_topology; mismatched_status[0].status = "SWITCHOVER_INITIATED"; - ok(invalid_topology_retains_available(admin, sim, deployment, mismatched_status, writer_hg), + ok(invalid_topology_retains_available( + admin, sim, deployment, mismatched_status, state.writer_hostgroup), "mismatched SOURCE/TARGET statuses do not replace AVAILABLE"); vector unknown_status = valid_topology; unknown_status[0].status = "UNSUPPORTED_STATUS"; unknown_status[1].status = "UNSUPPORTED_STATUS"; - ok(invalid_topology_retains_available(admin, sim, deployment, unknown_status, writer_hg), + ok(invalid_topology_retains_available( + admin, sim, deployment, unknown_status, state.writer_hostgroup), "unsupported topology status is not copied into runtime state"); - ok(runtime_production_unchanged(admin, writer_hg, reader_hg, deployment.production.members.size()), + ok(runtime_production_unchanged( + admin, state.writer_hostgroup, state.reader_hostgroup, + deployment.production.members.size()), "AVAILABLE discovery performs no production routing action"); + return EXIT_SUCCESS; +} - if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { +/** + * Discover a writer-only deployment without configured green hostgroups. + * + * - Reset the complete-cluster scenario. + * - Configure automatic discovery with writer-only production and target membership. + * - Verify the worker admits AVAILABLE and accepts the target snapshot. + */ +int test_writer_only_discovery(MYSQL* admin, BGD_Simulator& sim, TestState& state) { + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { diag("Error: failed to reset state before writer-only deployment"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } - Aurora_BGD_Test_Deployment writer_only = aurora_bgd_deployment_b_writer_only(); - if (aurora_bgd_publish(sim, writer_only) != EXIT_SUCCESS - || aurora_bgd_admin_setup(admin, writer_only, 1520, 1521, -1, -1, true) - != EXIT_SUCCESS) { + Aurora_BGD_Test_Deployment& deployment = state.writer_only; + if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, state.writer_only_writer_hostgroup, + state.writer_only_reader_hostgroup, -1, -1, true) != EXIT_SUCCESS) { diag("Error: failed to publish or configure writer-only deployment"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } - ok(aurora_bgd_wait_for_status(admin, 1520, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + + ok(aurora_bgd_wait_for_status( + admin, state.writer_only_writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, "auto-discovery admits a row without configured green hostgroups"); - { - vector writer_target { - writer_only.target.members.front().endpoint.backend() - }; - auto [rc, probe] = aurora_bgd_wait_for_replica_probe( - sim, 0, writer_target, Aurora_Replica_Probe_Kind::bgd_membership, - kProbeTimeoutMs, writer_only.target_replica_set); - ok(rc == EXIT_SUCCESS, "a writer-only production cluster accepts a writer-only target snapshot"); + + vector writer_target { deployment.target.members.front().endpoint.backend() }; + auto [probe_rc, probe] = aurora_bgd_wait_for_replica_probe( + sim, 0, writer_target, Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, deployment.target_replica_set); + ok(probe_rc == EXIT_SUCCESS, + "a writer-only production cluster accepts a writer-only target snapshot"); + return EXIT_SUCCESS; +} + +int main() { + plan(18); + + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: publish complete production and target membership. + // ProxySQL: configure explicit Aurora BGD hostgroups 1510-1513. + // Verify: discovery reaches AVAILABLE and hands probes to the selected target. + if (test_available_discovery(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: replace complete target membership with unusable observations. + // Verify: the worker retains its last complete target mapping. + if (test_target_membership_validation(sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish incomplete and unsupported topology rows. + // Verify: runtime state and production routing retain the last valid observation. + if (test_topology_validation(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: configure automatic discovery for a writer-only deployment. + // Verify: writer-only source and target snapshots are accepted. + if (test_writer_only_discovery(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; } +exit_cleanup: if (cleanup(admin, sim) != EXIT_SUCCESS) { diag("Error: failed to clean Aurora BGD discovery state"); return EXIT_FAILURE; diff --git a/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp b/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp new file mode 100644 index 0000000000..e79051ae67 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp @@ -0,0 +1,489 @@ +/** + * @file test_aurora_bgd_error_recovery-t.cpp + * @brief Aurora BGD active-state error retention, rollback, and cancellation. + * + * Steps: + * + * 1. Verify topology and membership errors retain INITIATED and POST_PROCESSING effects. + * 2. Move backward from IN_PROGRESS and restore canonical writer placement. + * 3. Cancel an active deployment through empty or absent topology and resume ordinary probes. + * 4. Roll back POST_PROCESSING pins while preserving configured green pools. + */ + +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const uint32_t kProbeTimeoutMs = 5000; +const char kOrdinaryAuroraQuery[] = + "SELECT SERVER_ID," + "IF(" + "SESSION_ID = 'MASTER_SESSION_ID' AND " + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " + "'probably_former_MASTER_SESSION_ID', SESSION_ID" + ") SESSION_ID, " + "LAST_UPDATE_TIMESTAMP, " + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " + "CPU " + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" + " ( " + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" + " OR SESSION_ID = 'MASTER_SESSION_ID'" + " ) " + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" + " ORDER BY SERVER_ID"; + +struct TestState { + Aurora_BGD_Test_Deployment initiated { aurora_bgd_deployment_b_writer_only() }; + int initiated_writer_hostgroup { 1590 }; + int initiated_reader_hostgroup { 1591 }; + int initiated_green_writer_hostgroup { 1592 }; + int initiated_green_reader_hostgroup { 1593 }; + Aurora_BGD_Test_Deployment progress { aurora_bgd_deployment_b_writer_only() }; + int progress_writer_hostgroup { 1600 }; + int progress_reader_hostgroup { 1601 }; + int progress_green_writer_hostgroup { 1602 }; + int progress_green_reader_hostgroup { 1603 }; + Aurora_BGD_Test_Deployment post { aurora_bgd_deployment_a() }; + int post_writer_hostgroup { 1610 }; + int post_reader_hostgroup { 1611 }; + int post_green_writer_hostgroup { 1612 }; + int post_green_reader_hostgroup { 1613 }; + vector post_route_hostgroups { 1614, 1615, 1616 }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + char username[] = "aurora1"; + char password[] = "pass1"; + if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS + || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS + || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to initialize the shared AWS simulator"); + return EXIT_FAILURE; + } + return aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " + "VALUES ('testuser','testuser',1,0,1)", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { + return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS + && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int reset_rc = reset_scenario(admin, sim); + int user_rc = aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); + mysql_close(admin); + return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int publish_initial( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return sim.replica_update( + deployment.production.replica_set_id, + deployment.production.replica_rows(), deployment.production.backends()) == EXIT_SUCCESS + && sim.replica_update( + deployment.target.replica_set_id, + deployment.target.replica_rows(), deployment.target.backends()) == EXIT_SUCCESS + && sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)) == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int publish_status( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)); +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + return aurora_bgd_execute_all(admin, { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +bool writer_placement( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, bool demoted +) { + string query = + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "0" : "1") + ") AND " + "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "1" : "0") + ")"; + return wait_for_cond(admin, query, kWaitSeconds) == EXIT_SUCCESS; +} + +bool wait_for_ordinary_probe( + BGD_Simulator& sim, uint64_t sequence, Aurora_BGD_Test_Deployment& deployment +) { + auto [rc, log] = aurora_bgd_wait_for_replica_probe( + sim, sequence, deployment.production.backends(), + Aurora_Replica_Probe_Kind::ordinary, kProbeTimeoutMs); + return rc == EXIT_SUCCESS; +} + +int add_routes( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, + const vector& route_hgs, int green_writer_hg +) { + vector queries; + for (size_t i = 0; i < route_hgs.size(); ++i) { + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(route_hgs[i]) + "," + + aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD rollback route')"); + } + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(green_writer_hg) + "," + + aurora_bgd_sql_quote(deployment.target.members.front().endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD rollback green pool')"); + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected) { + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS) { + return false; + } + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return false; + } + auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; + mysql_close(client); + if (query_rc != EXIT_SUCCESS) { + return false; + } + auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); + if (logs_rc != EXIT_SUCCESS) { + return false; + } + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary + && log.backend.host == expected.host && log.backend.port == expected.port) { + return true; + } + } + return false; +} + +bool route_members( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, const vector& route_hgs, bool target +) { + for (size_t i = 0; i < route_hgs.size(); ++i) { + if (set_default_hostgroup(admin, route_hgs[i]) != EXIT_SUCCESS) { + return false; + } + Endpoint expected = target + ? deployment.target.members[i].endpoint.backend() + : deployment.production.members[i].endpoint.backend(); + if (!route_to_backend(cl, sim, expected)) { + return false; + } + } + return true; +} + +int64_t pool_count(MYSQL* admin, int hostgroup) { + auto [rc, rows] = mysql_query_ext_rows( + admin, "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) " + "FROM stats_mysql_connection_pool WHERE hostgroup=" + to_string(hostgroup)); + return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 + ? strtoll(rows.front().front().c_str(), nullptr, 10) : -1; +} + +/** Retain INITIATED across topology and membership query errors. */ +int test_initiated_error_retention(MYSQL* admin, BGD_Simulator& sim, TestState& state) { + Aurora_BGD_Test_Deployment& deployment = state.initiated; + if (publish_initial(sim, deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, state.initiated_writer_hostgroup, + state.initiated_reader_hostgroup, state.initiated_green_writer_hostgroup, + state.initiated_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, state.initiated_writer_hostgroup, + "SWITCHOVER_INITIATED", kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: failed to configure INITIATED error retention"); + return EXIT_FAILURE; + } + + auto [topology_seq_rc, topology_sequence] = sim.probe_log_last_sequence(); + bool topology_error_seen = topology_seq_rc == EXIT_SUCCESS + && sim.topology_error( + aurora_bgd_topology_backends(deployment), 1205, + "simulated topology timeout") == EXIT_SUCCESS + && aurora_bgd_wait_for_topology_probe( + sim, topology_sequence, deployment.target.backends(), + BGD_Probe_Kind::metadata, kProbeTimeoutMs).first == EXIT_SUCCESS; + ok(topology_error_seen && aurora_bgd_wait_for_status( + admin, state.initiated_writer_hostgroup, "SWITCHOVER_INITIATED", 1) + == EXIT_SUCCESS, + "topology errors retain the active state"); + if (publish_status(sim, deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS) { + diag("Error: failed to restore topology after error"); + return EXIT_FAILURE; + } + + auto [membership_seq_rc, membership_sequence] = sim.replica_probe_log_last_sequence(); + bool membership_error_seen = membership_seq_rc == EXIT_SUCCESS + && sim.replica_error( + deployment.target.backends(), 1205, + "simulated membership timeout") == EXIT_SUCCESS + && aurora_bgd_wait_for_replica_probe( + sim, membership_sequence, deployment.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs).first + == EXIT_SUCCESS; + ok(membership_error_seen && aurora_bgd_wait_for_status( + admin, state.initiated_writer_hostgroup, "SWITCHOVER_INITIATED", 1) + == EXIT_SUCCESS, + "membership errors retain the last complete state"); + return EXIT_SUCCESS; +} + +/** Roll back IN_PROGRESS through an earlier status, then cancel and rearm. */ +int test_initiated_rollback_and_cancellation( + MYSQL* admin, BGD_Simulator& sim, TestState& state) +{ + Aurora_BGD_Test_Deployment& deployment = state.initiated; + if (sim.replica_update( + deployment.target.replica_set_id, deployment.target.replica_rows(), + deployment.target.backends()) != EXIT_SUCCESS + || publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, state.initiated_writer_hostgroup, + "SWITCHOVER_IN_PROGRESS", kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: failed to advance to IN_PROGRESS"); + return EXIT_FAILURE; + } + ok(writer_placement( + admin, state.initiated_writer_hostgroup, state.initiated_reader_hostgroup, + deployment.production.members.front().endpoint.hostname, true), + "IN_PROGRESS demotes the reconstructed production writer"); + ok(publish_status(sim, deployment, "SWITCHOVER_INITIATED") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, state.initiated_writer_hostgroup, + "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS + && writer_placement( + admin, state.initiated_writer_hostgroup, state.initiated_reader_hostgroup, + deployment.production.members.front().endpoint.hostname, false), + "a backward status rolls back writer placement before entering the earlier state"); + + auto [ordinary_seq_rc, ordinary_sequence] = sim.replica_probe_log_last_sequence(); + ok(sim.topology_delete(aurora_bgd_topology_backends(deployment)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, state.initiated_writer_hostgroup, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "a successful empty topology cancels the active deployment"); + ok(ordinary_seq_rc == EXIT_SUCCESS + && wait_for_ordinary_probe(sim, ordinary_sequence, deployment), + "cancellation resumes ordinary production probing"); + ok(publish_status(sim, deployment, "AVAILABLE") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, state.initiated_writer_hostgroup, "AVAILABLE", kWaitSeconds) + == EXIT_SUCCESS, + "the worker admits a repeated deployment after cancellation"); + return EXIT_SUCCESS; +} + +/** Roll back a first IN_PROGRESS observation after confirmed topology absence. */ +int test_in_progress_topology_absence(MYSQL* admin, BGD_Simulator& sim, TestState& state) { + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before IN_PROGRESS rollback"); + return EXIT_FAILURE; + } + Aurora_BGD_Test_Deployment& deployment = state.progress; + if (publish_initial(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, state.progress_writer_hostgroup, + state.progress_reader_hostgroup, state.progress_green_writer_hostgroup, + state.progress_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, state.progress_writer_hostgroup, + "SWITCHOVER_IN_PROGRESS", kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: failed to configure IN_PROGRESS rollback"); + return EXIT_FAILURE; + } + ok(sim.topology_drop(aurora_bgd_topology_backends(deployment)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, state.progress_writer_hostgroup, "NONE", kWaitSeconds) == EXIT_SUCCESS + && writer_placement( + admin, state.progress_writer_hostgroup, state.progress_reader_hostgroup, + deployment.production.members.front().endpoint.hostname, false), + "confirmed topology absence rolls back IN_PROGRESS to NONE"); + return EXIT_SUCCESS; +} + +/** Retain POST_PROCESSING pins across topology and membership query errors. */ +int test_post_processing_error_retention( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) +{ + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before POST_PROCESSING error retention"); + return EXIT_FAILURE; + } + Aurora_BGD_Test_Deployment& deployment = state.post; + if (publish_initial(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, state.post_writer_hostgroup, + state.post_reader_hostgroup, state.post_green_writer_hostgroup, + state.post_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS + || add_routes( + admin, deployment, state.post_route_hostgroups, + state.post_green_writer_hostgroup) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, state.post_writer_hostgroup, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: failed to configure POST_PROCESSING error retention"); + return EXIT_FAILURE; + } + + auto [topology_seq_rc, topology_sequence] = sim.probe_log_last_sequence(); + bool topology_error_seen = topology_seq_rc == EXIT_SUCCESS + && sim.topology_error( + aurora_bgd_topology_backends(deployment), 1205, + "simulated post timeout") == EXIT_SUCCESS + && aurora_bgd_wait_for_topology_probe( + sim, topology_sequence, deployment.target.backends(), + BGD_Probe_Kind::metadata, kProbeTimeoutMs).first == EXIT_SUCCESS; + ok(topology_error_seen && aurora_bgd_wait_for_status( + admin, state.post_writer_hostgroup, + "SWITCHOVER_IN_POST_PROCESSING", 1) == EXIT_SUCCESS + && route_members( + cl, admin, sim, deployment, state.post_route_hostgroups, true), + "topology errors do not roll back applied POST_PROCESSING pins"); + if (publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS) { + diag("Error: failed to restore POST_PROCESSING topology"); + return EXIT_FAILURE; + } + + auto [membership_seq_rc, membership_sequence] = sim.replica_probe_log_last_sequence(); + bool membership_error_seen = membership_seq_rc == EXIT_SUCCESS + && sim.replica_error( + deployment.target.backends(), 1205, + "simulated post membership timeout") == EXIT_SUCCESS + && aurora_bgd_wait_for_replica_probe( + sim, membership_sequence, deployment.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs).first + == EXIT_SUCCESS; + ok(membership_error_seen + && sim.replica_update( + deployment.target.replica_set_id, deployment.target.replica_rows(), + deployment.target.backends()) == EXIT_SUCCESS + && route_members( + cl, admin, sim, deployment, state.post_route_hostgroups, true), + "membership errors retain the last complete mapped routing"); + return EXIT_SUCCESS; +} + +/** Roll back POST_PROCESSING pins while preserving configured green pools. */ +int test_post_processing_rollback( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) +{ + Aurora_BGD_Test_Deployment& deployment = state.post; + ok(set_default_hostgroup(admin, state.post_green_writer_hostgroup) == EXIT_SUCCESS + && route_to_backend( + cl, sim, deployment.target.members.front().endpoint.backend()) + && pool_count(admin, state.post_green_writer_hostgroup) >= 1, + "a configured green pool is established before rollback"); + ok(publish_status(sim, deployment, "AVAILABLE") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, state.post_writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "a backward POST_PROCESSING status completes rollback before AVAILABLE"); + ok(writer_placement( + admin, state.post_writer_hostgroup, state.post_reader_hostgroup, + deployment.production.members.front().endpoint.hostname, false) + && route_members( + cl, admin, sim, deployment, state.post_route_hostgroups, false), + "rollback removes pins and restores canonical writer and member routing"); + ok(pool_count(admin, state.post_green_writer_hostgroup) >= 1, + "rollback preserves configured green pools instead of draining them"); + return EXIT_SUCCESS; +} + +int main() { + plan(14); + + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: inject topology and membership errors during INITIATED. + // Verify: the active state and last complete membership are retained. + if (test_initiated_error_retention(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: move backward from IN_PROGRESS, then publish empty topology. + // Verify: placement rolls back, ordinary probes resume, and the worker rearms. + if (test_initiated_rollback_and_cancellation(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: make IN_PROGRESS the first observation, then confirm absence. + // Verify: the worker returns to NONE and restores the writer. + if (test_in_progress_topology_absence(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: inject topology and membership errors during POST_PROCESSING. + // Verify: the applied target pins and last complete map remain active. + if (test_post_processing_error_retention(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: publish AVAILABLE after POST_PROCESSING. + // Verify: pins and placement roll back without draining configured green pools. + if (test_post_processing_rollback(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD error-recovery test data"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_late_entry-t.cpp b/test/tap/tests/test_aurora_bgd_late_entry-t.cpp new file mode 100644 index 0000000000..0119eac21a --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_late_entry-t.cpp @@ -0,0 +1,364 @@ +/** + * @file test_aurora_bgd_late_entry-t.cpp + * @brief Aurora BGD reconstruction from first observations in active states. + * + * Steps: + * + * 1. Start a worker from SWITCHOVER_INITIATED and reconstruct active probing. + * 2. Start a worker from SWITCHOVER_IN_PROGRESS and reconstruct writer demotion. + * 3. Start a worker from POST_PROCESSING and reconstruct every target pin. + */ + +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +using namespace std; + +const uint32_t kWaitSeconds = 5; +const char kOrdinaryAuroraQuery[] = + "SELECT SERVER_ID," + "IF(" + "SESSION_ID = 'MASTER_SESSION_ID' AND " + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " + "'probably_former_MASTER_SESSION_ID', SESSION_ID" + ") SESSION_ID, " + "LAST_UPDATE_TIMESTAMP, " + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " + "CPU " + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" + " ( " + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" + " OR SESSION_ID = 'MASTER_SESSION_ID'" + " ) " + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" + " ORDER BY SERVER_ID"; + +struct TestState { + Aurora_BGD_Test_Deployment initiated { aurora_bgd_deployment_b_writer_only() }; + int initiated_writer_hostgroup { 1590 }; + int initiated_reader_hostgroup { 1591 }; + int initiated_green_writer_hostgroup { 1592 }; + int initiated_green_reader_hostgroup { 1593 }; + Aurora_BGD_Test_Deployment progress { aurora_bgd_deployment_b_writer_only() }; + int progress_writer_hostgroup { 1600 }; + int progress_reader_hostgroup { 1601 }; + int progress_green_writer_hostgroup { 1602 }; + int progress_green_reader_hostgroup { 1603 }; + Aurora_BGD_Test_Deployment post { aurora_bgd_deployment_a() }; + int post_writer_hostgroup { 1610 }; + int post_reader_hostgroup { 1611 }; + int post_green_writer_hostgroup { 1612 }; + int post_green_reader_hostgroup { 1613 }; + vector post_route_hostgroups { 1614, 1615, 1616 }; +}; + +int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { + if (cl.getEnv()) { + return EXIT_FAILURE; + } + admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (admin == nullptr) { + return EXIT_FAILURE; + } + char username[] = "aurora1"; + char password[] = "pass1"; + if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS + || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS + || sim.cleanup() != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + return aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " + "VALUES ('testuser','testuser',1,0,1)", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { + return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS + && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int cleanup(MYSQL* admin, BGD_Simulator& sim) { + int reset_rc = reset_scenario(admin, sim); + int user_rc = aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); + mysql_close(admin); + return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int publish_initial( + BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return sim.replica_update( + deployment.production.replica_set_id, + deployment.production.replica_rows(), deployment.production.backends()) == EXIT_SUCCESS + && sim.replica_update( + deployment.target.replica_set_id, + deployment.target.replica_rows(), deployment.target.backends()) == EXIT_SUCCESS + && sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)) == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int set_default_hostgroup(MYSQL* admin, int hostgroup) { + return aurora_bgd_execute_all(admin, { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +bool writer_placement( + MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, bool demoted +) { + string query = + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "0" : "1") + ") AND " + "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "1" : "0") + ")"; + return wait_for_cond(admin, query, kWaitSeconds) == EXIT_SUCCESS; +} + +bool active_probe_policy( + BGD_Simulator& sim, uint64_t sequence, const string& target_replica_set, + uint32_t observation_ms +) { + usleep(observation_ms * 1000); + auto [rc, logs] = sim.replica_probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return false; + } + bool membership = false; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { + return false; + } + membership |= log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership + && log.replica_set_id == target_replica_set; + } + return membership; +} + +int add_routes( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, + const vector& route_hgs, int green_writer_hg +) { + vector queries; + for (size_t i = 0; i < route_hgs.size(); ++i) { + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(route_hgs[i]) + "," + + aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD rollback route')"); + } + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(green_writer_hg) + "," + + aurora_bgd_sql_quote(deployment.target.members.front().endpoint.hostname) + + ",3306,'ONLINE','Aurora BGD rollback green pool')"); + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected) { + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS) { + return false; + } + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return false; + } + auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; + mysql_close(client); + if (query_rc != EXIT_SUCCESS) { + return false; + } + auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); + if (logs_rc != EXIT_SUCCESS) { + return false; + } + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary + && log.backend.host == expected.host && log.backend.port == expected.port) { + return true; + } + } + return false; +} + +bool route_members( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, + Aurora_BGD_Test_Deployment& deployment, const vector& route_hgs, bool target +) { + for (size_t i = 0; i < route_hgs.size(); ++i) { + if (set_default_hostgroup(admin, route_hgs[i]) != EXIT_SUCCESS) { + return false; + } + Endpoint expected = target + ? deployment.target.members[i].endpoint.backend() + : deployment.production.members[i].endpoint.backend(); + if (!route_to_backend(cl, sim, expected)) { + return false; + } + } + return true; +} + +/** + * Start a worker from SWITCHOVER_INITIATED. + * + * - Publish INITIATED before the worker exists. + * - Keep canonical writer placement. + * - Reconstruct fast target-membership probing. + */ +int test_first_initiated(MYSQL* admin, BGD_Simulator& sim, TestState& state) { + Aurora_BGD_Test_Deployment& deployment = state.initiated; + if (publish_initial(sim, deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, state.initiated_writer_hostgroup, + state.initiated_reader_hostgroup, state.initiated_green_writer_hostgroup, + state.initiated_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS) { + diag("Error: failed to configure INITIATED late entry"); + return EXIT_FAILURE; + } + + ok(aurora_bgd_wait_for_status( + admin, state.initiated_writer_hostgroup, + "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, + "late entry at INITIATED publishes the observed state"); + ok(writer_placement( + admin, state.initiated_writer_hostgroup, state.initiated_reader_hostgroup, + deployment.production.members.front().endpoint.hostname, false), + "late INITIATED entry leaves writer placement unchanged"); + auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); + ok(sequence_rc == EXIT_SUCCESS && active_probe_policy( + sim, sequence, deployment.target_replica_set, 450), + "late INITIATED entry reconstructs fast membership probing"); + return EXIT_SUCCESS; +} + +/** + * Start a worker from SWITCHOVER_IN_PROGRESS. + * + * - Publish IN_PROGRESS before the worker exists. + * - Reconstruct the source snapshot. + * - Demote the production writer immediately. + */ +int test_first_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before IN_PROGRESS late entry"); + return EXIT_FAILURE; + } + + Aurora_BGD_Test_Deployment& deployment = state.progress; + if (publish_initial(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, state.progress_writer_hostgroup, + state.progress_reader_hostgroup, state.progress_green_writer_hostgroup, + state.progress_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS) { + diag("Error: failed to configure IN_PROGRESS late entry"); + return EXIT_FAILURE; + } + + ok(aurora_bgd_wait_for_status( + admin, state.progress_writer_hostgroup, + "SWITCHOVER_IN_PROGRESS", kWaitSeconds) == EXIT_SUCCESS + && writer_placement( + admin, state.progress_writer_hostgroup, state.progress_reader_hostgroup, + deployment.production.members.front().endpoint.hostname, true), + "late entry at IN_PROGRESS reconstructs and demotes the writer"); + return EXIT_SUCCESS; +} + +/** + * Start a worker from SWITCHOVER_IN_POST_PROCESSING. + * + * - Publish POST_PROCESSING before the worker exists. + * - Reconstruct the complete target map. + * - Apply every production-to-target traffic pin. + */ +int test_first_post_processing( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) +{ + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before POST_PROCESSING late entry"); + return EXIT_FAILURE; + } + + Aurora_BGD_Test_Deployment& deployment = state.post; + if (publish_initial(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, state.post_writer_hostgroup, + state.post_reader_hostgroup, state.post_green_writer_hostgroup, + state.post_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS + || add_routes( + admin, deployment, state.post_route_hostgroups, + state.post_green_writer_hostgroup) != EXIT_SUCCESS) { + diag("Error: failed to configure POST_PROCESSING late entry"); + return EXIT_FAILURE; + } + + ok(aurora_bgd_wait_for_status( + admin, state.post_writer_hostgroup, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + "late entry at POST_PROCESSING reconstructs the active phase"); + ok(route_members( + cl, admin, sim, deployment, state.post_route_hostgroups, true), + "late POST_PROCESSING entry reconstructs and applies every target pin"); + return EXIT_SUCCESS; +} + +int main() { + plan(6); + + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + + // Simulator: make INITIATED the first observed deployment state. + // Verify: canonical placement and fast target probing are reconstructed. + if (test_first_initiated(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: make IN_PROGRESS the first observed deployment state. + // Verify: the worker reconstructs and demotes the production writer. + if (test_first_in_progress(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: make POST_PROCESSING the first observed deployment state. + // Verify: the worker reconstructs and applies every target pin. + if (test_first_post_processing(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + +exit_cleanup: + if (cleanup(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to clean Aurora BGD late-entry test data"); + return EXIT_FAILURE; + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_resilience-t.cpp b/test/tap/tests/test_aurora_bgd_resilience-t.cpp deleted file mode 100644 index 9d11f19728..0000000000 --- a/test/tap/tests/test_aurora_bgd_resilience-t.cpp +++ /dev/null @@ -1,412 +0,0 @@ -/** - * @file test_aurora_bgd_resilience-t.cpp - * @brief Aurora BGD rollback, error retention, and active-state late entry. - */ - -#include -#include -#include -#include - -#include "aurora_bgd_tap.h" -#include "command_line.h" -#include "utils.h" - -using namespace std; - -const uint32_t kWaitSeconds = 5; -const uint32_t kProbeTimeoutMs = 5000; -const char kOrdinaryAuroraQuery[] = - "SELECT SERVER_ID," - "IF(" - "SESSION_ID = 'MASTER_SESSION_ID' AND " - "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " - "'probably_former_MASTER_SESSION_ID', SESSION_ID" - ") SESSION_ID, " - "LAST_UPDATE_TIMESTAMP, " - "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " - "CPU " - "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" - " ( " - "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" - " OR SESSION_ID = 'MASTER_SESSION_ID'" - " ) " - "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" - " ORDER BY SERVER_ID"; - -int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { - if (cl.getEnv()) { - return EXIT_FAILURE; - } - admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); - if (admin == nullptr) { - return EXIT_FAILURE; - } - char username[] = "aurora1"; - char password[] = "pass1"; - if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS - || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS - || sim.cleanup() != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - return aurora_bgd_execute_all(admin, { - "DELETE FROM mysql_users WHERE username='testuser'", - "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " - "VALUES ('testuser','testuser',1,0,1)", - "LOAD MYSQL USERS TO RUNTIME", - }); -} - -int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { - return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS - && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int cleanup(MYSQL* admin, BGD_Simulator& sim) { - int reset_rc = reset_scenario(admin, sim); - int user_rc = aurora_bgd_execute_all(admin, { - "DELETE FROM mysql_users WHERE username='testuser'", - "LOAD MYSQL USERS TO RUNTIME", - }); - mysql_close(admin); - return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS - ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int publish_initial( - BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status -) { - return sim.replica_update( - deployment.production.replica_set_id, - deployment.production.replica_rows(), deployment.production.backends()) == EXIT_SUCCESS - && sim.replica_update( - deployment.target.replica_set_id, - deployment.target.replica_rows(), deployment.target.backends()) == EXIT_SUCCESS - && sim.topology_update( - aurora_bgd_topology_backends(deployment), - aurora_bgd_topology(deployment, status)) == EXIT_SUCCESS - ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int publish_status( - BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status -) { - return sim.topology_update( - aurora_bgd_topology_backends(deployment), - aurora_bgd_topology(deployment, status)); -} - -int set_default_hostgroup(MYSQL* admin, int hostgroup) { - return aurora_bgd_execute_all(admin, { - "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + - " WHERE username='testuser'", - "LOAD MYSQL USERS TO RUNTIME", - }); -} - -bool writer_placement( - MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, bool demoted -) { - string query = - "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + - ")=" + (demoted ? "0" : "1") + ") AND " - "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + - ")=" + (demoted ? "1" : "0") + ")"; - return wait_for_cond(admin, query, kWaitSeconds) == EXIT_SUCCESS; -} - -bool active_probe_policy( - BGD_Simulator& sim, uint64_t sequence, const string& target_replica_set, - uint32_t observation_ms -) { - usleep(observation_ms * 1000); - auto [rc, logs] = sim.replica_probe_log_since(sequence); - if (rc != EXIT_SUCCESS) { - return false; - } - bool membership = false; - for (const Aurora_Replica_Probe_Log& log : logs) { - if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { - return false; - } - membership |= log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership - && log.replica_set_id == target_replica_set; - } - return membership; -} - -bool wait_for_ordinary_probe( - BGD_Simulator& sim, uint64_t sequence, Aurora_BGD_Test_Deployment& deployment -) { - auto [rc, log] = aurora_bgd_wait_for_replica_probe( - sim, sequence, deployment.production.backends(), - Aurora_Replica_Probe_Kind::ordinary, kProbeTimeoutMs); - return rc == EXIT_SUCCESS; -} - -int add_routes( - MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, - const vector& route_hgs, int green_writer_hg -) { - vector queries; - for (size_t i = 0; i < route_hgs.size(); ++i) { - queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + - to_string(route_hgs[i]) + "," + - aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + - ",3306,'ONLINE','Aurora BGD rollback route')"); - } - queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + - to_string(green_writer_hg) + "," + - aurora_bgd_sql_quote(deployment.target.members.front().endpoint.hostname) + - ",3306,'ONLINE','Aurora BGD rollback green pool')"); - queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); - return aurora_bgd_execute_all(admin, queries); -} - -bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected) { - auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); - if (sequence_rc != EXIT_SUCCESS) { - return false; - } - MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); - if (client == nullptr) { - return false; - } - auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); - (void)rows; - mysql_close(client); - if (query_rc != EXIT_SUCCESS) { - return false; - } - auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); - if (logs_rc != EXIT_SUCCESS) { - return false; - } - for (const Aurora_Replica_Probe_Log& log : logs) { - if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary - && log.backend.host == expected.host && log.backend.port == expected.port) { - return true; - } - } - return false; -} - -bool route_members( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, - Aurora_BGD_Test_Deployment& deployment, const vector& route_hgs, bool target -) { - for (size_t i = 0; i < route_hgs.size(); ++i) { - if (set_default_hostgroup(admin, route_hgs[i]) != EXIT_SUCCESS) { - return false; - } - Endpoint expected = target - ? deployment.target.members[i].endpoint.backend() - : deployment.production.members[i].endpoint.backend(); - if (!route_to_backend(cl, sim, expected)) { - return false; - } - } - return true; -} - -int64_t pool_count(MYSQL* admin, int hostgroup) { - auto [rc, rows] = mysql_query_ext_rows( - admin, "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) " - "FROM stats_mysql_connection_pool WHERE hostgroup=" + to_string(hostgroup)); - return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 - ? strtoll(rows.front().front().c_str(), nullptr, 10) : -1; -} - -int main() { - plan(20); - CommandLine cl {}; - MYSQL* admin = nullptr; - BGD_Simulator sim {}; - if (setup(cl, admin, sim) != EXIT_SUCCESS) { - return exit_status(); - } - - Aurora_BGD_Test_Deployment initiated = aurora_bgd_deployment_b_writer_only(); - const int initiated_writer_hg = 1590; - const int initiated_reader_hg = 1591; - if (publish_initial(sim, initiated, "SWITCHOVER_INITIATED") != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, initiated, initiated_writer_hg, initiated_reader_hg, - 1592, 1593, false, 300, false) != EXIT_SUCCESS) { - diag("Error: failed to configure INITIATED late entry"); - cleanup(admin, sim); - return exit_status(); - } - ok(aurora_bgd_wait_for_status( - admin, initiated_writer_hg, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, - "late entry at INITIATED publishes the observed state"); - ok(writer_placement( - admin, initiated_writer_hg, initiated_reader_hg, - initiated.production.members.front().endpoint.hostname, false), - "late INITIATED entry leaves writer placement unchanged"); - auto [initiated_seq_rc, initiated_sequence] = sim.replica_probe_log_last_sequence(); - ok(initiated_seq_rc == EXIT_SUCCESS && active_probe_policy( - sim, initiated_sequence, initiated.target_replica_set, 450), - "late INITIATED entry reconstructs fast membership probing"); - auto [topology_error_seq_rc, topology_error_sequence] = sim.probe_log_last_sequence(); - auto topology_error_seen = topology_error_seq_rc == EXIT_SUCCESS - && sim.topology_error( - aurora_bgd_topology_backends(initiated), 1205, "simulated topology timeout") - == EXIT_SUCCESS - && aurora_bgd_wait_for_topology_probe( - sim, topology_error_sequence, initiated.target.backends(), - BGD_Probe_Kind::metadata, kProbeTimeoutMs).first == EXIT_SUCCESS; - ok(topology_error_seen && aurora_bgd_wait_for_status( - admin, initiated_writer_hg, "SWITCHOVER_INITIATED", 1) == EXIT_SUCCESS, - "topology errors retain the active state"); - if (publish_status(sim, initiated, "SWITCHOVER_INITIATED") != EXIT_SUCCESS) { - diag("Error: failed to restore topology after error"); - cleanup(admin, sim); - return exit_status(); - } - auto [membership_error_seq_rc, membership_error_sequence] = - sim.replica_probe_log_last_sequence(); - auto membership_error_seen = membership_error_seq_rc == EXIT_SUCCESS - && sim.replica_error( - initiated.target.backends(), 1205, "simulated membership timeout") == EXIT_SUCCESS - && aurora_bgd_wait_for_replica_probe( - sim, membership_error_sequence, initiated.target.backends(), - Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs).first == EXIT_SUCCESS; - ok(membership_error_seen && aurora_bgd_wait_for_status( - admin, initiated_writer_hg, "SWITCHOVER_INITIATED", 1) == EXIT_SUCCESS, - "membership errors retain the last complete state"); - if (sim.replica_update( - initiated.target.replica_set_id, initiated.target.replica_rows(), - initiated.target.backends()) != EXIT_SUCCESS - || publish_status(sim, initiated, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, initiated_writer_hg, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) - != EXIT_SUCCESS) { - diag("Error: failed to advance to IN_PROGRESS"); - cleanup(admin, sim); - return exit_status(); - } - ok(writer_placement( - admin, initiated_writer_hg, initiated_reader_hg, - initiated.production.members.front().endpoint.hostname, true), - "IN_PROGRESS demotes the reconstructed production writer"); - ok(publish_status(sim, initiated, "SWITCHOVER_INITIATED") == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, initiated_writer_hg, "SWITCHOVER_INITIATED", kWaitSeconds) - == EXIT_SUCCESS - && writer_placement( - admin, initiated_writer_hg, initiated_reader_hg, - initiated.production.members.front().endpoint.hostname, false), - "a backward status rolls back writer placement before entering the earlier state"); - auto [ordinary_seq_rc, ordinary_sequence] = sim.replica_probe_log_last_sequence(); - ok(sim.topology_delete(aurora_bgd_topology_backends(initiated)) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, initiated_writer_hg, "NONE", kWaitSeconds) == EXIT_SUCCESS, - "a successful empty topology cancels the active deployment"); - ok(ordinary_seq_rc == EXIT_SUCCESS - && wait_for_ordinary_probe(sim, ordinary_sequence, initiated), - "cancellation resumes ordinary production probing"); - ok(publish_status(sim, initiated, "AVAILABLE") == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, initiated_writer_hg, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, - "the worker admits a repeated deployment after cancellation"); - - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to reset before IN_PROGRESS late entry"); - cleanup(admin, sim); - return exit_status(); - } - Aurora_BGD_Test_Deployment progress = aurora_bgd_deployment_b_writer_only(); - if (publish_initial(sim, progress, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS - || aurora_bgd_admin_setup(admin, progress, 1600, 1601, 1602, 1603, false, 300, false) - != EXIT_SUCCESS) { - diag("Error: failed to configure IN_PROGRESS late entry"); - cleanup(admin, sim); - return exit_status(); - } - ok(aurora_bgd_wait_for_status(admin, 1600, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) - == EXIT_SUCCESS - && writer_placement( - admin, 1600, 1601, progress.production.members.front().endpoint.hostname, true), - "late entry at IN_PROGRESS reconstructs and demotes the writer"); - ok(sim.topology_drop(aurora_bgd_topology_backends(progress)) == EXIT_SUCCESS - && aurora_bgd_wait_for_status(admin, 1600, "NONE", kWaitSeconds) == EXIT_SUCCESS - && writer_placement( - admin, 1600, 1601, progress.production.members.front().endpoint.hostname, false), - "confirmed topology absence rolls back IN_PROGRESS to NONE"); - - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to reset before POST_PROCESSING late entry"); - cleanup(admin, sim); - return exit_status(); - } - Aurora_BGD_Test_Deployment post = aurora_bgd_deployment_a(); - const vector route_hgs {1614, 1615, 1616}; - if (publish_initial(sim, post, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS - || aurora_bgd_admin_setup(admin, post, 1610, 1611, 1612, 1613, false, 300, false) - != EXIT_SUCCESS - || add_routes(admin, post, route_hgs, 1612) != EXIT_SUCCESS) { - diag("Error: failed to configure POST_PROCESSING late entry"); - cleanup(admin, sim); - return exit_status(); - } - ok(aurora_bgd_wait_for_status( - admin, 1610, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, - "late entry at POST_PROCESSING reconstructs the active phase"); - ok(route_members(cl, admin, sim, post, route_hgs, true), - "late POST_PROCESSING entry reconstructs and applies every target pin"); - ok(set_default_hostgroup(admin, 1612) == EXIT_SUCCESS - && route_to_backend(cl, sim, post.target.members.front().endpoint.backend()) - && pool_count(admin, 1612) >= 1, - "a configured green pool is established before rollback"); - auto [post_error_seq_rc, post_error_sequence] = sim.probe_log_last_sequence(); - bool post_error_seen = post_error_seq_rc == EXIT_SUCCESS - && sim.topology_error( - aurora_bgd_topology_backends(post), 1205, "simulated post timeout") == EXIT_SUCCESS - && aurora_bgd_wait_for_topology_probe( - sim, post_error_sequence, post.target.backends(), - BGD_Probe_Kind::metadata, kProbeTimeoutMs).first == EXIT_SUCCESS; - ok(post_error_seen && aurora_bgd_wait_for_status( - admin, 1610, "SWITCHOVER_IN_POST_PROCESSING", 1) == EXIT_SUCCESS - && route_members(cl, admin, sim, post, route_hgs, true), - "topology errors do not roll back applied POST_PROCESSING pins"); - if (publish_status(sim, post, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS) { - diag("Error: failed to restore POST_PROCESSING topology"); - cleanup(admin, sim); - return exit_status(); - } - auto [post_membership_seq_rc, post_membership_sequence] = - sim.replica_probe_log_last_sequence(); - bool post_membership_error = post_membership_seq_rc == EXIT_SUCCESS - && sim.replica_error(post.target.backends(), 1205, "simulated post membership timeout") - == EXIT_SUCCESS - && aurora_bgd_wait_for_replica_probe( - sim, post_membership_sequence, post.target.backends(), - Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs).first == EXIT_SUCCESS; - ok(post_membership_error - && sim.replica_update( - post.target.replica_set_id, post.target.replica_rows(), post.target.backends()) - == EXIT_SUCCESS - && route_members(cl, admin, sim, post, route_hgs, true), - "membership errors retain the last complete mapped routing"); - ok(publish_status(sim, post, "AVAILABLE") == EXIT_SUCCESS - && aurora_bgd_wait_for_status(admin, 1610, "AVAILABLE", kWaitSeconds) - == EXIT_SUCCESS, - "a backward POST_PROCESSING status completes rollback before AVAILABLE"); - ok(writer_placement( - admin, 1610, 1611, post.production.members.front().endpoint.hostname, false) - && route_members(cl, admin, sim, post, route_hgs, false), - "rollback removes pins and restores canonical writer and member routing"); - ok(pool_count(admin, 1612) >= 1, - "rollback preserves configured green pools instead of draining them"); - - if (cleanup(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to clean Aurora BGD resilience test data"); - return EXIT_FAILURE; - } - return exit_status(); -} diff --git a/test/tap/tests/test_aurora_bgd_lifecycle-t.cpp b/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp similarity index 50% rename from test/tap/tests/test_aurora_bgd_lifecycle-t.cpp rename to test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp index 77a2e210d0..1e69e26c61 100644 --- a/test/tap/tests/test_aurora_bgd_lifecycle-t.cpp +++ b/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp @@ -1,6 +1,13 @@ /** - * @file test_aurora_bgd_lifecycle-t.cpp - * @brief Aurora BGD reload, worker removal, and concurrent-cluster behavior. + * @file test_aurora_bgd_worker_lifecycle-t.cpp + * @brief Aurora BGD worker refresh, deactivation, and terminal rearming. + * + * Steps: + * + * 1. Refresh server, variable, and green-hostgroup configuration during POST_PROCESSING. + * 2. Deactivate the owning row and restore canonical production routing. + * 3. Refresh a completed worker without releasing its terminal latch. + * 4. Rearm the completed worker after a successful topology drain. */ #include @@ -32,6 +39,22 @@ const char kOrdinaryAuroraQuery[] = "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" " ORDER BY SERVER_ID"; +struct TestState { + Aurora_BGD_Test_Deployment reload { aurora_bgd_deployment_a() }; + int reload_writer_hostgroup { 1620 }; + int reload_reader_hostgroup { 1621 }; + int reload_green_writer_hostgroup { 1622 }; + int reload_green_reader_hostgroup { 1623 }; + int reload_route_hostgroup { 1624 }; + int refreshed_green_writer_hostgroup { 1626 }; + int refreshed_green_reader_hostgroup { 1627 }; + Aurora_BGD_Test_Deployment terminal { aurora_bgd_deployment_b_writer_only() }; + int terminal_writer_hostgroup { 1630 }; + int terminal_reader_hostgroup { 1631 }; + int terminal_green_writer_hostgroup { 1632 }; + int terminal_green_reader_hostgroup { 1633 }; +}; + int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { return EXIT_FAILURE; @@ -140,25 +163,6 @@ bool route_writer( return route_to_backend(cl, sim, expected); } -bool wait_for_writer_route( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, - Aurora_BGD_Test_Deployment& deployment, int route_hg, bool target -) { - if (set_default_hostgroup(admin, route_hg) != EXIT_SUCCESS) { - return false; - } - const Endpoint expected = target - ? deployment.target.members.front().endpoint.backend() - : deployment.production.members.front().endpoint.backend(); - for (uint32_t elapsed_ms = 0; elapsed_ms < kWaitSeconds * 1000; elapsed_ms += 100) { - if (route_to_backend(cl, sim, expected)) { - return true; - } - usleep(100000); - } - return false; -} - bool writer_placement( MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, bool demoted ) { @@ -180,112 +184,159 @@ bool wait_for_inactive_none(MYSQL* admin, int writer_hg) { kWaitSeconds) == EXIT_SUCCESS; } -bool wait_for_runtime_row_absent(MYSQL* admin, int writer_hg) { - return wait_for_cond( - admin, - "SELECT COUNT(*)=0 FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + - to_string(writer_hg), - kWaitSeconds) == EXIT_SUCCESS; -} - -int main() { - plan(28); - CommandLine cl {}; - MYSQL* admin = nullptr; - BGD_Simulator sim {}; - if (setup(cl, admin, sim) != EXIT_SUCCESS) { - return exit_status(); - } - - Aurora_BGD_Test_Deployment reload = aurora_bgd_deployment_a(); - if (aurora_bgd_publish(sim, reload) != EXIT_SUCCESS - || aurora_bgd_admin_setup(admin, reload, 1620, 1621, 1622, 1623, false, 300) - != EXIT_SUCCESS - || add_writer_route(admin, reload, 1624) != EXIT_SUCCESS) { +/** + * Refresh an active worker without resetting its FSM or traffic pins. + * + * - Reload unrelated server and monitor-variable configuration. + * - Refresh green staging hostgroups. + * - Preserve POST_PROCESSING status, cached membership, and target routing. + */ +int test_active_worker_refresh( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) +{ + Aurora_BGD_Test_Deployment& deployment = state.reload; + if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, state.reload_writer_hostgroup, + state.reload_reader_hostgroup, state.reload_green_writer_hostgroup, + state.reload_green_reader_hostgroup, false, 300) != EXIT_SUCCESS + || add_writer_route( + admin, deployment, state.reload_route_hostgroup) != EXIT_SUCCESS) { diag("Error: failed to configure the active reload scenario"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } - ok(aurora_bgd_wait_for_status(admin, 1620, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + + ok(aurora_bgd_wait_for_status( + admin, state.reload_writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, "reload scenario starts from AVAILABLE"); - ok(publish_status(sim, reload, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + ok(publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS && aurora_bgd_wait_for_status( - admin, 1620, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS - && route_writer(cl, admin, sim, reload, 1624, true), + admin, state.reload_writer_hostgroup, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS + && route_writer( + cl, admin, sim, deployment, state.reload_route_hostgroup, true), "active reload scenario pins writer traffic to the target"); - ok(aurora_bgd_execute_all(admin, {"LOAD MYSQL SERVERS TO RUNTIME"}) == EXIT_SUCCESS + ok(aurora_bgd_execute_all( + admin, { "LOAD MYSQL SERVERS TO RUNTIME" }) == EXIT_SUCCESS && aurora_bgd_wait_for_status( - admin, 1620, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + admin, state.reload_writer_hostgroup, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, "an unrelated server reload preserves active FSM status"); - ok(route_writer(cl, admin, sim, reload, 1624, true), + ok(route_writer( + cl, admin, sim, deployment, state.reload_route_hostgroup, true), "an unrelated server reload preserves the applied traffic pin"); ok(aurora_bgd_execute_all(admin, { "SET mysql-aws_blue_green_deployment_auto_discovery='false'", "LOAD MYSQL VARIABLES TO RUNTIME", }) == EXIT_SUCCESS && aurora_bgd_wait_for_status( - admin, 1620, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, + admin, state.reload_writer_hostgroup, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, "disabling auto-discovery does not abort an admitted deployment"); - ok(route_writer(cl, admin, sim, reload, 1624, true), + ok(route_writer( + cl, admin, sim, deployment, state.reload_route_hostgroup, true), "a variable refresh preserves cached membership and target routing"); ok(aurora_bgd_execute_all(admin, { - "UPDATE mysql_aws_aurora_hostgroups SET green_writer_hostgroup=1626," - "green_reader_hostgroup=1627 WHERE writer_hostgroup=1620", + "UPDATE mysql_aws_aurora_hostgroups SET green_writer_hostgroup=" + + to_string(state.refreshed_green_writer_hostgroup) + + ",green_reader_hostgroup=" + + to_string(state.refreshed_green_reader_hostgroup) + + " WHERE writer_hostgroup=" + to_string(state.reload_writer_hostgroup), "LOAD MYSQL SERVERS TO RUNTIME", }) == EXIT_SUCCESS && wait_for_cond( admin, "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups " - "WHERE writer_hostgroup=1620 AND green_writer_hostgroup=1626 " - "AND green_reader_hostgroup=1627 " - "AND bgd_status='SWITCHOVER_IN_POST_PROCESSING'", + "WHERE writer_hostgroup=" + to_string(state.reload_writer_hostgroup) + + " AND green_writer_hostgroup=" + + to_string(state.refreshed_green_writer_hostgroup) + + " AND green_reader_hostgroup=" + + to_string(state.refreshed_green_reader_hostgroup) + + " AND bgd_status='SWITCHOVER_IN_POST_PROCESSING'", kWaitSeconds) == EXIT_SUCCESS, "green hostgroup changes refresh staging references without resetting the FSM"); - ok(route_writer(cl, admin, sim, reload, 1624, true), + ok(route_writer( + cl, admin, sim, deployment, state.reload_route_hostgroup, true), "green hostgroup refresh preserves applied pins and cached target IPs"); + return EXIT_SUCCESS; +} + +/** + * Deactivate an active worker. + * + * - Stop the worker after publishing NONE. + * - Restore canonical writer placement and production routing. + * - Preserve administrator-owned configuration and server rows. + */ +int test_active_worker_deactivation( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) +{ + Aurora_BGD_Test_Deployment& deployment = state.reload; ok(aurora_bgd_execute_all(admin, { - "UPDATE mysql_aws_aurora_hostgroups SET active=0 WHERE writer_hostgroup=1620", + "UPDATE mysql_aws_aurora_hostgroups SET active=0 WHERE writer_hostgroup=" + + to_string(state.reload_writer_hostgroup), "LOAD MYSQL SERVERS TO RUNTIME", - }) == EXIT_SUCCESS && wait_for_inactive_none(admin, 1620), + }) == EXIT_SUCCESS && wait_for_inactive_none(admin, state.reload_writer_hostgroup), "deactivating the owning row stops the worker after publishing NONE"); ok(writer_placement( - admin, 1620, 1621, reload.production.members.front().endpoint.hostname, false), + admin, state.reload_writer_hostgroup, state.reload_reader_hostgroup, + deployment.production.members.front().endpoint.hostname, false), "worker teardown restores canonical writer placement"); - ok(route_writer(cl, admin, sim, reload, 1624, false), + ok(route_writer( + cl, admin, sim, deployment, state.reload_route_hostgroup, false), "worker teardown removes the applied traffic pin"); ok(wait_for_cond( admin, "SELECT ((SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups " - "WHERE writer_hostgroup=1620 AND active=0)=1) AND " + "WHERE writer_hostgroup=" + to_string(state.reload_writer_hostgroup) + + " AND active=0)=1) AND " "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostname=" + - aurora_bgd_sql_quote(reload.production.members.front().endpoint.hostname) + ")>=1)", + aurora_bgd_sql_quote(deployment.production.members.front().endpoint.hostname) + + ")>=1)", kWaitSeconds) == EXIT_SUCCESS, "teardown preserves user configuration and server rows"); + return EXIT_SUCCESS; +} +/** + * Refresh and rearm a worker in the completed latch. + * + * - Change configured green hostgroups and monitor variables. + * - Preserve SWITCHOVER_COMPLETED across the refresh. + * - Release the latch only after a successful topology drain. + */ +int test_terminal_worker_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { if (reset_scenario(admin, sim) != EXIT_SUCCESS) { diag("Error: failed to reset before terminal reload scenario"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } - Aurora_BGD_Test_Deployment terminal = aurora_bgd_deployment_b_writer_only(); - if (aurora_bgd_publish(sim, terminal) != EXIT_SUCCESS - || aurora_bgd_admin_setup(admin, terminal, 1630, 1631, -1, -1, true, 300) - != EXIT_SUCCESS) { + + Aurora_BGD_Test_Deployment& deployment = state.terminal; + if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, state.terminal_writer_hostgroup, + state.terminal_reader_hostgroup, -1, -1, true, 300) != EXIT_SUCCESS) { diag("Error: failed to configure the terminal reload scenario"); - cleanup(admin, sim); - return exit_status(); + return EXIT_FAILURE; } - ok(aurora_bgd_wait_for_status(admin, 1630, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + + ok(aurora_bgd_wait_for_status( + admin, state.terminal_writer_hostgroup, "AVAILABLE", kWaitSeconds) + == EXIT_SUCCESS, "terminal reload scenario starts from AVAILABLE"); ok(sim.topology_update( - aurora_bgd_topology_backends(terminal), aurora_bgd_completed_topology(terminal)) - == EXIT_SUCCESS + aurora_bgd_topology_backends(deployment), + aurora_bgd_completed_topology(deployment)) == EXIT_SUCCESS && aurora_bgd_wait_for_status( - admin, 1630, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS, + admin, state.terminal_writer_hostgroup, + "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS, "direct completion enters the terminal latch"); ok(aurora_bgd_execute_all(admin, { - "UPDATE mysql_aws_aurora_hostgroups SET green_writer_hostgroup=1632," - "green_reader_hostgroup=1633 WHERE writer_hostgroup=1630", + "UPDATE mysql_aws_aurora_hostgroups SET green_writer_hostgroup=" + + to_string(state.terminal_green_writer_hostgroup) + + ",green_reader_hostgroup=" + + to_string(state.terminal_green_reader_hostgroup) + + " WHERE writer_hostgroup=" + to_string(state.terminal_writer_hostgroup), "LOAD MYSQL SERVERS TO RUNTIME", "SET mysql-aws_blue_green_deployment_auto_discovery='false'", "LOAD MYSQL VARIABLES TO RUNTIME", @@ -293,104 +344,55 @@ int main() { && wait_for_cond( admin, "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups " - "WHERE writer_hostgroup=1630 AND green_writer_hostgroup=1632 " - "AND green_reader_hostgroup=1633 AND bgd_status='SWITCHOVER_COMPLETED'", + "WHERE writer_hostgroup=" + to_string(state.terminal_writer_hostgroup) + + " AND green_writer_hostgroup=" + + to_string(state.terminal_green_writer_hostgroup) + + " AND green_reader_hostgroup=" + + to_string(state.terminal_green_reader_hostgroup) + + " AND bgd_status='SWITCHOVER_COMPLETED'", kWaitSeconds) == EXIT_SUCCESS, "configuration and variable refresh preserve the terminal latch"); - ok(sim.topology_delete(aurora_bgd_topology_backends(terminal)) == EXIT_SUCCESS - && aurora_bgd_wait_for_status(admin, 1630, "NONE", kWaitSeconds) == EXIT_SUCCESS, + ok(sim.topology_delete(aurora_bgd_topology_backends(deployment)) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, state.terminal_writer_hostgroup, "NONE", kWaitSeconds) == EXIT_SUCCESS, "a successful topology drain rearms a refreshed terminal worker"); + return EXIT_SUCCESS; +} - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to reset before concurrent scenario"); - cleanup(admin, sim); +int main() { + plan(16); + + CommandLine cl {}; + MYSQL* admin = nullptr; + BGD_Simulator sim {}; + + if (setup(cl, admin, sim) != EXIT_SUCCESS) { return exit_status(); } - Aurora_BGD_Test_Deployment concurrent_a = aurora_bgd_deployment_a(); - Aurora_BGD_Test_Deployment concurrent_b = aurora_bgd_deployment_b_writer_only(); - Aurora_BGD_Test_Deployment concurrent_c = aurora_bgd_deployment_c_writer_only(); - if (aurora_bgd_publish(sim, concurrent_a) != EXIT_SUCCESS - || aurora_bgd_publish(sim, concurrent_b) != EXIT_SUCCESS - || aurora_bgd_publish(sim, concurrent_c) != EXIT_SUCCESS - || aurora_bgd_admin_setup(admin, concurrent_a, 1640, 1641, 1642, 1643, false, 300) - != EXIT_SUCCESS - || aurora_bgd_admin_setup(admin, concurrent_b, 1650, 1651, -1, -1, true, 300) - != EXIT_SUCCESS - || aurora_bgd_admin_setup(admin, concurrent_c, 1660, 1661, -1, -1, true, 300) - != EXIT_SUCCESS - || add_writer_route(admin, concurrent_a, 1644) != EXIT_SUCCESS - || add_writer_route(admin, concurrent_b, 1654) != EXIT_SUCCESS - || add_writer_route(admin, concurrent_c, 1664) != EXIT_SUCCESS) { - diag("Error: failed to configure three concurrent deployments"); - cleanup(admin, sim); - return exit_status(); + + TestState state {}; + + // ProxySQL: reload server, variable, and hostgroup configuration during POST_PROCESSING. + // Verify: the active FSM, cached target map, and applied pin remain intact. + if (test_active_worker_refresh(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: deactivate the owning Aurora BGD row. + // Verify: the worker stops and restores production routing without deleting configuration. + if (test_active_worker_deactivation(cl, admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // ProxySQL: refresh configuration while a worker is latched at completion. + // Verify: the terminal state survives refresh and rearms after topology drain. + if (test_terminal_worker_refresh(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; } - ok(aurora_bgd_wait_for_status(admin, 1640, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS - && aurora_bgd_wait_for_status(admin, 1650, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS - && aurora_bgd_wait_for_status(admin, 1660, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, - "three writer hostgroups discover deployments independently"); - ok(publish_status(sim, concurrent_a, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS - && publish_status(sim, concurrent_b, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS - && publish_status(sim, concurrent_c, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, 1640, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, 1650, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, 1660, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, - "three workers enter POST_PROCESSING without sharing FSM state"); - ok(route_writer(cl, admin, sim, concurrent_a, 1644, true), - "deployment A owns its target pin"); - ok(route_writer(cl, admin, sim, concurrent_b, 1654, true), - "deployment B owns its target pin"); - ok(route_writer(cl, admin, sim, concurrent_c, 1664, true), - "deployment C owns its target pin"); - ok(aurora_bgd_execute_all(admin, { - "UPDATE mysql_aws_aurora_hostgroups SET active=0 WHERE writer_hostgroup=1650", - "LOAD MYSQL SERVERS TO RUNTIME", - }) == EXIT_SUCCESS - && wait_for_inactive_none(admin, 1650) - && aurora_bgd_wait_for_status( - admin, 1640, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, 1660, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, - "deactivating deployment B cleans only its worker state"); - ok(writer_placement( - admin, 1650, 1651, concurrent_b.production.members.front().endpoint.hostname, false) - && route_writer(cl, admin, sim, concurrent_b, 1654, false), - "deployment B teardown restores only its production routing"); - ok(route_writer(cl, admin, sim, concurrent_a, 1644, true) - && route_writer(cl, admin, sim, concurrent_c, 1664, true), - "deployment B teardown leaves A and C pins intact"); - ok(sim.topology_update( - aurora_bgd_topology_backends(concurrent_a), - aurora_bgd_completed_topology(concurrent_a)) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, 1640, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, 1660, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, - "completing deployment A leaves deployment C active"); - ok(add_writer_route(admin, concurrent_a, 1645) == EXIT_SUCCESS - && route_writer(cl, admin, sim, concurrent_a, 1645, false) - && route_writer(cl, admin, sim, concurrent_c, 1664, true), - "deployment A cleanup removes only its pin"); - ok(aurora_bgd_execute_all(admin, { - "DELETE FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=1660", - "LOAD MYSQL SERVERS TO RUNTIME", - }) == EXIT_SUCCESS - && wait_for_runtime_row_absent(admin, 1660) - && writer_placement( - admin, 1660, 1661, - concurrent_c.production.members.front().endpoint.hostname, false) - && wait_for_writer_route(cl, admin, sim, concurrent_c, 1664, false), - "removing deployment C safely restores its production routing"); - ok(aurora_bgd_wait_for_status( - admin, 1640, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS - && route_writer(cl, admin, sim, concurrent_a, 1645, false), - "deployment C removal leaves deployment A terminal state unchanged"); +exit_cleanup: if (cleanup(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to clean Aurora BGD lifecycle test data"); + diag("Error: failed to clean Aurora BGD worker-lifecycle test data"); return EXIT_FAILURE; } return exit_status(); From d8b5e6b6bd0f4d98d577be533f4a159db0ed2ff3 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sun, 16 Aug 2026 12:54:29 +0000 Subject: [PATCH 28/34] fix: address Aurora BGD phase 3 review --- include/DNS_Cache.hpp | 1 + lib/DNS_Cache.cpp | 51 +-- lib/MySQL_HostGroups_Manager.cpp | 7 +- lib/MySQL_Monitor.cpp | 271 ++++++++++---- lib/ProxySQL_Cluster.cpp | 21 ++ test/infra/control/cluster-simulator-ci.bash | 5 + test/tap/groups/cluster_sim_aurora/add-hosts | 5 + test/tap/groups/groups.json | 18 +- test/tap/tap/aurora_bgd_tap.h | 49 ++- test/tap/tap/bgd_simulator.h | 3 +- test/tap/tests/test_aurora_bgd_active-t.cpp | 347 +++++++++++++++++- .../tests/test_aurora_bgd_cluster_sync-t.cpp | 54 +-- .../tests/test_aurora_bgd_completion-t.cpp | 115 ++++-- ...test_aurora_bgd_concurrent_isolation-t.cpp | 20 +- .../tap/tests/test_aurora_bgd_discovery-t.cpp | 85 ++++- .../test_aurora_bgd_error_recovery-t.cpp | 67 +++- .../tests/test_aurora_bgd_late_entry-t.cpp | 63 +++- .../test_aurora_bgd_worker_lifecycle-t.cpp | 18 +- .../tests/unit/admin_disk_upgrade_unit-t.cpp | 42 ++- .../tests/unit/aurora_bgd_config_unit-t.cpp | 2 +- test/tap/tests/unit/config_write_unit-t.cpp | 3 +- 21 files changed, 998 insertions(+), 249 deletions(-) diff --git a/include/DNS_Cache.hpp b/include/DNS_Cache.hpp index 44fb6dad44..b43ad1406e 100644 --- a/include/DNS_Cache.hpp +++ b/include/DNS_Cache.hpp @@ -81,6 +81,7 @@ class DNS_Cache { bool add(const std::string& hostname, std::vector&& ips); bool add_if_not_exist(const std::string& hostname, std::vector&& ips); void remove(const std::string& hostname); + // Clear ordinary resolved addresses while retaining explicit pins. void clear(); bool empty() const; bool is_ip_valid(const std::string& hostname, const std::string& ip) const; diff --git a/lib/DNS_Cache.cpp b/lib/DNS_Cache.cpp index 8face0834c..b18c1483c5 100644 --- a/lib/DNS_Cache.cpp +++ b/lib/DNS_Cache.cpp @@ -206,7 +206,7 @@ void* DNSResolverWorker::run() { } bool DNS_Cache::is_ip_valid(const std::string& hostname, const std::string& ip) const { - if (!enabled || hostname.empty() || ip.empty()) { + if (hostname.empty() || ip.empty()) { return false; } @@ -221,7 +221,7 @@ bool DNS_Cache::is_ip_valid(const std::string& hostname, const std::string& ip) && (itr->second.pinned_until == 0 || now <= itr->second.pinned_until); if (pin_active) { valid = ip == itr->second.pinned_ip; - } else { + } else if (enabled) { valid = std::find(itr->second.ips.begin(), itr->second.ips.end(), ip) != itr->second.ips.end(); } } @@ -245,16 +245,6 @@ bool DNS_Cache::add(const std::string& hostname, std::vector&& ips) auto& ip_addr = records[hostname]; ip_addr.ips = std::move(ips); - // Check if IP pinning is no longer necessary. - if (!ip_addr.pinned_ip.empty() && - std::find(ip_addr.ips.begin(), ip_addr.ips.end(), ip_addr.pinned_ip) != ip_addr.ips.end()) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, - "Unpinning DNS cache record because resolved IP matches pinned IP. (Hostname:[%s] IP:[%s])\n", - hostname.c_str(), ip_addr.pinned_ip.c_str()); - ip_addr.pinned_ip.clear(); - ip_addr.pinned_until = 0; - } - __sync_fetch_and_and(&ip_addr.counter, 0); rc = pthread_rwlock_unlock(&rwlock_); @@ -281,16 +271,6 @@ bool DNS_Cache::add_if_not_exist(const std::string& hostname, std::vectorsecond.pinned_ip.empty()) { + itr = records.erase(itr); + ++records_removed; + } else { + itr->second.ips.clear(); + itr->second.counter = 0; + ++itr; + } + } rc = pthread_rwlock_unlock(&rwlock_); assert(rc == 0); if (records_removed && counter_record_updated_) counter_record_updated_->fetch_add(records_removed, std::memory_order_relaxed); - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache was cleared.\n"); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, + "DNS cache resolved records were cleared; explicit pins were preserved.\n"); } bool DNS_Cache::empty() const { diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index fd572ef66c..0b446db149 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -7335,8 +7335,9 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer( char *error=NULL; q1 = (char *)"SELECT DISTINCT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers WHERE hostgroup_id IN (%d,%d) ORDER BY hostgroup_id, hostname, port"; q2 = (char *)"SELECT DISTINCT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming WHERE hostgroup_id IN (%d,%d) ORDER BY hostgroup_id, hostname, port"; - query = (char *)malloc(strlen(q2)+128); - sprintf(query,q1,_writer_hostgroup,_rhid); + const size_t query_size = strlen(q2) + 128; + query = (char *)malloc(query_size); + snprintf(query, query_size, q1, _writer_hostgroup, _rhid); mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset_servers); if (error == NULL) { if (resultset_servers) { @@ -7347,7 +7348,7 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer( delete resultset_servers; resultset_servers = NULL; } - sprintf(query,q2,_writer_hostgroup,_rhid); + snprintf(query, query_size, q2, _writer_hostgroup, _rhid); mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset_servers); if (error == NULL) { if (resultset_servers) { diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index d9adc334b5..92b9dfedf1 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6067,11 +6067,6 @@ struct AWS_Aurora_BGD_Topology_Observation { AWS_Aurora_BGD_Fingerprint fingerprint; }; -static std::string aws_aurora_bgd_normalize_server_id(const std::string& server_id) { - const size_t green_pos = server_id.find("-green-"); - return green_pos == std::string::npos ? server_id : server_id.substr(0, green_pos); -} - static std::string aws_aurora_bgd_member_hostname( const std::string& server_id, const std::string& domain_name ) { @@ -6083,6 +6078,21 @@ static std::string aws_aurora_bgd_member_hostname( : server_id + "." + domain_name; } +static std::string aws_aurora_bgd_server_id_from_hostname( + const std::string& hostname, const std::string& domain_name +) { + if (domain_name.empty()) { + return hostname; + } + const std::string suffix = domain_name.front() == '.' + ? domain_name : "." + domain_name; + if (hostname.size() <= suffix.size() + || strcasecmp(hostname.c_str() + hostname.size() - suffix.size(), suffix.c_str()) != 0) { + return {}; + } + return hostname.substr(0, hostname.size() - suffix.size()); +} + static bool aws_aurora_bgd_status_from_raw( const std::string& raw_status, AWS_Aurora_BGD_Status& status ) { @@ -6161,32 +6171,6 @@ static AWS_Aurora_BGD_Topology_Observation aws_aurora_bgd_validate_topology( return observation; } -static bool aws_aurora_bgd_select_reachable_host( - const std::vector& hosts, - unsigned int writer_hg, - AWS_RDS_BGD_Probe_Host& selected -) { - if (hosts.empty()) { - return false; - } - - const size_t first = static_cast(rand()) % hosts.size(); - for (size_t offset = 0; offset < hosts.size(); ++offset) { - const AWS_RDS_BGD_Probe_Host& host = hosts[(first + offset) % hosts.size()]; - if (GloMyMon->server_responds_to_ping( - const_cast(host.hostname.c_str()), host.port)) { - selected = host; - return true; - } - MyHGM->p_update_mysql_error_counter( - p_mysql_error_type::proxysql, writer_hg, - const_cast(host.hostname.c_str()), host.port, - ER_PROXYSQL_AWS_NO_PINGABLE_SRV - ); - } - return false; -} - static AWS_Aurora_BGD_Query_Result aws_aurora_bgd_query( const AWS_RDS_BGD_Probe_Host& host, unsigned int writer_hg, @@ -6235,6 +6219,48 @@ static AWS_Aurora_BGD_Query_Result aws_aurora_bgd_query( return out; } +static AWS_Aurora_BGD_Query_Result aws_aurora_bgd_query_candidates( + const std::vector& hosts, + unsigned int writer_hg, + unsigned int timeout_ms, + MySQL_Monitor_State_Data_Task_Type task_type, + const char* query, + std::atomic_bool& worker_stop, + AWS_RDS_BGD_Probe_Host* selected, + bool stop_on_missing_table +) { + AWS_Aurora_BGD_Query_Result last_result; + if (hosts.empty() || worker_stop.load()) { + return last_result; + } + + const size_t first = static_cast(rand_fast()) % hosts.size(); + for (size_t offset = 0; offset < hosts.size() && !worker_stop.load(); ++offset) { + const AWS_RDS_BGD_Probe_Host& host = hosts[(first + offset) % hosts.size()]; + if (!GloMyMon->server_responds_to_ping( + const_cast(host.hostname.c_str()), host.port)) { + MyHGM->p_update_mysql_error_counter( + p_mysql_error_type::proxysql, writer_hg, + const_cast(host.hostname.c_str()), host.port, + ER_PROXYSQL_AWS_NO_PINGABLE_SRV + ); + continue; + } + + last_result = aws_aurora_bgd_query( + host, writer_hg, timeout_ms, task_type, query, worker_stop); + if (last_result.rc == 0 + || (stop_on_missing_table && last_result.mysql_error == 1146)) { + if (selected != nullptr) { + *selected = host; + } + return last_result; + } + } + + return last_result; +} + static void aws_aurora_bgd_set_status( AWS_Aurora_BGD_State& st, AWS_Aurora_BGD_Status status ) { @@ -6259,7 +6285,10 @@ static bool aws_aurora_bgd_same_production_snapshot( for (const AWS_Aurora_BGD_Member& member : lhs) { auto found = std::find_if(rhs.begin(), rhs.end(), [&](const AWS_Aurora_BGD_Member& candidate) { return candidate.normalized_server_id == member.normalized_server_id - && candidate.is_writer == member.is_writer; + && candidate.is_writer == member.is_writer + && candidate.hostname == member.hostname + && candidate.port == member.port + && candidate.use_ssl == member.use_ssl; }); if (found == rhs.end()) { return false; @@ -6268,6 +6297,59 @@ static bool aws_aurora_bgd_same_production_snapshot( return true; } +static bool aws_aurora_bgd_rebuild_production_snapshot(AWS_Aurora_BGD_State& st) { + std::unordered_set writer_endpoints; + MyHGM->wrlock(); + MyHGC* writer_hgc = MyHGM->MyHGC_find(st.writer_hg); + if (writer_hgc != nullptr && writer_hgc->mysrvs != nullptr) { + for (unsigned int i = 0; i < writer_hgc->mysrvs->cnt(); ++i) { + MySrvC* server = writer_hgc->mysrvs->idx(i); + if (server->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD + || server->get_status() == MYSQL_SERVER_STATUS_OFFLINE_SOFT) { + continue; + } + writer_endpoints.insert( + std::string(server->address) + "\n" + std::to_string(server->port)); + } + } + MyHGM->wrunlock(); + + std::vector snapshot; + std::unordered_set member_ids; + unsigned int writers = 0; + for (const AWS_RDS_BGD_Probe_Host& host : st.production_probe_hosts) { + AWS_Aurora_BGD_Member member; + member.server_id = aws_aurora_bgd_server_id_from_hostname( + host.hostname, st.domain_name); + member.normalized_server_id = member.server_id; + if (member.server_id.empty() + || !member_ids.insert(member.normalized_server_id).second) { + return false; + } + member.hostname = host.hostname; + member.production_hostname = host.hostname; + member.port = host.port; + member.use_ssl = host.use_ssl; + member.is_writer = writer_endpoints.count( + host.hostname + "\n" + std::to_string(host.port)) != 0; + member.session_id = member.is_writer ? "MASTER_SESSION_ID" : ""; + writers += member.is_writer ? 1 : 0; + snapshot.push_back(std::move(member)); + } + if (writers != 1 || snapshot.empty()) { + return false; + } + + if (!aws_aurora_bgd_same_production_snapshot(st.production_members, snapshot)) { + st.target_snapshot_complete = false; + } + st.production_members = std::move(snapshot); + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: rebuilt production membership with %zu configured members\n", + st.writer_hg, st.reader_hg, st.production_members.size()); + return true; +} + static bool aws_aurora_bgd_parse_target_membership( AWS_Aurora_BGD_State& st, MYSQL_RES* result, std::vector& snapshot @@ -6337,14 +6419,22 @@ static bool aws_aurora_bgd_parse_target_membership( AWS_Aurora_BGD_Member member; member.server_id = row[server_id_idx]; - member.normalized_server_id = aws_aurora_bgd_normalize_server_id(member.server_id); + member.normalized_server_id = member.server_id; member.session_id = row[session_id_idx]; member.is_writer = strcasecmp(member.session_id.c_str(), "MASTER_SESSION_ID") == 0; - if (member.normalized_server_id.empty() || !target_ids.insert(member.normalized_server_id).second) { + auto production = production_by_id.find(member.normalized_server_id); + if (production == production_by_id.end()) { + const size_t green_pos = member.server_id.rfind("-green-"); + if (green_pos != std::string::npos + && green_pos + strlen("-green-") < member.server_id.size()) { + member.normalized_server_id = member.server_id.substr(0, green_pos); + production = production_by_id.find(member.normalized_server_id); + } + } + if (member.normalized_server_id.empty() + || !target_ids.insert(member.normalized_server_id).second) { return false; } - - auto production = production_by_id.find(member.normalized_server_id); if (production == production_by_id.end() || production->second->is_writer != member.is_writer) { return false; @@ -6384,12 +6474,15 @@ static bool aws_aurora_bgd_parse_target_membership( if (resolved_ip.empty()) { return false; } - auto previous_ip = previous_ip_by_id.find(member.normalized_server_id); - member.target_ip = previous_ip != previous_ip_by_id.end() && !previous_ip->second.empty() - ? previous_ip->second : resolved_ip; auto previous_action = previous_action_by_id.find(member.normalized_server_id); member.traffic_pin_applied = previous_action != previous_action_by_id.end() && previous_action->second; + auto previous_ip = previous_ip_by_id.find(member.normalized_server_id); + const bool preserve_pre_rename_ip = member.server_id == production->second->server_id; + member.target_ip = previous_ip != previous_ip_by_id.end() + && !previous_ip->second.empty() + && (member.traffic_pin_applied || preserve_pre_rename_ip) + ? previous_ip->second : resolved_ip; snapshot.push_back(std::move(member)); } @@ -6415,6 +6508,26 @@ static std::vector aws_aurora_bgd_target_probe_hosts( return hosts; } +static std::vector aws_aurora_bgd_membership_probe_hosts( + const AWS_Aurora_BGD_State& st +) { + std::vector hosts = aws_aurora_bgd_target_probe_hosts(st); + if (!st.fingerprint.empty()) { + auto endpoint = std::find_if(hosts.begin(), hosts.end(), [&](const AWS_RDS_BGD_Probe_Host& host) { + return host.hostname == st.fingerprint.target_endpoint + && host.port == st.fingerprint.target_port; + }); + if (endpoint == hosts.end()) { + hosts.push_back({ + st.fingerprint.target_endpoint, + st.fingerprint.target_port, + st.target_use_ssl, + }); + } + } + return hosts; +} + } // namespace void MySQL_Monitor::aws_aurora_bgd_refresh_production_snapshot( @@ -6435,7 +6548,7 @@ void MySQL_Monitor::aws_aurora_bgd_refresh_production_snapshot( } AWS_Aurora_BGD_Member member; member.server_id = row->server_id; - member.normalized_server_id = aws_aurora_bgd_normalize_server_id(member.server_id); + member.normalized_server_id = member.server_id; member.session_id = row->session_id; member.hostname = aws_aurora_bgd_member_hostname(member.server_id, st.domain_name); member.production_hostname = member.hostname; @@ -6465,6 +6578,12 @@ void MySQL_Monitor::aws_aurora_bgd_apply_active_actions( || st.status > AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING) { return; } + if (st.production_members.empty()) { + proxy_error( + "AWS Aurora BGD [wHG=%u rHG=%u]: cannot enter active handling without production membership\n", + st.writer_hg, st.reader_hg); + return; + } st.production_snapshot_frozen = true; st.production_probe_suspended = true; @@ -6675,15 +6794,12 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle( ? aws_aurora_bgd_target_probe_hosts(st) : st.production_probe_hosts; AWS_RDS_BGD_Probe_Host topology_host; - if (!aws_aurora_bgd_select_reachable_host(topology_hosts, st.writer_hg, topology_host)) { - proxy_error("No node is pingable for AWS Aurora BGD topology checks with writer HG %u\n", st.writer_hg); - return; - } if (st.topology_state == TOPOLOGY_TABLE_CHECK) { - AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query( - topology_host, st.writer_hg, st.check_timeout_ms, - MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK, worker_stop); + AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query_candidates( + topology_hosts, st.writer_hg, st.check_timeout_ms, + MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_TABLE_CHECK, worker_stop, + &topology_host, false); if (worker_stop.load()) { return; } @@ -6699,9 +6815,10 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle( } } } else { - AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query( - topology_host, st.writer_hg, st.check_timeout_ms, - MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY, worker_stop); + AWS_Aurora_BGD_Query_Result query = aws_aurora_bgd_query_candidates( + topology_hosts, st.writer_hg, st.check_timeout_ms, + MON_AWS_RDS_BGD, QUERY_AWS_RDS_TOPOLOGY_DISCOVERY, worker_stop, + &topology_host, true); if (worker_stop.load()) { return; } @@ -6722,14 +6839,33 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle( AWS_RDS_Topology_Result topology = parse_aws_rds_topology(query.mmsd->result); AWS_Aurora_BGD_Topology_Observation observation = aws_aurora_bgd_validate_topology(topology); + if (observation.valid + && st.status == AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED) { + if (st.fingerprint == observation.fingerprint) { + return; + } + st.fingerprint = AWS_Aurora_BGD_Fingerprint {}; + st.target_members.clear(); + st.target_snapshot_complete = false; + aws_aurora_bgd_set_status(st, AWS_Aurora_BGD_Status::NONE); + proxy_info( + "AWS Aurora BGD [wHG=%u rHG=%u]: new deployment fingerprint rearmed discovery\n", + st.writer_hg, st.reader_hg); + } if (observation.valid && observation.completed) { const bool same_active_deployment = st.fingerprint.empty() - || st.fingerprint == observation.fingerprint - || st.status == AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED; + || st.fingerprint == observation.fingerprint; if (same_active_deployment) { aws_aurora_bgd_apply_completion(st, observation.fingerprint); } } else if (observation.valid) { + if (st.production_members.empty() + && !aws_aurora_bgd_rebuild_production_snapshot(st)) { + proxy_error( + "AWS Aurora BGD [wHG=%u rHG=%u]: retaining current state until production membership is available\n", + st.writer_hg, st.reader_hg); + return; + } const bool same_deployment = st.fingerprint.empty() || st.fingerprint == observation.fingerprint; const bool rollback_transition = @@ -6755,16 +6891,12 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle( return; } - std::vector membership_hosts = aws_aurora_bgd_target_probe_hosts(st); - AWS_RDS_BGD_Probe_Host membership_host; - if (!aws_aurora_bgd_select_reachable_host(membership_hosts, st.writer_hg, membership_host)) { - proxy_error("No node is pingable for AWS Aurora BGD membership checks with writer HG %u\n", st.writer_hg); - return; - } - - AWS_Aurora_BGD_Query_Result membership = aws_aurora_bgd_query( - membership_host, st.writer_hg, st.check_timeout_ms, - MON_AWS_AURORA, QUERY_AWS_AURORA_BGD_REPLICA_HOST_STATUS, worker_stop); + std::vector membership_hosts = + aws_aurora_bgd_membership_probe_hosts(st); + AWS_Aurora_BGD_Query_Result membership = aws_aurora_bgd_query_candidates( + membership_hosts, st.writer_hg, st.check_timeout_ms, + MON_AWS_AURORA, QUERY_AWS_AURORA_BGD_REPLICA_HOST_STATUS, worker_stop, + nullptr, false); if (membership.rc != 0) { return; } @@ -6975,7 +7107,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { const size_t num_hosts = bgd_state.production_probe_hosts.size(); if (num_hosts != 0) { - rnd = static_cast(rand()) % num_hosts; + rnd = static_cast(rand_fast()) % num_hosts; for (size_t offset = 0; !found_pingable_host && offset < num_hosts; ++offset) { const size_t host_idx = (rnd + offset) % num_hosts; const AWS_RDS_BGD_Probe_Host& host = @@ -6983,7 +7115,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { rc_ping = GloMyMon->server_responds_to_ping( const_cast(host.hostname.c_str()), host.port); #ifdef TEST_AURORA_RANDOM - if (offset == 0 && rand() % 100 < 30) { + if (offset == 0 && rand_fast() % 100 < 30) { rc_ping = false; } #endif // TEST_AURORA_RANDOM @@ -7000,7 +7132,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { } #ifdef TEST_AURORA_RANDOM - if (rand() % 200 == 0) { + if (rand_fast() % 200 == 0) { // we randomly fail 0.5% of the requests found_pingable_host = false; } @@ -7016,7 +7148,7 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { continue; } #ifdef TEST_AURORA - if (rand() % 1000 == 0) { // suppress 99.9% of the output, too verbose + if (rand_fast() % 1000 == 0) { // suppress 99.9% of the output, too verbose const AWS_RDS_BGD_Probe_Host& host = bgd_state.production_probe_hosts[cur_host_idx]; proxy_info("Running check for AWS Aurora writer HG %u on %s:%d\n", wHG, host.hostname.c_str(), host.port); @@ -7277,12 +7409,6 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { if (mmsd) { delete (mmsd); mmsd = NULL; - for (unsigned int i=0; iIncrement(); + fetching_error = true; + fetch_failed = true; + for (MYSQL_RES*& result : results) { + if (result != nullptr) { + mysql_free_result(result); + } + result = nullptr; + } + } + if (fetching_error == false) { const uint64_t servers_hash = compute_servers_tables_raw_checksum(results, 8); // ignore runtime_mysql_servers in checksum calculation const string computed_checksum{ get_checksum_from_hash(servers_hash) }; diff --git a/test/infra/control/cluster-simulator-ci.bash b/test/infra/control/cluster-simulator-ci.bash index 2c14eac040..0cb82a7b8f 100755 --- a/test/infra/control/cluster-simulator-ci.bash +++ b/test/infra/control/cluster-simulator-ci.bash @@ -271,6 +271,11 @@ handle_stage() { install -D -m 0755 \ "${REPO_ROOT}/test/deps/cluster_simulator/cluster_simulator" \ "${STAGE_TEMP_DIR}/test/deps/cluster_simulator/cluster_simulator" + install -d "${STAGE_TEMP_DIR}/test/deps/cluster_simulator/obj" + cp -a "${REPO_ROOT}"/test/deps/cluster_simulator/*.gcno \ + "${STAGE_TEMP_DIR}/test/deps/cluster_simulator/" + cp -a "${REPO_ROOT}"/test/deps/cluster_simulator/obj/*.gcno \ + "${STAGE_TEMP_DIR}/test/deps/cluster_simulator/obj/" install -d "${STAGE_TEMP_DIR}/test/tap" cp -a "${REPO_ROOT}/test/tap/tap" "${STAGE_TEMP_DIR}/test/tap/" diff --git a/test/tap/groups/cluster_sim_aurora/add-hosts b/test/tap/groups/cluster_sim_aurora/add-hosts index b8853c07d1..714ef5ccf7 100644 --- a/test/tap/groups/cluster_sim_aurora/add-hosts +++ b/test/tap/groups/cluster_sim_aurora/add-hosts @@ -56,6 +56,9 @@ aurora-a-green-reader-2.a1.us-east-1.rds.amazonaws.com 127.0.11.23 aurora-a-writer-green-k7m2.a1.us-east-1.rds.amazonaws.com 127.0.11.21 aurora-a-reader-1-green-p4q8.a1.us-east-1.rds.amazonaws.com 127.0.11.22 aurora-a-reader-2-green-v9n3.a1.us-east-1.rds.amazonaws.com 127.0.11.23 +aurora-a-writer-green-r2.a1.us-east-1.rds.amazonaws.com 127.0.11.31 +aurora-a-reader-1-green-r2.a1.us-east-1.rds.amazonaws.com 127.0.11.32 +aurora-a-reader-2-green-r2.a1.us-east-1.rds.amazonaws.com 127.0.11.33 # Canonical names observed after target-member rename retain target addresses. aurora-a-canonical-writer.a1.us-east-1.rds.amazonaws.com 127.0.11.21 @@ -75,5 +78,7 @@ aurora-b-canonical-reader-1.b1.us-east-1.rds.amazonaws.com 127.0.12.22 # Aurora blue/green deployment C supports three-cluster isolation scenarios. aurora-c-writer.c1.us-east-1.rds.amazonaws.com 127.0.13.11 +aurora-c-green-writer.c1.us-east-1.rds.amazonaws.com 127.0.13.11 aurora-c-green.cluster-c1.us-east-1.rds.amazonaws.com 127.0.13.20 aurora-c-writer-green-m5n9.c1.us-east-1.rds.amazonaws.com 127.0.13.21 +aurora-c-green-writer-green-m5n9.c1.us-east-1.rds.amazonaws.com 127.0.13.21 diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 6dd6bc1909..1035f09df8 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -348,6 +348,14 @@ "test_PROXY_Protocol-t" : [ "legacy-g3","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3","mysql84-g3","mysql90-g3","mysql95-g3" ], "test_admin_stats-t" : [ "legacy-g7","mysql-auto_increment_delay_multiplex=0-g2","mysql-multiplexing=false-g2","mysql-query_digests=0-g2","mysql-query_digests_keep_comment=1-g2","mysql84-g7","mysql90-g2","mysql95-g2" ], "test_ansi_quotes_group_replication-t" : [ "mysql84-gr-g1","mysql90-gr-g1","mysql91-gr-g1","mysql92-gr-g1","mysql93-gr-g1","mysql95-gr-g1" ], + "test_aurora_bgd_active-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_cluster_sync-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_completion-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_concurrent_isolation-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_discovery-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_error_recovery-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_late_entry-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_worker_lifecycle-t" : [ "cluster_sim_aurora-g1" ], "test_auth_methods-t" : [ "mysql-auto_increment_delay_multiplex=0-g2","mysql-multiplexing=false-g2","mysql-query_digests=0-g2","mysql-query_digests_keep_comment=1-g2","mysql84-g7","mysql90-g2","mysql95-g2" ], "test_auto_increment_delay_multiplex-t" : [ "legacy-g7","mysql-auto_increment_delay_multiplex=0-g2","mysql-multiplexing=false-g2","mysql-query_digests=0-g2","mysql-query_digests_keep_comment=1-g2","mysql84-g7","mysql90-g2","mysql95-g2" ], "test_backend_conn_ping-t" : [ "legacy-g7","mysql-auto_increment_delay_multiplex=0-g2","mysql-multiplexing=false-g2","mysql-query_digests=0-g2","mysql-query_digests_keep_comment=1-g2","mysql84-g7","mysql90-g2","mysql95-g2" ], @@ -363,14 +371,6 @@ "test_client_limit_error-t" : [ "todo-g1" ], "test_cluster1-t" : [ "legacy-g5","mariadb10-galera-g5","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3","mysql84-g5","mysql84-gr-g5","mysql90-g5","mysql95-g5" ], "test_cluster_sim_aurora-t" : [ "cluster_sim_aurora-g1" ], - "test_aurora_bgd_discovery-t" : [ "cluster_sim_aurora-g1" ], - "test_aurora_bgd_active-t" : [ "cluster_sim_aurora-g1" ], - "test_aurora_bgd_completion-t" : [ "cluster_sim_aurora-g1" ], - "test_aurora_bgd_cluster_sync-t" : [ "cluster_sim_aurora-g1" ], - "test_aurora_bgd_late_entry-t" : [ "cluster_sim_aurora-g1" ], - "test_aurora_bgd_error_recovery-t" : [ "cluster_sim_aurora-g1" ], - "test_aurora_bgd_worker_lifecycle-t" : [ "cluster_sim_aurora-g1" ], - "test_aurora_bgd_concurrent_isolation-t" : [ "cluster_sim_aurora-g1" ], "test_cluster_sim_galera-t" : [ "cluster_sim_galera-g1" ], "test_cluster_sim_group_repl-t" : [ "cluster_sim_group_repl-g1" ], "test_cluster_sim_read_only-t" : [ "cluster_sim_read_only-g1" ], @@ -426,7 +426,7 @@ "test_frontend_x509_tier_gate-t" : [ "legacy-g6","mysql84-g6","mysql90-g1","mysql95-g1" ], "test_greeting_capabilities-t" : [ "legacy-g8","mariadb10-galera-g8","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3","mysql84-g8","mysql84-gr-g8","mysql90-g3","mysql95-g3" ], "test_gtid_forwarding-t" : [ "legacy-binlog-g1","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3","mysql84-g5","mysql90-g5","mysql95-g5" ], - "test_gtid_from_ok-t" : [ "legacy-binlog-g1", "mysql84-binlog-g1", "mysql90-binlog-g1", "mysql95-binlog-g1" ], + "test_gtid_from_ok-t" : [ "legacy-binlog-g1","mysql84-binlog-g1","mysql90-binlog-g1","mysql95-binlog-g1" ], "test_hostgroup_attributes_online_servers-t" : [ "legacy-g8","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3","mysql84-g8","mysql90-g3","mysql95-g3" ], "test_hostgroup_default_query_timeout-t" : [ "legacy-g8","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3","mysql84-g8","mysql90-g3","mysql95-g3" ], "test_ignore_min_gtid-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql90-g4","mysql95-g4" ], diff --git a/test/tap/tap/aurora_bgd_tap.h b/test/tap/tap/aurora_bgd_tap.h index c71d8100c1..647a2d4fcb 100644 --- a/test/tap/tap/aurora_bgd_tap.h +++ b/test/tap/tap/aurora_bgd_tap.h @@ -1,6 +1,7 @@ #ifndef TAP_TESTS_AURORA_BGD_TAP_H #define TAP_TESTS_AURORA_BGD_TAP_H +#include #include #include #include @@ -11,7 +12,10 @@ #include "aurora_bgd_simulator.h" #include "tap.h" -using namespace std; +using std::move; +using std::string; +using std::to_string; +using std::vector; struct Aurora_BGD_Test_Deployment { string name; @@ -127,8 +131,8 @@ inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_c_writer_only() { deployment.production = { deployment.blue_replica_set, { - aurora_bgd_member("aurora-c-writer", "MASTER_SESSION_ID", - {"aurora-c-writer.c1.us-east-1.rds.amazonaws.com", "127.0.13.11", 3306}), + aurora_bgd_member("aurora-c-green-writer", "MASTER_SESSION_ID", + {"aurora-c-green-writer.c1.us-east-1.rds.amazonaws.com", "127.0.13.11", 3306}), }, {} }; @@ -136,8 +140,8 @@ inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_c_writer_only() { deployment.target = { deployment.target_replica_set, { - aurora_bgd_member("aurora-c-writer-green-m5n9", "MASTER_SESSION_ID", - {"aurora-c-writer-green-m5n9.c1.us-east-1.rds.amazonaws.com", "127.0.13.21", 3306}), + aurora_bgd_member("aurora-c-green-writer-green-m5n9", "MASTER_SESSION_ID", + {"aurora-c-green-writer-green-m5n9.c1.us-east-1.rds.amazonaws.com", "127.0.13.21", 3306}), }, {deployment.target_cluster_endpoint} }; @@ -183,7 +187,7 @@ inline int aurora_bgd_admin_setup( MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, int writer_hg, int reader_hg, int green_writer_hg, int green_reader_hg, bool auto_discovery, int check_interval_ms = 100, - bool writer_is_also_reader = false + bool writer_is_also_reader = false, bool use_ssl = false ) { vector queries { "SET mysql-monitor_username='aurora1'", @@ -205,7 +209,8 @@ inline int aurora_bgd_admin_setup( "writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks," "autopurge_missing_checks,comment) VALUES (" + to_string(writer_hg) + "," + to_string(reader_hg) + "," + green_columns + - ",1,3306," + aurora_bgd_sql_quote(deployment.domain_name) + + ",1," + to_string(deployment.production.members.front().endpoint.port) + "," + + aurora_bgd_sql_quote(deployment.domain_name) + ",200," + to_string(check_interval_ms) + ",800," + to_string(writer_is_also_reader ? 1 : 0) + ",1,30,30,1,0," + aurora_bgd_sql_quote(deployment.name) + ")"); @@ -215,7 +220,9 @@ inline int aurora_bgd_admin_setup( queries.push_back( "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + to_string(hostgroup) + "," + aurora_bgd_sql_quote(member.endpoint.hostname) + - ",3306,'ONLINE',0,'Aurora BGD production member')"); + "," + to_string(member.endpoint.port) + + ",'ONLINE'," + string(use_ssl ? "1" : "0") + + ",'Aurora BGD production member')"); } queries.push_back("LOAD MYSQL VARIABLES TO RUNTIME"); queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); @@ -230,14 +237,14 @@ inline vector aurora_bgd_available_topology( { deployment.source_topology_id, deployment.production.members.front().endpoint.hostname, - 3306, + deployment.production.members.front().endpoint.port, "BLUE_GREEN_DEPLOYMENT_SOURCE", "AVAILABLE", }, { deployment.target_topology_id, deployment.target_cluster_endpoint.hostname, - 3306, + deployment.target_cluster_endpoint.port, "BLUE_GREEN_DEPLOYMENT_TARGET", "AVAILABLE", }, @@ -295,6 +302,28 @@ inline int aurora_bgd_publish( aurora_bgd_available_topology(deployment)); } +inline bool aurora_bgd_result_matches_membership( + const vector>& rows, const Aurora_BGD_Membership_Set& membership +) { + vector actual_ids; + actual_ids.reserve(rows.size()); + for (const vector& row : rows) { + if (row.empty()) { + return false; + } + actual_ids.push_back(row.front()); + } + + vector expected_ids; + expected_ids.reserve(membership.members.size()); + for (const Aurora_BGD_Member& member : membership.members) { + expected_ids.push_back(member.server_id); + } + std::sort(actual_ids.begin(), actual_ids.end()); + std::sort(expected_ids.begin(), expected_ids.end()); + return actual_ids == expected_ids; +} + inline int aurora_bgd_wait_for_status( MYSQL* admin, int writer_hg, const string& status, uint32_t timeout_seconds ) { diff --git a/test/tap/tap/bgd_simulator.h b/test/tap/tap/bgd_simulator.h index 30650da764..b4bc70e028 100644 --- a/test/tap/tap/bgd_simulator.h +++ b/test/tap/tap/bgd_simulator.h @@ -8,7 +8,8 @@ #include "cluster_simulator.h" #include "utils.h" -using namespace std; +using std::string; +using std::vector; /** One row returned by the simulated mysql.rds_topology table. */ struct BGD_Topology_Row { diff --git a/test/tap/tests/test_aurora_bgd_active-t.cpp b/test/tap/tests/test_aurora_bgd_active-t.cpp index b6a4a7a088..c5c48b315f 100644 --- a/test/tap/tests/test_aurora_bgd_active-t.cpp +++ b/test/tap/tests/test_aurora_bgd_active-t.cpp @@ -9,6 +9,7 @@ * 3. Enter IN_PROGRESS and verify one writer demotion with source routing intact. * 4. Enter POST_PROCESSING and verify restoration, pool retirement, and target pins. * 5. Complete a previously incomplete target snapshot during POST_PROCESSING. + * 6. Verify refreshed/renamed target identity with automatic multi-reader discovery. */ #include @@ -55,6 +56,11 @@ struct TestState { int gated_green_writer_hostgroup { 1542 }; int gated_green_reader_hostgroup { 1543 }; int gated_route_hostgroup { 1544 }; + Aurora_BGD_Test_Deployment refreshed { aurora_bgd_deployment_a() }; + int refreshed_writer_hostgroup { 1545 }; + int refreshed_reader_hostgroup { 1546 }; + vector refreshed_route_hostgroups { 1547, 1548, 1549 }; + MYSQL* held_client { nullptr }; }; int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { @@ -68,7 +74,7 @@ int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { return EXIT_FAILURE; } char simulator_username[] = "aurora1"; - char simulator_password[] = "pass1"; + char simulator_password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. if (sim.connect(cl.host, 3306, simulator_username, simulator_password) != EXIT_SUCCESS) { diag("Error: failed to connect to the shared AWS simulator"); mysql_close(admin); @@ -148,13 +154,18 @@ int publish_status( aurora_bgd_topology(deployment, status)); } -bool fast_membership_without_ordinary( +bool fast_bgd_without_ordinary( BGD_Simulator& sim, uint64_t sequence, const string& target_replica_set, uint32_t observation_ms, uint64_t minimum_membership_probes ) { + auto [topology_seq_rc, topology_sequence] = sim.probe_log_last_sequence(); + if (topology_seq_rc != EXIT_SUCCESS) { + return false; + } usleep(observation_ms * 1000); auto [rc, logs] = sim.replica_probe_log_since(sequence); - if (rc != EXIT_SUCCESS) { + auto [topology_rc, topology_logs] = sim.probe_log_since(topology_sequence); + if (rc != EXIT_SUCCESS || topology_rc != EXIT_SUCCESS) { return false; } uint64_t membership_probes = 0; @@ -167,13 +178,22 @@ bool fast_membership_without_ordinary( membership_probes++; } } - return membership_probes >= minimum_membership_probes; + uint64_t topology_probes = 0; + for (const BGD_Probe_Log& log : topology_logs) { + topology_probes += log.probe_kind == BGD_Probe_Kind::metadata ? 1 : 0; + } + return membership_probes >= minimum_membership_probes + && topology_probes >= minimum_membership_probes; } int add_member_routes( MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, const vector& route_hgs ) { + if (route_hgs.size() != deployment.production.members.size()) { + diag("Member-route hostgroup count does not match production membership"); + return EXIT_FAILURE; + } vector queries; for (size_t i = 0; i < deployment.production.members.size(); ++i) { queries.push_back( @@ -195,7 +215,8 @@ int set_default_hostgroup(MYSQL* admin, int hostgroup) { } bool route_to_expected_backend( - CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected_backend + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected_backend, + const Aurora_BGD_Membership_Set& expected_membership ) { auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); if (sequence_rc != EXIT_SUCCESS) { @@ -206,12 +227,14 @@ bool route_to_expected_backend( return false; } auto [rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); - (void)rows; - if (rc != EXIT_SUCCESS) { - diag("Backend routing query failed with MySQL error %d: %s", rc, mysql_error(client)); + const bool result_matches = rc == EXIT_SUCCESS + && aurora_bgd_result_matches_membership(rows, expected_membership); + if (!result_matches) { + diag("Backend routing query failed with MySQL error %d: %s", + mysql_errno(client), mysql_error(client)); } mysql_close(client); - if (rc != EXIT_SUCCESS) { + if (!result_matches) { return false; } @@ -250,7 +273,10 @@ bool route_members_to_expected_ips( Endpoint expected_backend = target ? deployment.target.members[i].endpoint.backend() : deployment.production.members[i].endpoint.backend(); - if (!route_to_expected_backend(cl, sim, expected_backend)) { + const Aurora_BGD_Membership_Set& expected_membership = target + ? deployment.target : deployment.production; + if (!route_to_expected_backend( + cl, sim, expected_backend, expected_membership)) { return false; } } @@ -292,6 +318,22 @@ int wait_for_member_route_pool_count( kWaitSeconds); } +int64_t member_route_used_count(MYSQL* admin, const vector& route_hgs) { + string hostgroups; + for (int hostgroup : route_hgs) { + if (!hostgroups.empty()) { + hostgroups += ","; + } + hostgroups += to_string(hostgroup); + } + auto [rc, rows] = mysql_query_ext_rows( + admin, + "SELECT COALESCE(SUM(ConnUsed),0) FROM stats_mysql_connection_pool " + "WHERE hostgroup IN (" + hostgroups + ")"); + return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 + ? strtoll(rows.front().front().c_str(), nullptr, 10) : -1; +} + /** Configure the active switchover scenario and reach AVAILABLE. */ int test_bgd_status_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { Aurora_BGD_Test_Deployment& deployment = state.deployment; @@ -335,7 +377,7 @@ int test_switchover_initiated(MYSQL* admin, BGD_Simulator& sim, TestState& state "INITIATED does not change writer placement"); auto [active_seq_rc, active_sequence] = sim.replica_probe_log_last_sequence(); - ok(active_seq_rc == EXIT_SUCCESS && fast_membership_without_ordinary( + ok(active_seq_rc == EXIT_SUCCESS && fast_bgd_without_ordinary( sim, active_sequence, deployment.target_replica_set, 650, 3), "INITIATED uses fast membership probes and suspends the ordinary Aurora query"); @@ -349,7 +391,7 @@ int test_switchover_initiated(MYSQL* admin, BGD_Simulator& sim, TestState& state diag("Error: failed to publish the competing source role observation"); return EXIT_FAILURE; } - ok(fast_membership_without_ordinary( + ok(fast_bgd_without_ordinary( sim, source_change_sequence, deployment.target_replica_set, 350, 2), "changed source roles cannot compete while production probing is suspended"); return EXIT_SUCCESS; @@ -392,7 +434,7 @@ int test_switchover_in_progress( diag("Error: failed to repeat SWITCHOVER_IN_PROGRESS"); return EXIT_FAILURE; } - ok(fast_membership_without_ordinary( + ok(fast_bgd_without_ordinary( sim, repeat_progress_sequence, deployment.target_replica_set, 350, 2) && wait_for_writer_placement( admin, state.writer_hostgroup, state.reader_hostgroup, @@ -404,6 +446,20 @@ int test_switchover_in_progress( && member_route_pool_count(admin, state.route_hostgroups) >= static_cast(state.route_hostgroups.size()), "all member routes use source IPs and hold pre-cutover pools"); + + if (set_default_hostgroup(admin, state.route_hostgroups.front()) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + state.held_client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + bool held_connection = state.held_client != nullptr + && mysql_query(state.held_client, "BEGIN") == 0; + if (held_connection) { + auto [query_rc, rows] = mysql_query_ext_rows(state.held_client, kOrdinaryAuroraQuery); + held_connection = query_rc == EXIT_SUCCESS + && aurora_bgd_result_matches_membership(rows, deployment.production); + } + ok(held_connection && member_route_used_count(admin, state.route_hostgroups) >= 1, + "a production connection remains in use across the cutover boundary"); return EXIT_SUCCESS; } @@ -418,7 +474,14 @@ int test_switchover_post_processing( CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) { Aurora_BGD_Test_Deployment& deployment = state.deployment; - if (publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + if (sim.read_only_update( + deployment.target.members.front().endpoint.host_endpoint(), true) != EXIT_SUCCESS + || aurora_bgd_execute_all(admin, { + "SET mysql-monitor_local_dns_cache_ttl=0", + "SET mysql-monitor_local_dns_cache_refresh_interval=0", + "LOAD MYSQL VARIABLES TO RUNTIME", + }) != EXIT_SUCCESS + || publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS || aurora_bgd_wait_for_status( admin, state.writer_hostgroup, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { @@ -438,12 +501,34 @@ int test_switchover_post_processing( admin, state.reader_hostgroup, deployment.production.members[2].endpoint.hostname, 1, "ONLINE"), "POST_PROCESSING leaves canonical readers ONLINE and eligible"); + ok(state.held_client != nullptr + && member_route_used_count(admin, state.route_hostgroups) >= 1, + "POST_PROCESSING advances without waiting for an in-use connection to close"); + if (state.held_client != nullptr) { + mysql_query(state.held_client, "ROLLBACK"); + mysql_close(state.held_client); + state.held_client = nullptr; + } ok(wait_for_member_route_pool_count( admin, state.route_hostgroups, "=0") == EXIT_SUCCESS, "POST_PROCESSING retires the pre-cutover member pools"); ok(route_members_to_expected_ips( cl, admin, sim, deployment, state.route_hostgroups, true), - "POST_PROCESSING pins every production hostname to its cached target IP"); + "POST_PROCESSING pins every production hostname despite disabled DNS caching and target read_only=1"); + ok(aurora_bgd_execute_all(admin, { + "LOAD MYSQL VARIABLES TO RUNTIME", + }) == EXIT_SUCCESS + && route_members_to_expected_ips( + cl, admin, sim, deployment, state.route_hostgroups, true), + "an active variables refresh preserves every explicit traffic pin"); + if (aurora_bgd_execute_all(admin, { + "SET mysql-monitor_local_dns_cache_ttl=300000", + "SET mysql-monitor_local_dns_cache_refresh_interval=60000", + "LOAD MYSQL VARIABLES TO RUNTIME", + }) != EXIT_SUCCESS) { + diag("Error: failed to restore DNS-cache variables"); + return EXIT_FAILURE; + } const int64_t target_pool_count = member_route_pool_count(admin, state.route_hostgroups); auto [repeat_post_rc, repeat_post_sequence] = sim.probe_log_last_sequence(); @@ -462,12 +547,216 @@ int test_switchover_post_processing( "repeated POST_PROCESSING does not replay completed member retirement"); auto [post_seq_rc, post_sequence] = sim.replica_probe_log_last_sequence(); - ok(post_seq_rc == EXIT_SUCCESS && fast_membership_without_ordinary( + ok(post_seq_rc == EXIT_SUCCESS && fast_bgd_without_ordinary( sim, post_sequence, deployment.target_replica_set, 350, 2), "POST_PROCESSING keeps fast BGD probes without ordinary Aurora queries"); return EXIT_SUCCESS; } +/** + * Refresh target IPs before rename and retain the last complete multi-reader map. + * + * Green hostgroups are intentionally NULL: auto-discovery must still pair all + * members, accept the canonical rename, and route with the last complete map. + */ +int test_auto_discovered_refresh_and_rename( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state +) { + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before refreshed-target scenario"); + return EXIT_FAILURE; + } + + Aurora_BGD_Test_Deployment& deployment = state.refreshed; + if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, state.refreshed_writer_hostgroup, + state.refreshed_reader_hostgroup, -1, -1, true, 300, false) != EXIT_SUCCESS + || add_member_routes( + admin, deployment, state.refreshed_route_hostgroups) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, state.refreshed_writer_hostgroup, "AVAILABLE", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to configure auto-discovered multi-reader deployment"); + return EXIT_FAILURE; + } + + const vector refreshed_ids { + "aurora-a-writer-green-r2", + "aurora-a-reader-1-green-r2", + "aurora-a-reader-2-green-r2", + }; + const vector refreshed_ips { "127.0.11.31", "127.0.11.32", "127.0.11.33" }; + for (size_t i = 0; i < deployment.target.members.size(); ++i) { + deployment.target.members[i].server_id = refreshed_ids[i]; + deployment.target.members[i].endpoint.hostname = + refreshed_ids[i] + deployment.domain_name; + deployment.target.members[i].endpoint.ip = refreshed_ips[i]; + } + deployment.target.serving_endpoints.clear(); + deployment.target.serving_endpoints.push_back(deployment.target_cluster_endpoint); + for (const Aurora_BGD_Member& member : deployment.target.members) { + deployment.target.serving_endpoints.push_back(member.endpoint); + } + + auto [refresh_seq_rc, refresh_sequence] = sim.replica_probe_log_last_sequence(); + if (refresh_seq_rc != EXIT_SUCCESS || sim.replica_update( + deployment.target_replica_set, deployment.target.replica_rows(), + deployment.target.backends()) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + vector refreshed_backends = deployment.target.backends(); + auto [refresh_probe_rc, refresh_probe] = aurora_bgd_wait_for_replica_probe( + sim, refresh_sequence, refreshed_backends, + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, + deployment.target_replica_set); + ok(refresh_probe_rc == EXIT_SUCCESS, + "AVAILABLE refreshes every target IP before the member rename"); + + vector canonical_rows = deployment.target.replica_rows(); + for (size_t i = 0; i < canonical_rows.size(); ++i) { + canonical_rows[i].server_id = deployment.production.members[i].server_id; + } + auto [rename_seq_rc, rename_sequence] = sim.replica_probe_log_last_sequence(); + if (rename_seq_rc != EXIT_SUCCESS || sim.replica_update( + deployment.target_replica_set, canonical_rows, + deployment.target.backends()) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + auto [rename_probe_rc, rename_probe] = aurora_bgd_wait_for_replica_probe( + sim, rename_sequence, refreshed_backends, + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, + deployment.target_replica_set); + ok(rename_probe_rc == EXIT_SUCCESS, + "stable reader sessions preserve identity across the canonical SERVER_ID rename"); + + vector incomplete_rows = canonical_rows; + incomplete_rows.pop_back(); + auto [incomplete_seq_rc, incomplete_sequence] = sim.replica_probe_log_last_sequence(); + if (incomplete_seq_rc != EXIT_SUCCESS || sim.replica_update( + deployment.target_replica_set, incomplete_rows, + deployment.target.backends()) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + auto [incomplete_probe_rc, incomplete_probe] = aurora_bgd_wait_for_replica_probe( + sim, incomplete_sequence, refreshed_backends, + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, + deployment.target_replica_set); + if (incomplete_probe_rc != EXIT_SUCCESS + || publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, state.refreshed_writer_hostgroup, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + Aurora_BGD_Membership_Set observed_target = deployment.target; + observed_target.members.pop_back(); + bool retained_routing = true; + for (size_t i = 0; i < state.refreshed_route_hostgroups.size(); ++i) { + retained_routing = retained_routing + && set_default_hostgroup(admin, state.refreshed_route_hostgroups[i]) == EXIT_SUCCESS + && route_to_expected_backend( + cl, sim, deployment.target.members[i].endpoint.backend(), observed_target); + } + ok(retained_routing, + "POST_PROCESSING routes all auto-discovered readers with the retained complete refreshed map"); + return EXIT_SUCCESS; +} + +/** Refresh production membership in AVAILABLE and freeze it after INITIATED. */ +int test_available_production_refresh_and_freeze( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state +) { + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + Aurora_BGD_Test_Deployment deployment = aurora_bgd_deployment_a(); + if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, state.refreshed_writer_hostgroup, + state.refreshed_reader_hostgroup, -1, -1, true, 300, false) != EXIT_SUCCESS + || add_member_routes( + admin, deployment, state.refreshed_route_hostgroups) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, state.refreshed_writer_hostgroup, "AVAILABLE", kWaitSeconds) + != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + vector reduced_production = deployment.production.replica_rows(); + reduced_production.pop_back(); + vector reduced_target = deployment.target.replica_rows(); + reduced_target.pop_back(); + auto [ordinary_seq_rc, ordinary_sequence] = sim.replica_probe_log_last_sequence(); + if (ordinary_seq_rc != EXIT_SUCCESS || sim.replica_update( + deployment.blue_replica_set, reduced_production, + deployment.production.backends()) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + auto [ordinary_probe_rc, ordinary_probe] = aurora_bgd_wait_for_replica_probe( + sim, ordinary_sequence, deployment.production.backends(), + Aurora_Replica_Probe_Kind::ordinary, kProbeTimeoutMs, + deployment.blue_replica_set); + auto [ordinary_retry_rc, ordinary_retry] = aurora_bgd_wait_for_replica_probe( + sim, ordinary_probe.sequence_id, deployment.production.backends(), + Aurora_Replica_Probe_Kind::ordinary, kProbeTimeoutMs, + deployment.blue_replica_set); + auto [target_seq_rc, target_sequence] = sim.replica_probe_log_last_sequence(); + if (ordinary_probe_rc != EXIT_SUCCESS || ordinary_retry_rc != EXIT_SUCCESS + || target_seq_rc != EXIT_SUCCESS + || sim.replica_update( + deployment.target_replica_set, reduced_target, + deployment.target.backends()) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + auto [target_probe_rc, target_probe] = aurora_bgd_wait_for_replica_probe( + sim, target_sequence, deployment.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, + deployment.target_replica_set); + auto [target_retry_rc, target_retry] = aurora_bgd_wait_for_replica_probe( + sim, target_probe.sequence_id, deployment.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, + deployment.target_replica_set); + ok(target_probe_rc == EXIT_SUCCESS && target_retry_rc == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + admin, state.refreshed_writer_hostgroup, "AVAILABLE", 1) == EXIT_SUCCESS, + "AVAILABLE refreshes production and target membership before the switchover"); + + if (publish_status(sim, deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, state.refreshed_writer_hostgroup, + "SWITCHOVER_INITIATED", kWaitSeconds) != EXIT_SUCCESS + || sim.replica_update( + deployment.blue_replica_set, deployment.production.replica_rows(), + deployment.production.backends()) != EXIT_SUCCESS + || sim.replica_update( + deployment.target_replica_set, deployment.target.replica_rows(), + deployment.target.backends()) != EXIT_SUCCESS + || publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, state.refreshed_writer_hostgroup, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + bool frozen_routes = true; + for (size_t i = 0; i < state.refreshed_route_hostgroups.size(); ++i) { + const bool target = i < reduced_production.size(); + const Endpoint expected = target + ? deployment.target.members[i].endpoint.backend() + : deployment.production.members[i].endpoint.backend(); + const Aurora_BGD_Membership_Set& expected_membership = target + ? deployment.target : deployment.production; + frozen_routes = frozen_routes + && set_default_hostgroup(admin, state.refreshed_route_hostgroups[i]) == EXIT_SUCCESS + && route_to_expected_backend(cl, sim, expected, expected_membership); + } + ok(frozen_routes, + "INITIATED freezes the refreshed production map despite later membership changes"); + return EXIT_SUCCESS; +} + /** * Complete target membership after entering POST_PROCESSING. * @@ -518,7 +807,8 @@ int test_post_processing_after_membership_completion( int default_rc = set_default_hostgroup(admin, state.gated_route_hostgroup); ok(default_rc == EXIT_SUCCESS && route_to_expected_backend( - cl, sim, deployment.production.members.front().endpoint.backend()), + cl, sim, deployment.production.members.front().endpoint.backend(), + deployment.production), "POST_PROCESSING leaves routing unchanged without a complete target snapshot"); auto [refresh_seq_rc, refresh_sequence] = sim.replica_probe_log_last_sequence(); @@ -538,7 +828,8 @@ int test_post_processing_after_membership_completion( admin, { state.gated_route_hostgroup }, "=0") == EXIT_SUCCESS && set_default_hostgroup(admin, state.gated_route_hostgroup) == EXIT_SUCCESS) { target_routing = route_to_expected_backend( - cl, sim, deployment.target.members.front().endpoint.backend()); + cl, sim, deployment.target.members.front().endpoint.backend(), + deployment.target); } ok(target_routing, "repeated POST_PROCESSING applies routing after membership becomes complete"); @@ -546,7 +837,7 @@ int test_post_processing_after_membership_completion( } int main() { - plan(20); + plan(28); CommandLine cl {}; MYSQL* admin = nullptr; @@ -590,7 +881,25 @@ int main() { goto exit_cleanup; } + // Simulator: refresh target IDs/IPs, publish canonical IDs, then an incomplete map. + // Verify: auto-discovered multi-reader routing uses the last complete refreshed map. + if (test_auto_discovered_refresh_and_rename(cl, admin, sim, state) + != EXIT_SUCCESS) { + goto exit_cleanup; + } + + // Simulator: change membership in AVAILABLE, then change it again after INITIATED. + // Verify: AVAILABLE refreshes the map and INITIATED freezes that exact snapshot. + if (test_available_production_refresh_and_freeze(cl, admin, sim, state) + != EXIT_SUCCESS) { + goto exit_cleanup; + } + exit_cleanup: + if (state.held_client != nullptr) { + mysql_close(state.held_client); + state.held_client = nullptr; + } if (cleanup(admin, sim) != EXIT_SUCCESS) { diag("Error: failed to clean Aurora BGD active-state test data"); return EXIT_FAILURE; diff --git a/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp b/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp index 961b064368..105f34f831 100644 --- a/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp +++ b/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp @@ -31,8 +31,11 @@ namespace fs = std::filesystem; const uint32_t kWaitSeconds = 10; const int kReplicaAdminPort = 16062; const int kReplicaMySQLPort = 16063; +const int kReplicaSQLiteProductionPort = 16064; +const int kReplicaSQLiteTargetPort = 16065; const char kReplicaHost[] = "127.0.0.1"; -const char kSQLiteInterfaces[] = "0.0.0.0:3306;0.0.0.0:3307"; +const char kPrimarySQLiteInterfaces[] = "0.0.0.0:3306;0.0.0.0:3307"; +const char kReplicaSQLiteInterfaces[] = "0.0.0.0:16064;0.0.0.0:16065"; struct Replica_Process { pid_t pid = -1; @@ -41,7 +44,8 @@ struct Replica_Process { string stderr_path; }; -Aurora_BGD_Test_Deployment peer_deployment(); +Aurora_BGD_Test_Deployment peer_deployment( + int production_port = 3306, int target_port = 3307); struct TestState { MYSQL* primary_admin { nullptr }; @@ -51,6 +55,9 @@ struct TestState { BGD_Simulator primary_simulator {}; BGD_Simulator replica_simulator {}; Aurora_BGD_Test_Deployment deployment { peer_deployment() }; + Aurora_BGD_Test_Deployment replica_deployment { + peer_deployment(kReplicaSQLiteProductionPort, kReplicaSQLiteTargetPort) + }; bool primary_simulator_connected { false }; bool replica_simulator_connected { false }; }; @@ -67,8 +74,8 @@ string config_quote(const string& value) { } int prepare_replica_config(const CommandLine& cl, Replica_Process& process) { - char directory_template[] = "/tmp/proxysql-aurora-bgd-sync-XXXXXX"; - char* directory = mkdtemp(directory_template); // NOSONAR: mkdtemp creates an owner-only directory. + char directory_template[] = "proxysql-aurora-bgd-sync-XXXXXX"; + char* directory = mkdtemp(directory_template); if (directory == nullptr) { diag("mkdtemp failed: %s", strerror(errno)); return EXIT_FAILURE; @@ -100,7 +107,7 @@ int prepare_replica_config(const CommandLine& cl, Replica_Process& process) { << "mysql_variables={\n" << " interfaces=\"0.0.0.0:" << kReplicaMySQLPort << "\"\n" << " monitor_username=\"aurora1\"\n" - << " monitor_password=\"pass1\"\n" + << " monitor_password=\"pass1\"\n" // NOSONAR: fixed simulator fixture credential. << " monitor_connect_timeout=500\n" << " monitor_ping_interval=10000\n" << "}\n" @@ -162,7 +169,7 @@ void stop_replica(MYSQL*& admin, Replica_Process& process, bool preserve_log) { } } -Aurora_BGD_Test_Deployment peer_deployment() { +Aurora_BGD_Test_Deployment peer_deployment(int production_port, int target_port) { Aurora_BGD_Test_Deployment deployment; deployment.name = "Aurora BGD peer-local status"; deployment.domain_name = ".localhost"; @@ -171,13 +178,13 @@ Aurora_BGD_Test_Deployment peer_deployment() { deployment.source_topology_id = "aurora-bgd-peer-source"; deployment.target_topology_id = "aurora-bgd-peer-target"; deployment.target_cluster_endpoint = { - "aurora-peer-writer-green-sync.localhost", "127.0.0.1", 3307 + "aurora-peer-writer-green-sync.localhost", "127.0.0.1", target_port }; deployment.production = { deployment.blue_replica_set, {aurora_bgd_member( "aurora-peer-writer", "MASTER_SESSION_ID", - {"aurora-peer-writer.localhost", "127.0.0.1", 3306})}, + {"aurora-peer-writer.localhost", "127.0.0.1", production_port})}, {} }; deployment.production.serving_endpoints.push_back( @@ -186,7 +193,7 @@ Aurora_BGD_Test_Deployment peer_deployment() { deployment.target_replica_set, {aurora_bgd_member( "aurora-peer-writer-green-sync", "MASTER_SESSION_ID", - {"aurora-peer-writer-green-sync.localhost", "127.0.0.1", 3307})}, + {"aurora-peer-writer-green-sync.localhost", "127.0.0.1", target_port})}, {deployment.target_cluster_endpoint} }; deployment.target.serving_endpoints.push_back( @@ -229,7 +236,7 @@ int setup(CommandLine& cl, TestState& state) { state.primary_admin, "sqliteserver-mysql_ifaces", state.primary_sqlite_interfaces) != EXIT_SUCCESS || configure_sqlite_interfaces( - state.primary_admin, kSQLiteInterfaces) != EXIT_SUCCESS + state.primary_admin, kPrimarySQLiteInterfaces) != EXIT_SUCCESS || launch_replica(cl, state.replica_process) != EXIT_SUCCESS) { diag("failed to prepare the two ProxySQL nodes"); return EXIT_FAILURE; @@ -240,24 +247,25 @@ int setup(CommandLine& cl, TestState& state) { kWaitSeconds); if (state.replica_admin == nullptr || configure_sqlite_interfaces( - state.replica_admin, kSQLiteInterfaces) != EXIT_SUCCESS) { + state.replica_admin, kReplicaSQLiteInterfaces) != EXIT_SUCCESS) { diag("failed to start the replica ProxySQL node"); return EXIT_FAILURE; } char username[] = "aurora1"; - char password[] = "pass1"; + char password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. + char replica_host[] = "127.0.0.1"; state.primary_simulator_connected = state.primary_simulator.connect( cl.host, 3306, username, password) == EXIT_SUCCESS; state.replica_simulator_connected = state.replica_simulator.connect( - const_cast(kReplicaHost), 3306, username, password) == EXIT_SUCCESS; + replica_host, kReplicaSQLiteProductionPort, username, password) == EXIT_SUCCESS; if (!state.primary_simulator_connected || !state.replica_simulator_connected || configure_peer( state.primary_admin, state.primary_simulator, state.deployment, "AVAILABLE") != EXIT_SUCCESS || configure_peer( state.replica_admin, state.replica_simulator, - state.deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS) { + state.replica_deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS) { diag("failed to publish the two node-local Aurora observations"); return EXIT_FAILURE; } @@ -311,27 +319,31 @@ int test_configuration_sync_preserves_local_status(CommandLine& cl, TestState& s } int cleanup(TestState& state) { + bool cleanup_ok = true; if (state.primary_simulator_connected) { - state.primary_simulator.cleanup(); + cleanup_ok = state.primary_simulator.cleanup() == EXIT_SUCCESS && cleanup_ok; } if (state.replica_simulator_connected) { - state.replica_simulator.cleanup(); + cleanup_ok = state.replica_simulator.cleanup() == EXIT_SUCCESS && cleanup_ok; } if (state.replica_admin != nullptr) { - aurora_bgd_admin_cleanup(state.replica_admin); + cleanup_ok = aurora_bgd_admin_cleanup(state.replica_admin) == EXIT_SUCCESS + && cleanup_ok; } if (state.primary_admin != nullptr) { - aurora_bgd_admin_cleanup(state.primary_admin); + cleanup_ok = aurora_bgd_admin_cleanup(state.primary_admin) == EXIT_SUCCESS + && cleanup_ok; if (!state.primary_sqlite_interfaces.empty()) { - configure_sqlite_interfaces( - state.primary_admin, state.primary_sqlite_interfaces); + cleanup_ok = configure_sqlite_interfaces( + state.primary_admin, state.primary_sqlite_interfaces) == EXIT_SUCCESS + && cleanup_ok; } mysql_close(state.primary_admin); state.primary_admin = nullptr; } stop_replica( state.replica_admin, state.replica_process, tests_failed() != 0); - return EXIT_SUCCESS; + return cleanup_ok ? EXIT_SUCCESS : EXIT_FAILURE; } int main() { diff --git a/test/tap/tests/test_aurora_bgd_completion-t.cpp b/test/tap/tests/test_aurora_bgd_completion-t.cpp index ce7ba1dfae..475a80c229 100644 --- a/test/tap/tests/test_aurora_bgd_completion-t.cpp +++ b/test/tap/tests/test_aurora_bgd_completion-t.cpp @@ -57,6 +57,7 @@ struct TestState { Aurora_BGD_Test_Deployment direct { aurora_bgd_deployment_b_writer_only() }; int direct_writer_hostgroup { 1580 }; int direct_reader_hostgroup { 1581 }; + int direct_route_hostgroup { 1582 }; }; int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { @@ -70,7 +71,7 @@ int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { return EXIT_FAILURE; } char simulator_username[] = "aurora1"; - char simulator_password[] = "pass1"; + char simulator_password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. if (sim.connect(cl.host, 3306, simulator_username, simulator_password) != EXIT_SUCCESS) { diag("Error: failed to connect to the shared AWS simulator"); mysql_close(admin); @@ -144,6 +145,10 @@ int add_route( int add_member_routes( MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, const vector& hostgroups ) { + if (hostgroups.size() != deployment.production.members.size()) { + diag("Member-route hostgroup count does not match production membership"); + return EXIT_FAILURE; + } vector queries; for (size_t i = 0; i < hostgroups.size(); ++i) { queries.push_back( @@ -172,12 +177,23 @@ int add_green_servers( ",3306," + aurora_bgd_sql_quote(status) + ",'Aurora BGD configured green member')"); } + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(green_reader_hg) + "," + + aurora_bgd_sql_quote(deployment.target.members.front().endpoint.hostname) + + ",3306,'SHUNNED','Aurora BGD configured green status matrix')"); + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + to_string(green_writer_hg) + "," + + aurora_bgd_sql_quote(deployment.target.members[1].endpoint.hostname) + + ",3306,'OFFLINE_HARD','Aurora BGD configured green status matrix')"); queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); return aurora_bgd_execute_all(admin, queries); } bool route_to_expected_backend( - CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected_backend + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected_backend, + const Aurora_BGD_Membership_Set& expected_membership ) { auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); if (sequence_rc != EXIT_SUCCESS) { @@ -188,13 +204,14 @@ bool route_to_expected_backend( return false; } auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); - (void)rows; - if (query_rc != EXIT_SUCCESS) { + const bool result_matches = query_rc == EXIT_SUCCESS + && aurora_bgd_result_matches_membership(rows, expected_membership); + if (!result_matches) { diag("Backend routing query failed with MySQL error %d: %s", mysql_errno(client), mysql_error(client)); } mysql_close(client); - if (query_rc != EXIT_SUCCESS) { + if (!result_matches) { return false; } @@ -223,7 +240,9 @@ bool route_members( const Endpoint expected = target ? deployment.target.members[i].endpoint.backend() : deployment.production.members[i].endpoint.backend(); - if (!route_to_expected_backend(cl, sim, expected)) { + const Aurora_BGD_Membership_Set& expected_membership = target + ? deployment.target : deployment.production; + if (!route_to_expected_backend(cl, sim, expected, expected_membership)) { return false; } } @@ -319,10 +338,16 @@ bool completion_probe_policy( bool wait_for_topology_observation( BGD_Simulator& sim, uint64_t sequence, Aurora_BGD_Test_Deployment& deployment ) { - auto [rc, probe] = aurora_bgd_wait_for_topology_probe( + auto [first_rc, first_probe] = aurora_bgd_wait_for_topology_probe( sim, sequence, deployment.production.backends(), BGD_Probe_Kind::metadata, kProbeTimeoutMs); - return rc == EXIT_SUCCESS; + if (first_rc != EXIT_SUCCESS) { + return false; + } + auto [second_rc, second_probe] = aurora_bgd_wait_for_topology_probe( + sim, first_probe.sequence_id, deployment.production.backends(), + BGD_Probe_Kind::metadata, kProbeTimeoutMs); + return second_rc == EXIT_SUCCESS; } /** @@ -453,7 +478,8 @@ int test_completion_after_post_processing( bool green_pool_ready = set_default_hostgroup(admin, state.green_writer_hostgroup) == EXIT_SUCCESS && route_to_expected_backend( - cl, sim, deployment.target.members.front().endpoint.backend()) + cl, sim, deployment.target.members.front().endpoint.backend(), + deployment.target) && pool_count(admin, state.green_writer_hostgroup) >= 1; ok(target_route_pool >= 1 && green_pool_ready, "pre-completion target and configured-green pools are established"); @@ -479,14 +505,16 @@ int test_completion_after_post_processing( admin, state.green_reader_hostgroup, deployment.target.members.back().endpoint.hostname, 1, "OFFLINE_SOFT"), "completion preserves configured green rows and OFFLINE status"); - ok(add_route( - admin, state.post_completion_route_hostgroup, - deployment.production.members.front().endpoint.hostname) == EXIT_SUCCESS - && set_default_hostgroup( - admin, state.post_completion_route_hostgroup) == EXIT_SUCCESS - && route_to_expected_backend( - cl, sim, deployment.production.members.front().endpoint.backend()), - "completion removes the production traffic pin without DNS verification"); + ok(server_count( + admin, state.green_reader_hostgroup, + deployment.target.members.front().endpoint.hostname, 1, "SHUNNED") + && server_count( + admin, state.green_writer_hostgroup, + deployment.target.members[1].endpoint.hostname, 1, "OFFLINE_HARD"), + "completion preserves configured SHUNNED and OFFLINE_HARD green rows"); + ok(route_members( + cl, admin, sim, deployment, state.route_hostgroups, false), + "completion removes every writer and reader traffic pin without DNS verification"); auto [replica_seq_rc, replica_sequence] = sim.replica_probe_log_last_sequence(); ok(replica_seq_rc == EXIT_SUCCESS && completion_probe_policy( @@ -496,7 +524,8 @@ int test_completion_after_post_processing( bool recreated_green_pool = set_default_hostgroup(admin, state.green_writer_hostgroup) == EXIT_SUCCESS && route_to_expected_backend( - cl, sim, deployment.target.members.front().endpoint.backend()) + cl, sim, deployment.target.members.front().endpoint.backend(), + deployment.target) && pool_count(admin, state.green_writer_hostgroup) >= 1; auto [same_seq_rc, same_sequence] = sim.probe_log_last_sequence(); bool same_completion_seen = same_seq_rc == EXIT_SUCCESS @@ -519,6 +548,26 @@ int test_completion_after_post_processing( "query errors neither release the latch nor repeat completion cleanup"); Aurora_BGD_Test_Deployment different = aurora_bgd_deployment_b_writer_only(); + auto [same_active_seq_rc, same_active_sequence] = sim.probe_log_last_sequence(); + bool same_active_seen = same_active_seq_rc == EXIT_SUCCESS + && publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") == EXIT_SUCCESS + && wait_for_topology_observation(sim, same_active_sequence, deployment); + ok(same_active_seen && aurora_bgd_wait_for_status( + admin, state.post_writer_hostgroup, "SWITCHOVER_COMPLETED", 1) + == EXIT_SUCCESS + && pool_count(admin, state.green_writer_hostgroup) >= 1, + "a stale active result for the completed fingerprint is ignored while latched"); + + if (sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(different, "SWITCHOVER_INITIATED")) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + admin, state.post_writer_hostgroup, "SWITCHOVER_INITIATED", kWaitSeconds) + == EXIT_SUCCESS, + "a different active deployment fingerprint rearms the completed latch"); + auto [different_seq_rc, different_sequence] = sim.probe_log_last_sequence(); bool different_seen = different_seq_rc == EXIT_SUCCESS && publish_completed(sim, deployment, different) == EXIT_SUCCESS @@ -530,7 +579,8 @@ int test_completion_after_post_processing( bool second_green_pool = set_default_hostgroup(admin, state.green_writer_hostgroup) == EXIT_SUCCESS && route_to_expected_backend( - cl, sim, deployment.target.members.front().endpoint.backend()); + cl, sim, deployment.target.members.front().endpoint.backend(), + deployment.target); auto [repeat_different_seq_rc, repeat_different_sequence] = sim.probe_log_last_sequence(); bool repeated_different = repeat_different_seq_rc == EXIT_SUCCESS && publish_completed(sim, deployment, different) == EXIT_SUCCESS @@ -552,7 +602,9 @@ int test_completion_after_post_processing( * - Keep canonical writer placement and avoid replaying active phases. * - Rearm only after a successful topology drain. */ -int test_first_completed_observation(MYSQL* admin, BGD_Simulator& sim, TestState& state) { +int test_first_completed_observation( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state +) { if (reset_scenario(admin, sim) != EXIT_SUCCESS) { diag("Error: failed to reset before direct-completion scenario"); return EXIT_FAILURE; @@ -568,6 +620,18 @@ int test_first_completed_observation(MYSQL* admin, BGD_Simulator& sim, TestState diag("Error: failed to publish direct completion inputs"); return EXIT_FAILURE; } + if (add_route( + admin, state.direct_route_hostgroup, + deployment.production.members.front().endpoint.hostname) != EXIT_SUCCESS + || set_default_hostgroup(admin, state.direct_route_hostgroup) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + bool direct_pool_ready = route_to_expected_backend( + cl, sim, deployment.production.members.front().endpoint.backend(), + deployment.production) + && pool_count(admin, state.direct_route_hostgroup) >= 1; + ok(direct_pool_ready, + "direct-completion setup has an observable production route and pool"); auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); if (aurora_bgd_admin_setup( admin, deployment, state.direct_writer_hostgroup, @@ -584,7 +648,14 @@ int test_first_completed_observation(MYSQL* admin, BGD_Simulator& sim, TestState admin, state.direct_writer_hostgroup, state.direct_reader_hostgroup, deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, "direct completion leaves canonical writer placement unchanged"); + ok(pool_count(admin, state.direct_route_hostgroup) >= 1 + && set_default_hostgroup(admin, state.direct_route_hostgroup) == EXIT_SUCCESS + && route_to_expected_backend( + cl, sim, deployment.production.members.front().endpoint.backend(), + deployment.production), + "direct completion leaves unrelated production routing effects untouched"); + usleep(750000); auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); bool membership_probe = false; for (const Aurora_Replica_Probe_Log& log : logs) { @@ -600,7 +671,7 @@ int test_first_completed_observation(MYSQL* admin, BGD_Simulator& sim, TestState } int main() { - plan(28); + plan(33); CommandLine cl {}; MYSQL* admin = nullptr; @@ -626,7 +697,7 @@ int main() { // Simulator: make SWITCHOVER_COMPLETED the first observed deployment state. // Verify: prior phase effects are not manufactured or replayed. - if (test_first_completed_observation(admin, sim, state) != EXIT_SUCCESS) { + if (test_first_completed_observation(cl, admin, sim, state) != EXIT_SUCCESS) { goto exit_cleanup; } diff --git a/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp b/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp index 03cf0b3576..f64a1bda71 100644 --- a/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp +++ b/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include "aurora_bgd_tap.h" #include "command_line.h" @@ -68,7 +69,7 @@ int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { return EXIT_FAILURE; } char username[] = "aurora1"; - char password[] = "pass1"; + char password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { @@ -123,7 +124,10 @@ int add_writer_route( }); } -bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected) { +bool route_to_backend( + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected, + const Aurora_BGD_Membership_Set& expected_membership +) { auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); if (sequence_rc != EXIT_SUCCESS) { return false; @@ -133,9 +137,9 @@ bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expec return false; } auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); - (void)rows; mysql_close(client); - if (query_rc != EXIT_SUCCESS) { + if (query_rc != EXIT_SUCCESS + || !aurora_bgd_result_matches_membership(rows, expected_membership)) { return false; } auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); @@ -161,7 +165,9 @@ bool route_writer( const Endpoint expected = target ? deployment.target.members.front().endpoint.backend() : deployment.production.members.front().endpoint.backend(); - return route_to_backend(cl, sim, expected); + const Aurora_BGD_Membership_Set& expected_membership = target + ? deployment.target : deployment.production; + return route_to_backend(cl, sim, expected, expected_membership); } bool wait_for_writer_route( @@ -175,7 +181,9 @@ bool wait_for_writer_route( ? deployment.target.members.front().endpoint.backend() : deployment.production.members.front().endpoint.backend(); for (uint32_t elapsed_ms = 0; elapsed_ms < kWaitSeconds * 1000; elapsed_ms += 100) { - if (route_to_backend(cl, sim, expected)) { + const Aurora_BGD_Membership_Set& expected_membership = target + ? deployment.target : deployment.production; + if (route_to_backend(cl, sim, expected, expected_membership)) { return true; } usleep(100000); diff --git a/test/tap/tests/test_aurora_bgd_discovery-t.cpp b/test/tap/tests/test_aurora_bgd_discovery-t.cpp index 90be2fb6a9..b4d442293b 100644 --- a/test/tap/tests/test_aurora_bgd_discovery-t.cpp +++ b/test/tap/tests/test_aurora_bgd_discovery-t.cpp @@ -9,6 +9,7 @@ * 3. Publish incomplete and inconsistent membership and retain the last complete map. * 4. Publish invalid topology rows and retain the last valid runtime state. * 5. Verify automatic discovery accepts a writer-only deployment. + * 6. Verify topology and membership probes inherit TLS from the Aurora row. */ #include @@ -34,6 +35,10 @@ struct TestState { Aurora_BGD_Test_Deployment writer_only { aurora_bgd_deployment_b_writer_only() }; int writer_only_writer_hostgroup { 1520 }; int writer_only_reader_hostgroup { 1521 }; + int tls_writer_hostgroup { 1525 }; + int tls_reader_hostgroup { 1526 }; + int tls_green_writer_hostgroup { 1527 }; + int tls_green_reader_hostgroup { 1528 }; }; int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { @@ -47,7 +52,7 @@ int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { return EXIT_FAILURE; } char simulator_username[] = "aurora1"; - char simulator_password[] = "pass1"; + char simulator_password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. if (sim.connect(cl.host, 3306, simulator_username, simulator_password) != EXIT_SUCCESS) { diag("Error: failed to connect to the shared AWS simulator"); mysql_close(admin); @@ -218,6 +223,9 @@ int test_target_membership_validation(BGD_Simulator& sim, TestState& state) { ok(retain_complete_target_after_membership(sim, deployment, incomplete_rows), "an incomplete target result retains the previous complete target selection"); + ok(retain_complete_target_after_membership(sim, deployment, {}), + "an empty target result retains the previous complete target selection"); + vector duplicate_writer_rows = complete_rows; duplicate_writer_rows[1].session_id = "MASTER_SESSION_ID"; ok(retain_complete_target_after_membership(sim, deployment, duplicate_writer_rows), @@ -271,6 +279,37 @@ int test_topology_validation(MYSQL* admin, BGD_Simulator& sim, TestState& state) admin, sim, deployment, source_only, state.writer_hostgroup), "SOURCE-only topology does not replace AVAILABLE"); + vector target_only { valid_topology.back() }; + ok(invalid_topology_retains_available( + admin, sim, deployment, target_only, state.writer_hostgroup), + "TARGET-only topology is accepted only for SWITCHOVER_COMPLETED"); + + vector duplicate_source = valid_topology; + duplicate_source.push_back(valid_topology.front()); + ok(invalid_topology_retains_available( + admin, sim, deployment, duplicate_source, state.writer_hostgroup), + "duplicate SOURCE rows do not replace AVAILABLE"); + + vector extra_role = valid_topology; + BGD_Topology_Row observer = valid_topology.front(); + observer.role = "BLUE_GREEN_DEPLOYMENT_OBSERVER"; + extra_role.push_back(observer); + ok(invalid_topology_retains_available( + admin, sim, deployment, extra_role, state.writer_hostgroup), + "an extra unsupported role does not replace AVAILABLE"); + + vector invalid_port = valid_topology; + invalid_port[1].port = 0; + ok(invalid_topology_retains_available( + admin, sim, deployment, invalid_port, state.writer_hostgroup), + "a TARGET row with an invalid port does not replace AVAILABLE"); + + vector empty_endpoint = valid_topology; + empty_endpoint[1].endpoint.clear(); + ok(invalid_topology_retains_available( + admin, sim, deployment, empty_endpoint, state.writer_hostgroup), + "a TARGET row with an empty endpoint does not replace AVAILABLE"); + vector mismatched_status = valid_topology; mismatched_status[0].status = "SWITCHOVER_INITIATED"; ok(invalid_topology_retains_available( @@ -291,6 +330,42 @@ int test_topology_validation(MYSQL* admin, BGD_Simulator& sim, TestState& state) return EXIT_SUCCESS; } +/** Configure SSL-enabled Aurora rows and verify both BGD probe types use TLS. */ +int test_tls_probe_policy(MYSQL* admin, BGD_Simulator& sim, TestState& state) { + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before the TLS scenario"); + return EXIT_FAILURE; + } + + Aurora_BGD_Test_Deployment& deployment = state.writer_only; + if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, state.tls_writer_hostgroup, state.tls_reader_hostgroup, + state.tls_green_writer_hostgroup, state.tls_green_reader_hostgroup, + false, 100, false, true) != EXIT_SUCCESS) { + diag("Error: failed to configure the TLS probe scenario"); + return EXIT_FAILURE; + } + + ok(aurora_bgd_wait_for_status( + admin, state.tls_writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "SSL-enabled Aurora rows reach AVAILABLE"); + + auto [topology_rc, topology_probe] = sim.wait_for_probe_log( + 0, deployment.production.members.front().endpoint.backend(), + BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1); + ok(topology_rc == EXIT_SUCCESS, + "topology discovery uses TLS when the Aurora row enables SSL"); + + vector target_backends = deployment.target.backends(); + auto [membership_rc, membership_probe] = aurora_bgd_wait_for_replica_probe( + sim, 0, target_backends, Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, deployment.target_replica_set); + ok(membership_rc == EXIT_SUCCESS && membership_probe.encrypted, + "target-membership discovery uses TLS when the Aurora row enables SSL"); + return EXIT_SUCCESS; +} + /** * Discover a writer-only deployment without configured green hostgroups. * @@ -327,7 +402,7 @@ int test_writer_only_discovery(MYSQL* admin, BGD_Simulator& sim, TestState& stat } int main() { - plan(18); + plan(27); CommandLine cl {}; MYSQL* admin = nullptr; @@ -363,6 +438,12 @@ int main() { goto exit_cleanup; } + // ProxySQL: configure SSL-enabled Aurora rows. + // Verify: topology and target-membership probes both use TLS. + if (test_tls_probe_policy(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + exit_cleanup: if (cleanup(admin, sim) != EXIT_SUCCESS) { diag("Error: failed to clean Aurora BGD discovery state"); diff --git a/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp b/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp index e79051ae67..77941af394 100644 --- a/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp +++ b/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp @@ -71,7 +71,7 @@ int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { return EXIT_FAILURE; } char username[] = "aurora1"; - char password[] = "pass1"; + char password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { @@ -176,7 +176,10 @@ int add_routes( return aurora_bgd_execute_all(admin, queries); } -bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected) { +bool route_to_backend( + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected, + const Aurora_BGD_Membership_Set& expected_membership +) { auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); if (sequence_rc != EXIT_SUCCESS) { return false; @@ -186,9 +189,9 @@ bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expec return false; } auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); - (void)rows; mysql_close(client); - if (query_rc != EXIT_SUCCESS) { + if (query_rc != EXIT_SUCCESS + || !aurora_bgd_result_matches_membership(rows, expected_membership)) { return false; } auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); @@ -215,7 +218,9 @@ bool route_members( Endpoint expected = target ? deployment.target.members[i].endpoint.backend() : deployment.production.members[i].endpoint.backend(); - if (!route_to_backend(cl, sim, expected)) { + const Aurora_BGD_Membership_Set& expected_membership = target + ? deployment.target : deployment.production; + if (!route_to_backend(cl, sim, expected, expected_membership)) { return false; } } @@ -230,6 +235,35 @@ int64_t pool_count(MYSQL* admin, int hostgroup) { ? strtoll(rows.front().front().c_str(), nullptr, 10) : -1; } +bool observe_two_topology_responses( + BGD_Simulator& sim, uint64_t sequence, const vector& backends +) { + auto [first_rc, first] = aurora_bgd_wait_for_topology_probe( + sim, sequence, backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); + if (first_rc != EXIT_SUCCESS) { + return false; + } + auto [second_rc, second] = aurora_bgd_wait_for_topology_probe( + sim, first.sequence_id, backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); + return second_rc == EXIT_SUCCESS; +} + +bool observe_two_membership_responses( + BGD_Simulator& sim, uint64_t sequence, + const vector& backends, const string& replica_set +) { + auto [first_rc, first] = aurora_bgd_wait_for_replica_probe( + sim, sequence, backends, Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, replica_set); + if (first_rc != EXIT_SUCCESS) { + return false; + } + auto [second_rc, second] = aurora_bgd_wait_for_replica_probe( + sim, first.sequence_id, backends, Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, replica_set); + return second_rc == EXIT_SUCCESS; +} + /** Retain INITIATED across topology and membership query errors. */ int test_initiated_error_retention(MYSQL* admin, BGD_Simulator& sim, TestState& state) { Aurora_BGD_Test_Deployment& deployment = state.initiated; @@ -250,9 +284,8 @@ int test_initiated_error_retention(MYSQL* admin, BGD_Simulator& sim, TestState& && sim.topology_error( aurora_bgd_topology_backends(deployment), 1205, "simulated topology timeout") == EXIT_SUCCESS - && aurora_bgd_wait_for_topology_probe( - sim, topology_sequence, deployment.target.backends(), - BGD_Probe_Kind::metadata, kProbeTimeoutMs).first == EXIT_SUCCESS; + && observe_two_topology_responses( + sim, topology_sequence, deployment.target.backends()); ok(topology_error_seen && aurora_bgd_wait_for_status( admin, state.initiated_writer_hostgroup, "SWITCHOVER_INITIATED", 1) == EXIT_SUCCESS, @@ -267,10 +300,9 @@ int test_initiated_error_retention(MYSQL* admin, BGD_Simulator& sim, TestState& && sim.replica_error( deployment.target.backends(), 1205, "simulated membership timeout") == EXIT_SUCCESS - && aurora_bgd_wait_for_replica_probe( + && observe_two_membership_responses( sim, membership_sequence, deployment.target.backends(), - Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs).first - == EXIT_SUCCESS; + deployment.target_replica_set); ok(membership_error_seen && aurora_bgd_wait_for_status( admin, state.initiated_writer_hostgroup, "SWITCHOVER_INITIATED", 1) == EXIT_SUCCESS, @@ -379,9 +411,8 @@ int test_post_processing_error_retention( && sim.topology_error( aurora_bgd_topology_backends(deployment), 1205, "simulated post timeout") == EXIT_SUCCESS - && aurora_bgd_wait_for_topology_probe( - sim, topology_sequence, deployment.target.backends(), - BGD_Probe_Kind::metadata, kProbeTimeoutMs).first == EXIT_SUCCESS; + && observe_two_topology_responses( + sim, topology_sequence, deployment.target.backends()); ok(topology_error_seen && aurora_bgd_wait_for_status( admin, state.post_writer_hostgroup, "SWITCHOVER_IN_POST_PROCESSING", 1) == EXIT_SUCCESS @@ -398,10 +429,9 @@ int test_post_processing_error_retention( && sim.replica_error( deployment.target.backends(), 1205, "simulated post membership timeout") == EXIT_SUCCESS - && aurora_bgd_wait_for_replica_probe( + && observe_two_membership_responses( sim, membership_sequence, deployment.target.backends(), - Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs).first - == EXIT_SUCCESS; + deployment.target_replica_set); ok(membership_error_seen && sim.replica_update( deployment.target.replica_set_id, deployment.target.replica_rows(), @@ -419,7 +449,8 @@ int test_post_processing_rollback( Aurora_BGD_Test_Deployment& deployment = state.post; ok(set_default_hostgroup(admin, state.post_green_writer_hostgroup) == EXIT_SUCCESS && route_to_backend( - cl, sim, deployment.target.members.front().endpoint.backend()) + cl, sim, deployment.target.members.front().endpoint.backend(), + deployment.target) && pool_count(admin, state.post_green_writer_hostgroup) >= 1, "a configured green pool is established before rollback"); ok(publish_status(sim, deployment, "AVAILABLE") == EXIT_SUCCESS diff --git a/test/tap/tests/test_aurora_bgd_late_entry-t.cpp b/test/tap/tests/test_aurora_bgd_late_entry-t.cpp index 0119eac21a..a9205abf4f 100644 --- a/test/tap/tests/test_aurora_bgd_late_entry-t.cpp +++ b/test/tap/tests/test_aurora_bgd_late_entry-t.cpp @@ -6,7 +6,8 @@ * * 1. Start a worker from SWITCHOVER_INITIATED and reconstruct active probing. * 2. Start a worker from SWITCHOVER_IN_PROGRESS and reconstruct writer demotion. - * 3. Start a worker from POST_PROCESSING and reconstruct every target pin. + * 3. Recover late IN_PROGRESS entry when the first ordinary probe fails. + * 4. Start a worker from POST_PROCESSING and reconstruct every target pin. */ #include @@ -67,7 +68,7 @@ int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { return EXIT_FAILURE; } char username[] = "aurora1"; - char password[] = "pass1"; + char password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { @@ -174,7 +175,10 @@ int add_routes( return aurora_bgd_execute_all(admin, queries); } -bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected) { +bool route_to_backend( + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected, + const Aurora_BGD_Membership_Set& expected_membership +) { auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); if (sequence_rc != EXIT_SUCCESS) { return false; @@ -184,9 +188,9 @@ bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expec return false; } auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); - (void)rows; mysql_close(client); - if (query_rc != EXIT_SUCCESS) { + if (query_rc != EXIT_SUCCESS + || !aurora_bgd_result_matches_membership(rows, expected_membership)) { return false; } auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); @@ -213,7 +217,9 @@ bool route_members( Endpoint expected = target ? deployment.target.members[i].endpoint.backend() : deployment.production.members[i].endpoint.backend(); - if (!route_to_backend(cl, sim, expected)) { + const Aurora_BGD_Membership_Set& expected_membership = target + ? deployment.target : deployment.production; + if (!route_to_backend(cl, sim, expected, expected_membership)) { return false; } } @@ -286,6 +292,43 @@ int test_first_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { return EXIT_SUCCESS; } +/** + * Reconstruct IN_PROGRESS while the first ordinary Aurora query is failing. + * + * The fixed production membership comes from the configured/runtime hostgroups, + * so entering an active phase never depends on a successful cutover-time query. + */ +int test_first_in_progress_after_ordinary_error( + MYSQL* admin, BGD_Simulator& sim, TestState& state +) { + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before failed-probe late entry"); + return EXIT_FAILURE; + } + + Aurora_BGD_Test_Deployment& deployment = state.progress; + if (publish_initial(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || sim.replica_error( + deployment.production.backends(), 1205, + "simulated initial ordinary Aurora timeout") != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, state.progress_writer_hostgroup, + state.progress_reader_hostgroup, state.progress_green_writer_hostgroup, + state.progress_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS) { + diag("Error: failed to configure failed-probe IN_PROGRESS late entry"); + return EXIT_FAILURE; + } + + ok(aurora_bgd_wait_for_status( + admin, state.progress_writer_hostgroup, + "SWITCHOVER_IN_PROGRESS", kWaitSeconds) == EXIT_SUCCESS + && writer_placement( + admin, state.progress_writer_hostgroup, state.progress_reader_hostgroup, + deployment.production.members.front().endpoint.hostname, true), + "late IN_PROGRESS entry reconstructs and demotes after an initial ordinary-query failure"); + return EXIT_SUCCESS; +} + /** * Start a worker from SWITCHOVER_IN_POST_PROCESSING. * @@ -325,7 +368,7 @@ int test_first_post_processing( } int main() { - plan(6); + plan(7); CommandLine cl {}; MYSQL* admin = nullptr; @@ -349,6 +392,12 @@ int main() { goto exit_cleanup; } + // Simulator: fail the initial ordinary query while publishing IN_PROGRESS. + // Verify: configured production membership still permits immediate demotion. + if (test_first_in_progress_after_ordinary_error(admin, sim, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + // Simulator: make POST_PROCESSING the first observed deployment state. // Verify: the worker reconstructs and applies every target pin. if (test_first_post_processing(cl, admin, sim, state) != EXIT_SUCCESS) { diff --git a/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp b/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp index 1e69e26c61..96b6cbf00e 100644 --- a/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp +++ b/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp @@ -57,17 +57,20 @@ struct TestState { int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); return EXIT_FAILURE; } admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); if (admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); return EXIT_FAILURE; } char username[] = "aurora1"; - char password[] = "pass1"; + char password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to initialize the shared AWS simulator"); return EXIT_FAILURE; } return aurora_bgd_execute_all(admin, { @@ -122,7 +125,10 @@ int add_writer_route( }); } -bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected) { +bool route_to_backend( + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected, + const Aurora_BGD_Membership_Set& expected_membership +) { auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); if (sequence_rc != EXIT_SUCCESS) { return false; @@ -132,9 +138,9 @@ bool route_to_backend(CommandLine& cl, BGD_Simulator& sim, const Endpoint& expec return false; } auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); - (void)rows; mysql_close(client); - if (query_rc != EXIT_SUCCESS) { + if (query_rc != EXIT_SUCCESS + || !aurora_bgd_result_matches_membership(rows, expected_membership)) { return false; } auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); @@ -160,7 +166,9 @@ bool route_writer( const Endpoint expected = target ? deployment.target.members.front().endpoint.backend() : deployment.production.members.front().endpoint.backend(); - return route_to_backend(cl, sim, expected); + const Aurora_BGD_Membership_Set& expected_membership = target + ? deployment.target : deployment.production; + return route_to_backend(cl, sim, expected, expected_membership); } bool writer_placement( diff --git a/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp b/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp index 26d2d35cc8..56be1817d7 100644 --- a/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp +++ b/test/tap/tests/unit/admin_disk_upgrade_unit-t.cpp @@ -586,55 +586,59 @@ static void test_mysql_servers_upgrade_multiple_rows_with_fixes() { } static void test_aurora_hostgroups_disk_roundtrip_uses_configured_projection() { - SQLite3DB* db = new SQLite3DB(); - db->open((char*)":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); - const bool schema_ready = db->execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS) && - db->execute("ATTACH DATABASE ':memory:' AS disk") && - create_table_in_schema(db, "disk", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + SQLite3DB db; + char memory_database[] = ":memory:"; + db.open(memory_database, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + const bool schema_ready = db.execute(ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS) && + db.execute("ATTACH DATABASE ':memory:' AS disk") && + create_table_in_schema(&db, "disk", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS) && + db.execute( + "ALTER TABLE disk.mysql_aws_aurora_hostgroups ADD COLUMN " + "bgd_status TEXT NOT NULL DEFAULT 'DISK_LOCAL'"); ok(schema_ready, "Aurora disk round trip uses the current schemas in main and disk"); - const bool rows_inserted = db->execute( + const bool rows_inserted = db.execute( "INSERT INTO mysql_aws_aurora_hostgroups (" "writer_hostgroup,reader_hostgroup,green_writer_hostgroup,green_reader_hostgroup,domain_name,comment" ") VALUES " "(600,601,602,603,'.disk.example','disk round trip')," "(610,611,NULL,NULL,'.disk-null.example',NULL)" ); - ok(rows_inserted && copy_mysql_aws_aurora_hostgroups_to_disk(db), + ok(rows_inserted && copy_mysql_aws_aurora_hostgroups_to_disk(&db), "Aurora disk SAVE copies the configured projection"); - ok(query_string(db, + ok(query_string(&db, "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " "FROM disk.mysql_aws_aurora_hostgroups WHERE writer_hostgroup=600") == "602,603", "Aurora disk SAVE preserves both configured green hostgroups"); - ok(query_int(db, + ok(query_int(&db, "SELECT COUNT(*) FROM disk.mysql_aws_aurora_hostgroups " "WHERE writer_hostgroup=610 AND green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL") == 1, "Aurora disk SAVE preserves paired NULL green hostgroups"); - ok(query_int(db, + ok(query_int(&db, "SELECT COUNT(*) FROM disk.mysql_aws_aurora_hostgroups " "WHERE writer_hostgroup=610 AND comment IS NULL") == 1, "Aurora disk SAVE preserves a NULL comment"); - ok(query_int(db, - "SELECT COUNT(*) FROM pragma_table_info('mysql_aws_aurora_hostgroups','disk') WHERE name='bgd_status'") == 0, - "Aurora disk SAVE excludes node-local bgd_status"); + ok(query_int(&db, + "SELECT COUNT(*) FROM disk.mysql_aws_aurora_hostgroups " + "WHERE bgd_status='DISK_LOCAL'") == 2, + "Aurora disk SAVE leaves node-local bgd_status at its destination default"); - ok(db->execute("DELETE FROM main.mysql_aws_aurora_hostgroups"), + ok(db.execute("DELETE FROM main.mysql_aws_aurora_hostgroups"), "Aurora disk LOAD test clears the configured source table"); - ok(copy_mysql_aws_aurora_hostgroups_from_disk(db), + ok(copy_mysql_aws_aurora_hostgroups_from_disk(&db), "Aurora disk LOAD copies the configured projection"); - ok(query_string(db, + ok(query_string(&db, "SELECT green_writer_hostgroup || ',' || green_reader_hostgroup " "FROM main.mysql_aws_aurora_hostgroups WHERE writer_hostgroup=600") == "602,603", "Aurora disk LOAD restores both configured green hostgroups"); - ok(query_int(db, + ok(query_int(&db, "SELECT COUNT(*) FROM main.mysql_aws_aurora_hostgroups " "WHERE writer_hostgroup=610 AND green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL") == 1, "Aurora disk LOAD restores paired NULL green hostgroups"); - ok(query_int(db, + ok(query_int(&db, "SELECT COUNT(*) FROM main.mysql_aws_aurora_hostgroups " "WHERE writer_hostgroup=610 AND comment IS NULL") == 1, "Aurora disk LOAD restores a NULL comment"); - delete db; } // ============================================================================ diff --git a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp index 84e1401608..e44524119d 100644 --- a/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp +++ b/test/tap/tests/unit/aurora_bgd_config_unit-t.cpp @@ -412,7 +412,7 @@ int main() { ok(test_init_hostgroups() == 0, "test_init_hostgroups() succeeds"); // 1 ok(test_init_monitor() == 0, "test_init_monitor() succeeds"); // 1 - test_runtime_ownership(); // 4 + test_runtime_ownership(); // 5 test_status_and_materialization(); // 8 test_reload_and_status_ordering(); // 4 test_runtime_removal(); // 2 diff --git a/test/tap/tests/unit/config_write_unit-t.cpp b/test/tap/tests/unit/config_write_unit-t.cpp index 1a12a85a7d..11c6e5cfc7 100644 --- a/test/tap/tests/unit/config_write_unit-t.cpp +++ b/test/tap/tests/unit/config_write_unit-t.cpp @@ -502,7 +502,8 @@ static void test_roundtrip_mysql_aws_aurora_hostgroups() { "RT Aurora hostgroups: green writer is exported"); ok(data.find("green_reader_hostgroup=503") != std::string::npos, "RT Aurora hostgroups: green reader is exported"); - ok(data.find("active=0") != std::string::npos && + ok(data.find("writer_hostgroup=500") != std::string::npos && + data.find("active=0") != std::string::npos && data.find("autopurge_missing_checks=9") != std::string::npos, "RT Aurora hostgroups: later configured columns retain their values"); ok(data.find("bgd_status") == std::string::npos, From cb6445b28d3b1e9ade9e2a872beff48eb3b83406 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sun, 16 Aug 2026 13:05:27 +0000 Subject: [PATCH 29/34] fix: harden Aurora BGD snapshot refresh --- lib/MySQL_Monitor.cpp | 13 ++- test/tap/tests/test_aurora_bgd_active-t.cpp | 40 ++++++++- .../tests/test_aurora_bgd_late_entry-t.cpp | 89 ++++++++++++++++++- 3 files changed, 136 insertions(+), 6 deletions(-) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 92b9dfedf1..8bf240e2e1 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6475,14 +6475,16 @@ static bool aws_aurora_bgd_parse_target_membership( return false; } auto previous_action = previous_action_by_id.find(member.normalized_server_id); - member.traffic_pin_applied = previous_action != previous_action_by_id.end() - && previous_action->second; auto previous_ip = previous_ip_by_id.find(member.normalized_server_id); const bool preserve_pre_rename_ip = member.server_id == production->second->server_id; member.target_ip = previous_ip != previous_ip_by_id.end() && !previous_ip->second.empty() - && (member.traffic_pin_applied || preserve_pre_rename_ip) + && preserve_pre_rename_ip ? previous_ip->second : resolved_ip; + member.traffic_pin_applied = previous_action != previous_action_by_id.end() + && previous_action->second + && previous_ip != previous_ip_by_id.end() + && previous_ip->second == member.target_ip; snapshot.push_back(std::move(member)); } @@ -6859,7 +6861,10 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle( aws_aurora_bgd_apply_completion(st, observation.fingerprint); } } else if (observation.valid) { - if (st.production_members.empty() + const bool entering_active = + observation.status >= AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED + && observation.status <= AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING; + if (entering_active && !st.production_snapshot_frozen && !aws_aurora_bgd_rebuild_production_snapshot(st)) { proxy_error( "AWS Aurora BGD [wHG=%u rHG=%u]: retaining current state until production membership is available\n", diff --git a/test/tap/tests/test_aurora_bgd_active-t.cpp b/test/tap/tests/test_aurora_bgd_active-t.cpp index c5c48b315f..0f3b369216 100644 --- a/test/tap/tests/test_aurora_bgd_active-t.cpp +++ b/test/tap/tests/test_aurora_bgd_active-t.cpp @@ -521,6 +521,44 @@ int test_switchover_post_processing( && route_members_to_expected_ips( cl, admin, sim, deployment, state.route_hostgroups, true), "an active variables refresh preserves every explicit traffic pin"); + + const vector moved_ids { + "aurora-a-writer-green-r2", + "aurora-a-reader-1-green-r2", + "aurora-a-reader-2-green-r2", + }; + const vector moved_ips { "127.0.11.31", "127.0.11.32", "127.0.11.33" }; + for (size_t i = 0; i < deployment.target.members.size(); ++i) { + deployment.target.members[i].server_id = moved_ids[i]; + deployment.target.members[i].endpoint.hostname = + moved_ids[i] + deployment.domain_name; + deployment.target.members[i].endpoint.ip = moved_ips[i]; + } + deployment.target.serving_endpoints.clear(); + deployment.target.serving_endpoints.push_back(deployment.target_cluster_endpoint); + for (const Aurora_BGD_Member& member : deployment.target.members) { + deployment.target.serving_endpoints.push_back(member.endpoint); + } + auto [move_seq_rc, move_sequence] = sim.replica_probe_log_last_sequence(); + bool moved_routing = move_seq_rc == EXIT_SUCCESS + && sim.replica_update( + deployment.target_replica_set, deployment.target.replica_rows(), + deployment.target.backends()) == EXIT_SUCCESS + && publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS; + if (moved_routing) { + auto [move_probe_rc, move_probe] = aurora_bgd_wait_for_replica_probe( + sim, move_sequence, deployment.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, + deployment.target_replica_set); + moved_routing = move_probe_rc == EXIT_SUCCESS + && wait_for_member_route_pool_count( + admin, state.route_hostgroups, "=0") == EXIT_SUCCESS + && route_members_to_expected_ips( + cl, admin, sim, deployment, state.route_hostgroups, true); + } + ok(moved_routing, + "POST_PROCESSING repins every member when target-shaped endpoints move"); + if (aurora_bgd_execute_all(admin, { "SET mysql-monitor_local_dns_cache_ttl=300000", "SET mysql-monitor_local_dns_cache_refresh_interval=60000", @@ -837,7 +875,7 @@ int test_post_processing_after_membership_completion( } int main() { - plan(28); + plan(29); CommandLine cl {}; MYSQL* admin = nullptr; diff --git a/test/tap/tests/test_aurora_bgd_late_entry-t.cpp b/test/tap/tests/test_aurora_bgd_late_entry-t.cpp index a9205abf4f..539e67a769 100644 --- a/test/tap/tests/test_aurora_bgd_late_entry-t.cpp +++ b/test/tap/tests/test_aurora_bgd_late_entry-t.cpp @@ -329,6 +329,86 @@ int test_first_in_progress_after_ordinary_error( return EXIT_SUCCESS; } +/** + * Reconstruct POST_PROCESSING after a non-empty but partial ordinary result. + * + * The configured/runtime hostgroups define the complete production membership, + * so a writer-only observation cannot make a writer-only target map actionable. + */ +int test_first_post_processing_after_partial_ordinary( + CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state +) { + if (reset_scenario(admin, sim) != EXIT_SUCCESS) { + diag("Error: failed to reset before partial-probe late entry"); + return EXIT_FAILURE; + } + + Aurora_BGD_Test_Deployment& deployment = state.post; + vector partial_production = deployment.production.replica_rows(); + partial_production.resize(1); + vector partial_target = deployment.target.replica_rows(); + partial_target.resize(1); + if (sim.replica_update( + deployment.production.replica_set_id, partial_production, + deployment.production.backends()) != EXIT_SUCCESS + || sim.replica_update( + deployment.target.replica_set_id, partial_target, + deployment.target.backends()) != EXIT_SUCCESS + || sim.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, "SWITCHOVER_IN_POST_PROCESSING")) + != EXIT_SUCCESS + || aurora_bgd_admin_setup( + admin, deployment, state.post_writer_hostgroup, + state.post_reader_hostgroup, state.post_green_writer_hostgroup, + state.post_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS + || add_routes( + admin, deployment, state.post_route_hostgroups, + state.post_green_writer_hostgroup) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + admin, state.post_writer_hostgroup, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: failed to configure partial-probe POST_PROCESSING late entry"); + return EXIT_FAILURE; + } + + auto [partial_seq_rc, partial_sequence] = sim.replica_probe_log_last_sequence(); + bool partial_map_blocked = partial_seq_rc == EXIT_SUCCESS + && sim.replica_update( + deployment.production.replica_set_id, + deployment.production.replica_rows(), deployment.production.backends()) + == EXIT_SUCCESS; + if (partial_map_blocked) { + auto [partial_probe_rc, partial_probe] = aurora_bgd_wait_for_replica_probe( + sim, partial_sequence, deployment.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, 5000, + deployment.target.replica_set_id); + partial_map_blocked = partial_probe_rc == EXIT_SUCCESS + && route_members( + cl, admin, sim, deployment, state.post_route_hostgroups, false); + } + ok(partial_map_blocked, + "a partial initial ordinary result cannot authorize partial target routing"); + + auto [complete_seq_rc, complete_sequence] = sim.replica_probe_log_last_sequence(); + bool complete_map_applied = complete_seq_rc == EXIT_SUCCESS + && sim.replica_update( + deployment.target.replica_set_id, deployment.target.replica_rows(), + deployment.target.backends()) == EXIT_SUCCESS; + if (complete_map_applied) { + auto [complete_probe_rc, complete_probe] = aurora_bgd_wait_for_replica_probe( + sim, complete_sequence, deployment.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, 5000, + deployment.target.replica_set_id); + complete_map_applied = complete_probe_rc == EXIT_SUCCESS + && route_members( + cl, admin, sim, deployment, state.post_route_hostgroups, true); + } + ok(complete_map_applied, + "complete target membership routes every configured production member"); + return EXIT_SUCCESS; +} + /** * Start a worker from SWITCHOVER_IN_POST_PROCESSING. * @@ -368,7 +448,7 @@ int test_first_post_processing( } int main() { - plan(7); + plan(9); CommandLine cl {}; MYSQL* admin = nullptr; @@ -398,6 +478,13 @@ int main() { goto exit_cleanup; } + // Simulator: return only the writer from the first ordinary and target probes. + // Verify: configured production membership blocks partial routing, then accepts the full map. + if (test_first_post_processing_after_partial_ordinary(cl, admin, sim, state) + != EXIT_SUCCESS) { + goto exit_cleanup; + } + // Simulator: make POST_PROCESSING the first observed deployment state. // Verify: the worker reconstructs and applies every target pin. if (test_first_post_processing(cl, admin, sim, state) != EXIT_SUCCESS) { From f957d0bd3a9637ad832c8e1a5857800dd3cd1662 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sun, 16 Aug 2026 14:17:15 +0000 Subject: [PATCH 30/34] fix: finalize Aurora BGD phase 3 review --- lib/MySQL_Monitor.cpp | 5 +- test/tap/tests/test_aurora_bgd_active-t.cpp | 143 ++++++++++-------- .../tests/test_aurora_bgd_completion-t.cpp | 67 ++++---- ...test_aurora_bgd_concurrent_isolation-t.cpp | 15 +- .../test_aurora_bgd_error_recovery-t.cpp | 14 +- .../tests/test_aurora_bgd_late_entry-t.cpp | 100 +----------- .../test_aurora_bgd_worker_lifecycle-t.cpp | 11 +- 7 files changed, 137 insertions(+), 218 deletions(-) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 8bf240e2e1..7fe7f73e3c 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6861,10 +6861,7 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle( aws_aurora_bgd_apply_completion(st, observation.fingerprint); } } else if (observation.valid) { - const bool entering_active = - observation.status >= AWS_Aurora_BGD_Status::SWITCHOVER_INITIATED - && observation.status <= AWS_Aurora_BGD_Status::SWITCHOVER_IN_POST_PROCESSING; - if (entering_active && !st.production_snapshot_frozen + if (st.production_members.empty() && !aws_aurora_bgd_rebuild_production_snapshot(st)) { proxy_error( "AWS Aurora BGD [wHG=%u rHG=%u]: retaining current state until production membership is available\n", diff --git a/test/tap/tests/test_aurora_bgd_active-t.cpp b/test/tap/tests/test_aurora_bgd_active-t.cpp index 0f3b369216..9192edf7d3 100644 --- a/test/tap/tests/test_aurora_bgd_active-t.cpp +++ b/test/tap/tests/test_aurora_bgd_active-t.cpp @@ -25,6 +25,7 @@ using namespace std; const uint32_t kWaitSeconds = 5; const uint32_t kProbeTimeoutMs = 5000; +const int kHeldConnectionRuleId = 153000; const char kOrdinaryAuroraQuery[] = "SELECT SERVER_ID," "IF(" @@ -86,6 +87,9 @@ int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { return EXIT_FAILURE; } if (aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_query_rules WHERE rule_id=" + + to_string(kHeldConnectionRuleId), + "LOAD MYSQL QUERY RULES TO RUNTIME", "DELETE FROM mysql_users WHERE username='testuser'", "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " "VALUES ('testuser','testuser',1,0,1)", @@ -100,6 +104,9 @@ int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { int cleanup(MYSQL* admin, BGD_Simulator& sim) { int admin_rc = aurora_bgd_admin_cleanup(admin); int user_rc = admin == nullptr ? EXIT_FAILURE : aurora_bgd_execute_all(admin, { + "DELETE FROM mysql_query_rules WHERE rule_id=" + + to_string(kHeldConnectionRuleId), + "LOAD MYSQL QUERY RULES TO RUNTIME", "DELETE FROM mysql_users WHERE username='testuser'", "LOAD MYSQL USERS TO RUNTIME", }); @@ -214,9 +221,24 @@ int set_default_hostgroup(MYSQL* admin, int hostgroup) { }); } +int set_held_connection_rule(MYSQL* admin, bool enabled) { + vector queries { + "DELETE FROM mysql_query_rules WHERE rule_id=" + + to_string(kHeldConnectionRuleId), + }; + if (enabled) { + queries.push_back( + "INSERT INTO mysql_query_rules" + "(rule_id,active,username,multiplex,apply,comment) VALUES (" + + to_string(kHeldConnectionRuleId) + + ",1,'testuser',0,1,'Aurora BGD held connection')"); + } + queries.push_back("LOAD MYSQL QUERY RULES TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + bool route_to_expected_backend( - CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected_backend, - const Aurora_BGD_Membership_Set& expected_membership + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected_backend ) { auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); if (sequence_rc != EXIT_SUCCESS) { @@ -227,16 +249,14 @@ bool route_to_expected_backend( return false; } auto [rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); - const bool result_matches = rc == EXIT_SUCCESS - && aurora_bgd_result_matches_membership(rows, expected_membership); - if (!result_matches) { + (void)rows; + if (rc != EXIT_SUCCESS) { diag("Backend routing query failed with MySQL error %d: %s", mysql_errno(client), mysql_error(client)); - } - mysql_close(client); - if (!result_matches) { + mysql_close(client); return false; } + mysql_close(client); auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); if (logs_rc != EXIT_SUCCESS) { @@ -246,19 +266,18 @@ bool route_to_expected_backend( for (const Aurora_Replica_Probe_Log& log : logs) { if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { routed_probe = &log; + if (log.backend.host == expected_backend.host + && log.backend.port == expected_backend.port) { + return true; + } } } - if (routed_probe == nullptr - || routed_probe->backend.host != expected_backend.host - || routed_probe->backend.port != expected_backend.port) { - diag( - "Ordinary Aurora query reached %s:%d; expected %s:%d", - routed_probe ? routed_probe->backend.host.c_str() : "", - routed_probe ? routed_probe->backend.port : 0, - expected_backend.host.c_str(), expected_backend.port); - return false; - } - return true; + diag( + "Ordinary Aurora query reached %s:%d; expected %s:%d", + routed_probe ? routed_probe->backend.host.c_str() : "", + routed_probe ? routed_probe->backend.port : 0, + expected_backend.host.c_str(), expected_backend.port); + return false; } bool route_members_to_expected_ips( @@ -273,10 +292,7 @@ bool route_members_to_expected_ips( Endpoint expected_backend = target ? deployment.target.members[i].endpoint.backend() : deployment.production.members[i].endpoint.backend(); - const Aurora_BGD_Membership_Set& expected_membership = target - ? deployment.target : deployment.production; - if (!route_to_expected_backend( - cl, sim, expected_backend, expected_membership)) { + if (!route_to_expected_backend(cl, sim, expected_backend)) { return false; } } @@ -358,7 +374,6 @@ int test_bgd_status_available(MYSQL* admin, BGD_Simulator& sim, TestState& state * * - Keep canonical writer placement unchanged. * - Replace ordinary Aurora probes with fast target-membership probes. - * - Ignore a competing production role observation. */ int test_switchover_initiated(MYSQL* admin, BGD_Simulator& sim, TestState& state) { Aurora_BGD_Test_Deployment& deployment = state.deployment; @@ -380,20 +395,6 @@ int test_switchover_initiated(MYSQL* admin, BGD_Simulator& sim, TestState& state ok(active_seq_rc == EXIT_SUCCESS && fast_bgd_without_ordinary( sim, active_sequence, deployment.target_replica_set, 650, 3), "INITIATED uses fast membership probes and suspends the ordinary Aurora query"); - - vector competing_source = deployment.production.replica_rows(); - competing_source[0].session_id = "source-observed-reader"; - competing_source[1].session_id = "MASTER_SESSION_ID"; - auto [source_change_rc, source_change_sequence] = sim.replica_probe_log_last_sequence(); - if (source_change_rc != EXIT_SUCCESS || sim.replica_update( - deployment.blue_replica_set, competing_source, deployment.production.backends()) - != EXIT_SUCCESS) { - diag("Error: failed to publish the competing source role observation"); - return EXIT_FAILURE; - } - ok(fast_bgd_without_ordinary( - sim, source_change_sequence, deployment.target_replica_set, 350, 2), - "changed source roles cannot compete while production probing is suspended"); return EXIT_SUCCESS; } @@ -421,13 +422,6 @@ int test_switchover_in_progress( admin, state.writer_hostgroup, state.reader_hostgroup, deployment.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, "IN_PROGRESS demotes the snapshotted production writer"); - ok(runtime_server_count( - admin, state.writer_hostgroup, deployment.production.members[1].endpoint.hostname, 0) - && runtime_server_count( - admin, state.reader_hostgroup, - deployment.production.members[1].endpoint.hostname, 1, "ONLINE"), - "the competing source observation does not promote a reader"); - auto [repeat_progress_rc, repeat_progress_sequence] = sim.replica_probe_log_last_sequence(); if (repeat_progress_rc != EXIT_SUCCESS || publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS) { @@ -447,19 +441,51 @@ int test_switchover_in_progress( >= static_cast(state.route_hostgroups.size()), "all member routes use source IPs and hold pre-cutover pools"); + vector competing_source = deployment.production.replica_rows(); + competing_source[0].session_id = "source-observed-reader"; + competing_source[1].session_id = "MASTER_SESSION_ID"; + auto [source_change_rc, source_change_sequence] = sim.replica_probe_log_last_sequence(); + bool source_change_ignored = source_change_rc == EXIT_SUCCESS + && sim.replica_update( + deployment.blue_replica_set, competing_source, deployment.production.backends()) + == EXIT_SUCCESS + && fast_bgd_without_ordinary( + sim, source_change_sequence, deployment.target_replica_set, 350, 2); + ok(source_change_ignored, + "changed source roles cannot compete while production probing is suspended"); + ok(runtime_server_count( + admin, state.writer_hostgroup, deployment.production.members[1].endpoint.hostname, 0) + && runtime_server_count( + admin, state.reader_hostgroup, + deployment.production.members[1].endpoint.hostname, 1, "ONLINE"), + "the competing source observation does not promote a reader"); + if (sim.replica_update( + deployment.blue_replica_set, deployment.production.replica_rows(), + deployment.production.backends()) != EXIT_SUCCESS) { + diag("Error: failed to restore the source membership fixture"); + return EXIT_FAILURE; + } + if (set_default_hostgroup(admin, state.route_hostgroups.front()) != EXIT_SUCCESS) { return EXIT_FAILURE; } - state.held_client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); - bool held_connection = state.held_client != nullptr - && mysql_query(state.held_client, "BEGIN") == 0; + bool held_connection = set_held_connection_rule(admin, true) == EXIT_SUCCESS; + state.held_client = held_connection + ? init_mysql_conn(cl.host, cl.port, cl.username, cl.password) : nullptr; + held_connection = state.held_client != nullptr + && mysql_query(state.held_client, "SELECT 1") == 0; if (held_connection) { - auto [query_rc, rows] = mysql_query_ext_rows(state.held_client, kOrdinaryAuroraQuery); - held_connection = query_rc == EXIT_SUCCESS - && aurora_bgd_result_matches_membership(rows, deployment.production); + MYSQL_RES* result = mysql_store_result(state.held_client); + held_connection = result != nullptr; + if (result != nullptr) { + mysql_free_result(result); + } } ok(held_connection && member_route_used_count(admin, state.route_hostgroups) >= 1, "a production connection remains in use across the cutover boundary"); + if (set_held_connection_rule(admin, false) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } return EXIT_SUCCESS; } @@ -505,7 +531,6 @@ int test_switchover_post_processing( && member_route_used_count(admin, state.route_hostgroups) >= 1, "POST_PROCESSING advances without waiting for an in-use connection to close"); if (state.held_client != nullptr) { - mysql_query(state.held_client, "ROLLBACK"); mysql_close(state.held_client); state.held_client = nullptr; } @@ -688,14 +713,12 @@ int test_auto_discovered_refresh_and_rename( return EXIT_FAILURE; } - Aurora_BGD_Membership_Set observed_target = deployment.target; - observed_target.members.pop_back(); bool retained_routing = true; for (size_t i = 0; i < state.refreshed_route_hostgroups.size(); ++i) { retained_routing = retained_routing && set_default_hostgroup(admin, state.refreshed_route_hostgroups[i]) == EXIT_SUCCESS && route_to_expected_backend( - cl, sim, deployment.target.members[i].endpoint.backend(), observed_target); + cl, sim, deployment.target.members[i].endpoint.backend()); } ok(retained_routing, "POST_PROCESSING routes all auto-discovered readers with the retained complete refreshed map"); @@ -784,11 +807,9 @@ int test_available_production_refresh_and_freeze( const Endpoint expected = target ? deployment.target.members[i].endpoint.backend() : deployment.production.members[i].endpoint.backend(); - const Aurora_BGD_Membership_Set& expected_membership = target - ? deployment.target : deployment.production; frozen_routes = frozen_routes && set_default_hostgroup(admin, state.refreshed_route_hostgroups[i]) == EXIT_SUCCESS - && route_to_expected_backend(cl, sim, expected, expected_membership); + && route_to_expected_backend(cl, sim, expected); } ok(frozen_routes, "INITIATED freezes the refreshed production map despite later membership changes"); @@ -845,8 +866,7 @@ int test_post_processing_after_membership_completion( int default_rc = set_default_hostgroup(admin, state.gated_route_hostgroup); ok(default_rc == EXIT_SUCCESS && route_to_expected_backend( - cl, sim, deployment.production.members.front().endpoint.backend(), - deployment.production), + cl, sim, deployment.production.members.front().endpoint.backend()), "POST_PROCESSING leaves routing unchanged without a complete target snapshot"); auto [refresh_seq_rc, refresh_sequence] = sim.replica_probe_log_last_sequence(); @@ -866,8 +886,7 @@ int test_post_processing_after_membership_completion( admin, { state.gated_route_hostgroup }, "=0") == EXIT_SUCCESS && set_default_hostgroup(admin, state.gated_route_hostgroup) == EXIT_SUCCESS) { target_routing = route_to_expected_backend( - cl, sim, deployment.target.members.front().endpoint.backend(), - deployment.target); + cl, sim, deployment.target.members.front().endpoint.backend()); } ok(target_routing, "repeated POST_PROCESSING applies routing after membership becomes complete"); diff --git a/test/tap/tests/test_aurora_bgd_completion-t.cpp b/test/tap/tests/test_aurora_bgd_completion-t.cpp index 475a80c229..515e1ef97e 100644 --- a/test/tap/tests/test_aurora_bgd_completion-t.cpp +++ b/test/tap/tests/test_aurora_bgd_completion-t.cpp @@ -53,7 +53,7 @@ struct TestState { int green_writer_hostgroup { 1562 }; int green_reader_hostgroup { 1563 }; vector route_hostgroups { 1564, 1565, 1566 }; - int post_completion_route_hostgroup { 1567 }; + vector post_completion_route_hostgroups { 1567, 1568, 1569 }; Aurora_BGD_Test_Deployment direct { aurora_bgd_deployment_b_writer_only() }; int direct_writer_hostgroup { 1580 }; int direct_reader_hostgroup { 1581 }; @@ -192,8 +192,7 @@ int add_green_servers( } bool route_to_expected_backend( - CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected_backend, - const Aurora_BGD_Membership_Set& expected_membership + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected_backend ) { auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); if (sequence_rc != EXIT_SUCCESS) { @@ -204,28 +203,37 @@ bool route_to_expected_backend( return false; } auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); - const bool result_matches = query_rc == EXIT_SUCCESS - && aurora_bgd_result_matches_membership(rows, expected_membership); - if (!result_matches) { + (void)rows; + if (query_rc != EXIT_SUCCESS) { diag("Backend routing query failed with MySQL error %d: %s", mysql_errno(client), mysql_error(client)); - } - mysql_close(client); - if (!result_matches) { + mysql_close(client); return false; } + mysql_close(client); auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); if (logs_rc != EXIT_SUCCESS) { return false; } + const Aurora_Replica_Probe_Log* ordinary_probe = nullptr; for (const Aurora_Replica_Probe_Log& log : logs) { - if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary - && log.backend.host == expected_backend.host - && log.backend.port == expected_backend.port) { - return true; + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { + ordinary_probe = &log; + if (log.backend.host == expected_backend.host + && log.backend.port == expected_backend.port) { + return true; + } } } + if (ordinary_probe == nullptr) { + diag("Routing query produced no ordinary Aurora backend probe"); + return false; + } + diag( + "Routing query reached backend %s:%d; expected %s:%d", + ordinary_probe->backend.host.c_str(), ordinary_probe->backend.port, + expected_backend.host.c_str(), expected_backend.port); return false; } @@ -240,9 +248,7 @@ bool route_members( const Endpoint expected = target ? deployment.target.members[i].endpoint.backend() : deployment.production.members[i].endpoint.backend(); - const Aurora_BGD_Membership_Set& expected_membership = target - ? deployment.target : deployment.production; - if (!route_to_expected_backend(cl, sim, expected, expected_membership)) { + if (!route_to_expected_backend(cl, sim, expected)) { return false; } } @@ -270,10 +276,10 @@ int wait_for_pool_count(MYSQL* admin, int hostgroup, const string& comparison) { bool server_count( MYSQL* admin, int hostgroup, const string& hostname, int expected, - const string& status = "" + const string& status = "", const string& table = "runtime_mysql_servers" ) { string query = - "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + "SELECT COUNT(*) FROM " + table + " WHERE hostgroup_id=" + to_string(hostgroup) + " AND hostname=" + aurora_bgd_sql_quote(hostname); if (!status.empty()) { query += " AND status=" + aurora_bgd_sql_quote(status); @@ -478,8 +484,7 @@ int test_completion_after_post_processing( bool green_pool_ready = set_default_hostgroup(admin, state.green_writer_hostgroup) == EXIT_SUCCESS && route_to_expected_backend( - cl, sim, deployment.target.members.front().endpoint.backend(), - deployment.target) + cl, sim, deployment.target.members.front().endpoint.backend()) && pool_count(admin, state.green_writer_hostgroup) >= 1; ok(target_route_pool >= 1 && green_pool_ready, "pre-completion target and configured-green pools are established"); @@ -510,10 +515,14 @@ int test_completion_after_post_processing( deployment.target.members.front().endpoint.hostname, 1, "SHUNNED") && server_count( admin, state.green_writer_hostgroup, - deployment.target.members[1].endpoint.hostname, 1, "OFFLINE_HARD"), + deployment.target.members[1].endpoint.hostname, 1, "OFFLINE_HARD", + "mysql_servers"), "completion preserves configured SHUNNED and OFFLINE_HARD green rows"); - ok(route_members( - cl, admin, sim, deployment, state.route_hostgroups, false), + ok(add_member_routes( + admin, deployment, state.post_completion_route_hostgroups) == EXIT_SUCCESS + && route_members( + cl, admin, sim, deployment, + state.post_completion_route_hostgroups, false), "completion removes every writer and reader traffic pin without DNS verification"); auto [replica_seq_rc, replica_sequence] = sim.replica_probe_log_last_sequence(); @@ -524,8 +533,7 @@ int test_completion_after_post_processing( bool recreated_green_pool = set_default_hostgroup(admin, state.green_writer_hostgroup) == EXIT_SUCCESS && route_to_expected_backend( - cl, sim, deployment.target.members.front().endpoint.backend(), - deployment.target) + cl, sim, deployment.target.members.front().endpoint.backend()) && pool_count(admin, state.green_writer_hostgroup) >= 1; auto [same_seq_rc, same_sequence] = sim.probe_log_last_sequence(); bool same_completion_seen = same_seq_rc == EXIT_SUCCESS @@ -579,8 +587,7 @@ int test_completion_after_post_processing( bool second_green_pool = set_default_hostgroup(admin, state.green_writer_hostgroup) == EXIT_SUCCESS && route_to_expected_backend( - cl, sim, deployment.target.members.front().endpoint.backend(), - deployment.target); + cl, sim, deployment.target.members.front().endpoint.backend()); auto [repeat_different_seq_rc, repeat_different_sequence] = sim.probe_log_last_sequence(); bool repeated_different = repeat_different_seq_rc == EXIT_SUCCESS && publish_completed(sim, deployment, different) == EXIT_SUCCESS @@ -627,8 +634,7 @@ int test_first_completed_observation( return EXIT_FAILURE; } bool direct_pool_ready = route_to_expected_backend( - cl, sim, deployment.production.members.front().endpoint.backend(), - deployment.production) + cl, sim, deployment.production.members.front().endpoint.backend()) && pool_count(admin, state.direct_route_hostgroup) >= 1; ok(direct_pool_ready, "direct-completion setup has an observable production route and pool"); @@ -651,8 +657,7 @@ int test_first_completed_observation( ok(pool_count(admin, state.direct_route_hostgroup) >= 1 && set_default_hostgroup(admin, state.direct_route_hostgroup) == EXIT_SUCCESS && route_to_expected_backend( - cl, sim, deployment.production.members.front().endpoint.backend(), - deployment.production), + cl, sim, deployment.production.members.front().endpoint.backend()), "direct completion leaves unrelated production routing effects untouched"); usleep(750000); diff --git a/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp b/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp index f64a1bda71..86ed95c067 100644 --- a/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp +++ b/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp @@ -125,8 +125,7 @@ int add_writer_route( } bool route_to_backend( - CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected, - const Aurora_BGD_Membership_Set& expected_membership + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected ) { auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); if (sequence_rc != EXIT_SUCCESS) { @@ -137,9 +136,9 @@ bool route_to_backend( return false; } auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; mysql_close(client); - if (query_rc != EXIT_SUCCESS - || !aurora_bgd_result_matches_membership(rows, expected_membership)) { + if (query_rc != EXIT_SUCCESS) { return false; } auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); @@ -165,9 +164,7 @@ bool route_writer( const Endpoint expected = target ? deployment.target.members.front().endpoint.backend() : deployment.production.members.front().endpoint.backend(); - const Aurora_BGD_Membership_Set& expected_membership = target - ? deployment.target : deployment.production; - return route_to_backend(cl, sim, expected, expected_membership); + return route_to_backend(cl, sim, expected); } bool wait_for_writer_route( @@ -181,9 +178,7 @@ bool wait_for_writer_route( ? deployment.target.members.front().endpoint.backend() : deployment.production.members.front().endpoint.backend(); for (uint32_t elapsed_ms = 0; elapsed_ms < kWaitSeconds * 1000; elapsed_ms += 100) { - const Aurora_BGD_Membership_Set& expected_membership = target - ? deployment.target : deployment.production; - if (route_to_backend(cl, sim, expected, expected_membership)) { + if (route_to_backend(cl, sim, expected)) { return true; } usleep(100000); diff --git a/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp b/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp index 77941af394..5192b04d9b 100644 --- a/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp +++ b/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp @@ -177,8 +177,7 @@ int add_routes( } bool route_to_backend( - CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected, - const Aurora_BGD_Membership_Set& expected_membership + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected ) { auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); if (sequence_rc != EXIT_SUCCESS) { @@ -189,9 +188,9 @@ bool route_to_backend( return false; } auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; mysql_close(client); - if (query_rc != EXIT_SUCCESS - || !aurora_bgd_result_matches_membership(rows, expected_membership)) { + if (query_rc != EXIT_SUCCESS) { return false; } auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); @@ -218,9 +217,7 @@ bool route_members( Endpoint expected = target ? deployment.target.members[i].endpoint.backend() : deployment.production.members[i].endpoint.backend(); - const Aurora_BGD_Membership_Set& expected_membership = target - ? deployment.target : deployment.production; - if (!route_to_backend(cl, sim, expected, expected_membership)) { + if (!route_to_backend(cl, sim, expected)) { return false; } } @@ -449,8 +446,7 @@ int test_post_processing_rollback( Aurora_BGD_Test_Deployment& deployment = state.post; ok(set_default_hostgroup(admin, state.post_green_writer_hostgroup) == EXIT_SUCCESS && route_to_backend( - cl, sim, deployment.target.members.front().endpoint.backend(), - deployment.target) + cl, sim, deployment.target.members.front().endpoint.backend()) && pool_count(admin, state.post_green_writer_hostgroup) >= 1, "a configured green pool is established before rollback"); ok(publish_status(sim, deployment, "AVAILABLE") == EXIT_SUCCESS diff --git a/test/tap/tests/test_aurora_bgd_late_entry-t.cpp b/test/tap/tests/test_aurora_bgd_late_entry-t.cpp index 539e67a769..b368d1389e 100644 --- a/test/tap/tests/test_aurora_bgd_late_entry-t.cpp +++ b/test/tap/tests/test_aurora_bgd_late_entry-t.cpp @@ -176,8 +176,7 @@ int add_routes( } bool route_to_backend( - CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected, - const Aurora_BGD_Membership_Set& expected_membership + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected ) { auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); if (sequence_rc != EXIT_SUCCESS) { @@ -188,9 +187,9 @@ bool route_to_backend( return false; } auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; mysql_close(client); - if (query_rc != EXIT_SUCCESS - || !aurora_bgd_result_matches_membership(rows, expected_membership)) { + if (query_rc != EXIT_SUCCESS) { return false; } auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); @@ -217,9 +216,7 @@ bool route_members( Endpoint expected = target ? deployment.target.members[i].endpoint.backend() : deployment.production.members[i].endpoint.backend(); - const Aurora_BGD_Membership_Set& expected_membership = target - ? deployment.target : deployment.production; - if (!route_to_backend(cl, sim, expected, expected_membership)) { + if (!route_to_backend(cl, sim, expected)) { return false; } } @@ -329,86 +326,6 @@ int test_first_in_progress_after_ordinary_error( return EXIT_SUCCESS; } -/** - * Reconstruct POST_PROCESSING after a non-empty but partial ordinary result. - * - * The configured/runtime hostgroups define the complete production membership, - * so a writer-only observation cannot make a writer-only target map actionable. - */ -int test_first_post_processing_after_partial_ordinary( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state -) { - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to reset before partial-probe late entry"); - return EXIT_FAILURE; - } - - Aurora_BGD_Test_Deployment& deployment = state.post; - vector partial_production = deployment.production.replica_rows(); - partial_production.resize(1); - vector partial_target = deployment.target.replica_rows(); - partial_target.resize(1); - if (sim.replica_update( - deployment.production.replica_set_id, partial_production, - deployment.production.backends()) != EXIT_SUCCESS - || sim.replica_update( - deployment.target.replica_set_id, partial_target, - deployment.target.backends()) != EXIT_SUCCESS - || sim.topology_update( - aurora_bgd_topology_backends(deployment), - aurora_bgd_topology(deployment, "SWITCHOVER_IN_POST_PROCESSING")) - != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.post_writer_hostgroup, - state.post_reader_hostgroup, state.post_green_writer_hostgroup, - state.post_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS - || add_routes( - admin, deployment, state.post_route_hostgroups, - state.post_green_writer_hostgroup) != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.post_writer_hostgroup, - "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { - diag("Error: failed to configure partial-probe POST_PROCESSING late entry"); - return EXIT_FAILURE; - } - - auto [partial_seq_rc, partial_sequence] = sim.replica_probe_log_last_sequence(); - bool partial_map_blocked = partial_seq_rc == EXIT_SUCCESS - && sim.replica_update( - deployment.production.replica_set_id, - deployment.production.replica_rows(), deployment.production.backends()) - == EXIT_SUCCESS; - if (partial_map_blocked) { - auto [partial_probe_rc, partial_probe] = aurora_bgd_wait_for_replica_probe( - sim, partial_sequence, deployment.target.backends(), - Aurora_Replica_Probe_Kind::bgd_membership, 5000, - deployment.target.replica_set_id); - partial_map_blocked = partial_probe_rc == EXIT_SUCCESS - && route_members( - cl, admin, sim, deployment, state.post_route_hostgroups, false); - } - ok(partial_map_blocked, - "a partial initial ordinary result cannot authorize partial target routing"); - - auto [complete_seq_rc, complete_sequence] = sim.replica_probe_log_last_sequence(); - bool complete_map_applied = complete_seq_rc == EXIT_SUCCESS - && sim.replica_update( - deployment.target.replica_set_id, deployment.target.replica_rows(), - deployment.target.backends()) == EXIT_SUCCESS; - if (complete_map_applied) { - auto [complete_probe_rc, complete_probe] = aurora_bgd_wait_for_replica_probe( - sim, complete_sequence, deployment.target.backends(), - Aurora_Replica_Probe_Kind::bgd_membership, 5000, - deployment.target.replica_set_id); - complete_map_applied = complete_probe_rc == EXIT_SUCCESS - && route_members( - cl, admin, sim, deployment, state.post_route_hostgroups, true); - } - ok(complete_map_applied, - "complete target membership routes every configured production member"); - return EXIT_SUCCESS; -} - /** * Start a worker from SWITCHOVER_IN_POST_PROCESSING. * @@ -448,7 +365,7 @@ int test_first_post_processing( } int main() { - plan(9); + plan(7); CommandLine cl {}; MYSQL* admin = nullptr; @@ -478,13 +395,6 @@ int main() { goto exit_cleanup; } - // Simulator: return only the writer from the first ordinary and target probes. - // Verify: configured production membership blocks partial routing, then accepts the full map. - if (test_first_post_processing_after_partial_ordinary(cl, admin, sim, state) - != EXIT_SUCCESS) { - goto exit_cleanup; - } - // Simulator: make POST_PROCESSING the first observed deployment state. // Verify: the worker reconstructs and applies every target pin. if (test_first_post_processing(cl, admin, sim, state) != EXIT_SUCCESS) { diff --git a/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp b/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp index 96b6cbf00e..841ea689da 100644 --- a/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp +++ b/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp @@ -126,8 +126,7 @@ int add_writer_route( } bool route_to_backend( - CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected, - const Aurora_BGD_Membership_Set& expected_membership + CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected ) { auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); if (sequence_rc != EXIT_SUCCESS) { @@ -138,9 +137,9 @@ bool route_to_backend( return false; } auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + (void)rows; mysql_close(client); - if (query_rc != EXIT_SUCCESS - || !aurora_bgd_result_matches_membership(rows, expected_membership)) { + if (query_rc != EXIT_SUCCESS) { return false; } auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); @@ -166,9 +165,7 @@ bool route_writer( const Endpoint expected = target ? deployment.target.members.front().endpoint.backend() : deployment.production.members.front().endpoint.backend(); - const Aurora_BGD_Membership_Set& expected_membership = target - ? deployment.target : deployment.production; - return route_to_backend(cl, sim, expected, expected_membership); + return route_to_backend(cl, sim, expected); } bool writer_placement( From 3c3877a4485fee5f4bfc3bc18a6b44d56eadd466 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Sun, 16 Aug 2026 15:00:42 +0000 Subject: [PATCH 31/34] test: correlate Aurora routing probes --- test/tap/tap/aurora_bgd_tap.h | 45 +++++++------ test/tap/tests/test_aurora_bgd_active-t.cpp | 45 ++----------- .../tests/test_aurora_bgd_completion-t.cpp | 64 ++++--------------- ...test_aurora_bgd_concurrent_isolation-t.cpp | 39 ++--------- .../test_aurora_bgd_error_recovery-t.cpp | 43 ++----------- .../tests/test_aurora_bgd_late_entry-t.cpp | 43 ++----------- .../test_aurora_bgd_worker_lifecycle-t.cpp | 39 ++--------- 7 files changed, 66 insertions(+), 252 deletions(-) diff --git a/test/tap/tap/aurora_bgd_tap.h b/test/tap/tap/aurora_bgd_tap.h index 647a2d4fcb..07dda5dc3d 100644 --- a/test/tap/tap/aurora_bgd_tap.h +++ b/test/tap/tap/aurora_bgd_tap.h @@ -302,26 +302,33 @@ inline int aurora_bgd_publish( aurora_bgd_available_topology(deployment)); } -inline bool aurora_bgd_result_matches_membership( - const vector>& rows, const Aurora_BGD_Membership_Set& membership -) { - vector actual_ids; - actual_ids.reserve(rows.size()); - for (const vector& row : rows) { - if (row.empty()) { - return false; - } - actual_ids.push_back(row.front()); - } +static const char kAuroraBGDRouteProbeQuery[] = + "SELECT SERVER_ID," + "IF(" + "SESSION_ID = 'MASTER_SESSION_ID' AND " + "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " + "'probably_former_MASTER_SESSION_ID', SESSION_ID" + ") SESSION_ID, " + "LAST_UPDATE_TIMESTAMP, " + "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " + "CPU " + "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" + " ( " + "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" + " OR SESSION_ID = 'MASTER_SESSION_ID'" + " ) " + "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" + " ORDER BY SERVER_ID"; - vector expected_ids; - expected_ids.reserve(membership.members.size()); - for (const Aurora_BGD_Member& member : membership.members) { - expected_ids.push_back(member.server_id); - } - std::sort(actual_ids.begin(), actual_ids.end()); - std::sort(expected_ids.begin(), expected_ids.end()); - return actual_ids == expected_ids; +inline bool aurora_bgd_routing_probe_reached( + BGD_Simulator& sim, uint64_t sequence, const Endpoint& expected_backend +) { + // Client-route fixtures use TLS while Aurora monitor fixtures use plaintext, + // so a background monitor probe cannot satisfy this client-query assertion. + auto [rc, log] = sim.wait_for_replica_probe_log( + sequence, expected_backend, Aurora_Replica_Probe_Kind::ordinary, + 1000, 1); + return rc == EXIT_SUCCESS; } inline int aurora_bgd_wait_for_status( diff --git a/test/tap/tests/test_aurora_bgd_active-t.cpp b/test/tap/tests/test_aurora_bgd_active-t.cpp index 9192edf7d3..41827ef641 100644 --- a/test/tap/tests/test_aurora_bgd_active-t.cpp +++ b/test/tap/tests/test_aurora_bgd_active-t.cpp @@ -26,23 +26,6 @@ using namespace std; const uint32_t kWaitSeconds = 5; const uint32_t kProbeTimeoutMs = 5000; const int kHeldConnectionRuleId = 153000; -const char kOrdinaryAuroraQuery[] = - "SELECT SERVER_ID," - "IF(" - "SESSION_ID = 'MASTER_SESSION_ID' AND " - "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " - "'probably_former_MASTER_SESSION_ID', SESSION_ID" - ") SESSION_ID, " - "LAST_UPDATE_TIMESTAMP, " - "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " - "CPU " - "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" - " ( " - "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" - " OR SESSION_ID = 'MASTER_SESSION_ID'" - " ) " - "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" - " ORDER BY SERVER_ID"; struct TestState { Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_a() }; @@ -204,10 +187,10 @@ int add_member_routes( vector queries; for (size_t i = 0; i < deployment.production.members.size(); ++i) { queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + to_string(route_hgs[i]) + "," + aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + - ",3306,'ONLINE','Aurora BGD member route')"); + ",3306,'ONLINE',1,'Aurora BGD member route')"); } queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); return aurora_bgd_execute_all(admin, queries); @@ -248,7 +231,7 @@ bool route_to_expected_backend( if (client == nullptr) { return false; } - auto [rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + auto [rc, rows] = mysql_query_ext_rows(client, kAuroraBGDRouteProbeQuery); (void)rows; if (rc != EXIT_SUCCESS) { diag("Backend routing query failed with MySQL error %d: %s", @@ -257,27 +240,7 @@ bool route_to_expected_backend( return false; } mysql_close(client); - - auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); - if (logs_rc != EXIT_SUCCESS) { - return false; - } - const Aurora_Replica_Probe_Log* routed_probe = nullptr; - for (const Aurora_Replica_Probe_Log& log : logs) { - if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { - routed_probe = &log; - if (log.backend.host == expected_backend.host - && log.backend.port == expected_backend.port) { - return true; - } - } - } - diag( - "Ordinary Aurora query reached %s:%d; expected %s:%d", - routed_probe ? routed_probe->backend.host.c_str() : "", - routed_probe ? routed_probe->backend.port : 0, - expected_backend.host.c_str(), expected_backend.port); - return false; + return aurora_bgd_routing_probe_reached(sim, sequence, expected_backend); } bool route_members_to_expected_ips( diff --git a/test/tap/tests/test_aurora_bgd_completion-t.cpp b/test/tap/tests/test_aurora_bgd_completion-t.cpp index 515e1ef97e..c21fce121b 100644 --- a/test/tap/tests/test_aurora_bgd_completion-t.cpp +++ b/test/tap/tests/test_aurora_bgd_completion-t.cpp @@ -23,23 +23,6 @@ using namespace std; const uint32_t kWaitSeconds = 5; const uint32_t kProbeTimeoutMs = 5000; -const char kOrdinaryAuroraQuery[] = - "SELECT SERVER_ID," - "IF(" - "SESSION_ID = 'MASTER_SESSION_ID' AND " - "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " - "'probably_former_MASTER_SESSION_ID', SESSION_ID" - ") SESSION_ID, " - "LAST_UPDATE_TIMESTAMP, " - "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " - "CPU " - "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" - " ( " - "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" - " OR SESSION_ID = 'MASTER_SESSION_ID'" - " ) " - "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" - " ORDER BY SERVER_ID"; struct TestState { Aurora_BGD_Test_Deployment progress { aurora_bgd_deployment_b_writer_only() }; @@ -135,9 +118,9 @@ int add_route( MYSQL* admin, int hostgroup, const string& hostname, const string& status = "ONLINE" ) { return aurora_bgd_execute_all(admin, { - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + to_string(hostgroup) + "," + aurora_bgd_sql_quote(hostname) + - ",3306," + aurora_bgd_sql_quote(status) + ",'Aurora BGD completion route')", + ",3306," + aurora_bgd_sql_quote(status) + ",1,'Aurora BGD completion route')", "LOAD MYSQL SERVERS TO RUNTIME", }); } @@ -152,10 +135,10 @@ int add_member_routes( vector queries; for (size_t i = 0; i < hostgroups.size(); ++i) { queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + to_string(hostgroups[i]) + "," + aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + - ",3306,'ONLINE','Aurora BGD completion member route')"); + ",3306,'ONLINE',1,'Aurora BGD completion member route')"); } queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); return aurora_bgd_execute_all(admin, queries); @@ -171,22 +154,22 @@ int add_green_servers( const string status = i + 1 == deployment.target.members.size() ? "OFFLINE_SOFT" : "ONLINE"; queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + to_string(hostgroup) + "," + aurora_bgd_sql_quote(deployment.target.members[i].endpoint.hostname) + - ",3306," + aurora_bgd_sql_quote(status) + + ",3306," + aurora_bgd_sql_quote(status) + ",1" + ",'Aurora BGD configured green member')"); } queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + to_string(green_reader_hg) + "," + aurora_bgd_sql_quote(deployment.target.members.front().endpoint.hostname) + - ",3306,'SHUNNED','Aurora BGD configured green status matrix')"); + ",3306,'SHUNNED',1,'Aurora BGD configured green status matrix')"); queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + to_string(green_writer_hg) + "," + aurora_bgd_sql_quote(deployment.target.members[1].endpoint.hostname) + - ",3306,'OFFLINE_HARD','Aurora BGD configured green status matrix')"); + ",3306,'OFFLINE_HARD',1,'Aurora BGD configured green status matrix')"); queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); return aurora_bgd_execute_all(admin, queries); } @@ -202,7 +185,7 @@ bool route_to_expected_backend( if (client == nullptr) { return false; } - auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + auto [query_rc, rows] = mysql_query_ext_rows(client, kAuroraBGDRouteProbeQuery); (void)rows; if (query_rc != EXIT_SUCCESS) { diag("Backend routing query failed with MySQL error %d: %s", @@ -211,30 +194,7 @@ bool route_to_expected_backend( return false; } mysql_close(client); - - auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); - if (logs_rc != EXIT_SUCCESS) { - return false; - } - const Aurora_Replica_Probe_Log* ordinary_probe = nullptr; - for (const Aurora_Replica_Probe_Log& log : logs) { - if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { - ordinary_probe = &log; - if (log.backend.host == expected_backend.host - && log.backend.port == expected_backend.port) { - return true; - } - } - } - if (ordinary_probe == nullptr) { - diag("Routing query produced no ordinary Aurora backend probe"); - return false; - } - diag( - "Routing query reached backend %s:%d; expected %s:%d", - ordinary_probe->backend.host.c_str(), ordinary_probe->backend.port, - expected_backend.host.c_str(), expected_backend.port); - return false; + return aurora_bgd_routing_probe_reached(sim, sequence, expected_backend); } bool route_members( diff --git a/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp b/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp index 86ed95c067..c5d9540cb8 100644 --- a/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp +++ b/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp @@ -22,23 +22,6 @@ using namespace std; const uint32_t kWaitSeconds = 5; -const char kOrdinaryAuroraQuery[] = - "SELECT SERVER_ID," - "IF(" - "SESSION_ID = 'MASTER_SESSION_ID' AND " - "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " - "'probably_former_MASTER_SESSION_ID', SESSION_ID" - ") SESSION_ID, " - "LAST_UPDATE_TIMESTAMP, " - "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " - "CPU " - "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" - " ( " - "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" - " OR SESSION_ID = 'MASTER_SESSION_ID'" - " ) " - "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" - " ORDER BY SERVER_ID"; struct TestState { Aurora_BGD_Test_Deployment deployment_a { aurora_bgd_deployment_a() }; @@ -116,10 +99,10 @@ int add_writer_route( MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, int hostgroup ) { return aurora_bgd_execute_all(admin, { - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + to_string(hostgroup) + "," + aurora_bgd_sql_quote(deployment.production.members.front().endpoint.hostname) + - ",3306,'ONLINE','Aurora BGD concurrent route')", + ",3306,'ONLINE',1,'Aurora BGD concurrent route')", "LOAD MYSQL SERVERS TO RUNTIME", }); } @@ -135,23 +118,11 @@ bool route_to_backend( if (client == nullptr) { return false; } - auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + auto [query_rc, rows] = mysql_query_ext_rows(client, kAuroraBGDRouteProbeQuery); (void)rows; mysql_close(client); - if (query_rc != EXIT_SUCCESS) { - return false; - } - auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); - if (logs_rc != EXIT_SUCCESS) { - return false; - } - for (const Aurora_Replica_Probe_Log& log : logs) { - if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary - && log.backend.host == expected.host && log.backend.port == expected.port) { - return true; - } - } - return false; + return query_rc == EXIT_SUCCESS + && aurora_bgd_routing_probe_reached(sim, sequence, expected); } bool route_writer( diff --git a/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp b/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp index 5192b04d9b..228ee58792 100644 --- a/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp +++ b/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp @@ -23,23 +23,6 @@ using namespace std; const uint32_t kWaitSeconds = 5; const uint32_t kProbeTimeoutMs = 5000; -const char kOrdinaryAuroraQuery[] = - "SELECT SERVER_ID," - "IF(" - "SESSION_ID = 'MASTER_SESSION_ID' AND " - "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " - "'probably_former_MASTER_SESSION_ID', SESSION_ID" - ") SESSION_ID, " - "LAST_UPDATE_TIMESTAMP, " - "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " - "CPU " - "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" - " ( " - "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" - " OR SESSION_ID = 'MASTER_SESSION_ID'" - " ) " - "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" - " ORDER BY SERVER_ID"; struct TestState { Aurora_BGD_Test_Deployment initiated { aurora_bgd_deployment_b_writer_only() }; @@ -162,16 +145,16 @@ int add_routes( vector queries; for (size_t i = 0; i < route_hgs.size(); ++i) { queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + to_string(route_hgs[i]) + "," + aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + - ",3306,'ONLINE','Aurora BGD rollback route')"); + ",3306,'ONLINE',1,'Aurora BGD rollback route')"); } queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + to_string(green_writer_hg) + "," + aurora_bgd_sql_quote(deployment.target.members.front().endpoint.hostname) + - ",3306,'ONLINE','Aurora BGD rollback green pool')"); + ",3306,'ONLINE',1,'Aurora BGD rollback green pool')"); queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); return aurora_bgd_execute_all(admin, queries); } @@ -187,23 +170,11 @@ bool route_to_backend( if (client == nullptr) { return false; } - auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + auto [query_rc, rows] = mysql_query_ext_rows(client, kAuroraBGDRouteProbeQuery); (void)rows; mysql_close(client); - if (query_rc != EXIT_SUCCESS) { - return false; - } - auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); - if (logs_rc != EXIT_SUCCESS) { - return false; - } - for (const Aurora_Replica_Probe_Log& log : logs) { - if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary - && log.backend.host == expected.host && log.backend.port == expected.port) { - return true; - } - } - return false; + return query_rc == EXIT_SUCCESS + && aurora_bgd_routing_probe_reached(sim, sequence, expected); } bool route_members( diff --git a/test/tap/tests/test_aurora_bgd_late_entry-t.cpp b/test/tap/tests/test_aurora_bgd_late_entry-t.cpp index b368d1389e..902f979864 100644 --- a/test/tap/tests/test_aurora_bgd_late_entry-t.cpp +++ b/test/tap/tests/test_aurora_bgd_late_entry-t.cpp @@ -22,23 +22,6 @@ using namespace std; const uint32_t kWaitSeconds = 5; -const char kOrdinaryAuroraQuery[] = - "SELECT SERVER_ID," - "IF(" - "SESSION_ID = 'MASTER_SESSION_ID' AND " - "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " - "'probably_former_MASTER_SESSION_ID', SESSION_ID" - ") SESSION_ID, " - "LAST_UPDATE_TIMESTAMP, " - "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " - "CPU " - "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" - " ( " - "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" - " OR SESSION_ID = 'MASTER_SESSION_ID'" - " ) " - "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" - " ORDER BY SERVER_ID"; struct TestState { Aurora_BGD_Test_Deployment initiated { aurora_bgd_deployment_b_writer_only() }; @@ -161,16 +144,16 @@ int add_routes( vector queries; for (size_t i = 0; i < route_hgs.size(); ++i) { queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + to_string(route_hgs[i]) + "," + aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + - ",3306,'ONLINE','Aurora BGD rollback route')"); + ",3306,'ONLINE',1,'Aurora BGD rollback route')"); } queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + to_string(green_writer_hg) + "," + aurora_bgd_sql_quote(deployment.target.members.front().endpoint.hostname) + - ",3306,'ONLINE','Aurora BGD rollback green pool')"); + ",3306,'ONLINE',1,'Aurora BGD rollback green pool')"); queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); return aurora_bgd_execute_all(admin, queries); } @@ -186,23 +169,11 @@ bool route_to_backend( if (client == nullptr) { return false; } - auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + auto [query_rc, rows] = mysql_query_ext_rows(client, kAuroraBGDRouteProbeQuery); (void)rows; mysql_close(client); - if (query_rc != EXIT_SUCCESS) { - return false; - } - auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); - if (logs_rc != EXIT_SUCCESS) { - return false; - } - for (const Aurora_Replica_Probe_Log& log : logs) { - if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary - && log.backend.host == expected.host && log.backend.port == expected.port) { - return true; - } - } - return false; + return query_rc == EXIT_SUCCESS + && aurora_bgd_routing_probe_reached(sim, sequence, expected); } bool route_members( diff --git a/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp b/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp index 841ea689da..143e57d7c1 100644 --- a/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp +++ b/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp @@ -21,23 +21,6 @@ using namespace std; const uint32_t kWaitSeconds = 5; -const char kOrdinaryAuroraQuery[] = - "SELECT SERVER_ID," - "IF(" - "SESSION_ID = 'MASTER_SESSION_ID' AND " - "SERVER_ID <> (SELECT SERVER_ID FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE SESSION_ID = 'MASTER_SESSION_ID' ORDER BY LAST_UPDATE_TIMESTAMP DESC LIMIT 1), " - "'probably_former_MASTER_SESSION_ID', SESSION_ID" - ") SESSION_ID, " - "LAST_UPDATE_TIMESTAMP, " - "IF(SESSION_ID = 'MASTER_SESSION_ID', 0, REPLICA_LAG_IN_MILLISECONDS) AS REPLICA_LAG_IN_MILLISECONDS, " - "CPU " - "FROM INFORMATION_SCHEMA.REPLICA_HOST_STATUS WHERE" - " ( " - "(REPLICA_LAG_IN_MILLISECONDS >= 0 AND REPLICA_LAG_IN_MILLISECONDS <= 600000)" - " OR SESSION_ID = 'MASTER_SESSION_ID'" - " ) " - "AND LAST_UPDATE_TIMESTAMP > NOW() - INTERVAL 180 SECOND" - " ORDER BY SERVER_ID"; struct TestState { Aurora_BGD_Test_Deployment reload { aurora_bgd_deployment_a() }; @@ -117,10 +100,10 @@ int add_writer_route( MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, int hostgroup ) { return aurora_bgd_execute_all(admin, { - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,comment) VALUES (" + + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + to_string(hostgroup) + "," + aurora_bgd_sql_quote(deployment.production.members.front().endpoint.hostname) + - ",3306,'ONLINE','Aurora BGD lifecycle route')", + ",3306,'ONLINE',1,'Aurora BGD lifecycle route')", "LOAD MYSQL SERVERS TO RUNTIME", }); } @@ -136,23 +119,11 @@ bool route_to_backend( if (client == nullptr) { return false; } - auto [query_rc, rows] = mysql_query_ext_rows(client, kOrdinaryAuroraQuery); + auto [query_rc, rows] = mysql_query_ext_rows(client, kAuroraBGDRouteProbeQuery); (void)rows; mysql_close(client); - if (query_rc != EXIT_SUCCESS) { - return false; - } - auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); - if (logs_rc != EXIT_SUCCESS) { - return false; - } - for (const Aurora_Replica_Probe_Log& log : logs) { - if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary - && log.backend.host == expected.host && log.backend.port == expected.port) { - return true; - } - } - return false; + return query_rc == EXIT_SUCCESS + && aurora_bgd_routing_probe_reached(sim, sequence, expected); } bool route_writer( From 529ab6f15d6ad60647ce2aa851d543fc75c37476 Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Mon, 17 Aug 2026 08:45:26 +0000 Subject: [PATCH 32/34] test: simplify Aurora BGD simulator coverage Split lifecycle coverage into focused scenarios. Keep each test scoped to one behavior. Reduce fixture complexity while preserving core behavior. Retain the ordinary Aurora regression suite. --- ...026-08-17-aws-aurora-bgd-implementation.md | 753 ++++++++++++++ test/tap/groups/groups.json | 24 +- test/tap/tap/aurora_bgd_scenario_tap.h | 344 +++++++ test/tap/tests/test_aurora_bgd_active-t.cpp | 928 ------------------ .../test_aurora_bgd_automatic_discovery-t.cpp | 134 +++ .../tests/test_aurora_bgd_cluster_sync-t.cpp | 372 ------- .../tests/test_aurora_bgd_completion-t.cpp | 675 ------------- ...test_aurora_bgd_concurrent_isolation-t.cpp | 421 ++------ ..._bgd_config_refresh_after_completion-t.cpp | 93 ++ ...aurora_bgd_disable_during_switchover-t.cpp | 89 ++ .../tap/tests/test_aurora_bgd_discovery-t.cpp | 453 --------- .../test_aurora_bgd_error_recovery-t.cpp | 487 --------- .../test_aurora_bgd_green_pool_cleanup-t.cpp | 184 ++++ .../tests/test_aurora_bgd_late_entry-t.cpp | 381 ------- ...test_aurora_bgd_late_entry_completed-t.cpp | 101 ++ ..._aurora_bgd_late_entry_writer_phases-t.cpp | 124 +++ .../tap/tests/test_aurora_bgd_probe_tls-t.cpp | 59 ++ .../tests/test_aurora_bgd_reader_policy-t.cpp | 80 ++ ...aurora_bgd_reader_switchover_cleanup-t.cpp | 124 +++ ..._aurora_bgd_remove_during_switchover-t.cpp | 90 ++ .../test_aurora_bgd_repeated_deployment-t.cpp | 142 +++ test/tap/tests/test_aurora_bgd_rollback-t.cpp | 132 +++ test/tap/tests/test_aurora_bgd_smoke-t.cpp | 69 ++ ...est_aurora_bgd_topology_empty_absent-t.cpp | 96 ++ .../test_aurora_bgd_topology_errors-t.cpp | 107 ++ ...est_aurora_bgd_worker_config_refresh-t.cpp | 104 ++ .../test_aurora_bgd_worker_lifecycle-t.cpp | 375 ------- .../test_aurora_bgd_writer_switchover-t.cpp | 163 +++ 28 files changed, 3103 insertions(+), 4001 deletions(-) create mode 100644 docs/superpowers/plans/2026-08-17-aws-aurora-bgd-implementation.md create mode 100644 test/tap/tap/aurora_bgd_scenario_tap.h delete mode 100644 test/tap/tests/test_aurora_bgd_active-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_automatic_discovery-t.cpp delete mode 100644 test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp delete mode 100644 test/tap/tests/test_aurora_bgd_completion-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_config_refresh_after_completion-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_disable_during_switchover-t.cpp delete mode 100644 test/tap/tests/test_aurora_bgd_discovery-t.cpp delete mode 100644 test/tap/tests/test_aurora_bgd_error_recovery-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_green_pool_cleanup-t.cpp delete mode 100644 test/tap/tests/test_aurora_bgd_late_entry-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_late_entry_completed-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_late_entry_writer_phases-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_probe_tls-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_reader_policy-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_reader_switchover_cleanup-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_remove_during_switchover-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_repeated_deployment-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_rollback-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_smoke-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_topology_empty_absent-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_topology_errors-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_worker_config_refresh-t.cpp delete mode 100644 test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp create mode 100644 test/tap/tests/test_aurora_bgd_writer_switchover-t.cpp diff --git a/docs/superpowers/plans/2026-08-17-aws-aurora-bgd-implementation.md b/docs/superpowers/plans/2026-08-17-aws-aurora-bgd-implementation.md new file mode 100644 index 0000000000..329e7c738c --- /dev/null +++ b/docs/superpowers/plans/2026-08-17-aws-aurora-bgd-implementation.md @@ -0,0 +1,753 @@ +# AWS Aurora Blue/Green Deployment Implementation Plan + +> **For agentic workers:** Phase 3 uses incremental test-after development with +> characterization coverage. Use `superpowers:receiving-code-review` +> before addressing review feedback. Use the `gh` CLI for every GitHub review, +> comment, reply, check, and review-trigger operation. + +This plan coordinates the implementation of the three approved specifications +under `docs/superpowers/specs/aws-aurora-blue-green/`. + +**Goal:** Implement AWS Aurora MySQL blue/green deployment handling through +three review-gated phases, complete the configuration and simulator +prerequisites first, implement and verify the monitor/FSM in gated slices, and finish with a +real-Aurora user acceptance test before the PR is made ready to merge. + +**Branch:** `feat/aws-aurora-bgd` + +**Base:** current `upstream/v3.0` + +**Pull request:** [sysown/proxysql#6044](https://github.com/sysown/proxysql/pull/6044), +from `feat/aws-aurora-bgd` to `v3.0`. Keep the PR draft until all three phases, +all review gates, and the real-Aurora acceptance gate pass. + +## Authoritative Inputs + +- `docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-configuration-runtime-cluster-sync-design.md` +- `docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-cluster-simulator-testing-design.md` +- `docs/superpowers/specs/aws-aurora-blue-green/2026-07-31-aurora-bgd-monitor-fsm-design.md` +- `RDS_Topology_metadata.pdf` (AWS-team-provided topology-status contract) +- `aurora-bgd-switchover-analysis/RESULTS-20260730T073724Z.md` +- `aurora-bgd-switchover-analysis/PLAN.md` +- `doc/AWS_Blue_Green/RDS_BGD_Monitor.md` +- `doc/AWS_Blue_Green/RDS_BGD_Simulator.md` + +The AWS-team-provided topology document is authoritative for status-to-routing +semantics. The recorded Aurora observation is the ground truth for the other +observed external behavior and corroborates those status semantics. Simulator +tests prove how ProxySQL responds to the contract and observations; they do not +prove that AWS produces the observed details. If a later real-Aurora result +contradicts either source, stop, update the design decision with the user, and +only then update simulator expectations and implementation. + +## Global Constraints + +- Every Aurora hostgroups row is user-created. +- Green hostgroups are either both configured or both `NULL`; do not introduce + explicit/automatic mode terminology or a mode column. +- Add only `green_writer_hostgroup` and `green_reader_hostgroup` as configured + columns and runtime-only `bgd_status` as monitor state. +- SAVE, disk/config export, and ProxySQL Cluster synchronization must exclude + `bgd_status`. +- Reuse RDS BGD behavior wherever its semantics match Aurora; retain the + Aurora-specific three-probe, all-member mapping, and completion-latch rules. +- `TEST_AURORA` enables ordinary Aurora and Aurora BGD simulation. Do not add a + separate simulator feature flag or CI group. +- Keep the existing ordinary Aurora JSON payload schema and all 33 scenarios. +- Keep all RDS Multi-AZ BGD behavior and all 22 RDS BGD TAP binaries green. +- Phase 3 implements one coherent behavior slice at a time, then adds focused + scenarios and runs the nearest characterization regressions before the next + slice. +- Use the current timestamp for every new commit. Do not rewrite dates. + +## Phase-Specific Development Protocol + +### Phases 1 and 2: prerequisite implementation and verification + +Configuration/runtime integration and the simulator are prerequisites for +test-driving the Aurora FSM. They do not use the strict red-green-refactor +cycle. Implement each coherent contract slice, run the existing consumer +regressions, and commit only after verification passes. The simulator is test +infrastructure and does not receive tests whose subject is the simulator itself. +Do not claim that these phases were developed through TDD. + +No Aurora BGD FSM behavior is implemented in Phase 1 or Phase 2. + +### Phase 3: incremental test-after development with characterization coverage + +Every Phase 3 implementation gate follows this sequence: + +1. Record the externally observable behaviors and assertions for the gate. +2. Implement the complete, coherent production slice for those behaviors. +3. Compile once after the slice is structurally complete. +4. Add focused interactive scenarios for the recorded behaviors. +5. Run the focused scenarios until they pass. +6. Run the nearest ordinary Aurora and RDS characterization regressions. +7. Fix any issue before starting the next gate, then commit the production and + scenario changes together with a narrow commit message. + +Tests are not deliberately run in an expected-failure state. They remain +contract-driven because their expected observations are recorded before the +implementation is written. Existing RDS BGD tests characterize shared +placement, DNS, connection-retirement, rollback, and cleanup behavior while +Aurora scenarios cover the Aurora-specific worker and FSM contract. + +## Pull Request and Review Workflow + +### Initial publication + +The feature branch contains only the three approved design specifications when +the draft PR is opened. Phase-one implementation begins only after the draft PR +exists. + +### GitHub writing style and attribution + +Every GitHub message drafted by Codex and posted through the user's account +must read like a concise maintainer response, not an agent activity report. +This applies to PR descriptions, submitted reviews, top-level comments, inline +replies, and reviewer-invocation comments. + +- Lead with the outcome. For an accepted review comment, use the form + `Fixed in . ` +- For a disputed comment, state the directly relevant technical reason and + evidence briefly. Do not narrate the investigation process. +- Do not list routine validation commands, internal workflow steps, tool names, + or exhaustive checks unless the reviewer asks for them or a specific result + is necessary to support the answer. +- Keep the tone natural and professional. Avoid status-report language such as + "documentation verification completed" and do not use an emoji in the + attribution. +- End every message with the following exact footer as its own final paragraph, + separated from the message by one blank line: + + ```html + [Drafted by Codex · AI agent] + ``` + +The footer requirement is prospective; do not edit existing GitHub messages +solely to add it. When invoking a reviewer, keep its required trigger at the +start of the message and put the footer in the final paragraph. + +### Gate after every phase + +After the phase acceptance commands pass: + +1. Push every commit created during the phase to + `upstream/feat/aws-aurora-bgd`. A phase normally contains multiple commits, + especially Phase 3's gated implementation slices. +2. Use `gh` to check whether Codex, CodeRabbit, and Gitar started automatically. + Manually invoke any missing reviewer. Codex and Gitar accept optional + natural-language instructions, so request a phase commit range, a full PR + review, or a focused review of named files, subsystems, contracts, or risks. + CodeRabbit uses its documented incremental or full-review command. +3. Record the phase's base and head commits locally. The base is the commit + immediately before the phase's first commit; the head is the last pushed + phase commit. Use those exact SHAs in phase-scoped review requests. +4. Keep the agent session live for one to two hours. Start a background review + watcher or a long-running `gh` command, poll it at bounded intervals, and + re-read the PR when it completes. Do not end the work session merely because + reviewers need time. +5. Use `gh` to inspect check results, submitted reviews, inline comments, and + unresolved review threads. +6. Classify every comment using the review dispositions below. A reviewer is + not presumed correct and may lack the full feature context. +7. Implement each accepted correction. Phase 3 behavior fixes return to the + affected gate, add or adjust focused coverage, and run the nearest + regressions. Phase 1 and Phase 2 fixes follow their focused verification + protocol. +8. Commit and push each coherent review-fix set. +9. Reply to accepted or disputed inline feedback in its original thread using + `gh api`. For a fix, name the commit and summarize the resulting behavior; + for a dispute, give the short technical reason no change is appropriate. + Follow the GitHub writing and attribution rules above. +10. Wait for the reviewer to resolve every actionable thread. A reply, pushed + commit, or green check does not count as resolution. +11. Proceed when required checks are green and no actionable review thread + remains unresolved. Intentionally deferred nitpicks or theoretical comments + listed in the local deferred-review file do not block the next phase. + +### Review dispositions + +- **Applicable correctness, security, compatibility, or spec issue:** verify it, + fix it, test it, reply with evidence, and wait for reviewer resolution. +- **Incorrect or not applicable:** reply in the inline thread with concrete + source, test, and specification evidence. Do not change correct code merely + to satisfy the comment. Wait for reviewer resolution; escalate to the user if + a technically answered blocking thread remains unresolved. +- **Already postponed or outside approved scope:** reply with the scope boundary + when useful and record the item in + `.todo/AWS_AURORA_BGD_DEFERRED_REVIEW_NOTES.md`. +- **Nitpick, speculative defense, or theoretical hardening with no present + failure mode:** it may remain unresolved without a reply. Record the comment, + rationale, and revisit condition in the local deferred-review file so the + user can audit the classification. +- **Ambiguous or potentially valuable but design-changing:** do not silently + defer or implement it. Ask the user for a decision. + +If a reviewer repeatedly leaves a corrected actionable thread unresolved, a +bot cannot re-evaluate, reviewers disagree about the contract, or a suggestion +requires a new design decision, stop and ask the user for direction. State the +exact thread, fix commit, verification, and remaining blocker. + +### Reviewer invocation rules + +- **Codex:** `@codex review` is the review trigger. Append natural-language + instructions for a commit range, a full PR pass, a subsystem, or a specific + risk. +- **CodeRabbit:** `@coderabbitai review` is incremental from CodeRabbit's last + review; `@coderabbitai full review` starts again across the complete PR. + `pause`, `resume`, and `help` are available controls. Do not use + `@coderabbitai resolve` to bypass the requirement that actionable feedback be + addressed and independently re-evaluated. +- **Gitar:** new PRs and pushes normally trigger it automatically. If it does + not start, mention the installed `@gitar-bot` account with a natural-language + request. Optional instructions may specify an exact commit range, ask for a + full review, and name the desired focus. Gitar's exact operational commands + (`unblock`, `auto-apply:on|off`, and `display:verbose|compact`) are not review + scope commands; do not substitute them for the natural-language request. + +### GitHub CLI commands + +```bash +export GH_REPO=sysown/proxysql +PR_NUMBER=6044 + +# Save these values at the phase boundary. PHASE_BASE is the commit immediately +# before the first phase commit; PHASE_HEAD is the final pushed phase commit. +PHASE_BASE='replace-with-full-base-sha' +PHASE_HEAD='replace-with-full-head-sha' + +# First inspect whether each reviewer already started. Do not send duplicate +# requests to reviewers that are already running or have reviewed PHASE_HEAD. +gh pr view "$PR_NUMBER" --comments +gh api --paginate "repos/{owner}/{repo}/pulls/$PR_NUMBER/reviews" + +# Codex: the exact trigger is `@codex review`; text after it can focus the pass. +gh pr comment "$PR_NUMBER" --body \ + "@codex review commits $PHASE_BASE through $PHASE_HEAD. Focus on the current phase's approved design contract, regressions, and missing tests. + +[Drafted by Codex · AI agent]" + +# Use this form when a fresh review of the entire PR is required. +gh pr comment "$PR_NUMBER" --body \ + '@codex review the full PR. Focus on cross-phase integration, regressions, and compliance with the approved specifications. + +[Drafted by Codex · AI agent]' + +# CodeRabbit has distinct formal incremental and full-review commands. Its +# incremental command reviews changes since its previous review, not an +# arbitrary SHA supplied by us. +gh pr comment "$PR_NUMBER" --body '@coderabbitai review + +[Drafted by Codex · AI agent]' +gh pr comment "$PR_NUMBER" --body '@coderabbitai full review + +[Drafted by Codex · AI agent]' + +# Gitar accepts natural-language instructions. The installed GitHub account on +# this repository is @gitar-bot. +gh pr comment "$PR_NUMBER" --body \ + "@gitar-bot review commits $PHASE_BASE through $PHASE_HEAD. Focus on the current phase's approved design contract, regressions, and missing tests. + +[Drafted by Codex · AI agent]" + +# Use this form when a fresh review of the entire PR is required. +gh pr comment "$PR_NUMBER" --body \ + '@gitar-bot perform a full review of the PR. Focus on cross-phase integration, regressions, and compliance with the approved specifications. + +[Drafted by Codex · AI agent]' + +# Wait for CI while the review watcher below monitors reviewer activity. +gh pr checks "$PR_NUMBER" --watch --interval 60 + +# Read PR conversation, formal reviews, and inline review comments. +gh pr view "$PR_NUMBER" --comments +gh api --paginate "repos/{owner}/{repo}/pulls/$PR_NUMBER/reviews" +gh api --paginate "repos/{owner}/{repo}/pulls/$PR_NUMBER/comments" + +# Read thread-level resolution state. Re-run after every review update. +gh api graphql \ + -F owner=sysown \ + -F name=proxysql \ + -F number="$PR_NUMBER" \ + -f query='query($owner:String!,$name:String!,$number:Int!){ + repository(owner:$owner,name:$name){ + pullRequest(number:$number){ + reviewThreads(first:100){ + nodes{ + id isResolved isOutdated + comments(first:100){nodes{id databaseId url author{login} body}} + } + } + } + } + }' + +# Reply to an inline review comment in its existing thread. +COMMENT_ID=123456789 +gh api --method POST \ + "repos/{owner}/{repo}/pulls/$PR_NUMBER/comments/$COMMENT_ID/replies" \ + -f body='Fixed in . + +[Drafted by Codex · AI agent]' + +# Keep the agent session live while reviewers work. Run this watcher in a +# persistent/background command, poll its PID/log at least once per minute, +# and perform the full reads above again when it exits. +REVIEW_WATCH_LOG=.todo/aws-aurora-bgd-review-watch.log +( + deadline=$(( $(date +%s) + 7200 )) + while [ "$(date +%s)" -lt "$deadline" ]; do + date -u +'%Y-%m-%dT%H:%M:%SZ' + gh pr checks "$PR_NUMBER" || true + gh api "repos/{owner}/{repo}/pulls/$PR_NUMBER/reviews" --jq 'length' + gh api "repos/{owner}/{repo}/pulls/$PR_NUMBER/comments" --jq 'length' + sleep 60 + done +) >"$REVIEW_WATCH_LOG" 2>&1 & +REVIEW_WATCH_PID=$! + +git status --short --branch +git log --oneline upstream/v3.0..HEAD +``` + +## Phase 1: Configuration, Runtime Status, and Cluster Sync + +### Deliverable + +The configured Aurora table contains the two nullable green hostgroups; the +runtime table contains those fields plus node-local `bgd_status`; every load, +save, upgrade, config-file, and cluster-sync path follows the approved column +projection. + +### Primary files + +- `include/ProxySQL_Admin_Tables_Definitions.h`: Admin, runtime, and upgrade + schema definitions. +- `include/MySQL_HostGroups_Manager.h`: HGM table schema, `AWS_Aurora_Info` + configured fields, and status publication API. +- `lib/MySQL_HostGroups_Manager.cpp`: row parsing, validation, runtime ownership, + resultset checksum, status preservation, and HGM table materialization. +- `lib/ProxySQL_Admin.cpp`: LOAD and SAVE projections and runtime table dump. +- `lib/ProxySQL_Config.cpp`: configuration-file import and export projections. +- ProxySQL Cluster Aurora table checksum/fetch/insert code located through + `mysql_aws_aurora_hostgroups` references in `lib/ProxySQL_Admin.cpp` and the + cluster synchronization TAP coverage. +- `test/tap/tests/unit/config_write_unit-t.cpp`: configuration export coverage. +- `test/tap/tests/test_cluster_sync-t.cpp`: Aurora configured-column cluster + synchronization coverage. + +### Interfaces produced for later phases + +```cpp +// SQL NULL is represented internally by -1. +int AWS_Aurora_Info::green_writer_hostgroup; +int AWS_Aurora_Info::green_reader_hostgroup; + +// Publishes only the node-local runtime value for one user-created Aurora row. +void MySQL_HostGroups_Manager::update_aws_aurora_bgd_status( + int writer_hostgroup, + const std::string& bgd_status +); +``` + +The Phase 3 Aurora FSM publishes only these values through the runtime status +API: + +```text +NONE +AVAILABLE +SWITCHOVER_INITIATED +SWITCHOVER_IN_PROGRESS +SWITCHOVER_IN_POST_PROCESSING +SWITCHOVER_COMPLETED +``` + +### Task 1.1: Schema and validation + +- [x] Add the new Admin/HGM/runtime definitions and in-place disk schema + migration on ProxySQL startup. +- [x] Add a focused schema test that asserts configured and runtime column + order, `NULL` defaults, paired-null validation, four-hostgroup uniqueness, + and runtime-only `bgd_status`. +- [x] Verify per-row isolation: report each invalid writer hostgroup and its + conflicting fields, atomically publish the filtered valid set, and safely + remove any previously active worker whose row becomes invalid. +- [x] Run the focused test and the Admin table unit suite. +- [x] Commit as `feat: add Aurora BGD hostgroup schema`. + +### Task 1.2: Runtime ownership and status publication + +- [x] Extend `AWS_Aurora_Info`, its constructor/update paths, HGM parsing, and + runtime materialization. Implement `update_aws_aurora_bgd_status()`. +- [x] Add focused tests for loading both configured values, preserving SQL + `NULL` as `-1`, initializing new runtime rows to `NONE`, preserving status on + an unrelated reload, covering reload-before-status and status-before-reload + ordering, and removing status with the owning row. +- [x] Run the focused tests and ordinary Aurora monitor regressions. +- [x] Commit as `feat: publish Aurora BGD runtime status`. + +### Task 1.3: Persistence and configuration-file projections + +- [x] Replace relevant `SELECT *` and positional copies with explicit + configured-column projections in Admin, HGM, and ProxySQL_Config paths. +- [x] Add round-trip tests proving both green hostgroups survive + memory/runtime/disk/config-file operations while `bgd_status` never enters + persistent configuration. +- [x] Run `config_write_unit-t` plus focused LOAD/SAVE round trips. +- [x] Commit as `feat: persist Aurora BGD configuration fields`. + +### Task 1.4: ProxySQL Cluster synchronization + +- [x] Include both green hostgroups in cluster checksum/fetch/insert projections + and exclude `bgd_status`. +- [x] Extend the Aurora block in `test_cluster_sync-t.cpp` to synchronize one + configured green pair with a non-NULL comment and one paired-NULL row with a + NULL comment. +- [x] Verify configured values converge. Defer peer-local status retention to + Phase 3, when worker-driven status transitions exist. +- [x] Commit as `feat: synchronize Aurora BGD configuration`. + +### Phase 1 finalization + +- [x] Require the canonical 17-column Aurora candidate projection and remove + the unsupported legacy-normalization test while retaining all row and + cross-row validation. +- [x] Keep `BQE1()` generic. Copy `mysql_aws_aurora_hostgroups` between main + and disk with dedicated, explicit configured-column statements. +- [x] Restore the established Aurora monitor lifecycle invariant. Test runtime + table merging through a helper with explicit dependencies rather than + bypassing monitor publication when `GloMyMon` is NULL. +- [x] Keep the Phase 1 status publisher as the specified string interface but + remove its duplicate vocabulary whitelist and defensive invalid-input test; + the Phase 3 Aurora FSM owns the typed status vocabulary. +- [x] Replace raw `calloc(sizeof(ProxySQL_Admin))` member-call fixtures and the + uncontrolled reload/status race with legitimate fixtures and deterministic + ordering coverage. +- [x] Cover disk persistence with current main/disk schemas, one configured + green pair, one paired-NULL row, and no ignored SQLite errors. +- [x] Consolidate Aurora cluster synchronization into one flow containing a + configured-green/non-NULL-comment row and a paired-NULL/NULL-comment row. + Remove the Admin-mirror-only status test. +- [x] In Phase 3, after worker-driven transitions exist, add the genuine + two-peer test: publish different HGM statuses, perform configuration sync, + and verify that both node-local statuses remain unchanged. +- [x] Run the focused unit tests, cluster integration test, ordinary Aurora + regression coverage, full Phase 1 CI-equivalent checks, and `git diff --check`. + +### Phase 1 acceptance + +- All schema, upgrade, LOAD/SAVE, config-file, and cluster tests pass. +- Ordinary Aurora configuration remains backward compatible with both new + values `NULL`. +- `git diff --check` is clean. +- Push, request reviews, and complete the phase review gate before Phase 2. + +## Phase 2: Shared AWS Simulator Services + +### Deliverable + +Both RDS BGD and Aurora BGD use the shared `AWS_BGD_*` topology service; +ordinary Aurora and Aurora BGD use the accepted-backend-to-replica-set service; +the existing ordinary Aurora JSON suite and RDS BGD TAP suite retain their +semantics. + +### Primary files + +- `include/SQLite3_Server.h` and `src/SQLite3_Server.cpp`: TEST-mode tables, + accepted backend extraction, production-query interception, response routing, + errors, and probe logs. +- `test/tap/tap/cluster_simulator.h` and `.cpp`: shared SQLite control client. +- Create `test/tap/tap/bgd_simulator.h` and `.cpp`: engine-neutral topology, + Aurora replica, read-only, probe-log, transaction, and cleanup operations. +- `test/tap/tap/rds_bgd_simulator.h` and `.cpp`: retain only RDS deployment + fixture types and consume `BGD_Simulator`. +- Create `test/tap/tap/aurora_bgd_simulator.h` and `.cpp`: Aurora endpoint, + member, membership-set, and rename fixture types. +- `test/deps/cluster_simulator/lib/aurora_utils.h` and `.cpp`: publish existing + JSON payload state into the new replica service without changing JSON. +- `test/deps/cluster_simulator/cluster_simulator.cpp`: pass + `CLUSTER_SIM_HOST_FILE` state publication through the new service. +- `test/tap/groups/cluster_sim_aurora/add-hosts`: fixed Aurora BGD aliases. +- `test/tap/tests/Makefile` and `test/tap/groups/groups.json`: build and register + focused contracts and later Aurora BGD tests in the existing group. + +### Interfaces produced for Phase 3 + +```cpp +struct BGD_Topology_Row { + std::string id; + std::string endpoint; + int port; + std::string role; + std::string status; +}; + +struct Aurora_Replica_Row { + std::string server_id; + std::string session_id; + double cpu; + std::string last_update_timestamp; + double replica_lag_in_milliseconds; + bool is_current; +}; + +class BGD_Simulator : public Cluster_Simulator { +public: + int topology_update(std::vector, std::vector); + int topology_delete(std::vector); + int topology_drop(std::vector); + int topology_error(std::vector, int, std::string); + int replica_update( + std::string replica_set_id, + std::vector rows, + std::vector backends + ); + int replica_drop(std::vector); + int replica_error(std::vector, int, std::string); + int cleanup(); +}; +``` + +Probe-log checkpoint/read/wait interfaces expose topology and replica probe +kind, accepted backend IP/port, TLS state, and replica-set identifier exactly +as defined by the simulator specification. + +### Task 2.1: Shared topology service rename + +- [x] Rename tables, query handling, and the generic helper to `AWS_BGD_*` and + `BGD_Simulator`; keep RDS deployment fixtures engine-specific. +- [x] Run all 22 `test_rds_bgd_*-t` binaries. +- [x] Commit as `refactor: share AWS BGD simulator topology`. + +### Task 2.2: Aurora replica service + +- [x] Implement `AWS_AURORA_REPLICA_CONTROL`, + `AWS_AURORA_REPLICA_PROBE_LOG`, and the new `REPLICA_HOST_STATUS` schema. +- [x] Intercept only recognized production Aurora monitor queries and retain + their requested result columns/filtering/order. +- [x] Verify the service through the unchanged ordinary Aurora production-query + scenarios and the existing RDS BGD consumer suite. +- [x] Commit as `feat: add Aurora replica simulator service`. + +### Task 2.3: Simulator helper and Aurora fixtures + +- [x] Implement the transactional `BGD_Simulator` replica, error, log, endpoint + predicate, and cleanup APIs plus the Aurora model types. +- [x] Build the helper APIs and model types for use by the Phase 3 interactive + Aurora BGD scenarios; do not add tests of the simulator helper itself. +- [x] Commit as `test: add Aurora BGD simulator controls`. + +### Task 2.4: Ordinary Aurora compatibility adapter + +- [x] Keep JSON unchanged; use each `DOMAIN_NAME` as its `REPLICA_SET_ID`, set + `IS_CURRENT=1`, and atomically map all member IPs from the host file. +- [x] Run `test_cluster_sim_aurora-t` and confirm all 11 payload files and 33 + scenario objects pass. +- [x] Commit as `test: migrate Aurora scenarios to replica sets`. + +### Phase 2 acceptance + +- Focused simulator contracts pass under the intended feature flags. +- All 33 ordinary Aurora cases pass with unchanged JSON schemas. +- All 22 RDS Multi-AZ BGD TAP binaries pass. +- Combined `testall` builds with no new simulator feature flag or group. +- `git diff --check` is clean. +- Push, request reviews, and complete the phase review gate before Phase 3. +- Phase 2 is the prerequisite completion point. Strict red-green-refactor does + not begin until this review gate is complete. + +## Phase 3: Aurora Monitor Loop and FSM — Gated Test-After Development + +### Deliverable + +The existing per-writer Aurora worker owns ordinary Aurora monitoring and the +Aurora BGD FSM, drives the three probes, maps every target member, applies +idempotent routing actions, and reaches the completed rearm latch exactly as +defined by the approved monitor/FSM specification. + +### Primary files + +- `include/MySQL_Monitor.hpp`: Aurora BGD state, snapshots, pairs, action flags, + probe state, and monitor method declarations. +- `lib/MySQL_Monitor.cpp`: worker integration, probe scheduling, topology and + membership parsing, transitions, routing, rollback, reload, and teardown. +- `include/MySQL_HostGroups_Manager.h` and + `lib/MySQL_HostGroups_Manager.cpp`: reuse existing writer/reader placement and + connection-drain behavior; add only narrowly required Aurora BGD actions. +- `include/DNS_Cache.hpp` and `lib/DNS_Cache.cpp`: reuse RDS BGD pin insertion + and explicit removal behavior. +- `test/tap/tap/aurora_bgd_tap.h`: Aurora deployment, simulator publication, + configuration, status, and probe primitives. +- `test/tap/tap/aurora_bgd_scenario_tap.h`: shared scenario setup, routing, + placement, runtime-row, status, and pool helpers. +- Focused `test/tap/tests/test_aurora_bgd_*-t.cpp` scenario binaries, registered + together with `test_cluster_sim_aurora-t` in `cluster_sim_aurora-g1`. + +### Simulator scenario organization + +The Aurora BGD scripts follow the corresponding RDS BGD scripts: one focused +domain responsibility per binary, coherent assertions grouped in named test +functions, setup failures reported as diagnostics, and an exact TAP plan in +`main()`. + +```cpp +struct TestState { + // One deployment and only the state needed by this script. +}; + +int test_(Context&, TestState&); + +int main() { + plan(); + // Connect once, run focused functions, clean up, return exit_status(). +} +``` + +Use descriptive snake_case function names. Name state members for their domain +objects and name booleans for the condition they represent. A function may use +several TAP assertions when they describe one coherent result. Compound SQL +checks may feed one assertion when the equivalent RDS test treats the combined +state as one outcome. + +The simulator suite stays within these boundaries: + +- Do not modify production behavior merely to satisfy a simulator scenario. If + a scenario reveals a production gap, stop and review it with the user. +- Do not test the simulator framework itself. +- Do not start a second ProxySQL process or add Aurora-specific ClusterSync + coverage; the generic ClusterSync unit and integration coverage is sufficient. +- Do not invent Aurora domain cases without an RDS BGD analogue. Adapt shared + RDS cases only where Aurora's three probes, all-member mapping, or completion + latch requires an Aurora-specific observation. +- Do not model membership changes after `SWITCHOVER_INITIATED`; AWS prevents + cluster membership changes after switchover begins. +- Do not duplicate persistence, generic ClusterSync, or simulator-service + behavior already covered by their owning tests. +- Keep exhaustive malformed-metadata matrices, endpoint rename/IP-churn chains, + forced ordinary-probe failures, held clients across cutover, and global + auto-discovery toggles outside this suite. The focused RDS-shaped scenarios + cover the approved Aurora domain contract. + +### Focused simulator scripts + +| Script | TAP plan | Focus and externally observable assertions | +| --- | ---: | --- | +| `test_aurora_bgd_smoke-t.cpp` | 3 | AVAILABLE is published while ordinary production probing remains active; the worker probes a target-cluster endpoint and its target replica set. | +| `test_aurora_bgd_automatic_discovery-t.cpp` | 5 | Paired-NULL green hostgroups admit discovery without generating configuration; absent topology remains `NONE`; AVAILABLE is later discovered; repeated polling keeps one runtime row. | +| `test_aurora_bgd_probe_tls-t.cpp` | 2 | TLS production rows produce encrypted topology and target-membership probes. | +| `test_aurora_bgd_writer_switchover-t.cpp` | 12 | AVAILABLE and INITIATED preserve placement; IN_PROGRESS demotes only the source writer; POST_PROCESSING pins mapped members, drains the old writer free pool, and routes writer traffic to the mapped target; repetition is idempotent. | +| `test_aurora_bgd_reader_policy-t.cpp` | 2 | Mapped source readers remain ONLINE and reader-hostgroup routes reach their mapped targets during POST_PROCESSING. | +| `test_aurora_bgd_green_pool_cleanup-t.cpp` | 5 | Rollback preserves green pools; completion drains ONLINE and SHUNNED green pools; OFFLINE_SOFT and OFFLINE_HARD pools and configured rows remain present. | +| `test_aurora_bgd_reader_switchover_cleanup-t.cpp` | 7 | Completion restores canonical writer placement, removes pins, resumes the ordinary probe, publishes `SWITCHOVER_COMPLETED`, and does not replay cleanup; a successful empty topology rearms to `NONE`. | +| `test_aurora_bgd_late_entry_writer_phases-t.cpp` | 7 | First observation of INITIATED, IN_PROGRESS, or POST_PROCESSING applies only the effects belonging to that observed phase. | +| `test_aurora_bgd_late_entry_completed-t.cpp` | 5 | First observation of COMPLETED performs no-op cleanup when no effects exist, creates no pins, publishes the completion latch, and rearms after a successful empty topology. | +| `test_aurora_bgd_rollback-t.cpp` | 8 | Rollback from INITIATED, IN_PROGRESS, and POST_PROCESSING restores the effects applied by each phase and resumes ordinary probing. | +| `test_aurora_bgd_topology_empty_absent-t.cpp` | 4 | Present-but-empty topology and a successful result without the deployment cancel an active pre-completion switchover, restore the writer, publish `NONE`, and keep probing live. | +| `test_aurora_bgd_topology_errors-t.cpp` | 2 | Topology and target-membership query errors retain active state, applied placement, and the last complete member map. | +| `test_aurora_bgd_worker_config_refresh-t.cpp` | 4 | An unrelated server reload and an in-place Aurora configuration refresh preserve active status, cached membership, and applied pins. | +| `test_aurora_bgd_config_refresh_after_completion-t.cpp` | 3 | Completion directly from IN_PROGRESS restores the writer; configuration refresh preserves the completion latch; an empty topology still rearms it. | +| `test_aurora_bgd_disable_during_switchover-t.cpp` | 4 | Deactivating the owning Aurora row during IN_PROGRESS restores canonical placement and publishes the inactive runtime row in `NONE`. | +| `test_aurora_bgd_remove_during_switchover-t.cpp` | 5 | Removing the owning Aurora row during IN_PROGRESS restores canonical placement and removes the runtime row. | +| `test_aurora_bgd_repeated_deployment-t.cpp` | 4 | A completed worker accepts a different deployment fingerprint, uses only the new member map and pins, routes to the new target, and cleans up without stale-map reuse. | +| `test_aurora_bgd_concurrent_isolation-t.cpp` | 9 | Three writer workers independently reach AVAILABLE, advance through different phases, and preserve each other's state, placement, and pins. | + +The group contains 91 Aurora BGD assertions across these 18 scripts plus the +unchanged ordinary Aurora regression. It uses the existing +`cluster_sim_aurora` environment and one ProxySQL process. + +### Gate 3.1: Discovery, mapping, and three-probe ownership + +- [x] Add per-writer Aurora BGD worker state and schedule the ordinary Aurora, + topology, and target-membership probes in `NONE`/`AVAILABLE`. +- [x] Implement the last-complete snapshot, normalized `SERVER_ID` pairing, + reader session continuity, cached IPs, and fail-closed validation. +- [x] Add the smoke, automatic-discovery, and probe-TLS scripts to characterize + AVAILABLE discovery, user-created paired-NULL configuration, probe ownership, + target membership, and TLS inheritance. +- [x] Keep ordinary Aurora's 33 scenarios as characterization coverage for + production membership parsing and placement. +- [x] Commit as `feat: add Aurora BGD discovery and member mapping`. + +### Gate 3.2: Active switchover routing + +- [x] Implement status publication, production-probe suspension, rollback-state + capture, fast cadence, and RDS-style idempotent writer demotion. +- [x] Implement idempotent per-member pinning, connection retirement, monitor + pool purge, canonical writer restoration, and unchanged reader eligibility. +- [x] Add the writer-switchover and reader-policy scripts to characterize + placement, pinning, pool retirement, target routing, reader eligibility, and + repeated POST_PROCESSING behavior. +- [x] Treat each per-member flag as action-applied bookkeeping; do not wait for + asynchronously retired used connections to close. +- [x] Assert no target-writability gate and no reader shun/unshun behavior. +- [x] Commit as `feat: handle Aurora BGD active switchover`. + +### Gate 3.3: Completion cleanup and terminal latch + +- [x] Implement immediate effect-driven cleanup using the existing map and + worker state, including writer reconciliation, DNS-entry removal, + production-probe resume, configured cadence, `SWITCHOVER_COMPLETED` + publication, fingerprint retention, and successful-drain transition to + `NONE`. +- [x] Add the green-pool-cleanup, reader-switchover-cleanup, + late-entry-completed, config-refresh-after-completion, and + repeated-deployment scripts to characterize effect-driven cleanup and rearm. +- [x] Verify cleanup replays no skipped phase, waits for neither DNS verification + nor physical connection closure, and never repeats for the same completed + result. +- [x] Commit as `feat: complete Aurora BGD switchover`. + +### Gate 3.4: Rollback, reload, removal, and concurrency + +- [x] Implement idempotent rollback and lifecycle preservation/cleanup using + per-writer state and per-member action flags. +- [x] Add the late-entry-writer-phases, rollback, topology-empty-absent, + topology-errors, worker-config-refresh, disable-during-switchover, + remove-during-switchover, and concurrent-isolation scripts. +- [x] Verify errors retain state and never masquerade as cancellation, + reader-less membership, topology drain, or completion. +- [x] Commit as `feat: harden Aurora BGD lifecycle handling`. + +### Phase 3 acceptance + +- Every monitor/FSM requirement maps to a focused scenario whose expected + observations were recorded before its implementation gate. +- The 18 Aurora BGD scripts and ordinary Aurora regression are registered only + in `cluster_sim_aurora-g1`, and the group runs with one ProxySQL process. +- All Aurora BGD, ordinary Aurora, RDS BGD, read-only, DNS, pool-retirement, + configuration, and cluster-sync regressions pass. +- Combined simulator CI passes and `git diff --check` is clean. +- Push, request reviews, and complete the phase review gate. + +## Final Real-Aurora User Acceptance Gate + +Run this only after Phase 3 reviews are resolved. Use the existing observation +scripts as the direct-Aurora evidence harness and add a separate ProxySQL-path +driver rather than changing the recorded 2026-07-30 result. + +The acceptance run must verify: + +1. ProxySQL publishes every expected `bgd_status` transition. +2. Ordinary production Aurora probes stop during the three active phases and + resume after cleanup. +3. The old writer is not used for writes after fencing and no dual-writer + routing interval is observed. +4. Writer and reader client traffic reaches promoted target members using the + cached mapping during canonical DNS disruption. +5. Established connections, free pools, and retired connections follow the + designed cutover behavior. +6. All traffic pins are removed at TARGET completion and the FSM returns to + `NONE` only after a successful topology drain observation. +7. A separate transaction-enabled run records application-visible transaction + errors and confirms ProxySQL does not transparently replay an active + transaction. + +Archive timestamps, ProxySQL logs, runtime/Admin snapshots, backend identities, +DNS results, and client outcomes. Compare invariants and ordering, not exact +durations, with `RESULTS-20260730T073724Z.md`. + +If the acceptance run passes, push the evidence-backed fixes, complete one final +review cycle, convert the draft PR to ready for review, and merge only after +required human approval and CI success. If it fails because AWS contradicts an +assumption, keep the PR draft and return to design review before changing code. diff --git a/test/tap/groups/groups.json b/test/tap/groups/groups.json index 1035f09df8..c6350c63af 100644 --- a/test/tap/groups/groups.json +++ b/test/tap/groups/groups.json @@ -348,14 +348,24 @@ "test_PROXY_Protocol-t" : [ "legacy-g3","mysql-auto_increment_delay_multiplex=0-g3","mysql-multiplexing=false-g3","mysql-query_digests=0-g3","mysql-query_digests_keep_comment=1-g3","mysql84-g3","mysql90-g3","mysql95-g3" ], "test_admin_stats-t" : [ "legacy-g7","mysql-auto_increment_delay_multiplex=0-g2","mysql-multiplexing=false-g2","mysql-query_digests=0-g2","mysql-query_digests_keep_comment=1-g2","mysql84-g7","mysql90-g2","mysql95-g2" ], "test_ansi_quotes_group_replication-t" : [ "mysql84-gr-g1","mysql90-gr-g1","mysql91-gr-g1","mysql92-gr-g1","mysql93-gr-g1","mysql95-gr-g1" ], - "test_aurora_bgd_active-t" : [ "cluster_sim_aurora-g1" ], - "test_aurora_bgd_cluster_sync-t" : [ "cluster_sim_aurora-g1" ], - "test_aurora_bgd_completion-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_automatic_discovery-t" : [ "cluster_sim_aurora-g1" ], "test_aurora_bgd_concurrent_isolation-t" : [ "cluster_sim_aurora-g1" ], - "test_aurora_bgd_discovery-t" : [ "cluster_sim_aurora-g1" ], - "test_aurora_bgd_error_recovery-t" : [ "cluster_sim_aurora-g1" ], - "test_aurora_bgd_late_entry-t" : [ "cluster_sim_aurora-g1" ], - "test_aurora_bgd_worker_lifecycle-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_config_refresh_after_completion-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_disable_during_switchover-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_green_pool_cleanup-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_late_entry_completed-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_late_entry_writer_phases-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_probe_tls-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_reader_policy-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_reader_switchover_cleanup-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_remove_during_switchover-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_repeated_deployment-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_rollback-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_smoke-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_topology_empty_absent-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_topology_errors-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_worker_config_refresh-t" : [ "cluster_sim_aurora-g1" ], + "test_aurora_bgd_writer_switchover-t" : [ "cluster_sim_aurora-g1" ], "test_auth_methods-t" : [ "mysql-auto_increment_delay_multiplex=0-g2","mysql-multiplexing=false-g2","mysql-query_digests=0-g2","mysql-query_digests_keep_comment=1-g2","mysql84-g7","mysql90-g2","mysql95-g2" ], "test_auto_increment_delay_multiplex-t" : [ "legacy-g7","mysql-auto_increment_delay_multiplex=0-g2","mysql-multiplexing=false-g2","mysql-query_digests=0-g2","mysql-query_digests_keep_comment=1-g2","mysql84-g7","mysql90-g2","mysql95-g2" ], "test_backend_conn_ping-t" : [ "legacy-g7","mysql-auto_increment_delay_multiplex=0-g2","mysql-multiplexing=false-g2","mysql-query_digests=0-g2","mysql-query_digests_keep_comment=1-g2","mysql84-g7","mysql90-g2","mysql95-g2" ], diff --git a/test/tap/tap/aurora_bgd_scenario_tap.h b/test/tap/tap/aurora_bgd_scenario_tap.h new file mode 100644 index 0000000000..821f4d8d1e --- /dev/null +++ b/test/tap/tap/aurora_bgd_scenario_tap.h @@ -0,0 +1,344 @@ +#ifndef TAP_TESTS_AURORA_BGD_SCENARIO_TAP_H +#define TAP_TESTS_AURORA_BGD_SCENARIO_TAP_H + +#include +#include +#include +#include + +#include "aurora_bgd_tap.h" +#include "command_line.h" +#include "utils.h" + +namespace aurora_bgd_scenario { + +using std::string; +using std::to_string; +using std::vector; + +const uint32_t kWaitSeconds = 5; +const uint32_t kProbeTimeoutMs = 5000; + +struct Context { + MYSQL* admin { nullptr }; + BGD_Simulator simulator; +}; + +inline bool scalar_is(MYSQL* admin, const string& query, const string& expected) { + auto [rc, rows] = mysql_query_ext_rows(admin, query); + return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 + && rows.front().front() == expected; +} + +inline int setup(CommandLine& cl, Context& context) { + if (cl.getEnv()) { + diag("Error: failed to load TAP environment"); + return EXIT_FAILURE; + } + context.admin = init_mysql_conn( + cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); + if (context.admin == nullptr) { + diag("Error: failed to connect to ProxySQL Admin"); + return EXIT_FAILURE; + } + char simulator_username[] = "aurora1"; + char simulator_password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. + if (context.simulator.connect( + cl.host, 3306, simulator_username, simulator_password) != EXIT_SUCCESS) { + diag("Error: failed to connect to the shared AWS simulator"); + mysql_close(context.admin); + context.admin = nullptr; + return EXIT_FAILURE; + } + if (aurora_bgd_admin_cleanup(context.admin) != EXIT_SUCCESS + || context.simulator.cleanup() != EXIT_SUCCESS) { + diag("Error: failed to clear prior Aurora BGD state"); + return EXIT_FAILURE; + } + return aurora_bgd_execute_all(context.admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " + "VALUES ('testuser','testuser',1,0,1)", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +inline int reset(Context& context) { + return aurora_bgd_admin_cleanup(context.admin) == EXIT_SUCCESS + && context.simulator.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; +} + +inline int cleanup(Context& context) { + int reset_rc = reset(context); + int user_rc = context.admin == nullptr ? EXIT_FAILURE : aurora_bgd_execute_all(context.admin, { + "DELETE FROM mysql_users WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); + if (context.admin != nullptr) { + mysql_close(context.admin); + context.admin = nullptr; + } + return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +inline int configure( + Context& context, Aurora_BGD_Test_Deployment& deployment, + int writer_hostgroup, int reader_hostgroup, + int green_writer_hostgroup, int green_reader_hostgroup, + bool automatic = false, int check_interval_ms = 100, + bool writer_is_also_reader = false, bool use_ssl = false +) { + return aurora_bgd_admin_setup( + context.admin, deployment, writer_hostgroup, reader_hostgroup, + green_writer_hostgroup, green_reader_hostgroup, automatic, + check_interval_ms, writer_is_also_reader, use_ssl); +} + +inline int publish_available( + Context& context, Aurora_BGD_Test_Deployment& deployment +) { + return aurora_bgd_publish(context.simulator, deployment); +} + +inline int publish_status( + Context& context, Aurora_BGD_Test_Deployment& deployment, const string& status +) { + return context.simulator.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)); +} + +inline int publish_initial( + Context& context, Aurora_BGD_Test_Deployment& deployment, + const string& status +) { + return context.simulator.replica_update( + deployment.production.replica_set_id, + deployment.production.replica_rows(), deployment.production.backends()) == EXIT_SUCCESS + && context.simulator.replica_update( + deployment.target.replica_set_id, + deployment.target.replica_rows(), deployment.target.backends()) == EXIT_SUCCESS + && context.simulator.topology_update( + aurora_bgd_topology_backends(deployment), + aurora_bgd_topology(deployment, status)) == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +inline int publish_completed( + Context& context, Aurora_BGD_Test_Deployment& serving_deployment, + Aurora_BGD_Test_Deployment& completed_deployment +) { + return context.simulator.topology_update( + aurora_bgd_topology_backends(serving_deployment), + aurora_bgd_completed_topology(completed_deployment)); +} + +inline bool runtime_status_is( + MYSQL* admin, int writer_hostgroup, const string& status +) { + return scalar_is( + admin, + "SELECT COUNT(*) FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hostgroup) + " AND bgd_status=" + aurora_bgd_sql_quote(status), + "1"); +} + +inline bool server_count( + MYSQL* admin, const string& table, int hostgroup, const string& hostname, + int expected, const string& status = "" +) { + string query = + "SELECT COUNT(*) FROM " + table + " WHERE hostgroup_id=" + + to_string(hostgroup) + " AND hostname=" + aurora_bgd_sql_quote(hostname); + if (!status.empty()) { + query += " AND status=" + aurora_bgd_sql_quote(status); + } + return scalar_is(admin, query, to_string(expected)); +} + +inline int wait_for_writer_placement( + MYSQL* admin, int writer_hostgroup, int reader_hostgroup, + const string& hostname, bool demoted +) { + string query = + "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(writer_hostgroup) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "0" : "1") + ") AND " + "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + + to_string(reader_hostgroup) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + + ")=" + (demoted ? "1" : "0") + ")"; + return wait_for_cond(admin, query, kWaitSeconds); +} + +inline bool writer_placement( + MYSQL* admin, int writer_hostgroup, int reader_hostgroup, + const string& hostname, bool demoted +) { + return wait_for_writer_placement( + admin, writer_hostgroup, reader_hostgroup, hostname, demoted) == EXIT_SUCCESS; +} + +inline int set_default_hostgroup(MYSQL* admin, int hostgroup) { + return aurora_bgd_execute_all(admin, { + "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + + " WHERE username='testuser'", + "LOAD MYSQL USERS TO RUNTIME", + }); +} + +inline int add_route( + MYSQL* admin, int hostgroup, const string& hostname, + const string& status = "ONLINE", bool use_ssl = true, + const string& comment = "Aurora BGD scenario route" +) { + return aurora_bgd_execute_all(admin, { + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + + to_string(hostgroup) + "," + aurora_bgd_sql_quote(hostname) + + ",3306," + aurora_bgd_sql_quote(status) + "," + + string(use_ssl ? "1" : "0") + "," + aurora_bgd_sql_quote(comment) + ")", + "LOAD MYSQL SERVERS TO RUNTIME", + }); +} + +inline int add_member_routes( + MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, + const vector& hostgroups +) { + if (hostgroups.size() != deployment.production.members.size()) { + diag("Member-route hostgroup count does not match production membership"); + return EXIT_FAILURE; + } + vector queries; + for (size_t i = 0; i < hostgroups.size(); ++i) { + queries.push_back( + "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + + to_string(hostgroups[i]) + "," + + aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + + ",3306,'ONLINE',1,'Aurora BGD scenario member route')"); + } + queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); + return aurora_bgd_execute_all(admin, queries); +} + +inline bool route_to_backend( + CommandLine& cl, Context& context, const Endpoint& expected_backend +) { + auto [sequence_rc, sequence] = context.simulator.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS) { + return false; + } + MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); + if (client == nullptr) { + return false; + } + auto [query_rc, rows] = mysql_query_ext_rows(client, kAuroraBGDRouteProbeQuery); + (void)rows; + if (query_rc != EXIT_SUCCESS) { + diag("Backend routing query failed with MySQL error %d: %s", + mysql_errno(client), mysql_error(client)); + mysql_close(client); + return false; + } + mysql_close(client); + return aurora_bgd_routing_probe_reached( + context.simulator, sequence, expected_backend); +} + +inline bool route_members( + CommandLine& cl, Context& context, Aurora_BGD_Test_Deployment& deployment, + const vector& hostgroups, bool target +) { + if (hostgroups.size() != deployment.production.members.size()) { + return false; + } + for (size_t i = 0; i < hostgroups.size(); ++i) { + if (set_default_hostgroup(context.admin, hostgroups[i]) != EXIT_SUCCESS) { + return false; + } + const Endpoint expected = target + ? deployment.target.members[i].endpoint.backend() + : deployment.production.members[i].endpoint.backend(); + if (!route_to_backend(cl, context, expected)) { + return false; + } + } + return true; +} + +inline int64_t pool_connections(MYSQL* admin, int hostgroup) { + auto [rc, rows] = mysql_query_ext_rows( + admin, + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool " + "WHERE hostgroup=" + to_string(hostgroup)); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { + return -1; + } + return strtoll(rows.front().front().c_str(), nullptr, 10); +} + +inline int64_t pool_connections_for_hostname(MYSQL* admin, const string& hostname) { + auto [rc, rows] = mysql_query_ext_rows( + admin, + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool " + "WHERE srv_host=" + aurora_bgd_sql_quote(hostname)); + if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { + return -1; + } + return strtoll(rows.front().front().c_str(), nullptr, 10); +} + +inline int wait_for_hostname_pool_count( + MYSQL* admin, const string& hostname, const string& comparison +) { + return wait_for_cond( + admin, + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0)" + comparison + + " FROM stats_mysql_connection_pool WHERE srv_host=" + + aurora_bgd_sql_quote(hostname), + kWaitSeconds); +} + +inline int wait_for_pool_count( + MYSQL* admin, int hostgroup, const string& comparison +) { + return wait_for_cond( + admin, + "SELECT COALESCE(SUM(ConnUsed+ConnFree),0)" + comparison + + " FROM stats_mysql_connection_pool WHERE hostgroup=" + to_string(hostgroup), + kWaitSeconds); +} + +inline bool ordinary_probe_reached( + Context& context, uint64_t sequence, + Aurora_BGD_Test_Deployment& deployment, int encrypted = -1 +) { + for (Aurora_BGD_Member& member : deployment.production.members) { + auto [rc, log] = context.simulator.wait_for_replica_probe_log( + sequence, member.endpoint.backend(), Aurora_Replica_Probe_Kind::ordinary, + kProbeTimeoutMs, encrypted, deployment.production.replica_set_id); + if (rc == EXIT_SUCCESS) { + return true; + } + } + return false; +} + +inline bool membership_probe_reached( + Context& context, uint64_t sequence, + Aurora_BGD_Test_Deployment& deployment, int encrypted = -1 +) { + for (Aurora_BGD_Member& member : deployment.target.members) { + auto [rc, log] = context.simulator.wait_for_replica_probe_log( + sequence, member.endpoint.backend(), Aurora_Replica_Probe_Kind::bgd_membership, + kProbeTimeoutMs, encrypted, deployment.target.replica_set_id); + if (rc == EXIT_SUCCESS) { + return true; + } + } + return false; +} + +} // namespace aurora_bgd_scenario + +#endif // TAP_TESTS_AURORA_BGD_SCENARIO_TAP_H diff --git a/test/tap/tests/test_aurora_bgd_active-t.cpp b/test/tap/tests/test_aurora_bgd_active-t.cpp deleted file mode 100644 index 41827ef641..0000000000 --- a/test/tap/tests/test_aurora_bgd_active-t.cpp +++ /dev/null @@ -1,928 +0,0 @@ -/** - * @file test_aurora_bgd_active-t.cpp - * @brief Aurora BGD active-state probe suspension and routing behavior. - * - * Steps: - * - * 1. Configure a complete deployment and reach AVAILABLE. - * 2. Enter INITIATED and verify placement suppression and fast target probing. - * 3. Enter IN_PROGRESS and verify one writer demotion with source routing intact. - * 4. Enter POST_PROCESSING and verify restoration, pool retirement, and target pins. - * 5. Complete a previously incomplete target snapshot during POST_PROCESSING. - * 6. Verify refreshed/renamed target identity with automatic multi-reader discovery. - */ - -#include -#include -#include -#include - -#include "aurora_bgd_tap.h" -#include "command_line.h" -#include "utils.h" - -using namespace std; - -const uint32_t kWaitSeconds = 5; -const uint32_t kProbeTimeoutMs = 5000; -const int kHeldConnectionRuleId = 153000; - -struct TestState { - Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_a() }; - int writer_hostgroup { 1530 }; - int reader_hostgroup { 1531 }; - int green_writer_hostgroup { 1532 }; - int green_reader_hostgroup { 1533 }; - vector route_hostgroups { 1534, 1535, 1536 }; - Aurora_BGD_Test_Deployment gated { aurora_bgd_deployment_b_writer_only() }; - int gated_writer_hostgroup { 1540 }; - int gated_reader_hostgroup { 1541 }; - int gated_green_writer_hostgroup { 1542 }; - int gated_green_reader_hostgroup { 1543 }; - int gated_route_hostgroup { 1544 }; - Aurora_BGD_Test_Deployment refreshed { aurora_bgd_deployment_a() }; - int refreshed_writer_hostgroup { 1545 }; - int refreshed_reader_hostgroup { 1546 }; - vector refreshed_route_hostgroups { 1547, 1548, 1549 }; - MYSQL* held_client { nullptr }; -}; - -int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { - if (cl.getEnv()) { - diag("Error: failed to load TAP environment"); - return EXIT_FAILURE; - } - admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); - if (admin == nullptr) { - diag("Error: failed to connect to ProxySQL Admin"); - return EXIT_FAILURE; - } - char simulator_username[] = "aurora1"; - char simulator_password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. - if (sim.connect(cl.host, 3306, simulator_username, simulator_password) != EXIT_SUCCESS) { - diag("Error: failed to connect to the shared AWS simulator"); - mysql_close(admin); - admin = nullptr; - return EXIT_FAILURE; - } - if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { - diag("Error: failed to clear prior Aurora BGD state"); - return EXIT_FAILURE; - } - if (aurora_bgd_execute_all(admin, { - "DELETE FROM mysql_query_rules WHERE rule_id=" + - to_string(kHeldConnectionRuleId), - "LOAD MYSQL QUERY RULES TO RUNTIME", - "DELETE FROM mysql_users WHERE username='testuser'", - "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " - "VALUES ('testuser','testuser',1,0,1)", - "LOAD MYSQL USERS TO RUNTIME", - }) != EXIT_SUCCESS) { - diag("Error: failed to configure the routing test user"); - return EXIT_FAILURE; - } - return EXIT_SUCCESS; -} - -int cleanup(MYSQL* admin, BGD_Simulator& sim) { - int admin_rc = aurora_bgd_admin_cleanup(admin); - int user_rc = admin == nullptr ? EXIT_FAILURE : aurora_bgd_execute_all(admin, { - "DELETE FROM mysql_query_rules WHERE rule_id=" + - to_string(kHeldConnectionRuleId), - "LOAD MYSQL QUERY RULES TO RUNTIME", - "DELETE FROM mysql_users WHERE username='testuser'", - "LOAD MYSQL USERS TO RUNTIME", - }); - int simulator_rc = sim.cleanup(); - if (admin) { - mysql_close(admin); - } - return admin_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS && simulator_rc == EXIT_SUCCESS - ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { - return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS - && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; -} - -bool runtime_server_count( - MYSQL* admin, int hostgroup, const string& hostname, int expected, - const string& status = "" -) { - string query = - "SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(hostgroup) + " AND hostname=" + aurora_bgd_sql_quote(hostname); - if (!status.empty()) { - query += " AND status=" + aurora_bgd_sql_quote(status); - } - auto [rc, rows] = mysql_query_ext_rows(admin, query); - return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 - && rows.front().front() == to_string(expected); -} - -int wait_for_writer_placement( - MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, - bool demoted -) { - string query = - "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + - ")=" + (demoted ? "0" : "1") + ") AND " - "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + - ")=" + (demoted ? "1" : "0") + ")"; - return wait_for_cond(admin, query, kWaitSeconds); -} - -int publish_status( - BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, - const string& status -) { - return sim.topology_update( - aurora_bgd_topology_backends(deployment), - aurora_bgd_topology(deployment, status)); -} - -bool fast_bgd_without_ordinary( - BGD_Simulator& sim, uint64_t sequence, const string& target_replica_set, - uint32_t observation_ms, uint64_t minimum_membership_probes -) { - auto [topology_seq_rc, topology_sequence] = sim.probe_log_last_sequence(); - if (topology_seq_rc != EXIT_SUCCESS) { - return false; - } - usleep(observation_ms * 1000); - auto [rc, logs] = sim.replica_probe_log_since(sequence); - auto [topology_rc, topology_logs] = sim.probe_log_since(topology_sequence); - if (rc != EXIT_SUCCESS || topology_rc != EXIT_SUCCESS) { - return false; - } - uint64_t membership_probes = 0; - for (const Aurora_Replica_Probe_Log& log : logs) { - if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { - return false; - } - if (log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership - && log.replica_set_id == target_replica_set) { - membership_probes++; - } - } - uint64_t topology_probes = 0; - for (const BGD_Probe_Log& log : topology_logs) { - topology_probes += log.probe_kind == BGD_Probe_Kind::metadata ? 1 : 0; - } - return membership_probes >= minimum_membership_probes - && topology_probes >= minimum_membership_probes; -} - -int add_member_routes( - MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, - const vector& route_hgs -) { - if (route_hgs.size() != deployment.production.members.size()) { - diag("Member-route hostgroup count does not match production membership"); - return EXIT_FAILURE; - } - vector queries; - for (size_t i = 0; i < deployment.production.members.size(); ++i) { - queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + - to_string(route_hgs[i]) + "," + - aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + - ",3306,'ONLINE',1,'Aurora BGD member route')"); - } - queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); - return aurora_bgd_execute_all(admin, queries); -} - -int set_default_hostgroup(MYSQL* admin, int hostgroup) { - return aurora_bgd_execute_all(admin, { - "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + - " WHERE username='testuser'", - "LOAD MYSQL USERS TO RUNTIME", - }); -} - -int set_held_connection_rule(MYSQL* admin, bool enabled) { - vector queries { - "DELETE FROM mysql_query_rules WHERE rule_id=" + - to_string(kHeldConnectionRuleId), - }; - if (enabled) { - queries.push_back( - "INSERT INTO mysql_query_rules" - "(rule_id,active,username,multiplex,apply,comment) VALUES (" + - to_string(kHeldConnectionRuleId) + - ",1,'testuser',0,1,'Aurora BGD held connection')"); - } - queries.push_back("LOAD MYSQL QUERY RULES TO RUNTIME"); - return aurora_bgd_execute_all(admin, queries); -} - -bool route_to_expected_backend( - CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected_backend -) { - auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); - if (sequence_rc != EXIT_SUCCESS) { - return false; - } - MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); - if (client == nullptr) { - return false; - } - auto [rc, rows] = mysql_query_ext_rows(client, kAuroraBGDRouteProbeQuery); - (void)rows; - if (rc != EXIT_SUCCESS) { - diag("Backend routing query failed with MySQL error %d: %s", - mysql_errno(client), mysql_error(client)); - mysql_close(client); - return false; - } - mysql_close(client); - return aurora_bgd_routing_probe_reached(sim, sequence, expected_backend); -} - -bool route_members_to_expected_ips( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, - Aurora_BGD_Test_Deployment& deployment, - const vector& route_hgs, bool target -) { - for (size_t i = 0; i < route_hgs.size(); ++i) { - if (set_default_hostgroup(admin, route_hgs[i]) != EXIT_SUCCESS) { - return false; - } - Endpoint expected_backend = target - ? deployment.target.members[i].endpoint.backend() - : deployment.production.members[i].endpoint.backend(); - if (!route_to_expected_backend(cl, sim, expected_backend)) { - return false; - } - } - return true; -} - -int64_t member_route_pool_count(MYSQL* admin, const vector& route_hgs) { - string hostgroups; - for (int hostgroup : route_hgs) { - if (!hostgroups.empty()) { - hostgroups += ","; - } - hostgroups += to_string(hostgroup); - } - auto [rc, rows] = mysql_query_ext_rows( - admin, - "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool " - "WHERE hostgroup IN (" + hostgroups + ")"); - if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { - return -1; - } - return strtoll(rows.front().front().c_str(), nullptr, 10); -} - -int wait_for_member_route_pool_count( - MYSQL* admin, const vector& route_hgs, const string& comparison -) { - string hostgroups; - for (int hostgroup : route_hgs) { - if (!hostgroups.empty()) { - hostgroups += ","; - } - hostgroups += to_string(hostgroup); - } - return wait_for_cond( - admin, - "SELECT COALESCE(SUM(ConnUsed+ConnFree),0)" + comparison + - " FROM stats_mysql_connection_pool WHERE hostgroup IN (" + hostgroups + ")", - kWaitSeconds); -} - -int64_t member_route_used_count(MYSQL* admin, const vector& route_hgs) { - string hostgroups; - for (int hostgroup : route_hgs) { - if (!hostgroups.empty()) { - hostgroups += ","; - } - hostgroups += to_string(hostgroup); - } - auto [rc, rows] = mysql_query_ext_rows( - admin, - "SELECT COALESCE(SUM(ConnUsed),0) FROM stats_mysql_connection_pool " - "WHERE hostgroup IN (" + hostgroups + ")"); - return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 - ? strtoll(rows.front().front().c_str(), nullptr, 10) : -1; -} - -/** Configure the active switchover scenario and reach AVAILABLE. */ -int test_bgd_status_available(MYSQL* admin, BGD_Simulator& sim, TestState& state) { - Aurora_BGD_Test_Deployment& deployment = state.deployment; - if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.writer_hostgroup, state.reader_hostgroup, - state.green_writer_hostgroup, state.green_reader_hostgroup, - false, 1000, false) != EXIT_SUCCESS - || add_member_routes(admin, deployment, state.route_hostgroups) != EXIT_SUCCESS) { - diag("Error: failed to configure the active switchover scenario"); - return EXIT_FAILURE; - } - - ok(aurora_bgd_wait_for_status( - admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, - "active scenario starts from a complete AVAILABLE snapshot"); - return EXIT_SUCCESS; -} - -/** - * Enter SWITCHOVER_INITIATED. - * - * - Keep canonical writer placement unchanged. - * - Replace ordinary Aurora probes with fast target-membership probes. - */ -int test_switchover_initiated(MYSQL* admin, BGD_Simulator& sim, TestState& state) { - Aurora_BGD_Test_Deployment& deployment = state.deployment; - if (publish_status(sim, deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.writer_hostgroup, "SWITCHOVER_INITIATED", kWaitSeconds) - != EXIT_SUCCESS) { - diag("Error: worker did not enter SWITCHOVER_INITIATED"); - return EXIT_FAILURE; - } - - ok(true, "INITIATED is published"); - ok(wait_for_writer_placement( - admin, state.writer_hostgroup, state.reader_hostgroup, - deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, - "INITIATED does not change writer placement"); - - auto [active_seq_rc, active_sequence] = sim.replica_probe_log_last_sequence(); - ok(active_seq_rc == EXIT_SUCCESS && fast_bgd_without_ordinary( - sim, active_sequence, deployment.target_replica_set, 650, 3), - "INITIATED uses fast membership probes and suspends the ordinary Aurora query"); - return EXIT_SUCCESS; -} - -/** - * Enter SWITCHOVER_IN_PROGRESS. - * - * - Demote the snapshotted production writer exactly once. - * - Keep the competing source reader out of the writer hostgroup. - * - Preserve source-backed routing pools. - */ -int test_switchover_in_progress( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) -{ - Aurora_BGD_Test_Deployment& deployment = state.deployment; - if (publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.writer_hostgroup, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) - != EXIT_SUCCESS) { - diag("Error: worker did not enter SWITCHOVER_IN_PROGRESS"); - return EXIT_FAILURE; - } - - ok(true, "IN_PROGRESS is published"); - ok(wait_for_writer_placement( - admin, state.writer_hostgroup, state.reader_hostgroup, - deployment.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, - "IN_PROGRESS demotes the snapshotted production writer"); - auto [repeat_progress_rc, repeat_progress_sequence] = sim.replica_probe_log_last_sequence(); - if (repeat_progress_rc != EXIT_SUCCESS - || publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS) { - diag("Error: failed to repeat SWITCHOVER_IN_PROGRESS"); - return EXIT_FAILURE; - } - ok(fast_bgd_without_ordinary( - sim, repeat_progress_sequence, deployment.target_replica_set, 350, 2) - && wait_for_writer_placement( - admin, state.writer_hostgroup, state.reader_hostgroup, - deployment.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, - "repeated IN_PROGRESS retains the one demotion and active probe policy"); - - ok(route_members_to_expected_ips( - cl, admin, sim, deployment, state.route_hostgroups, false) - && member_route_pool_count(admin, state.route_hostgroups) - >= static_cast(state.route_hostgroups.size()), - "all member routes use source IPs and hold pre-cutover pools"); - - vector competing_source = deployment.production.replica_rows(); - competing_source[0].session_id = "source-observed-reader"; - competing_source[1].session_id = "MASTER_SESSION_ID"; - auto [source_change_rc, source_change_sequence] = sim.replica_probe_log_last_sequence(); - bool source_change_ignored = source_change_rc == EXIT_SUCCESS - && sim.replica_update( - deployment.blue_replica_set, competing_source, deployment.production.backends()) - == EXIT_SUCCESS - && fast_bgd_without_ordinary( - sim, source_change_sequence, deployment.target_replica_set, 350, 2); - ok(source_change_ignored, - "changed source roles cannot compete while production probing is suspended"); - ok(runtime_server_count( - admin, state.writer_hostgroup, deployment.production.members[1].endpoint.hostname, 0) - && runtime_server_count( - admin, state.reader_hostgroup, - deployment.production.members[1].endpoint.hostname, 1, "ONLINE"), - "the competing source observation does not promote a reader"); - if (sim.replica_update( - deployment.blue_replica_set, deployment.production.replica_rows(), - deployment.production.backends()) != EXIT_SUCCESS) { - diag("Error: failed to restore the source membership fixture"); - return EXIT_FAILURE; - } - - if (set_default_hostgroup(admin, state.route_hostgroups.front()) != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - bool held_connection = set_held_connection_rule(admin, true) == EXIT_SUCCESS; - state.held_client = held_connection - ? init_mysql_conn(cl.host, cl.port, cl.username, cl.password) : nullptr; - held_connection = state.held_client != nullptr - && mysql_query(state.held_client, "SELECT 1") == 0; - if (held_connection) { - MYSQL_RES* result = mysql_store_result(state.held_client); - held_connection = result != nullptr; - if (result != nullptr) { - mysql_free_result(result); - } - } - ok(held_connection && member_route_used_count(admin, state.route_hostgroups) >= 1, - "a production connection remains in use across the cutover boundary"); - if (set_held_connection_rule(admin, false) != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - return EXIT_SUCCESS; -} - -/** - * Enter SWITCHOVER_IN_POST_PROCESSING. - * - * - Restore canonical placement and keep readers eligible. - * - Retire pre-cutover pools and pin every production hostname to its target IP. - * - Avoid replaying retirement on repeated POST_PROCESSING. - */ -int test_switchover_post_processing( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) -{ - Aurora_BGD_Test_Deployment& deployment = state.deployment; - if (sim.read_only_update( - deployment.target.members.front().endpoint.host_endpoint(), true) != EXIT_SUCCESS - || aurora_bgd_execute_all(admin, { - "SET mysql-monitor_local_dns_cache_ttl=0", - "SET mysql-monitor_local_dns_cache_refresh_interval=0", - "LOAD MYSQL VARIABLES TO RUNTIME", - }) != EXIT_SUCCESS - || publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.writer_hostgroup, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) - != EXIT_SUCCESS) { - diag("Error: worker did not enter SWITCHOVER_IN_POST_PROCESSING"); - return EXIT_FAILURE; - } - - ok(true, "POST_PROCESSING is published without a target-writability gate"); - ok(wait_for_writer_placement( - admin, state.writer_hostgroup, state.reader_hostgroup, - deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, - "POST_PROCESSING restores the canonical writer placement"); - ok(runtime_server_count( - admin, state.reader_hostgroup, - deployment.production.members[1].endpoint.hostname, 1, "ONLINE") - && runtime_server_count( - admin, state.reader_hostgroup, - deployment.production.members[2].endpoint.hostname, 1, "ONLINE"), - "POST_PROCESSING leaves canonical readers ONLINE and eligible"); - ok(state.held_client != nullptr - && member_route_used_count(admin, state.route_hostgroups) >= 1, - "POST_PROCESSING advances without waiting for an in-use connection to close"); - if (state.held_client != nullptr) { - mysql_close(state.held_client); - state.held_client = nullptr; - } - ok(wait_for_member_route_pool_count( - admin, state.route_hostgroups, "=0") == EXIT_SUCCESS, - "POST_PROCESSING retires the pre-cutover member pools"); - ok(route_members_to_expected_ips( - cl, admin, sim, deployment, state.route_hostgroups, true), - "POST_PROCESSING pins every production hostname despite disabled DNS caching and target read_only=1"); - ok(aurora_bgd_execute_all(admin, { - "LOAD MYSQL VARIABLES TO RUNTIME", - }) == EXIT_SUCCESS - && route_members_to_expected_ips( - cl, admin, sim, deployment, state.route_hostgroups, true), - "an active variables refresh preserves every explicit traffic pin"); - - const vector moved_ids { - "aurora-a-writer-green-r2", - "aurora-a-reader-1-green-r2", - "aurora-a-reader-2-green-r2", - }; - const vector moved_ips { "127.0.11.31", "127.0.11.32", "127.0.11.33" }; - for (size_t i = 0; i < deployment.target.members.size(); ++i) { - deployment.target.members[i].server_id = moved_ids[i]; - deployment.target.members[i].endpoint.hostname = - moved_ids[i] + deployment.domain_name; - deployment.target.members[i].endpoint.ip = moved_ips[i]; - } - deployment.target.serving_endpoints.clear(); - deployment.target.serving_endpoints.push_back(deployment.target_cluster_endpoint); - for (const Aurora_BGD_Member& member : deployment.target.members) { - deployment.target.serving_endpoints.push_back(member.endpoint); - } - auto [move_seq_rc, move_sequence] = sim.replica_probe_log_last_sequence(); - bool moved_routing = move_seq_rc == EXIT_SUCCESS - && sim.replica_update( - deployment.target_replica_set, deployment.target.replica_rows(), - deployment.target.backends()) == EXIT_SUCCESS - && publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS; - if (moved_routing) { - auto [move_probe_rc, move_probe] = aurora_bgd_wait_for_replica_probe( - sim, move_sequence, deployment.target.backends(), - Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, - deployment.target_replica_set); - moved_routing = move_probe_rc == EXIT_SUCCESS - && wait_for_member_route_pool_count( - admin, state.route_hostgroups, "=0") == EXIT_SUCCESS - && route_members_to_expected_ips( - cl, admin, sim, deployment, state.route_hostgroups, true); - } - ok(moved_routing, - "POST_PROCESSING repins every member when target-shaped endpoints move"); - - if (aurora_bgd_execute_all(admin, { - "SET mysql-monitor_local_dns_cache_ttl=300000", - "SET mysql-monitor_local_dns_cache_refresh_interval=60000", - "LOAD MYSQL VARIABLES TO RUNTIME", - }) != EXIT_SUCCESS) { - diag("Error: failed to restore DNS-cache variables"); - return EXIT_FAILURE; - } - - const int64_t target_pool_count = member_route_pool_count(admin, state.route_hostgroups); - auto [repeat_post_rc, repeat_post_sequence] = sim.probe_log_last_sequence(); - if (target_pool_count < static_cast(state.route_hostgroups.size()) - || repeat_post_rc != EXIT_SUCCESS - || publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS) { - diag("Error: failed to prepare repeated POST_PROCESSING"); - return EXIT_FAILURE; - } - vector target_backends = deployment.target.backends(); - auto [repeat_probe_rc, repeat_probe] = aurora_bgd_wait_for_topology_probe( - sim, repeat_post_sequence, target_backends, BGD_Probe_Kind::metadata, - kProbeTimeoutMs); - ok(repeat_probe_rc == EXIT_SUCCESS - && member_route_pool_count(admin, state.route_hostgroups) >= target_pool_count, - "repeated POST_PROCESSING does not replay completed member retirement"); - - auto [post_seq_rc, post_sequence] = sim.replica_probe_log_last_sequence(); - ok(post_seq_rc == EXIT_SUCCESS && fast_bgd_without_ordinary( - sim, post_sequence, deployment.target_replica_set, 350, 2), - "POST_PROCESSING keeps fast BGD probes without ordinary Aurora queries"); - return EXIT_SUCCESS; -} - -/** - * Refresh target IPs before rename and retain the last complete multi-reader map. - * - * Green hostgroups are intentionally NULL: auto-discovery must still pair all - * members, accept the canonical rename, and route with the last complete map. - */ -int test_auto_discovered_refresh_and_rename( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state -) { - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to reset before refreshed-target scenario"); - return EXIT_FAILURE; - } - - Aurora_BGD_Test_Deployment& deployment = state.refreshed; - if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.refreshed_writer_hostgroup, - state.refreshed_reader_hostgroup, -1, -1, true, 300, false) != EXIT_SUCCESS - || add_member_routes( - admin, deployment, state.refreshed_route_hostgroups) != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.refreshed_writer_hostgroup, "AVAILABLE", kWaitSeconds) - != EXIT_SUCCESS) { - diag("Error: failed to configure auto-discovered multi-reader deployment"); - return EXIT_FAILURE; - } - - const vector refreshed_ids { - "aurora-a-writer-green-r2", - "aurora-a-reader-1-green-r2", - "aurora-a-reader-2-green-r2", - }; - const vector refreshed_ips { "127.0.11.31", "127.0.11.32", "127.0.11.33" }; - for (size_t i = 0; i < deployment.target.members.size(); ++i) { - deployment.target.members[i].server_id = refreshed_ids[i]; - deployment.target.members[i].endpoint.hostname = - refreshed_ids[i] + deployment.domain_name; - deployment.target.members[i].endpoint.ip = refreshed_ips[i]; - } - deployment.target.serving_endpoints.clear(); - deployment.target.serving_endpoints.push_back(deployment.target_cluster_endpoint); - for (const Aurora_BGD_Member& member : deployment.target.members) { - deployment.target.serving_endpoints.push_back(member.endpoint); - } - - auto [refresh_seq_rc, refresh_sequence] = sim.replica_probe_log_last_sequence(); - if (refresh_seq_rc != EXIT_SUCCESS || sim.replica_update( - deployment.target_replica_set, deployment.target.replica_rows(), - deployment.target.backends()) != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - vector refreshed_backends = deployment.target.backends(); - auto [refresh_probe_rc, refresh_probe] = aurora_bgd_wait_for_replica_probe( - sim, refresh_sequence, refreshed_backends, - Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, - deployment.target_replica_set); - ok(refresh_probe_rc == EXIT_SUCCESS, - "AVAILABLE refreshes every target IP before the member rename"); - - vector canonical_rows = deployment.target.replica_rows(); - for (size_t i = 0; i < canonical_rows.size(); ++i) { - canonical_rows[i].server_id = deployment.production.members[i].server_id; - } - auto [rename_seq_rc, rename_sequence] = sim.replica_probe_log_last_sequence(); - if (rename_seq_rc != EXIT_SUCCESS || sim.replica_update( - deployment.target_replica_set, canonical_rows, - deployment.target.backends()) != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - auto [rename_probe_rc, rename_probe] = aurora_bgd_wait_for_replica_probe( - sim, rename_sequence, refreshed_backends, - Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, - deployment.target_replica_set); - ok(rename_probe_rc == EXIT_SUCCESS, - "stable reader sessions preserve identity across the canonical SERVER_ID rename"); - - vector incomplete_rows = canonical_rows; - incomplete_rows.pop_back(); - auto [incomplete_seq_rc, incomplete_sequence] = sim.replica_probe_log_last_sequence(); - if (incomplete_seq_rc != EXIT_SUCCESS || sim.replica_update( - deployment.target_replica_set, incomplete_rows, - deployment.target.backends()) != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - auto [incomplete_probe_rc, incomplete_probe] = aurora_bgd_wait_for_replica_probe( - sim, incomplete_sequence, refreshed_backends, - Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, - deployment.target_replica_set); - if (incomplete_probe_rc != EXIT_SUCCESS - || publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.refreshed_writer_hostgroup, - "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - - bool retained_routing = true; - for (size_t i = 0; i < state.refreshed_route_hostgroups.size(); ++i) { - retained_routing = retained_routing - && set_default_hostgroup(admin, state.refreshed_route_hostgroups[i]) == EXIT_SUCCESS - && route_to_expected_backend( - cl, sim, deployment.target.members[i].endpoint.backend()); - } - ok(retained_routing, - "POST_PROCESSING routes all auto-discovered readers with the retained complete refreshed map"); - return EXIT_SUCCESS; -} - -/** Refresh production membership in AVAILABLE and freeze it after INITIATED. */ -int test_available_production_refresh_and_freeze( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state -) { - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - Aurora_BGD_Test_Deployment deployment = aurora_bgd_deployment_a(); - if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.refreshed_writer_hostgroup, - state.refreshed_reader_hostgroup, -1, -1, true, 300, false) != EXIT_SUCCESS - || add_member_routes( - admin, deployment, state.refreshed_route_hostgroups) != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.refreshed_writer_hostgroup, "AVAILABLE", kWaitSeconds) - != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - - vector reduced_production = deployment.production.replica_rows(); - reduced_production.pop_back(); - vector reduced_target = deployment.target.replica_rows(); - reduced_target.pop_back(); - auto [ordinary_seq_rc, ordinary_sequence] = sim.replica_probe_log_last_sequence(); - if (ordinary_seq_rc != EXIT_SUCCESS || sim.replica_update( - deployment.blue_replica_set, reduced_production, - deployment.production.backends()) != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - auto [ordinary_probe_rc, ordinary_probe] = aurora_bgd_wait_for_replica_probe( - sim, ordinary_sequence, deployment.production.backends(), - Aurora_Replica_Probe_Kind::ordinary, kProbeTimeoutMs, - deployment.blue_replica_set); - auto [ordinary_retry_rc, ordinary_retry] = aurora_bgd_wait_for_replica_probe( - sim, ordinary_probe.sequence_id, deployment.production.backends(), - Aurora_Replica_Probe_Kind::ordinary, kProbeTimeoutMs, - deployment.blue_replica_set); - auto [target_seq_rc, target_sequence] = sim.replica_probe_log_last_sequence(); - if (ordinary_probe_rc != EXIT_SUCCESS || ordinary_retry_rc != EXIT_SUCCESS - || target_seq_rc != EXIT_SUCCESS - || sim.replica_update( - deployment.target_replica_set, reduced_target, - deployment.target.backends()) != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - auto [target_probe_rc, target_probe] = aurora_bgd_wait_for_replica_probe( - sim, target_sequence, deployment.target.backends(), - Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, - deployment.target_replica_set); - auto [target_retry_rc, target_retry] = aurora_bgd_wait_for_replica_probe( - sim, target_probe.sequence_id, deployment.target.backends(), - Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, - deployment.target_replica_set); - ok(target_probe_rc == EXIT_SUCCESS && target_retry_rc == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.refreshed_writer_hostgroup, "AVAILABLE", 1) == EXIT_SUCCESS, - "AVAILABLE refreshes production and target membership before the switchover"); - - if (publish_status(sim, deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.refreshed_writer_hostgroup, - "SWITCHOVER_INITIATED", kWaitSeconds) != EXIT_SUCCESS - || sim.replica_update( - deployment.blue_replica_set, deployment.production.replica_rows(), - deployment.production.backends()) != EXIT_SUCCESS - || sim.replica_update( - deployment.target_replica_set, deployment.target.replica_rows(), - deployment.target.backends()) != EXIT_SUCCESS - || publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.refreshed_writer_hostgroup, - "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - - bool frozen_routes = true; - for (size_t i = 0; i < state.refreshed_route_hostgroups.size(); ++i) { - const bool target = i < reduced_production.size(); - const Endpoint expected = target - ? deployment.target.members[i].endpoint.backend() - : deployment.production.members[i].endpoint.backend(); - frozen_routes = frozen_routes - && set_default_hostgroup(admin, state.refreshed_route_hostgroups[i]) == EXIT_SUCCESS - && route_to_expected_backend(cl, sim, expected); - } - ok(frozen_routes, - "INITIATED freezes the refreshed production map despite later membership changes"); - return EXIT_SUCCESS; -} - -/** - * Complete target membership after entering POST_PROCESSING. - * - * - Leave routing unchanged while the target snapshot is incomplete. - * - Publish complete membership. - * - Apply target routing on the next POST_PROCESSING observation. - */ -int test_post_processing_after_membership_completion( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) -{ - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to reset before the incomplete-snapshot scenario"); - return EXIT_FAILURE; - } - - Aurora_BGD_Test_Deployment& deployment = state.gated; - vector invalid_target = deployment.target.replica_rows(); - invalid_target.front().server_id = "unpaired-target-writer"; - if (sim.replica_update( - deployment.blue_replica_set, deployment.production.replica_rows(), - deployment.production.backends()) != EXIT_SUCCESS - || sim.replica_update( - deployment.target_replica_set, invalid_target, deployment.target.backends()) - != EXIT_SUCCESS - || sim.topology_update( - aurora_bgd_topology_backends(deployment), - aurora_bgd_available_topology(deployment)) != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.gated_writer_hostgroup, - state.gated_reader_hostgroup, state.gated_green_writer_hostgroup, - state.gated_green_reader_hostgroup, false, 1000, false) != EXIT_SUCCESS - || add_member_routes( - admin, deployment, { state.gated_route_hostgroup }) != EXIT_SUCCESS) { - diag("Error: failed to configure incomplete target membership"); - return EXIT_FAILURE; - } - - ok(aurora_bgd_wait_for_status( - admin, state.gated_writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, - "an incomplete target snapshot can publish AVAILABLE without routing"); - if (publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.gated_writer_hostgroup, - "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { - diag("Error: incomplete scenario did not publish POST_PROCESSING"); - return EXIT_FAILURE; - } - - int default_rc = set_default_hostgroup(admin, state.gated_route_hostgroup); - ok(default_rc == EXIT_SUCCESS && route_to_expected_backend( - cl, sim, deployment.production.members.front().endpoint.backend()), - "POST_PROCESSING leaves routing unchanged without a complete target snapshot"); - - auto [refresh_seq_rc, refresh_sequence] = sim.replica_probe_log_last_sequence(); - if (refresh_seq_rc != EXIT_SUCCESS || sim.replica_update( - deployment.target_replica_set, deployment.target.replica_rows(), - deployment.target.backends()) != EXIT_SUCCESS) { - diag("Error: failed to restore complete target membership"); - return EXIT_FAILURE; - } - auto [probe_rc, probe] = aurora_bgd_wait_for_replica_probe( - sim, refresh_sequence, deployment.target.backends(), - Aurora_Replica_Probe_Kind::bgd_membership, - kProbeTimeoutMs, deployment.target_replica_set); - bool target_routing = false; - if (probe_rc == EXIT_SUCCESS - && wait_for_member_route_pool_count( - admin, { state.gated_route_hostgroup }, "=0") == EXIT_SUCCESS - && set_default_hostgroup(admin, state.gated_route_hostgroup) == EXIT_SUCCESS) { - target_routing = route_to_expected_backend( - cl, sim, deployment.target.members.front().endpoint.backend()); - } - ok(target_routing, - "repeated POST_PROCESSING applies routing after membership becomes complete"); - return EXIT_SUCCESS; -} - -int main() { - plan(29); - - CommandLine cl {}; - MYSQL* admin = nullptr; - BGD_Simulator sim {}; - - if (setup(cl, admin, sim) != EXIT_SUCCESS) { - return exit_status(); - } - - TestState state {}; - - // Simulator: publish a complete deployment. - // ProxySQL: configure Aurora BGD and per-member routing hostgroups. - // Verify: the worker reaches AVAILABLE. - if (test_bgd_status_available(admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: publish INITIATED and a competing production role observation. - // Verify: placement is suppressed and only fast target membership remains active. - if (test_switchover_initiated(admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: publish IN_PROGRESS twice. - // Verify: the snapshotted writer is demoted once and source routes remain active. - if (test_switchover_in_progress(cl, admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: publish POST_PROCESSING twice. - // Verify: canonical placement, target pins, and one-time pool retirement. - if (test_switchover_post_processing(cl, admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: enter POST_PROCESSING with incomplete membership, then complete it. - // Verify: routing changes only after a complete target map exists. - if (test_post_processing_after_membership_completion(cl, admin, sim, state) - != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: refresh target IDs/IPs, publish canonical IDs, then an incomplete map. - // Verify: auto-discovered multi-reader routing uses the last complete refreshed map. - if (test_auto_discovered_refresh_and_rename(cl, admin, sim, state) - != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: change membership in AVAILABLE, then change it again after INITIATED. - // Verify: AVAILABLE refreshes the map and INITIATED freezes that exact snapshot. - if (test_available_production_refresh_and_freeze(cl, admin, sim, state) - != EXIT_SUCCESS) { - goto exit_cleanup; - } - -exit_cleanup: - if (state.held_client != nullptr) { - mysql_close(state.held_client); - state.held_client = nullptr; - } - if (cleanup(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to clean Aurora BGD active-state test data"); - return EXIT_FAILURE; - } - return exit_status(); -} diff --git a/test/tap/tests/test_aurora_bgd_automatic_discovery-t.cpp b/test/tap/tests/test_aurora_bgd_automatic_discovery-t.cpp new file mode 100644 index 0000000000..c5b5f979de --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_automatic_discovery-t.cpp @@ -0,0 +1,134 @@ +/** + * @file test_aurora_bgd_automatic_discovery-t.cpp + * @brief Aurora BGD runtime derivation for NULL green hostgroups. + */ + +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +struct TestState { + Aurora_BGD_Test_Deployment first { aurora_bgd_deployment_b_writer_only() }; + int first_writer_hostgroup { 2020 }; + int first_reader_hostgroup { 2021 }; + Aurora_BGD_Test_Deployment second { aurora_bgd_deployment_c_writer_only() }; + int second_writer_hostgroup { 2024 }; + int second_reader_hostgroup { 2025 }; +}; + +bool runtime_row_stays_null( + Context& context, int writer_hostgroup, int reader_hostgroup, + const string& status +) { + return scalar_is( + context.admin, + "SELECT COUNT(*) FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hostgroup) + " AND reader_hostgroup=" + to_string(reader_hostgroup) + + " AND green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL " + "AND bgd_status=" + aurora_bgd_sql_quote(status), + "1"); +} + +bool persistent_row_stays_null(Context& context, int writer_hostgroup) { + return scalar_is( + context.admin, + "SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + + to_string(writer_hostgroup) + " " + "AND green_writer_hostgroup IS NULL AND green_reader_hostgroup IS NULL", + "1"); +} + +int test_null_green_hostgroups(Context& context, TestState& state) { + if (publish_available(context, state.first) != EXIT_SUCCESS + || configure( + context, state.first, state.first_writer_hostgroup, state.first_reader_hostgroup, + -1, -1, true) != EXIT_SUCCESS) { + diag("Error: failed to configure NULL green hostgroups"); + return EXIT_FAILURE; + } + + ok(wait_for_cond( + context.admin, + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=2020 AND bgd_status='AVAILABLE'", + kWaitSeconds) == EXIT_SUCCESS + && runtime_row_stays_null(context, 2020, 2021, "AVAILABLE"), + "AVAILABLE admits BGD discovery while runtime green hostgroups remain NULL"); + ok(persistent_row_stays_null(context, 2020), + "discovery keeps the user-created green hostgroups NULL"); + return EXIT_SUCCESS; +} + +int test_absent_then_available(Context& context, TestState& state) { + if (context.simulator.replica_update( + state.second.production.replica_set_id, + state.second.production.replica_rows(), + state.second.production.backends()) != EXIT_SUCCESS + || context.simulator.replica_update( + state.second.target.replica_set_id, + state.second.target.replica_rows(), + state.second.target.backends()) != EXIT_SUCCESS + || context.simulator.topology_update( + aurora_bgd_topology_backends(state.second), {}) != EXIT_SUCCESS + || configure( + context, state.second, state.second_writer_hostgroup, state.second_reader_hostgroup, + -1, -1, true) != EXIT_SUCCESS) { + diag("Error: failed to configure the absent-topology discovery case"); + return EXIT_FAILURE; + } + + ok(aurora_bgd_wait_for_status( + context.admin, state.second_writer_hostgroup, "NONE", kWaitSeconds) == EXIT_SUCCESS + && persistent_row_stays_null(context, state.second_writer_hostgroup), + "absent topology keeps one user-created row in NONE without persisting derived hostgroups"); + + if (context.simulator.topology_update( + aurora_bgd_topology_backends(state.second), + aurora_bgd_available_topology(state.second)) != EXIT_SUCCESS) { + diag("Error: failed to publish AVAILABLE topology"); + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, state.second_writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS + && runtime_row_stays_null(context, 2024, 2025, "AVAILABLE") + && membership_probe_reached(context, 0, state.second), + "AVAILABLE later discovers an internal target map without generating green hostgroups"); + return EXIT_SUCCESS; +} + +int test_repeated_discovery(Context& context, TestState& state) { + usleep(300000); + ok(runtime_row_stays_null(context, 2024, 2025, "AVAILABLE") + && scalar_is( + context.admin, + "SELECT COUNT(*) FROM runtime_mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=2024", + "1"), + "repeated AVAILABLE polling keeps one runtime row with NULL green hostgroups"); + return EXIT_SUCCESS; +} + +int main() { + plan(5); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + if (test_null_green_hostgroups(context, state) != EXIT_SUCCESS + || test_absent_then_available(context, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + test_repeated_discovery(context, state); + +exit_cleanup: + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the automatic-discovery fixture"); + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp b/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp deleted file mode 100644 index 105f34f831..0000000000 --- a/test/tap/tests/test_aurora_bgd_cluster_sync-t.cpp +++ /dev/null @@ -1,372 +0,0 @@ -/** - * @file test_aurora_bgd_cluster_sync-t.cpp - * @brief Aurora BGD configuration sync preserves worker-owned status per node. - * - * Steps: - * - * 1. Start two ProxySQL nodes with independent simulator endpoints. - * 2. Publish AVAILABLE on the primary and INITIATED on the replica. - * 3. Synchronize the configured Aurora hostgroup row from primary to replica. - * 4. Verify configured fields synchronize while runtime status remains node-local. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "aurora_bgd_tap.h" -#include "command_line.h" -#include "utils.h" - -using namespace std; -namespace fs = std::filesystem; - -const uint32_t kWaitSeconds = 10; -const int kReplicaAdminPort = 16062; -const int kReplicaMySQLPort = 16063; -const int kReplicaSQLiteProductionPort = 16064; -const int kReplicaSQLiteTargetPort = 16065; -const char kReplicaHost[] = "127.0.0.1"; -const char kPrimarySQLiteInterfaces[] = "0.0.0.0:3306;0.0.0.0:3307"; -const char kReplicaSQLiteInterfaces[] = "0.0.0.0:16064;0.0.0.0:16065"; - -struct Replica_Process { - pid_t pid = -1; - string directory; - string config_path; - string stderr_path; -}; - -Aurora_BGD_Test_Deployment peer_deployment( - int production_port = 3306, int target_port = 3307); - -struct TestState { - MYSQL* primary_admin { nullptr }; - MYSQL* replica_admin { nullptr }; - Replica_Process replica_process {}; - string primary_sqlite_interfaces; - BGD_Simulator primary_simulator {}; - BGD_Simulator replica_simulator {}; - Aurora_BGD_Test_Deployment deployment { peer_deployment() }; - Aurora_BGD_Test_Deployment replica_deployment { - peer_deployment(kReplicaSQLiteProductionPort, kReplicaSQLiteTargetPort) - }; - bool primary_simulator_connected { false }; - bool replica_simulator_connected { false }; -}; - -string config_quote(const string& value) { - string quoted; - for (char c : value) { - if (c == '\\' || c == '"') { - quoted += '\\'; - } - quoted += c; - } - return quoted; -} - -int prepare_replica_config(const CommandLine& cl, Replica_Process& process) { - char directory_template[] = "proxysql-aurora-bgd-sync-XXXXXX"; - char* directory = mkdtemp(directory_template); - if (directory == nullptr) { - diag("mkdtemp failed: %s", strerror(errno)); - return EXIT_FAILURE; - } - process.directory = directory; - process.config_path = process.directory + "/proxysql.cnf"; - process.stderr_path = process.directory + "/proxysql.stderr"; - - ofstream config(process.config_path); - if (!config.is_open()) { - diag("failed to create replica config: %s", process.config_path.c_str()); - return EXIT_FAILURE; - } - config - << "datadir=\"" << config_quote(process.directory) << "\"\n" - << "admin_variables={\n" - << " admin_credentials=\"" << config_quote(cl.admin_username) << ":" - << config_quote(cl.admin_password) << ";radmin:radmin\"\n" - << " mysql_ifaces=\"0.0.0.0:" << kReplicaAdminPort << "\"\n" - << " cluster_username=\"radmin\"\n" - << " cluster_password=\"radmin\"\n" - << " cluster_check_interval_ms=200\n" - << " cluster_check_status_frequency=100\n" - << " cluster_admin_variables_diffs_before_sync=0\n" - << " cluster_mysql_servers_diffs_before_sync=1\n" - << " cluster_mysql_servers_save_to_disk=false\n" - << " cluster_mysql_servers_sync_algorithm=3\n" - << "}\n" - << "mysql_variables={\n" - << " interfaces=\"0.0.0.0:" << kReplicaMySQLPort << "\"\n" - << " monitor_username=\"aurora1\"\n" - << " monitor_password=\"pass1\"\n" // NOSONAR: fixed simulator fixture credential. - << " monitor_connect_timeout=500\n" - << " monitor_ping_interval=10000\n" - << "}\n" - << "proxysql_servers=()\n"; - config.close(); - return config.fail() ? EXIT_FAILURE : EXIT_SUCCESS; -} - -int launch_replica(const CommandLine& cl, Replica_Process& process) { - if (prepare_replica_config(cl, process) != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - const string binary = string(cl.workdir) + "../../../src/proxysql"; - process.pid = fork(); - if (process.pid == -1) { - diag("fork failed: %s", strerror(errno)); - return EXIT_FAILURE; - } - if (process.pid == 0) { - int stderr_fd = open(process.stderr_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); - if (stderr_fd >= 0) { - dup2(stderr_fd, STDOUT_FILENO); - dup2(stderr_fd, STDERR_FILENO); - close(stderr_fd); - } - execl( - binary.c_str(), "proxysql", "--sqlite3-server", "-f", "-c", - process.config_path.c_str(), static_cast(nullptr)); - _exit(127); - } - return EXIT_SUCCESS; -} - -void stop_replica(MYSQL*& admin, Replica_Process& process, bool preserve_log) { - if (admin != nullptr) { - mysql_query(admin, "PROXYSQL SHUTDOWN"); - mysql_close(admin); - admin = nullptr; - } - if (process.pid > 0) { - bool exited = false; - for (int i = 0; i < 50; ++i) { - pid_t rc = waitpid(process.pid, nullptr, WNOHANG); - if (rc == process.pid || rc == -1) { - exited = true; - break; - } - usleep(100000); - } - if (!exited) { - kill(process.pid, SIGKILL); - waitpid(process.pid, nullptr, 0); - } - } - if (!preserve_log && !process.directory.empty()) { - fs::remove_all(process.directory); - } else if (preserve_log) { - diag("replica ProxySQL log retained at %s", process.stderr_path.c_str()); - } -} - -Aurora_BGD_Test_Deployment peer_deployment(int production_port, int target_port) { - Aurora_BGD_Test_Deployment deployment; - deployment.name = "Aurora BGD peer-local status"; - deployment.domain_name = ".localhost"; - deployment.blue_replica_set = "aurora-bgd-peer-blue"; - deployment.target_replica_set = "aurora-bgd-peer-target"; - deployment.source_topology_id = "aurora-bgd-peer-source"; - deployment.target_topology_id = "aurora-bgd-peer-target"; - deployment.target_cluster_endpoint = { - "aurora-peer-writer-green-sync.localhost", "127.0.0.1", target_port - }; - deployment.production = { - deployment.blue_replica_set, - {aurora_bgd_member( - "aurora-peer-writer", "MASTER_SESSION_ID", - {"aurora-peer-writer.localhost", "127.0.0.1", production_port})}, - {} - }; - deployment.production.serving_endpoints.push_back( - deployment.production.members.front().endpoint); - deployment.target = { - deployment.target_replica_set, - {aurora_bgd_member( - "aurora-peer-writer-green-sync", "MASTER_SESSION_ID", - {"aurora-peer-writer-green-sync.localhost", "127.0.0.1", target_port})}, - {deployment.target_cluster_endpoint} - }; - deployment.target.serving_endpoints.push_back( - deployment.target.members.front().endpoint); - return deployment; -} - -int configure_sqlite_interfaces(MYSQL* admin, const string& interfaces) { - return aurora_bgd_execute_all(admin, { - "SET sqliteserver-mysql_ifaces=" + aurora_bgd_sql_quote(interfaces), - "LOAD SQLITESERVER VARIABLES TO RUNTIME", - }); -} - -int configure_peer( - MYSQL* admin, BGD_Simulator& simulator, Aurora_BGD_Test_Deployment& deployment, - const string& status -) { - return simulator.cleanup() == EXIT_SUCCESS - && aurora_bgd_publish(simulator, deployment) == EXIT_SUCCESS - && simulator.topology_update( - aurora_bgd_topology_backends(deployment), aurora_bgd_topology(deployment, status)) - == EXIT_SUCCESS - && aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS - && aurora_bgd_admin_setup(admin, deployment, 1800, 1801, 1802, 1803, false) - == EXIT_SUCCESS - ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int setup(CommandLine& cl, TestState& state) { - if (cl.getEnv()) { - diag("failed to load TAP environment"); - return EXIT_FAILURE; - } - - state.primary_admin = init_mysql_conn( - cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); - if (state.primary_admin == nullptr - || get_variable_value( - state.primary_admin, "sqliteserver-mysql_ifaces", - state.primary_sqlite_interfaces) != EXIT_SUCCESS - || configure_sqlite_interfaces( - state.primary_admin, kPrimarySQLiteInterfaces) != EXIT_SUCCESS - || launch_replica(cl, state.replica_process) != EXIT_SUCCESS) { - diag("failed to prepare the two ProxySQL nodes"); - return EXIT_FAILURE; - } - - state.replica_admin = wait_for_proxysql( - {kReplicaHost, cl.admin_username, cl.admin_password, kReplicaAdminPort}, - kWaitSeconds); - if (state.replica_admin == nullptr - || configure_sqlite_interfaces( - state.replica_admin, kReplicaSQLiteInterfaces) != EXIT_SUCCESS) { - diag("failed to start the replica ProxySQL node"); - return EXIT_FAILURE; - } - - char username[] = "aurora1"; - char password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. - char replica_host[] = "127.0.0.1"; - state.primary_simulator_connected = state.primary_simulator.connect( - cl.host, 3306, username, password) == EXIT_SUCCESS; - state.replica_simulator_connected = state.replica_simulator.connect( - replica_host, kReplicaSQLiteProductionPort, username, password) == EXIT_SUCCESS; - if (!state.primary_simulator_connected || !state.replica_simulator_connected - || configure_peer( - state.primary_admin, state.primary_simulator, - state.deployment, "AVAILABLE") != EXIT_SUCCESS - || configure_peer( - state.replica_admin, state.replica_simulator, - state.replica_deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS) { - diag("failed to publish the two node-local Aurora observations"); - return EXIT_FAILURE; - } - return EXIT_SUCCESS; -} - -/** - * Synchronize configured Aurora fields without synchronizing worker-owned status. - * - * - Verify each node publishes its local topology observation. - * - Synchronize the primary's configured hostgroup comment to the replica. - * - Verify AVAILABLE and INITIATED remain local to their respective workers. - */ -int test_configuration_sync_preserves_local_status(CommandLine& cl, TestState& state) { - ok(aurora_bgd_wait_for_status( - state.primary_admin, 1800, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, - "primary worker publishes its local AVAILABLE observation"); - ok(aurora_bgd_wait_for_status( - state.replica_admin, 1800, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, - "replica worker publishes its local SWITCHOVER_INITIATED observation"); - - if (aurora_bgd_execute_all(state.replica_admin, { - "DELETE FROM proxysql_servers", - "INSERT INTO proxysql_servers(hostname,port,weight,comment) VALUES (" + - aurora_bgd_sql_quote(cl.admin_host) + "," + to_string(cl.admin_port) + - ",0,'Aurora BGD sync primary')", - "LOAD PROXYSQL SERVERS TO RUNTIME", - }) != EXIT_SUCCESS - || aurora_bgd_execute_all(state.primary_admin, { - "UPDATE mysql_aws_aurora_hostgroups SET comment='peer-config-synced' " - "WHERE writer_hostgroup=1800", - "LOAD MYSQL SERVERS TO RUNTIME", - }) != EXIT_SUCCESS) { - diag("failed to initiate Aurora configuration synchronization"); - return EXIT_FAILURE; - } - - ok(wait_for_cond( - state.replica_admin, - "SELECT COUNT(*)=1 FROM mysql_aws_aurora_hostgroups " - "WHERE writer_hostgroup=1800 AND comment='peer-config-synced'", - kWaitSeconds) == EXIT_SUCCESS, - "Aurora BGD configured fields synchronize to the peer"); - ok(aurora_bgd_wait_for_status( - state.primary_admin, 1800, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, - "configuration sync preserves the primary's local status"); - ok(aurora_bgd_wait_for_status( - state.replica_admin, 1800, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, - "configuration sync preserves the replica's local status"); - return EXIT_SUCCESS; -} - -int cleanup(TestState& state) { - bool cleanup_ok = true; - if (state.primary_simulator_connected) { - cleanup_ok = state.primary_simulator.cleanup() == EXIT_SUCCESS && cleanup_ok; - } - if (state.replica_simulator_connected) { - cleanup_ok = state.replica_simulator.cleanup() == EXIT_SUCCESS && cleanup_ok; - } - if (state.replica_admin != nullptr) { - cleanup_ok = aurora_bgd_admin_cleanup(state.replica_admin) == EXIT_SUCCESS - && cleanup_ok; - } - if (state.primary_admin != nullptr) { - cleanup_ok = aurora_bgd_admin_cleanup(state.primary_admin) == EXIT_SUCCESS - && cleanup_ok; - if (!state.primary_sqlite_interfaces.empty()) { - cleanup_ok = configure_sqlite_interfaces( - state.primary_admin, state.primary_sqlite_interfaces) == EXIT_SUCCESS - && cleanup_ok; - } - mysql_close(state.primary_admin); - state.primary_admin = nullptr; - } - stop_replica( - state.replica_admin, state.replica_process, tests_failed() != 0); - return cleanup_ok ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int main() { - plan(5); - - CommandLine cl {}; - TestState state {}; - - if (setup(cl, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: publish different Aurora states to two real ProxySQL nodes. - // ProxySQL: synchronize the configured Aurora hostgroup row between them. - // Verify: configured fields synchronize while runtime status remains node-local. - if (test_configuration_sync_preserves_local_status(cl, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - -exit_cleanup: - if (cleanup(state) != EXIT_SUCCESS) { - diag("failed to clean the Aurora BGD cluster-sync state"); - return EXIT_FAILURE; - } - return exit_status(); -} diff --git a/test/tap/tests/test_aurora_bgd_completion-t.cpp b/test/tap/tests/test_aurora_bgd_completion-t.cpp deleted file mode 100644 index c21fce121b..0000000000 --- a/test/tap/tests/test_aurora_bgd_completion-t.cpp +++ /dev/null @@ -1,675 +0,0 @@ -/** - * @file test_aurora_bgd_completion-t.cpp - * @brief Aurora BGD completion cleanup and terminal-latch behavior. - * - * Steps: - * - * 1. Complete directly from IN_PROGRESS and reconcile only the writer effect. - * 2. Complete after POST_PROCESSING and clean pins and eligible pools once. - * 3. Rearm completion only for a new fingerprint or confirmed topology drain. - * 4. Start directly at completion and verify no earlier phase is replayed. - */ - -#include -#include -#include -#include - -#include "aurora_bgd_tap.h" -#include "command_line.h" -#include "utils.h" - -using namespace std; - -const uint32_t kWaitSeconds = 5; -const uint32_t kProbeTimeoutMs = 5000; - -struct TestState { - Aurora_BGD_Test_Deployment progress { aurora_bgd_deployment_b_writer_only() }; - int progress_writer_hostgroup { 1550 }; - int progress_reader_hostgroup { 1551 }; - int progress_green_writer_hostgroup { 1552 }; - int progress_green_reader_hostgroup { 1553 }; - Aurora_BGD_Test_Deployment post { aurora_bgd_deployment_a() }; - int post_writer_hostgroup { 1560 }; - int post_reader_hostgroup { 1561 }; - int green_writer_hostgroup { 1562 }; - int green_reader_hostgroup { 1563 }; - vector route_hostgroups { 1564, 1565, 1566 }; - vector post_completion_route_hostgroups { 1567, 1568, 1569 }; - Aurora_BGD_Test_Deployment direct { aurora_bgd_deployment_b_writer_only() }; - int direct_writer_hostgroup { 1580 }; - int direct_reader_hostgroup { 1581 }; - int direct_route_hostgroup { 1582 }; -}; - -int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { - if (cl.getEnv()) { - diag("Error: failed to load TAP environment"); - return EXIT_FAILURE; - } - admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); - if (admin == nullptr) { - diag("Error: failed to connect to ProxySQL Admin"); - return EXIT_FAILURE; - } - char simulator_username[] = "aurora1"; - char simulator_password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. - if (sim.connect(cl.host, 3306, simulator_username, simulator_password) != EXIT_SUCCESS) { - diag("Error: failed to connect to the shared AWS simulator"); - mysql_close(admin); - admin = nullptr; - return EXIT_FAILURE; - } - if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { - diag("Error: failed to clear prior Aurora BGD state"); - return EXIT_FAILURE; - } - return aurora_bgd_execute_all(admin, { - "DELETE FROM mysql_users WHERE username='testuser'", - "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " - "VALUES ('testuser','testuser',1,0,1)", - "LOAD MYSQL USERS TO RUNTIME", - }); -} - -int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { - return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS - && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int cleanup(MYSQL* admin, BGD_Simulator& sim) { - int reset_rc = reset_scenario(admin, sim); - int user_rc = aurora_bgd_execute_all(admin, { - "DELETE FROM mysql_users WHERE username='testuser'", - "LOAD MYSQL USERS TO RUNTIME", - }); - mysql_close(admin); - return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS - ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int publish_status( - BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status -) { - return sim.topology_update( - aurora_bgd_topology_backends(deployment), - aurora_bgd_topology(deployment, status)); -} - -int publish_completed( - BGD_Simulator& sim, Aurora_BGD_Test_Deployment& serving_deployment, - Aurora_BGD_Test_Deployment& completed_deployment -) { - return sim.topology_update( - aurora_bgd_topology_backends(serving_deployment), - aurora_bgd_completed_topology(completed_deployment)); -} - -int set_default_hostgroup(MYSQL* admin, int hostgroup) { - return aurora_bgd_execute_all(admin, { - "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + - " WHERE username='testuser'", - "LOAD MYSQL USERS TO RUNTIME", - }); -} - -int add_route( - MYSQL* admin, int hostgroup, const string& hostname, const string& status = "ONLINE" -) { - return aurora_bgd_execute_all(admin, { - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + - to_string(hostgroup) + "," + aurora_bgd_sql_quote(hostname) + - ",3306," + aurora_bgd_sql_quote(status) + ",1,'Aurora BGD completion route')", - "LOAD MYSQL SERVERS TO RUNTIME", - }); -} - -int add_member_routes( - MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, const vector& hostgroups -) { - if (hostgroups.size() != deployment.production.members.size()) { - diag("Member-route hostgroup count does not match production membership"); - return EXIT_FAILURE; - } - vector queries; - for (size_t i = 0; i < hostgroups.size(); ++i) { - queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + - to_string(hostgroups[i]) + "," + - aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + - ",3306,'ONLINE',1,'Aurora BGD completion member route')"); - } - queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); - return aurora_bgd_execute_all(admin, queries); -} - -int add_green_servers( - MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, - int green_writer_hg, int green_reader_hg -) { - vector queries; - for (size_t i = 0; i < deployment.target.members.size(); ++i) { - const int hostgroup = i == 0 ? green_writer_hg : green_reader_hg; - const string status = i + 1 == deployment.target.members.size() - ? "OFFLINE_SOFT" : "ONLINE"; - queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + - to_string(hostgroup) + "," + - aurora_bgd_sql_quote(deployment.target.members[i].endpoint.hostname) + - ",3306," + aurora_bgd_sql_quote(status) + ",1" + - ",'Aurora BGD configured green member')"); - } - queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + - to_string(green_reader_hg) + "," + - aurora_bgd_sql_quote(deployment.target.members.front().endpoint.hostname) + - ",3306,'SHUNNED',1,'Aurora BGD configured green status matrix')"); - queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + - to_string(green_writer_hg) + "," + - aurora_bgd_sql_quote(deployment.target.members[1].endpoint.hostname) + - ",3306,'OFFLINE_HARD',1,'Aurora BGD configured green status matrix')"); - queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); - return aurora_bgd_execute_all(admin, queries); -} - -bool route_to_expected_backend( - CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected_backend -) { - auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); - if (sequence_rc != EXIT_SUCCESS) { - return false; - } - MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); - if (client == nullptr) { - return false; - } - auto [query_rc, rows] = mysql_query_ext_rows(client, kAuroraBGDRouteProbeQuery); - (void)rows; - if (query_rc != EXIT_SUCCESS) { - diag("Backend routing query failed with MySQL error %d: %s", - mysql_errno(client), mysql_error(client)); - mysql_close(client); - return false; - } - mysql_close(client); - return aurora_bgd_routing_probe_reached(sim, sequence, expected_backend); -} - -bool route_members( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, - Aurora_BGD_Test_Deployment& deployment, const vector& hostgroups, bool target -) { - for (size_t i = 0; i < hostgroups.size(); ++i) { - if (set_default_hostgroup(admin, hostgroups[i]) != EXIT_SUCCESS) { - return false; - } - const Endpoint expected = target - ? deployment.target.members[i].endpoint.backend() - : deployment.production.members[i].endpoint.backend(); - if (!route_to_expected_backend(cl, sim, expected)) { - return false; - } - } - return true; -} - -int64_t pool_count(MYSQL* admin, int hostgroup) { - auto [rc, rows] = mysql_query_ext_rows( - admin, - "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) FROM stats_mysql_connection_pool " - "WHERE hostgroup=" + to_string(hostgroup)); - if (rc != EXIT_SUCCESS || rows.size() != 1 || rows.front().size() != 1) { - return -1; - } - return strtoll(rows.front().front().c_str(), nullptr, 10); -} - -int wait_for_pool_count(MYSQL* admin, int hostgroup, const string& comparison) { - return wait_for_cond( - admin, - "SELECT COALESCE(SUM(ConnUsed+ConnFree),0)" + comparison + - " FROM stats_mysql_connection_pool WHERE hostgroup=" + to_string(hostgroup), - kWaitSeconds); -} - -bool server_count( - MYSQL* admin, int hostgroup, const string& hostname, int expected, - const string& status = "", const string& table = "runtime_mysql_servers" -) { - string query = - "SELECT COUNT(*) FROM " + table + " WHERE hostgroup_id=" + - to_string(hostgroup) + " AND hostname=" + aurora_bgd_sql_quote(hostname); - if (!status.empty()) { - query += " AND status=" + aurora_bgd_sql_quote(status); - } - auto [rc, rows] = mysql_query_ext_rows(admin, query); - return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 - && rows.front().front() == to_string(expected); -} - -int wait_for_writer_policy( - MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, - bool writer_is_also_reader -) { - return wait_for_cond( - admin, - "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + - ")=1) AND ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + - ")=" + (writer_is_also_reader ? "1" : "0") + ")", - kWaitSeconds); -} - -int wait_for_writer_demotion( - MYSQL* admin, int writer_hg, int reader_hg, const string& hostname -) { - return wait_for_cond( - admin, - "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + - ")=0) AND ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + - ")=1)", - kWaitSeconds); -} - -bool completion_probe_policy( - BGD_Simulator& sim, uint64_t sequence, - Aurora_BGD_Test_Deployment& deployment, uint32_t observation_ms -) { - usleep(observation_ms * 1000); - auto [rc, logs] = sim.replica_probe_log_since(sequence); - if (rc != EXIT_SUCCESS) { - return false; - } - bool ordinary_on_production = false; - for (const Aurora_Replica_Probe_Log& log : logs) { - if (log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership) { - return false; - } - if (log.probe_kind != Aurora_Replica_Probe_Kind::ordinary) { - continue; - } - for (const Endpoint& backend : deployment.production.backends()) { - ordinary_on_production |= log.backend.host == backend.host - && log.backend.port == backend.port; - } - } - return ordinary_on_production; -} - -bool wait_for_topology_observation( - BGD_Simulator& sim, uint64_t sequence, Aurora_BGD_Test_Deployment& deployment -) { - auto [first_rc, first_probe] = aurora_bgd_wait_for_topology_probe( - sim, sequence, deployment.production.backends(), - BGD_Probe_Kind::metadata, kProbeTimeoutMs); - if (first_rc != EXIT_SUCCESS) { - return false; - } - auto [second_rc, second_probe] = aurora_bgd_wait_for_topology_probe( - sim, first_probe.sequence_id, deployment.production.backends(), - BGD_Probe_Kind::metadata, kProbeTimeoutMs); - return second_rc == EXIT_SUCCESS; -} - -/** - * Complete a deployment directly from IN_PROGRESS. - * - * - Restore the demoted writer without replaying POST_PROCESSING. - * - Resume ordinary production probing while retaining a terminal latch. - * - Release the latch only after confirmed topology absence. - */ -int test_completion_from_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { - Aurora_BGD_Test_Deployment& deployment = state.progress; - if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.progress_writer_hostgroup, - state.progress_reader_hostgroup, state.progress_green_writer_hostgroup, - state.progress_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS) { - diag("Error: failed to configure completion-from-IN_PROGRESS scenario"); - return EXIT_FAILURE; - } - - ok(aurora_bgd_wait_for_status( - admin, state.progress_writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, - "completion-from-IN_PROGRESS scenario reaches AVAILABLE"); - if (publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.progress_writer_hostgroup, - "SWITCHOVER_IN_PROGRESS", kWaitSeconds) != EXIT_SUCCESS) { - diag("Error: failed to reach IN_PROGRESS"); - return EXIT_FAILURE; - } - ok(wait_for_writer_demotion( - admin, state.progress_writer_hostgroup, state.progress_reader_hostgroup, - deployment.production.members.front().endpoint.hostname) == EXIT_SUCCESS, - "IN_PROGRESS demotes the writer before completion"); - - if (publish_completed(sim, deployment, deployment) != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.progress_writer_hostgroup, - "SWITCHOVER_COMPLETED", kWaitSeconds) != EXIT_SUCCESS) { - diag("Error: failed to enter the completed latch from IN_PROGRESS"); - return EXIT_FAILURE; - } - ok(true, "TARGET-only completion publishes SWITCHOVER_COMPLETED"); - ok(wait_for_writer_policy( - admin, state.progress_writer_hostgroup, state.progress_reader_hostgroup, - deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, - "completion restores the demoted writer without replaying POST_PROCESSING"); - - auto [replica_seq_rc, replica_sequence] = sim.replica_probe_log_last_sequence(); - ok(replica_seq_rc == EXIT_SUCCESS && completion_probe_policy( - sim, replica_sequence, deployment, 750), - "completion resumes ordinary production probing and stops membership probing"); - - auto [repeat_seq_rc, repeat_sequence] = sim.probe_log_last_sequence(); - bool repeated = repeat_seq_rc == EXIT_SUCCESS - && publish_completed(sim, deployment, deployment) == EXIT_SUCCESS - && wait_for_topology_observation(sim, repeat_sequence, deployment); - ok(repeated && wait_for_writer_policy( - admin, state.progress_writer_hostgroup, state.progress_reader_hostgroup, - deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, - "repeated completion is a no-op while latched"); - - auto [error_seq_rc, error_sequence] = sim.probe_log_last_sequence(); - bool error_retained = error_seq_rc == EXIT_SUCCESS - && sim.topology_error( - aurora_bgd_topology_backends(deployment), 1205, - "simulated completion timeout") == EXIT_SUCCESS - && wait_for_topology_observation(sim, error_sequence, deployment); - ok(error_retained && aurora_bgd_wait_for_status( - admin, state.progress_writer_hostgroup, "SWITCHOVER_COMPLETED", 1) - == EXIT_SUCCESS, - "topology query errors retain the completed latch"); - ok(sim.topology_drop(aurora_bgd_topology_backends(deployment)) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.progress_writer_hostgroup, "NONE", kWaitSeconds) == EXIT_SUCCESS, - "confirmed topology absence releases the completed latch to NONE"); - return EXIT_SUCCESS; -} - -/** - * Complete a deployment after POST_PROCESSING. - * - * - Remove production pins and drain eligible configured-green pools once. - * - Preserve configured rows, public status, and post-cutover production pools. - * - Rearm cleanup only for a different deployment fingerprint or topology drain. - */ -int test_completion_after_post_processing( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) -{ - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to reset before POST_PROCESSING completion scenario"); - return EXIT_FAILURE; - } - - Aurora_BGD_Test_Deployment& deployment = state.post; - if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.post_writer_hostgroup, state.post_reader_hostgroup, - state.green_writer_hostgroup, state.green_reader_hostgroup, - false, 300, true) != EXIT_SUCCESS - || add_green_servers( - admin, deployment, state.green_writer_hostgroup, - state.green_reader_hostgroup) != EXIT_SUCCESS - || add_member_routes(admin, deployment, state.route_hostgroups) != EXIT_SUCCESS) { - diag("Error: failed to configure POST_PROCESSING completion scenario"); - return EXIT_FAILURE; - } - - ok(aurora_bgd_wait_for_status( - admin, state.post_writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, - "POST_PROCESSING completion scenario reaches AVAILABLE"); - if (publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.post_writer_hostgroup, - "SWITCHOVER_IN_PROGRESS", kWaitSeconds) != EXIT_SUCCESS - || publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.post_writer_hostgroup, - "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { - diag("Error: failed to advance through POST_PROCESSING"); - return EXIT_FAILURE; - } - ok(true, "active deployment advances through POST_PROCESSING"); - ok(route_members(cl, admin, sim, deployment, state.route_hostgroups, true), - "POST_PROCESSING routes every production member to its target IP"); - - const int64_t target_route_pool = pool_count(admin, state.route_hostgroups.front()); - bool green_pool_ready = set_default_hostgroup(admin, state.green_writer_hostgroup) - == EXIT_SUCCESS - && route_to_expected_backend( - cl, sim, deployment.target.members.front().endpoint.backend()) - && pool_count(admin, state.green_writer_hostgroup) >= 1; - ok(target_route_pool >= 1 && green_pool_ready, - "pre-completion target and configured-green pools are established"); - - if (publish_completed(sim, deployment, deployment) != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.post_writer_hostgroup, - "SWITCHOVER_COMPLETED", kWaitSeconds) != EXIT_SUCCESS) { - diag("Error: failed to complete the POST_PROCESSING scenario"); - return EXIT_FAILURE; - } - ok(true, "completion after POST_PROCESSING enters the terminal latch"); - ok(wait_for_writer_policy( - admin, state.post_writer_hostgroup, state.post_reader_hostgroup, - deployment.production.members.front().endpoint.hostname, true) == EXIT_SUCCESS, - "completion preserves canonical writer_is_also_reader placement"); - ok(pool_count(admin, state.route_hostgroups.front()) >= target_route_pool, - "completion does not repeat retirement of post-cutover production pools"); - ok(wait_for_pool_count( - admin, state.green_writer_hostgroup, "=0") == EXIT_SUCCESS, - "completion drains eligible configured-green pools immediately"); - ok(server_count( - admin, state.green_reader_hostgroup, - deployment.target.members.back().endpoint.hostname, 1, "OFFLINE_SOFT"), - "completion preserves configured green rows and OFFLINE status"); - ok(server_count( - admin, state.green_reader_hostgroup, - deployment.target.members.front().endpoint.hostname, 1, "SHUNNED") - && server_count( - admin, state.green_writer_hostgroup, - deployment.target.members[1].endpoint.hostname, 1, "OFFLINE_HARD", - "mysql_servers"), - "completion preserves configured SHUNNED and OFFLINE_HARD green rows"); - ok(add_member_routes( - admin, deployment, state.post_completion_route_hostgroups) == EXIT_SUCCESS - && route_members( - cl, admin, sim, deployment, - state.post_completion_route_hostgroups, false), - "completion removes every writer and reader traffic pin without DNS verification"); - - auto [replica_seq_rc, replica_sequence] = sim.replica_probe_log_last_sequence(); - ok(replica_seq_rc == EXIT_SUCCESS && completion_probe_policy( - sim, replica_sequence, deployment, 750), - "the completed latch uses configured cadence and canonical production probes"); - - bool recreated_green_pool = set_default_hostgroup(admin, state.green_writer_hostgroup) - == EXIT_SUCCESS - && route_to_expected_backend( - cl, sim, deployment.target.members.front().endpoint.backend()) - && pool_count(admin, state.green_writer_hostgroup) >= 1; - auto [same_seq_rc, same_sequence] = sim.probe_log_last_sequence(); - bool same_completion_seen = same_seq_rc == EXIT_SUCCESS - && publish_completed(sim, deployment, deployment) == EXIT_SUCCESS - && wait_for_topology_observation(sim, same_sequence, deployment); - ok(recreated_green_pool && same_completion_seen - && pool_count(admin, state.green_writer_hostgroup) >= 1, - "repeated completion does not drain a pool created while latched"); - - auto [error_seq_rc, error_sequence] = sim.probe_log_last_sequence(); - bool error_seen = error_seq_rc == EXIT_SUCCESS - && sim.topology_error( - aurora_bgd_topology_backends(deployment), 1205, - "simulated latched timeout") == EXIT_SUCCESS - && wait_for_topology_observation(sim, error_sequence, deployment); - ok(error_seen && aurora_bgd_wait_for_status( - admin, state.post_writer_hostgroup, "SWITCHOVER_COMPLETED", 1) - == EXIT_SUCCESS - && pool_count(admin, state.green_writer_hostgroup) >= 1, - "query errors neither release the latch nor repeat completion cleanup"); - - Aurora_BGD_Test_Deployment different = aurora_bgd_deployment_b_writer_only(); - auto [same_active_seq_rc, same_active_sequence] = sim.probe_log_last_sequence(); - bool same_active_seen = same_active_seq_rc == EXIT_SUCCESS - && publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") == EXIT_SUCCESS - && wait_for_topology_observation(sim, same_active_sequence, deployment); - ok(same_active_seen && aurora_bgd_wait_for_status( - admin, state.post_writer_hostgroup, "SWITCHOVER_COMPLETED", 1) - == EXIT_SUCCESS - && pool_count(admin, state.green_writer_hostgroup) >= 1, - "a stale active result for the completed fingerprint is ignored while latched"); - - if (sim.topology_update( - aurora_bgd_topology_backends(deployment), - aurora_bgd_topology(different, "SWITCHOVER_INITIATED")) != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - ok(aurora_bgd_wait_for_status( - admin, state.post_writer_hostgroup, "SWITCHOVER_INITIATED", kWaitSeconds) - == EXIT_SUCCESS, - "a different active deployment fingerprint rearms the completed latch"); - - auto [different_seq_rc, different_sequence] = sim.probe_log_last_sequence(); - bool different_seen = different_seq_rc == EXIT_SUCCESS - && publish_completed(sim, deployment, different) == EXIT_SUCCESS - && wait_for_topology_observation(sim, different_sequence, deployment); - ok(different_seen && wait_for_pool_count( - admin, state.green_writer_hostgroup, "=0") == EXIT_SUCCESS, - "a different completed deployment fingerprint rearms and runs its cleanup"); - - bool second_green_pool = set_default_hostgroup(admin, state.green_writer_hostgroup) - == EXIT_SUCCESS - && route_to_expected_backend( - cl, sim, deployment.target.members.front().endpoint.backend()); - auto [repeat_different_seq_rc, repeat_different_sequence] = sim.probe_log_last_sequence(); - bool repeated_different = repeat_different_seq_rc == EXIT_SUCCESS - && publish_completed(sim, deployment, different) == EXIT_SUCCESS - && wait_for_topology_observation(sim, repeat_different_sequence, deployment); - ok(second_green_pool && repeated_different - && pool_count(admin, state.green_writer_hostgroup) >= 1, - "the new fingerprint is retained and its repeated completion is a no-op"); - ok(sim.topology_delete(aurora_bgd_topology_backends(deployment)) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.post_writer_hostgroup, "NONE", kWaitSeconds) == EXIT_SUCCESS, - "successful empty topology releases the rearmed completed latch"); - return EXIT_SUCCESS; -} - -/** - * Start a worker from its first SWITCHOVER_COMPLETED observation. - * - * - Enter the terminal latch without a cached target map. - * - Keep canonical writer placement and avoid replaying active phases. - * - Rearm only after a successful topology drain. - */ -int test_first_completed_observation( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state -) { - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to reset before direct-completion scenario"); - return EXIT_FAILURE; - } - - Aurora_BGD_Test_Deployment& deployment = state.direct; - if (sim.replica_update( - deployment.blue_replica_set, deployment.production.replica_rows(), - deployment.production.backends()) != EXIT_SUCCESS - || sim.topology_update( - aurora_bgd_topology_backends(deployment), - aurora_bgd_completed_topology(deployment)) != EXIT_SUCCESS) { - diag("Error: failed to publish direct completion inputs"); - return EXIT_FAILURE; - } - if (add_route( - admin, state.direct_route_hostgroup, - deployment.production.members.front().endpoint.hostname) != EXIT_SUCCESS - || set_default_hostgroup(admin, state.direct_route_hostgroup) != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - bool direct_pool_ready = route_to_expected_backend( - cl, sim, deployment.production.members.front().endpoint.backend()) - && pool_count(admin, state.direct_route_hostgroup) >= 1; - ok(direct_pool_ready, - "direct-completion setup has an observable production route and pool"); - auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); - if (aurora_bgd_admin_setup( - admin, deployment, state.direct_writer_hostgroup, - state.direct_reader_hostgroup, -1, -1, true, 300, false) != EXIT_SUCCESS) { - diag("Error: failed to configure direct completion"); - return EXIT_FAILURE; - } - - ok(aurora_bgd_wait_for_status( - admin, state.direct_writer_hostgroup, - "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS, - "late entry directly at completion enters the terminal latch"); - ok(wait_for_writer_policy( - admin, state.direct_writer_hostgroup, state.direct_reader_hostgroup, - deployment.production.members.front().endpoint.hostname, false) == EXIT_SUCCESS, - "direct completion leaves canonical writer placement unchanged"); - ok(pool_count(admin, state.direct_route_hostgroup) >= 1 - && set_default_hostgroup(admin, state.direct_route_hostgroup) == EXIT_SUCCESS - && route_to_expected_backend( - cl, sim, deployment.production.members.front().endpoint.backend()), - "direct completion leaves unrelated production routing effects untouched"); - - usleep(750000); - auto [logs_rc, logs] = sim.replica_probe_log_since(sequence); - bool membership_probe = false; - for (const Aurora_Replica_Probe_Log& log : logs) { - membership_probe |= log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership; - } - ok(sequence_rc == EXIT_SUCCESS && logs_rc == EXIT_SUCCESS && !membership_probe, - "direct completion does not manufacture target membership or replay active phases"); - ok(sim.topology_delete(aurora_bgd_topology_backends(deployment)) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.direct_writer_hostgroup, "NONE", kWaitSeconds) == EXIT_SUCCESS, - "direct-completion latch rearms only after topology drain"); - return EXIT_SUCCESS; -} - -int main() { - plan(33); - - CommandLine cl {}; - MYSQL* admin = nullptr; - BGD_Simulator sim {}; - - if (setup(cl, admin, sim) != EXIT_SUCCESS) { - return exit_status(); - } - - TestState state {}; - - // Simulator: advance a deployment to IN_PROGRESS, then publish completion. - // Verify: only the writer effect is reconciled and the completed latch is retained. - if (test_completion_from_in_progress(admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: advance through POST_PROCESSING, then publish completion repeatedly. - // Verify: pins and eligible pools are cleaned once per deployment fingerprint. - if (test_completion_after_post_processing(cl, admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: make SWITCHOVER_COMPLETED the first observed deployment state. - // Verify: prior phase effects are not manufactured or replayed. - if (test_first_completed_observation(cl, admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - -exit_cleanup: - if (cleanup(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to clean Aurora BGD completion test data"); - return EXIT_FAILURE; - } - return exit_status(); -} diff --git a/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp b/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp index c5d9540cb8..57fd351546 100644 --- a/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp +++ b/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp @@ -1,365 +1,140 @@ /** * @file test_aurora_bgd_concurrent_isolation-t.cpp - * @brief Isolation across three concurrent Aurora BGD workers. - * - * Steps: - * - * 1. Configure three deployments with independent workers and route hostgroups. - * 2. Enter POST_PROCESSING on all workers and verify each owns its target pin. - * 3. Deactivate one worker without changing the other two. - * 4. Complete and remove different workers without leaking lifecycle effects. + * @brief Independent Aurora BGD state for three writer hostgroups. */ -#include #include -#include -#include -#include "aurora_bgd_tap.h" -#include "command_line.h" -#include "utils.h" +#include "aurora_bgd_scenario_tap.h" -using namespace std; +using namespace aurora_bgd_scenario; -const uint32_t kWaitSeconds = 5; +struct Worker { + Aurora_BGD_Test_Deployment deployment; + int writer_hostgroup; + int reader_hostgroup; + int green_writer_hostgroup; + int green_reader_hostgroup; + int route_hostgroup; +}; struct TestState { - Aurora_BGD_Test_Deployment deployment_a { aurora_bgd_deployment_a() }; - int writer_hostgroup_a { 1640 }; - int reader_hostgroup_a { 1641 }; - int green_writer_hostgroup_a { 1642 }; - int green_reader_hostgroup_a { 1643 }; - int route_hostgroup_a { 1644 }; - int post_completion_route_hostgroup_a { 1645 }; - Aurora_BGD_Test_Deployment deployment_b { aurora_bgd_deployment_b_writer_only() }; - int writer_hostgroup_b { 1650 }; - int reader_hostgroup_b { 1651 }; - int route_hostgroup_b { 1654 }; - Aurora_BGD_Test_Deployment deployment_c { aurora_bgd_deployment_c_writer_only() }; - int writer_hostgroup_c { 1660 }; - int reader_hostgroup_c { 1661 }; - int route_hostgroup_c { 1664 }; + Worker first { aurora_bgd_deployment_a(), 2250, 2251, 2252, 2253, 2254 }; + Worker second { aurora_bgd_deployment_b_writer_only(), 2260, 2261, 2262, 2263, 2264 }; + Worker third { aurora_bgd_deployment_c_writer_only(), 2270, 2271, 2272, 2273, 2274 }; }; -int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { - if (cl.getEnv()) { - diag("Error: failed to load TAP environment"); - return EXIT_FAILURE; - } - admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); - if (admin == nullptr) { - diag("Error: failed to connect to ProxySQL Admin"); - return EXIT_FAILURE; - } - char username[] = "aurora1"; - char password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. - if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS - || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS - || sim.cleanup() != EXIT_SUCCESS) { - diag("Error: failed to initialize the shared AWS simulator"); - return EXIT_FAILURE; - } - return aurora_bgd_execute_all(admin, { - "DELETE FROM mysql_users WHERE username='testuser'", - "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " - "VALUES ('testuser','testuser',1,0,1)", - "LOAD MYSQL USERS TO RUNTIME", - }); -} - -int cleanup(MYSQL* admin, BGD_Simulator& sim) { - int admin_rc = aurora_bgd_admin_cleanup(admin); - int user_rc = aurora_bgd_execute_all(admin, { - "DELETE FROM mysql_users WHERE username='testuser'", - "LOAD MYSQL USERS TO RUNTIME", - }); - int simulator_rc = sim.cleanup(); - mysql_close(admin); - return admin_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS - && simulator_rc == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int publish_status( - BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status -) { - return sim.topology_update( - aurora_bgd_topology_backends(deployment), - aurora_bgd_topology(deployment, status)); -} - -int set_default_hostgroup(MYSQL* admin, int hostgroup) { - return aurora_bgd_execute_all(admin, { - "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + - " WHERE username='testuser'", - "LOAD MYSQL USERS TO RUNTIME", - }); +int configure_available(Context& context, Worker& worker, bool use_ssl) { + return publish_available(context, worker.deployment) == EXIT_SUCCESS + && configure( + context, worker.deployment, worker.writer_hostgroup, + worker.reader_hostgroup, worker.green_writer_hostgroup, + worker.green_reader_hostgroup, false, 300, false, use_ssl) == EXIT_SUCCESS + && add_route( + context.admin, worker.route_hostgroup, + worker.deployment.production.members.front().endpoint.hostname) == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; } -int add_writer_route( - MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, int hostgroup +bool worker_matches( + Context& context, Worker& worker, const string& status, bool demoted ) { - return aurora_bgd_execute_all(admin, { - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + - to_string(hostgroup) + "," + - aurora_bgd_sql_quote(deployment.production.members.front().endpoint.hostname) + - ",3306,'ONLINE',1,'Aurora BGD concurrent route')", - "LOAD MYSQL SERVERS TO RUNTIME", - }); -} - -bool route_to_backend( - CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected -) { - auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); - if (sequence_rc != EXIT_SUCCESS) { - return false; - } - MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); - if (client == nullptr) { - return false; - } - auto [query_rc, rows] = mysql_query_ext_rows(client, kAuroraBGDRouteProbeQuery); - (void)rows; - mysql_close(client); - return query_rc == EXIT_SUCCESS - && aurora_bgd_routing_probe_reached(sim, sequence, expected); + return runtime_status_is(context.admin, worker.writer_hostgroup, status) + && writer_placement( + context.admin, worker.writer_hostgroup, worker.reader_hostgroup, + worker.deployment.production.members.front().endpoint.hostname, demoted); } -bool route_writer( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, - Aurora_BGD_Test_Deployment& deployment, int route_hg, bool target -) { - if (set_default_hostgroup(admin, route_hg) != EXIT_SUCCESS) { - return false; +int test_three_workers_available(Context& context, TestState& state) { + if (configure_available(context, state.first, false) != EXIT_SUCCESS + || configure_available(context, state.second, true) != EXIT_SUCCESS + || configure_available(context, state.third, false) != EXIT_SUCCESS) { + diag("Error: failed to configure three Aurora BGD workers"); + return EXIT_FAILURE; } - const Endpoint expected = target - ? deployment.target.members.front().endpoint.backend() - : deployment.production.members.front().endpoint.backend(); - return route_to_backend(cl, sim, expected); + ok(aurora_bgd_wait_for_status( + context.admin, state.first.writer_hostgroup, "AVAILABLE", kWaitSeconds) + == EXIT_SUCCESS, + "BGD wHG 2250 independently reaches AVAILABLE"); + ok(aurora_bgd_wait_for_status( + context.admin, state.second.writer_hostgroup, "AVAILABLE", kWaitSeconds) + == EXIT_SUCCESS, + "BGD wHG 2260 independently reaches AVAILABLE"); + ok(aurora_bgd_wait_for_status( + context.admin, state.third.writer_hostgroup, "AVAILABLE", kWaitSeconds) + == EXIT_SUCCESS, + "BGD wHG 2270 independently reaches AVAILABLE"); + return EXIT_SUCCESS; } -bool wait_for_writer_route( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, - Aurora_BGD_Test_Deployment& deployment, int route_hg, bool target +int test_independent_phase_changes( + CommandLine& cl, Context& context, TestState& state ) { - if (set_default_hostgroup(admin, route_hg) != EXIT_SUCCESS) { - return false; + if (publish_status( + context, state.first.deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.first.writer_hostgroup, "SWITCHOVER_IN_PROGRESS", + kWaitSeconds) != EXIT_SUCCESS) { + return EXIT_FAILURE; } - const Endpoint expected = target - ? deployment.target.members.front().endpoint.backend() - : deployment.production.members.front().endpoint.backend(); - for (uint32_t elapsed_ms = 0; elapsed_ms < kWaitSeconds * 1000; elapsed_ms += 100) { - if (route_to_backend(cl, sim, expected)) { - return true; - } - usleep(100000); + ok(worker_matches(context, state.first, "SWITCHOVER_IN_PROGRESS", true), + "advancing wHG 2250 changes only its writer placement and state"); + ok(worker_matches(context, state.second, "AVAILABLE", false) + && worker_matches(context, state.third, "AVAILABLE", false), + "advancing wHG 2250 leaves wHG 2260 and wHG 2270 unchanged"); + + if (publish_status( + context, state.second.deployment, + "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.second.writer_hostgroup, + "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS + || set_default_hostgroup( + context.admin, state.second.route_hostgroup) != EXIT_SUCCESS) { + return EXIT_FAILURE; } - return false; -} - -bool writer_placement( - MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, bool demoted -) { - string query = - "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + - ")=" + (demoted ? "0" : "1") + ") AND " - "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + - ")=" + (demoted ? "1" : "0") + ")"; - return wait_for_cond(admin, query, kWaitSeconds) == EXIT_SUCCESS; -} - -bool wait_for_inactive_none(MYSQL* admin, int writer_hg) { - return wait_for_cond( - admin, - "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + - to_string(writer_hg) + " AND active=0 AND bgd_status='NONE'", - kWaitSeconds) == EXIT_SUCCESS; -} - -bool wait_for_runtime_row_absent(MYSQL* admin, int writer_hg) { - return wait_for_cond( - admin, - "SELECT COUNT(*)=0 FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + - to_string(writer_hg), - kWaitSeconds) == EXIT_SUCCESS; -} - -/** Configure three workers and move each independently to POST_PROCESSING. */ -int test_three_workers_post_processing( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) -{ - if (aurora_bgd_publish(sim, state.deployment_a) != EXIT_SUCCESS - || aurora_bgd_publish(sim, state.deployment_b) != EXIT_SUCCESS - || aurora_bgd_publish(sim, state.deployment_c) != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, state.deployment_a, state.writer_hostgroup_a, - state.reader_hostgroup_a, state.green_writer_hostgroup_a, - state.green_reader_hostgroup_a, false, 300) != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, state.deployment_b, state.writer_hostgroup_b, - state.reader_hostgroup_b, -1, -1, true, 300) != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, state.deployment_c, state.writer_hostgroup_c, - state.reader_hostgroup_c, -1, -1, true, 300) != EXIT_SUCCESS - || add_writer_route( - admin, state.deployment_a, state.route_hostgroup_a) != EXIT_SUCCESS - || add_writer_route( - admin, state.deployment_b, state.route_hostgroup_b) != EXIT_SUCCESS - || add_writer_route( - admin, state.deployment_c, state.route_hostgroup_c) != EXIT_SUCCESS) { - diag("Error: failed to configure three concurrent deployments"); + ok(route_to_backend( + cl, context, + state.second.deployment.target.members.front().endpoint.backend()), + "advancing wHG 2260 applies only its own target pin"); + ok(worker_matches(context, state.first, "SWITCHOVER_IN_PROGRESS", true) + && worker_matches(context, state.third, "AVAILABLE", false), + "advancing wHG 2260 preserves wHG 2250 progress and wHG 2270 availability"); + + if (publish_status( + context, state.third.deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.third.writer_hostgroup, "SWITCHOVER_INITIATED", + kWaitSeconds) != EXIT_SUCCESS) { return EXIT_FAILURE; } - - ok(aurora_bgd_wait_for_status( - admin, state.writer_hostgroup_a, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.writer_hostgroup_b, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.writer_hostgroup_c, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, - "three writer hostgroups discover deployments independently"); - ok(publish_status( - sim, state.deployment_a, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS - && publish_status( - sim, state.deployment_b, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS - && publish_status( - sim, state.deployment_c, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.writer_hostgroup_a, - "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.writer_hostgroup_b, - "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.writer_hostgroup_c, - "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, - "three workers enter POST_PROCESSING without sharing FSM state"); - ok(route_writer( - cl, admin, sim, state.deployment_a, state.route_hostgroup_a, true), - "deployment A owns its target pin"); - ok(route_writer( - cl, admin, sim, state.deployment_b, state.route_hostgroup_b, true), - "deployment B owns its target pin"); - ok(route_writer( - cl, admin, sim, state.deployment_c, state.route_hostgroup_c, true), - "deployment C owns its target pin"); - return EXIT_SUCCESS; -} - -/** Deactivate deployment B without changing deployments A or C. */ -int test_deactivate_one_worker( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) -{ - ok(aurora_bgd_execute_all(admin, { - "UPDATE mysql_aws_aurora_hostgroups SET active=0 WHERE writer_hostgroup=" + - to_string(state.writer_hostgroup_b), - "LOAD MYSQL SERVERS TO RUNTIME", - }) == EXIT_SUCCESS - && wait_for_inactive_none(admin, state.writer_hostgroup_b) - && aurora_bgd_wait_for_status( - admin, state.writer_hostgroup_a, - "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.writer_hostgroup_c, - "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, - "deactivating deployment B cleans only its worker state"); - ok(writer_placement( - admin, state.writer_hostgroup_b, state.reader_hostgroup_b, - state.deployment_b.production.members.front().endpoint.hostname, false) - && route_writer( - cl, admin, sim, state.deployment_b, state.route_hostgroup_b, false), - "deployment B teardown restores only its production routing"); - ok(route_writer( - cl, admin, sim, state.deployment_a, state.route_hostgroup_a, true) - && route_writer( - cl, admin, sim, state.deployment_c, state.route_hostgroup_c, true), - "deployment B teardown leaves A and C pins intact"); - return EXIT_SUCCESS; -} - -/** Complete deployment A and remove deployment C without cross-worker effects. */ -int test_complete_and_remove_independent_workers( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) -{ - ok(sim.topology_update( - aurora_bgd_topology_backends(state.deployment_a), - aurora_bgd_completed_topology(state.deployment_a)) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.writer_hostgroup_a, - "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.writer_hostgroup_c, - "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, - "completing deployment A leaves deployment C active"); - ok(add_writer_route( - admin, state.deployment_a, state.post_completion_route_hostgroup_a) - == EXIT_SUCCESS - && route_writer( - cl, admin, sim, state.deployment_a, - state.post_completion_route_hostgroup_a, false) - && route_writer( - cl, admin, sim, state.deployment_c, state.route_hostgroup_c, true), - "deployment A cleanup removes only its pin"); - ok(aurora_bgd_execute_all(admin, { - "DELETE FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + - to_string(state.writer_hostgroup_c), - "LOAD MYSQL SERVERS TO RUNTIME", - }) == EXIT_SUCCESS - && wait_for_runtime_row_absent(admin, state.writer_hostgroup_c) - && writer_placement( - admin, state.writer_hostgroup_c, state.reader_hostgroup_c, - state.deployment_c.production.members.front().endpoint.hostname, false) - && wait_for_writer_route( - cl, admin, sim, state.deployment_c, state.route_hostgroup_c, false), - "removing deployment C safely restores its production routing"); - ok(aurora_bgd_wait_for_status( - admin, state.writer_hostgroup_a, - "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS - && route_writer( - cl, admin, sim, state.deployment_a, - state.post_completion_route_hostgroup_a, false), - "deployment C removal leaves deployment A terminal state unchanged"); + ok(worker_matches(context, state.third, "SWITCHOVER_INITIATED", false), + "advancing wHG 2270 records INITIATED without changing writer placement"); + ok(worker_matches(context, state.first, "SWITCHOVER_IN_PROGRESS", true) + && worker_matches( + context, state.second, "SWITCHOVER_IN_POST_PROCESSING", false), + "advancing wHG 2270 preserves the independent states of wHG 2250 and wHG 2260"); return EXIT_SUCCESS; } int main() { - plan(12); + plan(9); CommandLine cl {}; - MYSQL* admin = nullptr; - BGD_Simulator sim {}; - - if (setup(cl, admin, sim) != EXIT_SUCCESS) { + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { return exit_status(); } TestState state {}; - - // Simulator: publish three independent deployments and POST_PROCESSING states. - // Verify: each worker owns only its target pin and FSM state. - if (test_three_workers_post_processing(cl, admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // ProxySQL: deactivate deployment B. - // Verify: only deployment B returns to production routing. - if (test_deactivate_one_worker(cl, admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator and ProxySQL: complete A and remove C. - // Verify: their cleanup effects remain isolated from each other. - if (test_complete_and_remove_independent_workers(cl, admin, sim, state) - != EXIT_SUCCESS) { + if (test_three_workers_available(context, state) != EXIT_SUCCESS) { goto exit_cleanup; } + test_independent_phase_changes(cl, context, state); exit_cleanup: - if (cleanup(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to clean Aurora BGD concurrent-isolation test data"); - return EXIT_FAILURE; + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the concurrent-isolation fixture"); } return exit_status(); } diff --git a/test/tap/tests/test_aurora_bgd_config_refresh_after_completion-t.cpp b/test/tap/tests/test_aurora_bgd_config_refresh_after_completion-t.cpp new file mode 100644 index 0000000000..cc18f8e0a1 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_config_refresh_after_completion-t.cpp @@ -0,0 +1,93 @@ +/** + * @file test_aurora_bgd_config_refresh_after_completion-t.cpp + * @brief Aurora BGD configuration refresh while completion is latched. + */ + +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +struct TestState { + Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_b_writer_only() }; + int writer_hostgroup { 2210 }; + int reader_hostgroup { 2211 }; +}; + +int test_completed_latch(Context& context, TestState& state) { + if (publish_available(context, state.deployment) != EXIT_SUCCESS + || configure( + context, state.deployment, state.writer_hostgroup, state.reader_hostgroup, + -1, -1, true, 300) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) != EXIT_SUCCESS + || publish_status( + context, state.deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_IN_PROGRESS", + kWaitSeconds) != EXIT_SUCCESS + || wait_for_writer_placement( + context.admin, state.writer_hostgroup, state.reader_hostgroup, + state.deployment.production.members.front().endpoint.hostname, + true) != EXIT_SUCCESS + || publish_completed(context, state.deployment, state.deployment) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_COMPLETED", + kWaitSeconds) == EXIT_SUCCESS + && writer_placement( + context.admin, state.writer_hostgroup, state.reader_hostgroup, + state.deployment.production.members.front().endpoint.hostname, false), + "completion from IN_PROGRESS restores the writer and enters the terminal latch"); + return EXIT_SUCCESS; +} + +int test_config_refresh_after_completion(Context& context, TestState& state) { + if (aurora_bgd_execute_all(context.admin, { + "UPDATE mysql_aws_aurora_hostgroups SET check_timeout_ms=950 " + "WHERE writer_hostgroup=2210", + "LOAD MYSQL SERVERS TO RUNTIME", + }) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(wait_for_cond( + context.admin, + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=2210 AND check_timeout_ms=950 " + "AND bgd_status='SWITCHOVER_COMPLETED'", + kWaitSeconds) == EXIT_SUCCESS, + "configuration refresh preserves the completed latch for wHG 2210"); + + if (context.simulator.topology_delete( + aurora_bgd_topology_backends(state.deployment)) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "empty topology rearms the refreshed terminal worker to NONE"); + return EXIT_SUCCESS; +} + +int main() { + plan(3); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + if (test_completed_latch(context, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + test_config_refresh_after_completion(context, state); + +exit_cleanup: + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the completion-refresh fixture"); + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_disable_during_switchover-t.cpp b/test/tap/tests/test_aurora_bgd_disable_during_switchover-t.cpp new file mode 100644 index 0000000000..869c88348d --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_disable_during_switchover-t.cpp @@ -0,0 +1,89 @@ +/** + * @file test_aurora_bgd_disable_during_switchover-t.cpp + * @brief Aurora BGD cleanup when the owning row is deactivated. + */ + +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +struct TestState { + Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_b_writer_only() }; + int writer_hostgroup { 2220 }; + int reader_hostgroup { 2221 }; + int green_writer_hostgroup { 2222 }; + int green_reader_hostgroup { 2223 }; +}; + +int test_bgd_status_available(Context& context, TestState& state) { + if (publish_available(context, state.deployment) != EXIT_SUCCESS + || configure( + context, state.deployment, state.writer_hostgroup, state.reader_hostgroup, + state.green_writer_hostgroup, state.green_reader_hostgroup) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "BGD status for wHG 2220 reports AVAILABLE"); + return EXIT_SUCCESS; +} + +int test_writer_switchover_in_progress(Context& context, TestState& state) { + if (publish_status( + context, state.deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_IN_PROGRESS", + kWaitSeconds) == EXIT_SUCCESS, + "BGD status for wHG 2220 reports SWITCHOVER_IN_PROGRESS"); + ok(writer_placement( + context.admin, state.writer_hostgroup, state.reader_hostgroup, + state.deployment.production.members.front().endpoint.hostname, true), + "SWITCHOVER_IN_PROGRESS demotes the source writer for wHG 2220"); + return EXIT_SUCCESS; +} + +int test_disable_during_switchover(Context& context, TestState& state) { + if (aurora_bgd_execute_all(context.admin, { + "UPDATE mysql_aws_aurora_hostgroups SET active=0 WHERE writer_hostgroup=2220", + "LOAD MYSQL SERVERS TO RUNTIME", + }) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(writer_placement( + context.admin, state.writer_hostgroup, state.reader_hostgroup, + state.deployment.production.members.front().endpoint.hostname, false) + && wait_for_cond( + context.admin, + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=2220 AND active=0 AND bgd_status='NONE'", + kWaitSeconds) == EXIT_SUCCESS, + "setting active=0 restores the source writer and publishes inactive NONE"); + return EXIT_SUCCESS; +} + +int main() { + plan(4); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + if (test_bgd_status_available(context, state) != EXIT_SUCCESS + || test_writer_switchover_in_progress(context, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + test_disable_during_switchover(context, state); + +exit_cleanup: + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the disable-during-switchover fixture"); + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_discovery-t.cpp b/test/tap/tests/test_aurora_bgd_discovery-t.cpp deleted file mode 100644 index b4d442293b..0000000000 --- a/test/tap/tests/test_aurora_bgd_discovery-t.cpp +++ /dev/null @@ -1,453 +0,0 @@ -/** - * @file test_aurora_bgd_discovery-t.cpp - * @brief Aurora BGD AVAILABLE discovery, three probes, and fail-closed mapping. - * - * Steps: - * - * 1. Publish a complete SOURCE/TARGET deployment and verify AVAILABLE discovery. - * 2. Verify topology and target-membership probes move to the selected target. - * 3. Publish incomplete and inconsistent membership and retain the last complete map. - * 4. Publish invalid topology rows and retain the last valid runtime state. - * 5. Verify automatic discovery accepts a writer-only deployment. - * 6. Verify topology and membership probes inherit TLS from the Aurora row. - */ - -#include -#include -#include -#include - -#include "aurora_bgd_tap.h" -#include "command_line.h" -#include "utils.h" - -using namespace std; - -const uint32_t kWaitSeconds = 5; -const uint32_t kProbeTimeoutMs = 5000; - -struct TestState { - Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_a() }; - int writer_hostgroup { 1510 }; - int reader_hostgroup { 1511 }; - int green_writer_hostgroup { 1512 }; - int green_reader_hostgroup { 1513 }; - Aurora_BGD_Test_Deployment writer_only { aurora_bgd_deployment_b_writer_only() }; - int writer_only_writer_hostgroup { 1520 }; - int writer_only_reader_hostgroup { 1521 }; - int tls_writer_hostgroup { 1525 }; - int tls_reader_hostgroup { 1526 }; - int tls_green_writer_hostgroup { 1527 }; - int tls_green_reader_hostgroup { 1528 }; -}; - -int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { - if (cl.getEnv()) { - diag("Error: failed to load TAP environment"); - return EXIT_FAILURE; - } - admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); - if (admin == nullptr) { - diag("Error: failed to connect to ProxySQL Admin"); - return EXIT_FAILURE; - } - char simulator_username[] = "aurora1"; - char simulator_password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. - if (sim.connect(cl.host, 3306, simulator_username, simulator_password) != EXIT_SUCCESS) { - diag("Error: failed to connect to the shared AWS simulator"); - mysql_close(admin); - admin = nullptr; - return EXIT_FAILURE; - } - if (aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS || sim.cleanup() != EXIT_SUCCESS) { - diag("Error: failed to clear prior Aurora BGD state"); - return EXIT_FAILURE; - } - return EXIT_SUCCESS; -} - -int cleanup(MYSQL* admin, BGD_Simulator& sim) { - int admin_rc = aurora_bgd_admin_cleanup(admin); - int simulator_rc = sim.cleanup(); - if (admin) { - mysql_close(admin); - } - return admin_rc == EXIT_SUCCESS && simulator_rc == EXIT_SUCCESS - ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { - return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS - && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; -} - -bool runtime_status_is(MYSQL* admin, int writer_hg, const string& status) { - auto [rc, rows] = mysql_query_ext_rows( - admin, - "SELECT bgd_status FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + - to_string(writer_hg)); - return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 - && rows.front().front() == status; -} - -bool runtime_production_unchanged(MYSQL* admin, int writer_hg, int reader_hg, size_t members) { - auto [rc, rows] = mysql_query_ext_rows( - admin, - "SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id IN (" + - to_string(writer_hg) + "," + to_string(reader_hg) + ")"); - return rc == EXIT_SUCCESS && rows.size() == members; -} - -bool retain_complete_target_after_membership( - BGD_Simulator& sim, - Aurora_BGD_Test_Deployment& deployment, - const vector& rows -) { - auto [seq_rc, sequence] = sim.replica_probe_log_last_sequence(); - if (seq_rc != EXIT_SUCCESS || sim.replica_update( - deployment.target_replica_set, rows, deployment.target.backends()) != EXIT_SUCCESS) { - return false; - } - - vector target_members; - for (Aurora_BGD_Member& member : deployment.target.members) { - target_members.push_back(member.endpoint.backend()); - } - auto [first_rc, first_probe] = aurora_bgd_wait_for_replica_probe( - sim, sequence, target_members, Aurora_Replica_Probe_Kind::bgd_membership, - kProbeTimeoutMs, deployment.target_replica_set); - if (first_rc != EXIT_SUCCESS) { - return false; - } - auto [next_rc, next_probe] = aurora_bgd_wait_for_replica_probe( - sim, first_probe.sequence_id, target_members, - Aurora_Replica_Probe_Kind::bgd_membership, - kProbeTimeoutMs, deployment.target_replica_set); - return next_rc == EXIT_SUCCESS; -} - -bool invalid_topology_retains_available( - MYSQL* admin, - BGD_Simulator& sim, - Aurora_BGD_Test_Deployment& deployment, - const vector& rows, - int writer_hg -) { - auto [seq_rc, sequence] = sim.probe_log_last_sequence(); - vector target_members; - for (Aurora_BGD_Member& member : deployment.target.members) { - target_members.push_back(member.endpoint.backend()); - } - if (seq_rc != EXIT_SUCCESS || sim.topology_update( - aurora_bgd_topology_backends(deployment), rows) != EXIT_SUCCESS) { - return false; - } - auto [probe_rc, probe] = aurora_bgd_wait_for_topology_probe( - sim, sequence, target_members, BGD_Probe_Kind::metadata, kProbeTimeoutMs); - return probe_rc == EXIT_SUCCESS && runtime_status_is(admin, writer_hg, "AVAILABLE"); -} - -/** - * Discover a complete Aurora blue/green deployment. - * - * - Publish complete production and target membership. - * - Verify the worker reaches AVAILABLE without interrupting ordinary Aurora probes. - * - Verify topology discovery, target bootstrap, and steady probes use the expected endpoints. - */ -int test_available_discovery(MYSQL* admin, BGD_Simulator& sim, TestState& state) { - Aurora_BGD_Test_Deployment& deployment = state.deployment; - if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.writer_hostgroup, state.reader_hostgroup, - state.green_writer_hostgroup, state.green_reader_hostgroup, false) != EXIT_SUCCESS) { - diag("Error: failed to publish or configure deployment A"); - return EXIT_FAILURE; - } - - ok(aurora_bgd_wait_for_status( - admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, - "valid SOURCE/TARGET topology publishes AVAILABLE"); - - auto [logs_rc, logs] = sim.replica_probe_log_since(0); - bool ordinary_seen = false; - for (const Aurora_Replica_Probe_Log& log : logs) { - if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary - && log.replica_set_id == deployment.blue_replica_set) { - ordinary_seen = true; - } - } - ok(logs_rc == EXIT_SUCCESS && ordinary_seen, - "the existing worker continues ordinary Aurora probing in AVAILABLE"); - - vector blue_backends = deployment.production.backends(); - auto [topology_rc, topology_probe] = aurora_bgd_wait_for_topology_probe( - sim, 0, blue_backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); - ok(topology_rc == EXIT_SUCCESS, - "topology discovery starts on a reachable production member"); - - auto [bootstrap_rc, bootstrap_probe] = sim.wait_for_replica_probe_log( - 0, deployment.target_cluster_endpoint.backend(), - Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, - 0, deployment.target_replica_set); - ok(bootstrap_rc == EXIT_SUCCESS, - "TARGET cluster endpoint bootstraps target membership"); - - vector target_backends; - for (Aurora_BGD_Member& member : deployment.target.members) { - target_backends.push_back(member.endpoint.backend()); - } - auto [target_topology_rc, target_topology_probe] = aurora_bgd_wait_for_topology_probe( - sim, 0, target_backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); - ok(target_topology_rc == EXIT_SUCCESS, - "complete membership moves topology probes to target members"); - - auto [target_membership_rc, target_membership_probe] = aurora_bgd_wait_for_replica_probe( - sim, 0, target_backends, Aurora_Replica_Probe_Kind::bgd_membership, - kProbeTimeoutMs, deployment.target_replica_set); - ok(target_membership_rc == EXIT_SUCCESS, - "complete membership moves membership probes to target members"); - return EXIT_SUCCESS; -} - -/** - * Retain the last complete target membership after unusable observations. - * - * - Publish incomplete, ambiguous, unresolved, and mismatched target snapshots. - * - Ignore stale non-current rows. - * - Verify the worker continues probing the last complete target selection. - */ -int test_target_membership_validation(BGD_Simulator& sim, TestState& state) { - Aurora_BGD_Test_Deployment& deployment = state.deployment; - vector complete_rows = deployment.target.replica_rows(); - vector incomplete_rows { complete_rows[0], complete_rows[1] }; - ok(retain_complete_target_after_membership(sim, deployment, incomplete_rows), - "an incomplete target result retains the previous complete target selection"); - - ok(retain_complete_target_after_membership(sim, deployment, {}), - "an empty target result retains the previous complete target selection"); - - vector duplicate_writer_rows = complete_rows; - duplicate_writer_rows[1].session_id = "MASTER_SESSION_ID"; - ok(retain_complete_target_after_membership(sim, deployment, duplicate_writer_rows), - "multiple current target writers retain the previous complete snapshot"); - - vector unresolved_rows = complete_rows; - unresolved_rows[2].server_id = "aurora-a-reader-2-green-missing"; - ok(retain_complete_target_after_membership(sim, deployment, unresolved_rows), - "an unresolved target member retains the previous complete snapshot"); - - vector mismatched_session_rows = complete_rows; - mismatched_session_rows[1].session_id = "reader-a-1-replaced"; - ok(retain_complete_target_after_membership(sim, deployment, mismatched_session_rows), - "a reader SESSION_ID mismatch retains the previous complete snapshot"); - - vector stale_extra_rows = complete_rows; - Aurora_Replica_Row stale_writer = complete_rows.front(); - stale_writer.server_id = "unrelated-stale-writer"; - stale_writer.is_current = false; - stale_extra_rows.push_back(stale_writer); - ok(retain_complete_target_after_membership(sim, deployment, stale_extra_rows), - "IS_CURRENT=0 rows are excluded from the complete target snapshot"); - - int restore_rc = sim.replica_update( - deployment.target_replica_set, complete_rows, deployment.target.backends()); - if (restore_rc != EXIT_SUCCESS) { - diag("Error: failed to restore complete target membership"); - return EXIT_FAILURE; - } - return EXIT_SUCCESS; -} - -/** - * Reject incomplete or unsupported topology observations. - * - * - Publish missing identity, SOURCE-only, mismatched-status, and unknown-status rows. - * - Verify the last AVAILABLE state and production routing remain unchanged. - */ -int test_topology_validation(MYSQL* admin, BGD_Simulator& sim, TestState& state) { - Aurora_BGD_Test_Deployment& deployment = state.deployment; - vector valid_topology = aurora_bgd_available_topology(deployment); - - vector missing_identity = valid_topology; - missing_identity[1].id.clear(); - ok(invalid_topology_retains_available( - admin, sim, deployment, missing_identity, state.writer_hostgroup), - "missing TARGET identity does not replace AVAILABLE"); - - vector source_only { valid_topology.front() }; - ok(invalid_topology_retains_available( - admin, sim, deployment, source_only, state.writer_hostgroup), - "SOURCE-only topology does not replace AVAILABLE"); - - vector target_only { valid_topology.back() }; - ok(invalid_topology_retains_available( - admin, sim, deployment, target_only, state.writer_hostgroup), - "TARGET-only topology is accepted only for SWITCHOVER_COMPLETED"); - - vector duplicate_source = valid_topology; - duplicate_source.push_back(valid_topology.front()); - ok(invalid_topology_retains_available( - admin, sim, deployment, duplicate_source, state.writer_hostgroup), - "duplicate SOURCE rows do not replace AVAILABLE"); - - vector extra_role = valid_topology; - BGD_Topology_Row observer = valid_topology.front(); - observer.role = "BLUE_GREEN_DEPLOYMENT_OBSERVER"; - extra_role.push_back(observer); - ok(invalid_topology_retains_available( - admin, sim, deployment, extra_role, state.writer_hostgroup), - "an extra unsupported role does not replace AVAILABLE"); - - vector invalid_port = valid_topology; - invalid_port[1].port = 0; - ok(invalid_topology_retains_available( - admin, sim, deployment, invalid_port, state.writer_hostgroup), - "a TARGET row with an invalid port does not replace AVAILABLE"); - - vector empty_endpoint = valid_topology; - empty_endpoint[1].endpoint.clear(); - ok(invalid_topology_retains_available( - admin, sim, deployment, empty_endpoint, state.writer_hostgroup), - "a TARGET row with an empty endpoint does not replace AVAILABLE"); - - vector mismatched_status = valid_topology; - mismatched_status[0].status = "SWITCHOVER_INITIATED"; - ok(invalid_topology_retains_available( - admin, sim, deployment, mismatched_status, state.writer_hostgroup), - "mismatched SOURCE/TARGET statuses do not replace AVAILABLE"); - - vector unknown_status = valid_topology; - unknown_status[0].status = "UNSUPPORTED_STATUS"; - unknown_status[1].status = "UNSUPPORTED_STATUS"; - ok(invalid_topology_retains_available( - admin, sim, deployment, unknown_status, state.writer_hostgroup), - "unsupported topology status is not copied into runtime state"); - - ok(runtime_production_unchanged( - admin, state.writer_hostgroup, state.reader_hostgroup, - deployment.production.members.size()), - "AVAILABLE discovery performs no production routing action"); - return EXIT_SUCCESS; -} - -/** Configure SSL-enabled Aurora rows and verify both BGD probe types use TLS. */ -int test_tls_probe_policy(MYSQL* admin, BGD_Simulator& sim, TestState& state) { - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to reset before the TLS scenario"); - return EXIT_FAILURE; - } - - Aurora_BGD_Test_Deployment& deployment = state.writer_only; - if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.tls_writer_hostgroup, state.tls_reader_hostgroup, - state.tls_green_writer_hostgroup, state.tls_green_reader_hostgroup, - false, 100, false, true) != EXIT_SUCCESS) { - diag("Error: failed to configure the TLS probe scenario"); - return EXIT_FAILURE; - } - - ok(aurora_bgd_wait_for_status( - admin, state.tls_writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, - "SSL-enabled Aurora rows reach AVAILABLE"); - - auto [topology_rc, topology_probe] = sim.wait_for_probe_log( - 0, deployment.production.members.front().endpoint.backend(), - BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1); - ok(topology_rc == EXIT_SUCCESS, - "topology discovery uses TLS when the Aurora row enables SSL"); - - vector target_backends = deployment.target.backends(); - auto [membership_rc, membership_probe] = aurora_bgd_wait_for_replica_probe( - sim, 0, target_backends, Aurora_Replica_Probe_Kind::bgd_membership, - kProbeTimeoutMs, deployment.target_replica_set); - ok(membership_rc == EXIT_SUCCESS && membership_probe.encrypted, - "target-membership discovery uses TLS when the Aurora row enables SSL"); - return EXIT_SUCCESS; -} - -/** - * Discover a writer-only deployment without configured green hostgroups. - * - * - Reset the complete-cluster scenario. - * - Configure automatic discovery with writer-only production and target membership. - * - Verify the worker admits AVAILABLE and accepts the target snapshot. - */ -int test_writer_only_discovery(MYSQL* admin, BGD_Simulator& sim, TestState& state) { - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to reset state before writer-only deployment"); - return EXIT_FAILURE; - } - - Aurora_BGD_Test_Deployment& deployment = state.writer_only; - if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.writer_only_writer_hostgroup, - state.writer_only_reader_hostgroup, -1, -1, true) != EXIT_SUCCESS) { - diag("Error: failed to publish or configure writer-only deployment"); - return EXIT_FAILURE; - } - - ok(aurora_bgd_wait_for_status( - admin, state.writer_only_writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, - "auto-discovery admits a row without configured green hostgroups"); - - vector writer_target { deployment.target.members.front().endpoint.backend() }; - auto [probe_rc, probe] = aurora_bgd_wait_for_replica_probe( - sim, 0, writer_target, Aurora_Replica_Probe_Kind::bgd_membership, - kProbeTimeoutMs, deployment.target_replica_set); - ok(probe_rc == EXIT_SUCCESS, - "a writer-only production cluster accepts a writer-only target snapshot"); - return EXIT_SUCCESS; -} - -int main() { - plan(27); - - CommandLine cl {}; - MYSQL* admin = nullptr; - BGD_Simulator sim {}; - if (setup(cl, admin, sim) != EXIT_SUCCESS) { - return exit_status(); - } - - TestState state {}; - - // Simulator: publish complete production and target membership. - // ProxySQL: configure explicit Aurora BGD hostgroups 1510-1513. - // Verify: discovery reaches AVAILABLE and hands probes to the selected target. - if (test_available_discovery(admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: replace complete target membership with unusable observations. - // Verify: the worker retains its last complete target mapping. - if (test_target_membership_validation(sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: publish incomplete and unsupported topology rows. - // Verify: runtime state and production routing retain the last valid observation. - if (test_topology_validation(admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // ProxySQL: configure automatic discovery for a writer-only deployment. - // Verify: writer-only source and target snapshots are accepted. - if (test_writer_only_discovery(admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // ProxySQL: configure SSL-enabled Aurora rows. - // Verify: topology and target-membership probes both use TLS. - if (test_tls_probe_policy(admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - -exit_cleanup: - if (cleanup(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to clean Aurora BGD discovery state"); - return EXIT_FAILURE; - } - return exit_status(); -} diff --git a/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp b/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp deleted file mode 100644 index 228ee58792..0000000000 --- a/test/tap/tests/test_aurora_bgd_error_recovery-t.cpp +++ /dev/null @@ -1,487 +0,0 @@ -/** - * @file test_aurora_bgd_error_recovery-t.cpp - * @brief Aurora BGD active-state error retention, rollback, and cancellation. - * - * Steps: - * - * 1. Verify topology and membership errors retain INITIATED and POST_PROCESSING effects. - * 2. Move backward from IN_PROGRESS and restore canonical writer placement. - * 3. Cancel an active deployment through empty or absent topology and resume ordinary probes. - * 4. Roll back POST_PROCESSING pins while preserving configured green pools. - */ - -#include -#include -#include -#include - -#include "aurora_bgd_tap.h" -#include "command_line.h" -#include "utils.h" - -using namespace std; - -const uint32_t kWaitSeconds = 5; -const uint32_t kProbeTimeoutMs = 5000; - -struct TestState { - Aurora_BGD_Test_Deployment initiated { aurora_bgd_deployment_b_writer_only() }; - int initiated_writer_hostgroup { 1590 }; - int initiated_reader_hostgroup { 1591 }; - int initiated_green_writer_hostgroup { 1592 }; - int initiated_green_reader_hostgroup { 1593 }; - Aurora_BGD_Test_Deployment progress { aurora_bgd_deployment_b_writer_only() }; - int progress_writer_hostgroup { 1600 }; - int progress_reader_hostgroup { 1601 }; - int progress_green_writer_hostgroup { 1602 }; - int progress_green_reader_hostgroup { 1603 }; - Aurora_BGD_Test_Deployment post { aurora_bgd_deployment_a() }; - int post_writer_hostgroup { 1610 }; - int post_reader_hostgroup { 1611 }; - int post_green_writer_hostgroup { 1612 }; - int post_green_reader_hostgroup { 1613 }; - vector post_route_hostgroups { 1614, 1615, 1616 }; -}; - -int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { - if (cl.getEnv()) { - diag("Error: failed to load TAP environment"); - return EXIT_FAILURE; - } - admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); - if (admin == nullptr) { - diag("Error: failed to connect to ProxySQL Admin"); - return EXIT_FAILURE; - } - char username[] = "aurora1"; - char password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. - if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS - || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS - || sim.cleanup() != EXIT_SUCCESS) { - diag("Error: failed to initialize the shared AWS simulator"); - return EXIT_FAILURE; - } - return aurora_bgd_execute_all(admin, { - "DELETE FROM mysql_users WHERE username='testuser'", - "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " - "VALUES ('testuser','testuser',1,0,1)", - "LOAD MYSQL USERS TO RUNTIME", - }); -} - -int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { - return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS - && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int cleanup(MYSQL* admin, BGD_Simulator& sim) { - int reset_rc = reset_scenario(admin, sim); - int user_rc = aurora_bgd_execute_all(admin, { - "DELETE FROM mysql_users WHERE username='testuser'", - "LOAD MYSQL USERS TO RUNTIME", - }); - mysql_close(admin); - return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS - ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int publish_initial( - BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status -) { - return sim.replica_update( - deployment.production.replica_set_id, - deployment.production.replica_rows(), deployment.production.backends()) == EXIT_SUCCESS - && sim.replica_update( - deployment.target.replica_set_id, - deployment.target.replica_rows(), deployment.target.backends()) == EXIT_SUCCESS - && sim.topology_update( - aurora_bgd_topology_backends(deployment), - aurora_bgd_topology(deployment, status)) == EXIT_SUCCESS - ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int publish_status( - BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status -) { - return sim.topology_update( - aurora_bgd_topology_backends(deployment), - aurora_bgd_topology(deployment, status)); -} - -int set_default_hostgroup(MYSQL* admin, int hostgroup) { - return aurora_bgd_execute_all(admin, { - "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + - " WHERE username='testuser'", - "LOAD MYSQL USERS TO RUNTIME", - }); -} - -bool writer_placement( - MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, bool demoted -) { - string query = - "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + - ")=" + (demoted ? "0" : "1") + ") AND " - "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + - ")=" + (demoted ? "1" : "0") + ")"; - return wait_for_cond(admin, query, kWaitSeconds) == EXIT_SUCCESS; -} - -bool wait_for_ordinary_probe( - BGD_Simulator& sim, uint64_t sequence, Aurora_BGD_Test_Deployment& deployment -) { - auto [rc, log] = aurora_bgd_wait_for_replica_probe( - sim, sequence, deployment.production.backends(), - Aurora_Replica_Probe_Kind::ordinary, kProbeTimeoutMs); - return rc == EXIT_SUCCESS; -} - -int add_routes( - MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, - const vector& route_hgs, int green_writer_hg -) { - vector queries; - for (size_t i = 0; i < route_hgs.size(); ++i) { - queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + - to_string(route_hgs[i]) + "," + - aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + - ",3306,'ONLINE',1,'Aurora BGD rollback route')"); - } - queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + - to_string(green_writer_hg) + "," + - aurora_bgd_sql_quote(deployment.target.members.front().endpoint.hostname) + - ",3306,'ONLINE',1,'Aurora BGD rollback green pool')"); - queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); - return aurora_bgd_execute_all(admin, queries); -} - -bool route_to_backend( - CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected -) { - auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); - if (sequence_rc != EXIT_SUCCESS) { - return false; - } - MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); - if (client == nullptr) { - return false; - } - auto [query_rc, rows] = mysql_query_ext_rows(client, kAuroraBGDRouteProbeQuery); - (void)rows; - mysql_close(client); - return query_rc == EXIT_SUCCESS - && aurora_bgd_routing_probe_reached(sim, sequence, expected); -} - -bool route_members( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, - Aurora_BGD_Test_Deployment& deployment, const vector& route_hgs, bool target -) { - for (size_t i = 0; i < route_hgs.size(); ++i) { - if (set_default_hostgroup(admin, route_hgs[i]) != EXIT_SUCCESS) { - return false; - } - Endpoint expected = target - ? deployment.target.members[i].endpoint.backend() - : deployment.production.members[i].endpoint.backend(); - if (!route_to_backend(cl, sim, expected)) { - return false; - } - } - return true; -} - -int64_t pool_count(MYSQL* admin, int hostgroup) { - auto [rc, rows] = mysql_query_ext_rows( - admin, "SELECT COALESCE(SUM(ConnUsed+ConnFree),0) " - "FROM stats_mysql_connection_pool WHERE hostgroup=" + to_string(hostgroup)); - return rc == EXIT_SUCCESS && rows.size() == 1 && rows.front().size() == 1 - ? strtoll(rows.front().front().c_str(), nullptr, 10) : -1; -} - -bool observe_two_topology_responses( - BGD_Simulator& sim, uint64_t sequence, const vector& backends -) { - auto [first_rc, first] = aurora_bgd_wait_for_topology_probe( - sim, sequence, backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); - if (first_rc != EXIT_SUCCESS) { - return false; - } - auto [second_rc, second] = aurora_bgd_wait_for_topology_probe( - sim, first.sequence_id, backends, BGD_Probe_Kind::metadata, kProbeTimeoutMs); - return second_rc == EXIT_SUCCESS; -} - -bool observe_two_membership_responses( - BGD_Simulator& sim, uint64_t sequence, - const vector& backends, const string& replica_set -) { - auto [first_rc, first] = aurora_bgd_wait_for_replica_probe( - sim, sequence, backends, Aurora_Replica_Probe_Kind::bgd_membership, - kProbeTimeoutMs, replica_set); - if (first_rc != EXIT_SUCCESS) { - return false; - } - auto [second_rc, second] = aurora_bgd_wait_for_replica_probe( - sim, first.sequence_id, backends, Aurora_Replica_Probe_Kind::bgd_membership, - kProbeTimeoutMs, replica_set); - return second_rc == EXIT_SUCCESS; -} - -/** Retain INITIATED across topology and membership query errors. */ -int test_initiated_error_retention(MYSQL* admin, BGD_Simulator& sim, TestState& state) { - Aurora_BGD_Test_Deployment& deployment = state.initiated; - if (publish_initial(sim, deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.initiated_writer_hostgroup, - state.initiated_reader_hostgroup, state.initiated_green_writer_hostgroup, - state.initiated_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.initiated_writer_hostgroup, - "SWITCHOVER_INITIATED", kWaitSeconds) != EXIT_SUCCESS) { - diag("Error: failed to configure INITIATED error retention"); - return EXIT_FAILURE; - } - - auto [topology_seq_rc, topology_sequence] = sim.probe_log_last_sequence(); - bool topology_error_seen = topology_seq_rc == EXIT_SUCCESS - && sim.topology_error( - aurora_bgd_topology_backends(deployment), 1205, - "simulated topology timeout") == EXIT_SUCCESS - && observe_two_topology_responses( - sim, topology_sequence, deployment.target.backends()); - ok(topology_error_seen && aurora_bgd_wait_for_status( - admin, state.initiated_writer_hostgroup, "SWITCHOVER_INITIATED", 1) - == EXIT_SUCCESS, - "topology errors retain the active state"); - if (publish_status(sim, deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS) { - diag("Error: failed to restore topology after error"); - return EXIT_FAILURE; - } - - auto [membership_seq_rc, membership_sequence] = sim.replica_probe_log_last_sequence(); - bool membership_error_seen = membership_seq_rc == EXIT_SUCCESS - && sim.replica_error( - deployment.target.backends(), 1205, - "simulated membership timeout") == EXIT_SUCCESS - && observe_two_membership_responses( - sim, membership_sequence, deployment.target.backends(), - deployment.target_replica_set); - ok(membership_error_seen && aurora_bgd_wait_for_status( - admin, state.initiated_writer_hostgroup, "SWITCHOVER_INITIATED", 1) - == EXIT_SUCCESS, - "membership errors retain the last complete state"); - return EXIT_SUCCESS; -} - -/** Roll back IN_PROGRESS through an earlier status, then cancel and rearm. */ -int test_initiated_rollback_and_cancellation( - MYSQL* admin, BGD_Simulator& sim, TestState& state) -{ - Aurora_BGD_Test_Deployment& deployment = state.initiated; - if (sim.replica_update( - deployment.target.replica_set_id, deployment.target.replica_rows(), - deployment.target.backends()) != EXIT_SUCCESS - || publish_status(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.initiated_writer_hostgroup, - "SWITCHOVER_IN_PROGRESS", kWaitSeconds) != EXIT_SUCCESS) { - diag("Error: failed to advance to IN_PROGRESS"); - return EXIT_FAILURE; - } - ok(writer_placement( - admin, state.initiated_writer_hostgroup, state.initiated_reader_hostgroup, - deployment.production.members.front().endpoint.hostname, true), - "IN_PROGRESS demotes the reconstructed production writer"); - ok(publish_status(sim, deployment, "SWITCHOVER_INITIATED") == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.initiated_writer_hostgroup, - "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS - && writer_placement( - admin, state.initiated_writer_hostgroup, state.initiated_reader_hostgroup, - deployment.production.members.front().endpoint.hostname, false), - "a backward status rolls back writer placement before entering the earlier state"); - - auto [ordinary_seq_rc, ordinary_sequence] = sim.replica_probe_log_last_sequence(); - ok(sim.topology_delete(aurora_bgd_topology_backends(deployment)) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.initiated_writer_hostgroup, "NONE", kWaitSeconds) == EXIT_SUCCESS, - "a successful empty topology cancels the active deployment"); - ok(ordinary_seq_rc == EXIT_SUCCESS - && wait_for_ordinary_probe(sim, ordinary_sequence, deployment), - "cancellation resumes ordinary production probing"); - ok(publish_status(sim, deployment, "AVAILABLE") == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.initiated_writer_hostgroup, "AVAILABLE", kWaitSeconds) - == EXIT_SUCCESS, - "the worker admits a repeated deployment after cancellation"); - return EXIT_SUCCESS; -} - -/** Roll back a first IN_PROGRESS observation after confirmed topology absence. */ -int test_in_progress_topology_absence(MYSQL* admin, BGD_Simulator& sim, TestState& state) { - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to reset before IN_PROGRESS rollback"); - return EXIT_FAILURE; - } - Aurora_BGD_Test_Deployment& deployment = state.progress; - if (publish_initial(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.progress_writer_hostgroup, - state.progress_reader_hostgroup, state.progress_green_writer_hostgroup, - state.progress_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.progress_writer_hostgroup, - "SWITCHOVER_IN_PROGRESS", kWaitSeconds) != EXIT_SUCCESS) { - diag("Error: failed to configure IN_PROGRESS rollback"); - return EXIT_FAILURE; - } - ok(sim.topology_drop(aurora_bgd_topology_backends(deployment)) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.progress_writer_hostgroup, "NONE", kWaitSeconds) == EXIT_SUCCESS - && writer_placement( - admin, state.progress_writer_hostgroup, state.progress_reader_hostgroup, - deployment.production.members.front().endpoint.hostname, false), - "confirmed topology absence rolls back IN_PROGRESS to NONE"); - return EXIT_SUCCESS; -} - -/** Retain POST_PROCESSING pins across topology and membership query errors. */ -int test_post_processing_error_retention( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) -{ - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to reset before POST_PROCESSING error retention"); - return EXIT_FAILURE; - } - Aurora_BGD_Test_Deployment& deployment = state.post; - if (publish_initial(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.post_writer_hostgroup, - state.post_reader_hostgroup, state.post_green_writer_hostgroup, - state.post_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS - || add_routes( - admin, deployment, state.post_route_hostgroups, - state.post_green_writer_hostgroup) != EXIT_SUCCESS - || aurora_bgd_wait_for_status( - admin, state.post_writer_hostgroup, - "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) != EXIT_SUCCESS) { - diag("Error: failed to configure POST_PROCESSING error retention"); - return EXIT_FAILURE; - } - - auto [topology_seq_rc, topology_sequence] = sim.probe_log_last_sequence(); - bool topology_error_seen = topology_seq_rc == EXIT_SUCCESS - && sim.topology_error( - aurora_bgd_topology_backends(deployment), 1205, - "simulated post timeout") == EXIT_SUCCESS - && observe_two_topology_responses( - sim, topology_sequence, deployment.target.backends()); - ok(topology_error_seen && aurora_bgd_wait_for_status( - admin, state.post_writer_hostgroup, - "SWITCHOVER_IN_POST_PROCESSING", 1) == EXIT_SUCCESS - && route_members( - cl, admin, sim, deployment, state.post_route_hostgroups, true), - "topology errors do not roll back applied POST_PROCESSING pins"); - if (publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS) { - diag("Error: failed to restore POST_PROCESSING topology"); - return EXIT_FAILURE; - } - - auto [membership_seq_rc, membership_sequence] = sim.replica_probe_log_last_sequence(); - bool membership_error_seen = membership_seq_rc == EXIT_SUCCESS - && sim.replica_error( - deployment.target.backends(), 1205, - "simulated post membership timeout") == EXIT_SUCCESS - && observe_two_membership_responses( - sim, membership_sequence, deployment.target.backends(), - deployment.target_replica_set); - ok(membership_error_seen - && sim.replica_update( - deployment.target.replica_set_id, deployment.target.replica_rows(), - deployment.target.backends()) == EXIT_SUCCESS - && route_members( - cl, admin, sim, deployment, state.post_route_hostgroups, true), - "membership errors retain the last complete mapped routing"); - return EXIT_SUCCESS; -} - -/** Roll back POST_PROCESSING pins while preserving configured green pools. */ -int test_post_processing_rollback( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) -{ - Aurora_BGD_Test_Deployment& deployment = state.post; - ok(set_default_hostgroup(admin, state.post_green_writer_hostgroup) == EXIT_SUCCESS - && route_to_backend( - cl, sim, deployment.target.members.front().endpoint.backend()) - && pool_count(admin, state.post_green_writer_hostgroup) >= 1, - "a configured green pool is established before rollback"); - ok(publish_status(sim, deployment, "AVAILABLE") == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.post_writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, - "a backward POST_PROCESSING status completes rollback before AVAILABLE"); - ok(writer_placement( - admin, state.post_writer_hostgroup, state.post_reader_hostgroup, - deployment.production.members.front().endpoint.hostname, false) - && route_members( - cl, admin, sim, deployment, state.post_route_hostgroups, false), - "rollback removes pins and restores canonical writer and member routing"); - ok(pool_count(admin, state.post_green_writer_hostgroup) >= 1, - "rollback preserves configured green pools instead of draining them"); - return EXIT_SUCCESS; -} - -int main() { - plan(14); - - CommandLine cl {}; - MYSQL* admin = nullptr; - BGD_Simulator sim {}; - - if (setup(cl, admin, sim) != EXIT_SUCCESS) { - return exit_status(); - } - - TestState state {}; - - // Simulator: inject topology and membership errors during INITIATED. - // Verify: the active state and last complete membership are retained. - if (test_initiated_error_retention(admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: move backward from IN_PROGRESS, then publish empty topology. - // Verify: placement rolls back, ordinary probes resume, and the worker rearms. - if (test_initiated_rollback_and_cancellation(admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: make IN_PROGRESS the first observation, then confirm absence. - // Verify: the worker returns to NONE and restores the writer. - if (test_in_progress_topology_absence(admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: inject topology and membership errors during POST_PROCESSING. - // Verify: the applied target pins and last complete map remain active. - if (test_post_processing_error_retention(cl, admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: publish AVAILABLE after POST_PROCESSING. - // Verify: pins and placement roll back without draining configured green pools. - if (test_post_processing_rollback(cl, admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - -exit_cleanup: - if (cleanup(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to clean Aurora BGD error-recovery test data"); - return EXIT_FAILURE; - } - return exit_status(); -} diff --git a/test/tap/tests/test_aurora_bgd_green_pool_cleanup-t.cpp b/test/tap/tests/test_aurora_bgd_green_pool_cleanup-t.cpp new file mode 100644 index 0000000000..82c35da7cc --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_green_pool_cleanup-t.cpp @@ -0,0 +1,184 @@ +/** + * @file test_aurora_bgd_green_pool_cleanup-t.cpp + * @brief Aurora BGD rollback and completion behavior for green status pools. + */ + +#include +#include +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +struct GreenServer { + int hostgroup; + Aurora_BGD_Endpoint endpoint; + string status; + int route_hostgroup; +}; + +struct TestState { + Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_a() }; + Aurora_BGD_Test_Deployment extra { aurora_bgd_deployment_c_writer_only() }; + int writer_hostgroup { 2060 }; + int reader_hostgroup { 2061 }; + int green_writer_hostgroup { 2062 }; + int green_reader_hostgroup { 2063 }; + vector servers { + { green_writer_hostgroup, deployment.target.members[0].endpoint, "ONLINE", 2064 }, + { green_reader_hostgroup, deployment.target.members[1].endpoint, "SHUNNED", 2065 }, + { green_reader_hostgroup, deployment.target.members[2].endpoint, "OFFLINE_SOFT", 2066 }, + { green_reader_hostgroup, extra.target.members[0].endpoint, "OFFLINE_HARD", 2067 }, + }; + vector pools_before; + vector pools_after; +}; + +int add_status_matrix(Context& context, TestState& state) { + if (context.simulator.replica_update( + state.extra.target.replica_set_id, state.extra.target.replica_rows(), + state.extra.target.backends()) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + for (GreenServer& server : state.servers) { + if (add_route( + context.admin, server.hostgroup, server.endpoint.hostname, + server.status, true, "Aurora BGD configured green status") != EXIT_SUCCESS + || add_route( + context.admin, server.route_hostgroup, server.endpoint.hostname, + "ONLINE", true, "Aurora BGD green pool route") != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; +} + +int create_green_pools(CommandLine& cl, Context& context, TestState& state) { + state.pools_before.clear(); + for (GreenServer& server : state.servers) { + if (set_default_hostgroup(context.admin, server.route_hostgroup) != EXIT_SUCCESS + || !route_to_backend(cl, context, server.endpoint.backend())) { + return EXIT_FAILURE; + } + int64_t pool = pool_connections_for_hostname( + context.admin, server.endpoint.hostname); + if (pool < 1) { + return EXIT_FAILURE; + } + state.pools_before.push_back(pool); + } + return set_default_hostgroup(context.admin, state.writer_hostgroup); +} + +bool configured_rows_preserved(Context& context, TestState& state) { + for (GreenServer& server : state.servers) { + if (!server_count( + context.admin, "mysql_servers", server.hostgroup, + server.endpoint.hostname, 1, server.status)) { + return false; + } + } + return true; +} + +int test_rollback_preserves_green_pools( + CommandLine& cl, Context& context, TestState& state +) { + if (publish_available(context, state.deployment) != EXIT_SUCCESS + || configure( + context, state.deployment, state.writer_hostgroup, state.reader_hostgroup, + state.green_writer_hostgroup, state.green_reader_hostgroup) != EXIT_SUCCESS + || add_status_matrix(context, state) != EXIT_SUCCESS + || create_green_pools(cl, context, state) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) != EXIT_SUCCESS + || publish_status( + context, state.deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_IN_PROGRESS", + kWaitSeconds) != EXIT_SUCCESS + || publish_status(context, state.deployment, "AVAILABLE") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: failed to run the green-pool rollback lifecycle"); + return EXIT_FAILURE; + } + + bool pools_preserved = state.pools_before.size() == state.servers.size(); + for (size_t i = 0; i < state.servers.size(); ++i) { + pools_preserved = pools_preserved + && pool_connections_for_hostname( + context.admin, state.servers[i].endpoint.hostname) >= state.pools_before[i]; + } + ok(pools_preserved, + "AVAILABLE rollback preserves all four green status pools for wHG 2060"); + ok(configured_rows_preserved(context, state), + "AVAILABLE rollback preserves configured green rows and public statuses for wHG 2060"); + return EXIT_SUCCESS; +} + +int test_successful_cleanup_drains_non_offline( + Context& context, TestState& state +) { + if (publish_status( + context, state.deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_IN_POST_PROCESSING", + kWaitSeconds) != EXIT_SUCCESS + || publish_completed(context, state.deployment, state.deployment) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_COMPLETED", + kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: failed to complete the green-pool lifecycle"); + return EXIT_FAILURE; + } + + bool drained = wait_for_hostname_pool_count( + context.admin, state.servers[0].endpoint.hostname, "=0") == EXIT_SUCCESS + && wait_for_hostname_pool_count( + context.admin, state.servers[1].endpoint.hostname, "=0") == EXIT_SUCCESS; + ok(drained, + "completion drains ONLINE and SHUNNED green pools for wHG 2060"); + ok(configured_rows_preserved(context, state), + "completion retains configured green rows and public statuses for wHG 2060"); + return EXIT_SUCCESS; +} + +int test_cleanup_preserves_offline_pools(Context& context, TestState& state) { + state.pools_after.clear(); + for (GreenServer& server : state.servers) { + state.pools_after.push_back(pool_connections_for_hostname( + context.admin, server.endpoint.hostname)); + } + bool offline_preserved = state.pools_before.size() == 4 + && state.pools_after.size() == 4 + && state.pools_after[2] >= state.pools_before[2] + && state.pools_after[3] >= state.pools_before[3]; + ok(offline_preserved, + "completion preserves OFFLINE_SOFT and OFFLINE_HARD green pools for wHG 2060"); + return EXIT_SUCCESS; +} + +int main() { + plan(5); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + if (test_rollback_preserves_green_pools(cl, context, state) != EXIT_SUCCESS + || test_successful_cleanup_drains_non_offline(context, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + test_cleanup_preserves_offline_pools(context, state); + +exit_cleanup: + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the green-pool fixture"); + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_late_entry-t.cpp b/test/tap/tests/test_aurora_bgd_late_entry-t.cpp deleted file mode 100644 index 902f979864..0000000000 --- a/test/tap/tests/test_aurora_bgd_late_entry-t.cpp +++ /dev/null @@ -1,381 +0,0 @@ -/** - * @file test_aurora_bgd_late_entry-t.cpp - * @brief Aurora BGD reconstruction from first observations in active states. - * - * Steps: - * - * 1. Start a worker from SWITCHOVER_INITIATED and reconstruct active probing. - * 2. Start a worker from SWITCHOVER_IN_PROGRESS and reconstruct writer demotion. - * 3. Recover late IN_PROGRESS entry when the first ordinary probe fails. - * 4. Start a worker from POST_PROCESSING and reconstruct every target pin. - */ - -#include -#include -#include -#include - -#include "aurora_bgd_tap.h" -#include "command_line.h" -#include "utils.h" - -using namespace std; - -const uint32_t kWaitSeconds = 5; - -struct TestState { - Aurora_BGD_Test_Deployment initiated { aurora_bgd_deployment_b_writer_only() }; - int initiated_writer_hostgroup { 1590 }; - int initiated_reader_hostgroup { 1591 }; - int initiated_green_writer_hostgroup { 1592 }; - int initiated_green_reader_hostgroup { 1593 }; - Aurora_BGD_Test_Deployment progress { aurora_bgd_deployment_b_writer_only() }; - int progress_writer_hostgroup { 1600 }; - int progress_reader_hostgroup { 1601 }; - int progress_green_writer_hostgroup { 1602 }; - int progress_green_reader_hostgroup { 1603 }; - Aurora_BGD_Test_Deployment post { aurora_bgd_deployment_a() }; - int post_writer_hostgroup { 1610 }; - int post_reader_hostgroup { 1611 }; - int post_green_writer_hostgroup { 1612 }; - int post_green_reader_hostgroup { 1613 }; - vector post_route_hostgroups { 1614, 1615, 1616 }; -}; - -int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { - if (cl.getEnv()) { - return EXIT_FAILURE; - } - admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); - if (admin == nullptr) { - return EXIT_FAILURE; - } - char username[] = "aurora1"; - char password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. - if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS - || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS - || sim.cleanup() != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - return aurora_bgd_execute_all(admin, { - "DELETE FROM mysql_users WHERE username='testuser'", - "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " - "VALUES ('testuser','testuser',1,0,1)", - "LOAD MYSQL USERS TO RUNTIME", - }); -} - -int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { - return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS - && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int cleanup(MYSQL* admin, BGD_Simulator& sim) { - int reset_rc = reset_scenario(admin, sim); - int user_rc = aurora_bgd_execute_all(admin, { - "DELETE FROM mysql_users WHERE username='testuser'", - "LOAD MYSQL USERS TO RUNTIME", - }); - mysql_close(admin); - return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS - ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int publish_initial( - BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status -) { - return sim.replica_update( - deployment.production.replica_set_id, - deployment.production.replica_rows(), deployment.production.backends()) == EXIT_SUCCESS - && sim.replica_update( - deployment.target.replica_set_id, - deployment.target.replica_rows(), deployment.target.backends()) == EXIT_SUCCESS - && sim.topology_update( - aurora_bgd_topology_backends(deployment), - aurora_bgd_topology(deployment, status)) == EXIT_SUCCESS - ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int set_default_hostgroup(MYSQL* admin, int hostgroup) { - return aurora_bgd_execute_all(admin, { - "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + - " WHERE username='testuser'", - "LOAD MYSQL USERS TO RUNTIME", - }); -} - -bool writer_placement( - MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, bool demoted -) { - string query = - "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + - ")=" + (demoted ? "0" : "1") + ") AND " - "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + - ")=" + (demoted ? "1" : "0") + ")"; - return wait_for_cond(admin, query, kWaitSeconds) == EXIT_SUCCESS; -} - -bool active_probe_policy( - BGD_Simulator& sim, uint64_t sequence, const string& target_replica_set, - uint32_t observation_ms -) { - usleep(observation_ms * 1000); - auto [rc, logs] = sim.replica_probe_log_since(sequence); - if (rc != EXIT_SUCCESS) { - return false; - } - bool membership = false; - for (const Aurora_Replica_Probe_Log& log : logs) { - if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { - return false; - } - membership |= log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership - && log.replica_set_id == target_replica_set; - } - return membership; -} - -int add_routes( - MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, - const vector& route_hgs, int green_writer_hg -) { - vector queries; - for (size_t i = 0; i < route_hgs.size(); ++i) { - queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + - to_string(route_hgs[i]) + "," + - aurora_bgd_sql_quote(deployment.production.members[i].endpoint.hostname) + - ",3306,'ONLINE',1,'Aurora BGD rollback route')"); - } - queries.push_back( - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + - to_string(green_writer_hg) + "," + - aurora_bgd_sql_quote(deployment.target.members.front().endpoint.hostname) + - ",3306,'ONLINE',1,'Aurora BGD rollback green pool')"); - queries.push_back("LOAD MYSQL SERVERS TO RUNTIME"); - return aurora_bgd_execute_all(admin, queries); -} - -bool route_to_backend( - CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected -) { - auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); - if (sequence_rc != EXIT_SUCCESS) { - return false; - } - MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); - if (client == nullptr) { - return false; - } - auto [query_rc, rows] = mysql_query_ext_rows(client, kAuroraBGDRouteProbeQuery); - (void)rows; - mysql_close(client); - return query_rc == EXIT_SUCCESS - && aurora_bgd_routing_probe_reached(sim, sequence, expected); -} - -bool route_members( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, - Aurora_BGD_Test_Deployment& deployment, const vector& route_hgs, bool target -) { - for (size_t i = 0; i < route_hgs.size(); ++i) { - if (set_default_hostgroup(admin, route_hgs[i]) != EXIT_SUCCESS) { - return false; - } - Endpoint expected = target - ? deployment.target.members[i].endpoint.backend() - : deployment.production.members[i].endpoint.backend(); - if (!route_to_backend(cl, sim, expected)) { - return false; - } - } - return true; -} - -/** - * Start a worker from SWITCHOVER_INITIATED. - * - * - Publish INITIATED before the worker exists. - * - Keep canonical writer placement. - * - Reconstruct fast target-membership probing. - */ -int test_first_initiated(MYSQL* admin, BGD_Simulator& sim, TestState& state) { - Aurora_BGD_Test_Deployment& deployment = state.initiated; - if (publish_initial(sim, deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.initiated_writer_hostgroup, - state.initiated_reader_hostgroup, state.initiated_green_writer_hostgroup, - state.initiated_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS) { - diag("Error: failed to configure INITIATED late entry"); - return EXIT_FAILURE; - } - - ok(aurora_bgd_wait_for_status( - admin, state.initiated_writer_hostgroup, - "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, - "late entry at INITIATED publishes the observed state"); - ok(writer_placement( - admin, state.initiated_writer_hostgroup, state.initiated_reader_hostgroup, - deployment.production.members.front().endpoint.hostname, false), - "late INITIATED entry leaves writer placement unchanged"); - auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); - ok(sequence_rc == EXIT_SUCCESS && active_probe_policy( - sim, sequence, deployment.target_replica_set, 450), - "late INITIATED entry reconstructs fast membership probing"); - return EXIT_SUCCESS; -} - -/** - * Start a worker from SWITCHOVER_IN_PROGRESS. - * - * - Publish IN_PROGRESS before the worker exists. - * - Reconstruct the source snapshot. - * - Demote the production writer immediately. - */ -int test_first_in_progress(MYSQL* admin, BGD_Simulator& sim, TestState& state) { - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to reset before IN_PROGRESS late entry"); - return EXIT_FAILURE; - } - - Aurora_BGD_Test_Deployment& deployment = state.progress; - if (publish_initial(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.progress_writer_hostgroup, - state.progress_reader_hostgroup, state.progress_green_writer_hostgroup, - state.progress_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS) { - diag("Error: failed to configure IN_PROGRESS late entry"); - return EXIT_FAILURE; - } - - ok(aurora_bgd_wait_for_status( - admin, state.progress_writer_hostgroup, - "SWITCHOVER_IN_PROGRESS", kWaitSeconds) == EXIT_SUCCESS - && writer_placement( - admin, state.progress_writer_hostgroup, state.progress_reader_hostgroup, - deployment.production.members.front().endpoint.hostname, true), - "late entry at IN_PROGRESS reconstructs and demotes the writer"); - return EXIT_SUCCESS; -} - -/** - * Reconstruct IN_PROGRESS while the first ordinary Aurora query is failing. - * - * The fixed production membership comes from the configured/runtime hostgroups, - * so entering an active phase never depends on a successful cutover-time query. - */ -int test_first_in_progress_after_ordinary_error( - MYSQL* admin, BGD_Simulator& sim, TestState& state -) { - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to reset before failed-probe late entry"); - return EXIT_FAILURE; - } - - Aurora_BGD_Test_Deployment& deployment = state.progress; - if (publish_initial(sim, deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS - || sim.replica_error( - deployment.production.backends(), 1205, - "simulated initial ordinary Aurora timeout") != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.progress_writer_hostgroup, - state.progress_reader_hostgroup, state.progress_green_writer_hostgroup, - state.progress_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS) { - diag("Error: failed to configure failed-probe IN_PROGRESS late entry"); - return EXIT_FAILURE; - } - - ok(aurora_bgd_wait_for_status( - admin, state.progress_writer_hostgroup, - "SWITCHOVER_IN_PROGRESS", kWaitSeconds) == EXIT_SUCCESS - && writer_placement( - admin, state.progress_writer_hostgroup, state.progress_reader_hostgroup, - deployment.production.members.front().endpoint.hostname, true), - "late IN_PROGRESS entry reconstructs and demotes after an initial ordinary-query failure"); - return EXIT_SUCCESS; -} - -/** - * Start a worker from SWITCHOVER_IN_POST_PROCESSING. - * - * - Publish POST_PROCESSING before the worker exists. - * - Reconstruct the complete target map. - * - Apply every production-to-target traffic pin. - */ -int test_first_post_processing( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) -{ - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to reset before POST_PROCESSING late entry"); - return EXIT_FAILURE; - } - - Aurora_BGD_Test_Deployment& deployment = state.post; - if (publish_initial(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.post_writer_hostgroup, - state.post_reader_hostgroup, state.post_green_writer_hostgroup, - state.post_green_reader_hostgroup, false, 300, false) != EXIT_SUCCESS - || add_routes( - admin, deployment, state.post_route_hostgroups, - state.post_green_writer_hostgroup) != EXIT_SUCCESS) { - diag("Error: failed to configure POST_PROCESSING late entry"); - return EXIT_FAILURE; - } - - ok(aurora_bgd_wait_for_status( - admin, state.post_writer_hostgroup, - "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, - "late entry at POST_PROCESSING reconstructs the active phase"); - ok(route_members( - cl, admin, sim, deployment, state.post_route_hostgroups, true), - "late POST_PROCESSING entry reconstructs and applies every target pin"); - return EXIT_SUCCESS; -} - -int main() { - plan(7); - - CommandLine cl {}; - MYSQL* admin = nullptr; - BGD_Simulator sim {}; - - if (setup(cl, admin, sim) != EXIT_SUCCESS) { - return exit_status(); - } - - TestState state {}; - - // Simulator: make INITIATED the first observed deployment state. - // Verify: canonical placement and fast target probing are reconstructed. - if (test_first_initiated(admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: make IN_PROGRESS the first observed deployment state. - // Verify: the worker reconstructs and demotes the production writer. - if (test_first_in_progress(admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: fail the initial ordinary query while publishing IN_PROGRESS. - // Verify: configured production membership still permits immediate demotion. - if (test_first_in_progress_after_ordinary_error(admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // Simulator: make POST_PROCESSING the first observed deployment state. - // Verify: the worker reconstructs and applies every target pin. - if (test_first_post_processing(cl, admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - -exit_cleanup: - if (cleanup(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to clean Aurora BGD late-entry test data"); - return EXIT_FAILURE; - } - return exit_status(); -} diff --git a/test/tap/tests/test_aurora_bgd_late_entry_completed-t.cpp b/test/tap/tests/test_aurora_bgd_late_entry_completed-t.cpp new file mode 100644 index 0000000000..42dc34d8e4 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_late_entry_completed-t.cpp @@ -0,0 +1,101 @@ +/** + * @file test_aurora_bgd_late_entry_completed-t.cpp + * @brief First observation of Aurora BGD completion and later rearm. + */ + +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +struct TestState { + Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_b_writer_only() }; + int writer_hostgroup { 2120 }; + int reader_hostgroup { 2121 }; + int route_hostgroup { 2122 }; +}; + +int test_first_completed( + CommandLine& cl, Context& context, TestState& state +) { + if (context.simulator.replica_update( + state.deployment.production.replica_set_id, + state.deployment.production.replica_rows(), + state.deployment.production.backends()) != EXIT_SUCCESS + || context.simulator.topology_update( + aurora_bgd_topology_backends(state.deployment), + aurora_bgd_completed_topology(state.deployment)) != EXIT_SUCCESS + || add_route( + context.admin, state.route_hostgroup, + state.deployment.production.members.front().endpoint.hostname) != EXIT_SUCCESS) { + diag("Error: failed to publish direct-completion inputs"); + return EXIT_FAILURE; + } + auto [sequence_rc, sequence] = context.simulator.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS + || configure( + context, state.deployment, state.writer_hostgroup, state.reader_hostgroup, + -1, -1, true, 300) != EXIT_SUCCESS + || set_default_hostgroup( + context.admin, state.route_hostgroup) != EXIT_SUCCESS) { + diag("Error: failed to configure direct completion"); + return EXIT_FAILURE; + } + + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_COMPLETED", + kWaitSeconds) == EXIT_SUCCESS, + "first COMPLETED observation reports SWITCHOVER_COMPLETED for wHG 2120"); + ok(writer_placement( + context.admin, state.writer_hostgroup, state.reader_hostgroup, + state.deployment.production.members.front().endpoint.hostname, false), + "first COMPLETED observation leaves canonical writer placement unchanged"); + ok(route_to_backend( + cl, context, state.deployment.production.members.front().endpoint.backend()), + "first COMPLETED observation creates no target traffic pin"); + + usleep(500000); + auto [logs_rc, logs] = context.simulator.replica_probe_log_since(sequence); + bool membership_seen = false; + for (const Aurora_Replica_Probe_Log& log : logs) { + membership_seen = membership_seen + || log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership; + } + ok(logs_rc == EXIT_SUCCESS && !membership_seen, + "first COMPLETED observation does not replay target membership discovery"); + return EXIT_SUCCESS; +} + +int test_completed_empty_topology(Context& context, TestState& state) { + if (context.simulator.topology_delete( + aurora_bgd_topology_backends(state.deployment)) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "empty topology rearms the direct-completion latch to NONE"); + return EXIT_SUCCESS; +} + +int main() { + plan(5); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + if (test_first_completed(cl, context, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + test_completed_empty_topology(context, state); + +exit_cleanup: + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the late-entry completion fixture"); + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_late_entry_writer_phases-t.cpp b/test/tap/tests/test_aurora_bgd_late_entry_writer_phases-t.cpp new file mode 100644 index 0000000000..f064fde453 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_late_entry_writer_phases-t.cpp @@ -0,0 +1,124 @@ +/** + * @file test_aurora_bgd_late_entry_writer_phases-t.cpp + * @brief First observation of each active Aurora BGD writer phase. + */ + +#include +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +struct TestState { + Aurora_BGD_Test_Deployment initiated { aurora_bgd_deployment_b_writer_only() }; + Aurora_BGD_Test_Deployment progress { aurora_bgd_deployment_b_writer_only() }; + Aurora_BGD_Test_Deployment post { aurora_bgd_deployment_a() }; + vector post_routes { 2114, 2115, 2116 }; +}; + +bool active_probe_policy( + Context& context, uint64_t sequence, const string& target_replica_set +) { + usleep(450000); + auto [rc, logs] = context.simulator.replica_probe_log_since(sequence); + if (rc != EXIT_SUCCESS) { + return false; + } + bool membership_seen = false; + for (const Aurora_Replica_Probe_Log& log : logs) { + if (log.probe_kind == Aurora_Replica_Probe_Kind::ordinary) { + return false; + } + membership_seen = membership_seen + || (log.probe_kind == Aurora_Replica_Probe_Kind::bgd_membership + && log.replica_set_id == target_replica_set); + } + return membership_seen; +} + +int test_first_initiated(Context& context, TestState& state) { + if (publish_initial( + context, state.initiated, "SWITCHOVER_INITIATED") != EXIT_SUCCESS + || configure(context, state.initiated, 2090, 2091, 2092, 2093, + false, 300) != EXIT_SUCCESS) { + diag("Error: failed to configure INITIATED late entry"); + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, 2090, "SWITCHOVER_INITIATED", kWaitSeconds) == EXIT_SUCCESS, + "first INITIATED observation reports SWITCHOVER_INITIATED for wHG 2090"); + ok(writer_placement( + context.admin, 2090, 2091, + state.initiated.production.members.front().endpoint.hostname, false), + "first INITIATED observation leaves source placement unchanged"); + auto [sequence_rc, sequence] = context.simulator.replica_probe_log_last_sequence(); + ok(sequence_rc == EXIT_SUCCESS && active_probe_policy( + context, sequence, state.initiated.target_replica_set), + "first INITIATED observation reconstructs active target probing"); + return EXIT_SUCCESS; +} + +int test_first_in_progress(Context& context, TestState& state) { + if (reset(context) != EXIT_SUCCESS + || publish_initial( + context, state.progress, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || configure(context, state.progress, 2100, 2101, 2102, 2103, + false, 300) != EXIT_SUCCESS) { + diag("Error: failed to configure IN_PROGRESS late entry"); + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, 2100, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) == EXIT_SUCCESS, + "first IN_PROGRESS observation reports SWITCHOVER_IN_PROGRESS for wHG 2100"); + ok(writer_placement( + context.admin, 2100, 2101, + state.progress.production.members.front().endpoint.hostname, true), + "first IN_PROGRESS observation demotes the source writer"); + return EXIT_SUCCESS; +} + +int test_first_post_processing( + CommandLine& cl, Context& context, TestState& state +) { + if (reset(context) != EXIT_SUCCESS + || publish_initial( + context, state.post, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || configure(context, state.post, 2110, 2111, 2112, 2113, + false, 300) != EXIT_SUCCESS + || add_member_routes( + context.admin, state.post, state.post_routes) != EXIT_SUCCESS) { + diag("Error: failed to configure POST_PROCESSING late entry"); + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, 2110, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) + == EXIT_SUCCESS, + "first POST_PROCESSING observation reports SWITCHOVER_IN_POST_PROCESSING for wHG 2110"); + ok(route_members(cl, context, state.post, state.post_routes, true), + "first POST_PROCESSING observation reconstructs every target pin"); + return EXIT_SUCCESS; +} + +int main() { + plan(7); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + if (test_first_initiated(context, state) != EXIT_SUCCESS + || test_first_in_progress(context, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + test_first_post_processing(cl, context, state); + +exit_cleanup: + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the late-entry writer-phase fixture"); + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_probe_tls-t.cpp b/test/tap/tests/test_aurora_bgd_probe_tls-t.cpp new file mode 100644 index 0000000000..bba132d235 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_probe_tls-t.cpp @@ -0,0 +1,59 @@ +/** + * @file test_aurora_bgd_probe_tls-t.cpp + * @brief Aurora BGD topology and target-membership probe TLS policy. + */ + +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +struct TestState { + Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_b_writer_only() }; + int writer_hostgroup { 2030 }; + int reader_hostgroup { 2031 }; + int green_writer_hostgroup { 2032 }; + int green_reader_hostgroup { 2033 }; +}; + +int test_tls_probe_policy(Context& context, TestState& state) { + if (publish_available(context, state.deployment) != EXIT_SUCCESS + || configure( + context, state.deployment, state.writer_hostgroup, state.reader_hostgroup, + state.green_writer_hostgroup, state.green_reader_hostgroup, + false, 100, false, true) != EXIT_SUCCESS) { + diag("Error: failed to configure the TLS probe case"); + return EXIT_FAILURE; + } + + auto [topology_rc, topology_probe] = context.simulator.wait_for_probe_log( + 0, state.deployment.production.members.front().endpoint.backend(), + BGD_Probe_Kind::metadata, kProbeTimeoutMs, 1); + (void)topology_probe; + ok(topology_rc == EXIT_SUCCESS, + "TLS Aurora rows use TLS topology probes"); + + ok(membership_probe_reached(context, 0, state.deployment, 1), + "TLS Aurora rows use TLS target-membership probes"); + return EXIT_SUCCESS; +} + +int main() { + plan(2); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + test_tls_probe_policy(context, state); + +exit_cleanup: + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the probe-TLS fixture"); + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_reader_policy-t.cpp b/test/tap/tests/test_aurora_bgd_reader_policy-t.cpp new file mode 100644 index 0000000000..1703ea6086 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_reader_policy-t.cpp @@ -0,0 +1,80 @@ +/** + * @file test_aurora_bgd_reader_policy-t.cpp + * @brief Aurora BGD mapped-reader eligibility and routing. + */ + +#include +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +struct TestState { + Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_a() }; + int writer_hostgroup { 2050 }; + int reader_hostgroup { 2051 }; + int green_writer_hostgroup { 2052 }; + int green_reader_hostgroup { 2053 }; + vector reader_routes { 2054, 2055 }; +}; + +int test_mapped_readers(CommandLine& cl, Context& context, TestState& state) { + if (publish_available(context, state.deployment) != EXIT_SUCCESS + || configure( + context, state.deployment, state.writer_hostgroup, state.reader_hostgroup, + state.green_writer_hostgroup, state.green_reader_hostgroup) != EXIT_SUCCESS + || add_route( + context.admin, state.reader_routes[0], + state.deployment.production.members[1].endpoint.hostname) != EXIT_SUCCESS + || add_route( + context.admin, state.reader_routes[1], + state.deployment.production.members[2].endpoint.hostname) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) != EXIT_SUCCESS + || publish_status( + context, state.deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_IN_POST_PROCESSING", + kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: failed to prepare the mapped-reader policy case"); + return EXIT_FAILURE; + } + + ok(server_count( + context.admin, "runtime_mysql_servers", state.reader_hostgroup, + state.deployment.production.members[1].endpoint.hostname, 1, "ONLINE") + && server_count( + context.admin, "runtime_mysql_servers", state.reader_hostgroup, + state.deployment.production.members[2].endpoint.hostname, 1, "ONLINE"), + "POST_PROCESSING keeps mapped source readers ONLINE in hostgroup 2051"); + + bool routed = true; + for (size_t i = 0; i < state.reader_routes.size(); ++i) { + routed = routed + && set_default_hostgroup(context.admin, state.reader_routes[i]) == EXIT_SUCCESS + && route_to_backend( + cl, context, state.deployment.target.members[i + 1].endpoint.backend()); + } + ok(routed, + "POST_PROCESSING routes each reader hostname to its mapped target reader"); + return EXIT_SUCCESS; +} + +int main() { + plan(2); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + test_mapped_readers(cl, context, state); + + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the reader-policy fixture"); + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_reader_switchover_cleanup-t.cpp b/test/tap/tests/test_aurora_bgd_reader_switchover_cleanup-t.cpp new file mode 100644 index 0000000000..af245a02d0 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_reader_switchover_cleanup-t.cpp @@ -0,0 +1,124 @@ +/** + * @file test_aurora_bgd_reader_switchover_cleanup-t.cpp + * @brief Aurora BGD immediate completion cleanup and topology-drain rearm. + */ + +#include +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +struct TestState { + Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_a() }; + int writer_hostgroup { 2070 }; + int reader_hostgroup { 2071 }; + int green_writer_hostgroup { 2072 }; + int green_reader_hostgroup { 2073 }; + vector target_routes { 2074, 2075, 2076 }; + vector source_routes { 2077, 2078, 2079 }; +}; + +int test_completion_cleanup( + CommandLine& cl, Context& context, TestState& state +) { + if (publish_available(context, state.deployment) != EXIT_SUCCESS + || configure( + context, state.deployment, state.writer_hostgroup, state.reader_hostgroup, + state.green_writer_hostgroup, state.green_reader_hostgroup) != EXIT_SUCCESS + || add_member_routes( + context.admin, state.deployment, state.target_routes) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) != EXIT_SUCCESS + || publish_status( + context, state.deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_IN_PROGRESS", + kWaitSeconds) != EXIT_SUCCESS + || publish_status( + context, state.deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_IN_POST_PROCESSING", + kWaitSeconds) != EXIT_SUCCESS + || !route_members( + cl, context, state.deployment, state.target_routes, true)) { + diag("Error: failed to reach POST_PROCESSING with target routes"); + return EXIT_FAILURE; + } + + auto [sequence_rc, sequence] = context.simulator.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS + || publish_completed(context, state.deployment, state.deployment) != EXIT_SUCCESS) { + diag("Error: failed to publish completion"); + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_COMPLETED", + kWaitSeconds) == EXIT_SUCCESS, + "completion reports SWITCHOVER_COMPLETED for wHG 2070"); + ok(writer_placement( + context.admin, state.writer_hostgroup, state.reader_hostgroup, + state.deployment.production.members.front().endpoint.hostname, false), + "completion restores canonical writer placement"); + + if (add_member_routes( + context.admin, state.deployment, state.source_routes) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(route_members( + cl, context, state.deployment, state.source_routes, false), + "completion removes writer and reader traffic pins"); + ok(ordinary_probe_reached(context, sequence, state.deployment), + "completion resumes the ordinary Aurora probe"); + + if (publish_completed(context, state.deployment, state.deployment) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + usleep(300000); + ok(runtime_status_is(context.admin, state.writer_hostgroup, "SWITCHOVER_COMPLETED") + && route_members(cl, context, state.deployment, state.source_routes, false), + "repeated completion preserves the completed cleanup result"); + return EXIT_SUCCESS; +} + +int test_topology_drain(Context& context, TestState& state) { + if (context.simulator.topology_delete( + aurora_bgd_topology_backends(state.deployment)) != EXIT_SUCCESS) { + diag("Error: failed to publish empty topology"); + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "NONE", kWaitSeconds) == EXIT_SUCCESS, + "successful empty topology rearms wHG 2070 to NONE"); + ok(server_count( + context.admin, "runtime_mysql_servers", state.writer_hostgroup, + state.deployment.production.members.front().endpoint.hostname, 1) + && server_count( + context.admin, "runtime_mysql_servers", state.reader_hostgroup, + state.deployment.production.members[1].endpoint.hostname, 1), + "topology rearm retains configured source rows"); + return EXIT_SUCCESS; +} + +int main() { + plan(7); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + if (test_completion_cleanup(cl, context, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + test_topology_drain(context, state); + +exit_cleanup: + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the completion-cleanup fixture"); + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_remove_during_switchover-t.cpp b/test/tap/tests/test_aurora_bgd_remove_during_switchover-t.cpp new file mode 100644 index 0000000000..c175df1340 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_remove_during_switchover-t.cpp @@ -0,0 +1,90 @@ +/** + * @file test_aurora_bgd_remove_during_switchover-t.cpp + * @brief Aurora BGD cleanup when the owning row is deleted. + */ + +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +struct TestState { + Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_b_writer_only() }; + int writer_hostgroup { 2230 }; + int reader_hostgroup { 2231 }; + int green_writer_hostgroup { 2232 }; + int green_reader_hostgroup { 2233 }; +}; + +int test_bgd_status_available(Context& context, TestState& state) { + if (publish_available(context, state.deployment) != EXIT_SUCCESS + || configure( + context, state.deployment, state.writer_hostgroup, state.reader_hostgroup, + state.green_writer_hostgroup, state.green_reader_hostgroup) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "BGD status for wHG 2230 reports AVAILABLE"); + return EXIT_SUCCESS; +} + +int test_writer_switchover_in_progress(Context& context, TestState& state) { + if (publish_status( + context, state.deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_IN_PROGRESS", + kWaitSeconds) == EXIT_SUCCESS, + "BGD status for wHG 2230 reports SWITCHOVER_IN_PROGRESS"); + ok(writer_placement( + context.admin, state.writer_hostgroup, state.reader_hostgroup, + state.deployment.production.members.front().endpoint.hostname, true), + "SWITCHOVER_IN_PROGRESS demotes the source writer for wHG 2230"); + return EXIT_SUCCESS; +} + +int test_remove_during_switchover(Context& context, TestState& state) { + if (aurora_bgd_execute_all(context.admin, { + "DELETE FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=2230", + "LOAD MYSQL SERVERS TO RUNTIME", + }) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(writer_placement( + context.admin, state.writer_hostgroup, state.reader_hostgroup, + state.deployment.production.members.front().endpoint.hostname, false), + "deleting BGD configuration restores the source writer to hostgroup 2230"); + ok(wait_for_cond( + context.admin, + "SELECT COUNT(*)=0 FROM runtime_mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=2230", + kWaitSeconds) == EXIT_SUCCESS, + "deleting wHG 2230 removes it from runtime_mysql_aws_aurora_hostgroups"); + return EXIT_SUCCESS; +} + +int main() { + plan(5); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + if (test_bgd_status_available(context, state) != EXIT_SUCCESS + || test_writer_switchover_in_progress(context, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + test_remove_during_switchover(context, state); + +exit_cleanup: + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the remove-during-switchover fixture"); + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_repeated_deployment-t.cpp b/test/tap/tests/test_aurora_bgd_repeated_deployment-t.cpp new file mode 100644 index 0000000000..461b00efc3 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_repeated_deployment-t.cpp @@ -0,0 +1,142 @@ +/** + * @file test_aurora_bgd_repeated_deployment-t.cpp + * @brief Reuse one Aurora worker for a later deployment fingerprint. + */ + +#include +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +Aurora_BGD_Test_Deployment deployment_b() { + Aurora_BGD_Test_Deployment deployment = aurora_bgd_deployment_a(); + deployment.name = "Aurora BGD deployment A second target"; + deployment.target_replica_set = "aurora-bgd-target-a-r2"; + deployment.target_topology_id = "aurora-bgd-target-a-r2"; + const vector ids { + "aurora-a-writer-green-r2", + "aurora-a-reader-1-green-r2", + "aurora-a-reader-2-green-r2", + }; + const vector ips { "127.0.11.31", "127.0.11.32", "127.0.11.33" }; + for (size_t i = 0; i < deployment.target.members.size(); ++i) { + deployment.target.members[i].server_id = ids[i]; + deployment.target.members[i].endpoint.hostname = ids[i] + deployment.domain_name; + deployment.target.members[i].endpoint.ip = ips[i]; + } + deployment.target.replica_set_id = deployment.target_replica_set; + deployment.target.serving_endpoints.clear(); + deployment.target.serving_endpoints.push_back(deployment.target_cluster_endpoint); + for (Aurora_BGD_Member& member : deployment.target.members) { + deployment.target.serving_endpoints.push_back(member.endpoint); + } + return deployment; +} + +struct TestState { + Aurora_BGD_Test_Deployment first { aurora_bgd_deployment_a() }; + Aurora_BGD_Test_Deployment second { deployment_b() }; + int writer_hostgroup { 2240 }; + int reader_hostgroup { 2241 }; + int green_writer_hostgroup { 2242 }; + int green_reader_hostgroup { 2243 }; + vector routes { 2244, 2245, 2246 }; + vector completion_routes { 2247, 2248, 2249 }; +}; + +int complete_deployment(Context& context, Aurora_BGD_Test_Deployment& deployment) { + return publish_status(context, deployment, "SWITCHOVER_IN_PROGRESS") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + context.admin, 2240, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) == EXIT_SUCCESS + && publish_status( + context, deployment, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + context.admin, 2240, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) + == EXIT_SUCCESS + && publish_completed(context, deployment, deployment) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + context.admin, 2240, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int test_deployment_a(Context& context, TestState& state) { + if (publish_available(context, state.first) != EXIT_SUCCESS + || configure( + context, state.first, state.writer_hostgroup, state.reader_hostgroup, + state.green_writer_hostgroup, state.green_reader_hostgroup) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) + != EXIT_SUCCESS + || complete_deployment(context, state.first) != EXIT_SUCCESS) { + diag("Error: failed to complete deployment A"); + return EXIT_FAILURE; + } + ok(writer_placement( + context.admin, state.writer_hostgroup, state.reader_hostgroup, + state.first.production.members.front().endpoint.hostname, false), + "deployment A completion latches and restores canonical writer placement"); + return EXIT_SUCCESS; +} + +int test_deployment_b( + CommandLine& cl, Context& context, TestState& state +) { + auto [sequence_rc, sequence] = context.simulator.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS + || publish_available(context, state.second) != EXIT_SUCCESS + || add_member_routes( + context.admin, state.second, state.routes) != EXIT_SUCCESS) { + diag("Error: failed to publish deployment B"); + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS + && membership_probe_reached(context, sequence, state.second), + "deployment B reuses wHG 2240 with only its new target membership"); + + if (publish_status( + context, state.second, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_IN_POST_PROCESSING", + kWaitSeconds) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(route_members(cl, context, state.second, state.routes, true), + "deployment B POST_PROCESSING routes new connections to deployment B"); + + if (publish_completed(context, state.second, state.second) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_COMPLETED", + kWaitSeconds) != EXIT_SUCCESS + || add_member_routes( + context.admin, state.second, state.completion_routes) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(route_members(cl, context, state.second, state.completion_routes, false), + "deployment B completion removes its pins without restoring deployment A's stale map"); + return EXIT_SUCCESS; +} + +int main() { + plan(4); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + if (test_deployment_a(context, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + test_deployment_b(cl, context, state); + +exit_cleanup: + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the repeated-deployment fixture"); + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_rollback-t.cpp b/test/tap/tests/test_aurora_bgd_rollback-t.cpp new file mode 100644 index 0000000000..998d8e04c3 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_rollback-t.cpp @@ -0,0 +1,132 @@ +/** + * @file test_aurora_bgd_rollback-t.cpp + * @brief Aurora BGD rollback from each active writer phase. + */ + +#include +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +struct TestState { + Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_a() }; + vector routes { 2154, 2155, 2156 }; +}; + +int prepare_available( + Context& context, TestState& state, + int writer_hostgroup, int reader_hostgroup +) { + return publish_available(context, state.deployment) == EXIT_SUCCESS + && configure( + context, state.deployment, writer_hostgroup, reader_hostgroup, + writer_hostgroup + 2, reader_hostgroup + 2) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + context.admin, writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int test_initiated_rollback(Context& context, TestState& state) { + if (prepare_available(context, state, 2130, 2131) != EXIT_SUCCESS + || publish_status( + context, state.deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, 2130, "SWITCHOVER_INITIATED", kWaitSeconds) != EXIT_SUCCESS + || publish_status(context, state.deployment, "AVAILABLE") != EXIT_SUCCESS) { + diag("Error: failed to run INITIATED rollback"); + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, 2130, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "INITIATED rollback returns wHG 2130 to AVAILABLE"); + ok(writer_placement( + context.admin, 2130, 2131, + state.deployment.production.members.front().endpoint.hostname, false), + "INITIATED rollback leaves canonical writer placement unchanged"); + return EXIT_SUCCESS; +} + +int test_in_progress_rollback(Context& context, TestState& state) { + if (reset(context) != EXIT_SUCCESS + || prepare_available(context, state, 2140, 2141) != EXIT_SUCCESS + || publish_status( + context, state.deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, 2140, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) != EXIT_SUCCESS) { + diag("Error: failed to enter IN_PROGRESS rollback"); + return EXIT_FAILURE; + } + ok(writer_placement( + context.admin, 2140, 2141, + state.deployment.production.members.front().endpoint.hostname, true), + "IN_PROGRESS demotes the source writer before rollback"); + auto [sequence_rc, sequence] = context.simulator.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS + || publish_status(context, state.deployment, "AVAILABLE") != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, 2140, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS + && writer_placement( + context.admin, 2140, 2141, + state.deployment.production.members.front().endpoint.hostname, false), + "IN_PROGRESS rollback restores AVAILABLE and canonical writer placement"); + ok(ordinary_probe_reached(context, sequence, state.deployment), + "IN_PROGRESS rollback resumes the ordinary Aurora probe"); + return EXIT_SUCCESS; +} + +int test_post_processing_rollback( + CommandLine& cl, Context& context, TestState& state +) { + if (reset(context) != EXIT_SUCCESS + || prepare_available(context, state, 2150, 2151) != EXIT_SUCCESS + || add_member_routes(context.admin, state.deployment, state.routes) != EXIT_SUCCESS + || publish_status( + context, state.deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, 2150, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) + != EXIT_SUCCESS) { + diag("Error: failed to enter POST_PROCESSING rollback"); + return EXIT_FAILURE; + } + ok(route_members(cl, context, state.deployment, state.routes, true), + "POST_PROCESSING applies the complete target map before rollback"); + if (publish_status(context, state.deployment, "AVAILABLE") != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, 2150, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS + && writer_placement( + context.admin, 2150, 2151, + state.deployment.production.members.front().endpoint.hostname, false), + "POST_PROCESSING rollback restores AVAILABLE and canonical writer placement"); + ok(route_members(cl, context, state.deployment, state.routes, false), + "POST_PROCESSING rollback removes every target traffic pin"); + return EXIT_SUCCESS; +} + +int main() { + plan(8); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + if (test_initiated_rollback(context, state) != EXIT_SUCCESS + || test_in_progress_rollback(context, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + test_post_processing_rollback(cl, context, state); + +exit_cleanup: + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the rollback fixture"); + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_smoke-t.cpp b/test/tap/tests/test_aurora_bgd_smoke-t.cpp new file mode 100644 index 0000000000..231f26649e --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_smoke-t.cpp @@ -0,0 +1,69 @@ +/** + * @file test_aurora_bgd_smoke-t.cpp + * @brief Basic Aurora BGD AVAILABLE discovery. + */ + +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +struct TestState { + Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_a() }; + int writer_hostgroup { 2010 }; + int reader_hostgroup { 2011 }; + int green_writer_hostgroup { 2012 }; + int green_reader_hostgroup { 2013 }; +}; + +int test_available_topology(Context& context, TestState& state) { + if (publish_available(context, state.deployment) != EXIT_SUCCESS + || configure( + context, state.deployment, state.writer_hostgroup, state.reader_hostgroup, + state.green_writer_hostgroup, state.green_reader_hostgroup) != EXIT_SUCCESS) { + diag("Error: failed to publish or configure the smoke deployment"); + return EXIT_FAILURE; + } + + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS + && ordinary_probe_reached(context, 0, state.deployment), + "AVAILABLE topology publishes AVAILABLE and keeps ordinary production probing active"); + + auto [probe_rc, probe] = aurora_bgd_wait_for_topology_probe( + context.simulator, 0, state.deployment.target.backends(), + BGD_Probe_Kind::metadata, kProbeTimeoutMs); + (void)probe; + ok(probe_rc == EXIT_SUCCESS, + "AVAILABLE discovery moves topology probing to a target member"); + return EXIT_SUCCESS; +} + +int test_target_membership(Context& context, TestState& state) { + ok(membership_probe_reached(context, 0, state.deployment), + "AVAILABLE discovery probes the target replica set"); + return EXIT_SUCCESS; +} + +int main() { + plan(3); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + if (test_available_topology(context, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + test_target_membership(context, state); + +exit_cleanup: + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the Aurora BGD smoke fixture"); + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_topology_empty_absent-t.cpp b/test/tap/tests/test_aurora_bgd_topology_empty_absent-t.cpp new file mode 100644 index 0000000000..e5d3fbf451 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_topology_empty_absent-t.cpp @@ -0,0 +1,96 @@ +/** + * @file test_aurora_bgd_topology_empty_absent-t.cpp + * @brief Aurora BGD cancellation on successful empty or absent topology. + */ + +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +struct TestState { + Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_b_writer_only() }; +}; + +int prepare_in_progress( + Context& context, TestState& state, int writer_hostgroup, int reader_hostgroup +) { + return publish_available(context, state.deployment) == EXIT_SUCCESS + && configure( + context, state.deployment, writer_hostgroup, reader_hostgroup, + writer_hostgroup + 2, reader_hostgroup + 2) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + context.admin, writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS + && publish_status( + context, state.deployment, "SWITCHOVER_IN_PROGRESS") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + context.admin, writer_hostgroup, "SWITCHOVER_IN_PROGRESS", + kWaitSeconds) == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int test_empty_before_completion(Context& context, TestState& state) { + if (prepare_in_progress(context, state, 2160, 2161) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + auto [sequence_rc, sequence] = context.simulator.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS + || context.simulator.topology_delete( + aurora_bgd_topology_backends(state.deployment)) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, 2160, "NONE", kWaitSeconds) == EXIT_SUCCESS + && writer_placement( + context.admin, 2160, 2161, + state.deployment.production.members.front().endpoint.hostname, false), + "empty topology restores the source writer and sets wHG 2160 to NONE"); + ok(ordinary_probe_reached(context, sequence, state.deployment), + "empty-topology cancellation resumes ordinary Aurora probing"); + return EXIT_SUCCESS; +} + +int test_absent_before_completion(Context& context, TestState& state) { + if (reset(context) != EXIT_SUCCESS + || prepare_in_progress(context, state, 2170, 2171) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + auto [sequence_rc, sequence] = context.simulator.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS + || context.simulator.topology_drop( + aurora_bgd_topology_backends(state.deployment)) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, 2170, "NONE", kWaitSeconds) == EXIT_SUCCESS + && writer_placement( + context.admin, 2170, 2171, + state.deployment.production.members.front().endpoint.hostname, false), + "absent topology restores the source writer and sets wHG 2170 to NONE"); + ok(ordinary_probe_reached(context, sequence, state.deployment), + "absent-topology cancellation resumes ordinary Aurora probing"); + return EXIT_SUCCESS; +} + +int main() { + plan(4); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + if (test_empty_before_completion(context, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + test_absent_before_completion(context, state); + +exit_cleanup: + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the topology-empty/absent fixture"); + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_topology_errors-t.cpp b/test/tap/tests/test_aurora_bgd_topology_errors-t.cpp new file mode 100644 index 0000000000..353ccb5170 --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_topology_errors-t.cpp @@ -0,0 +1,107 @@ +/** + * @file test_aurora_bgd_topology_errors-t.cpp + * @brief Aurora BGD state retention across generic metadata errors. + */ + +#include +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +struct TestState { + Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_a() }; + vector routes { 2194, 2195, 2196 }; +}; + +int test_generic_topology_error(Context& context, TestState& state) { + if (publish_initial( + context, state.deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS + || configure(context, state.deployment, 2180, 2181, 2182, 2183, + false, 300) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, 2180, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) + != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + auto [sequence_rc, sequence] = context.simulator.probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS + || context.simulator.topology_error( + aurora_bgd_topology_backends(state.deployment), 1205, + "simulated topology timeout") != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + auto [probe_rc, probe] = aurora_bgd_wait_for_topology_probe( + context.simulator, sequence, state.deployment.target.backends(), + BGD_Probe_Kind::metadata, kProbeTimeoutMs); + (void)probe; + ok(probe_rc == EXIT_SUCCESS + && runtime_status_is(context.admin, 2180, "SWITCHOVER_IN_PROGRESS") + && writer_placement( + context.admin, 2180, 2181, + state.deployment.production.members.front().endpoint.hostname, true), + "generic topology errors retain active state and writer placement"); + return EXIT_SUCCESS; +} + +int test_generic_membership_error( + CommandLine& cl, Context& context, TestState& state +) { + if (reset(context) != EXIT_SUCCESS + || publish_initial( + context, state.deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || configure(context, state.deployment, 2190, 2191, 2192, 2193, + false, 300) != EXIT_SUCCESS + || add_member_routes(context.admin, state.deployment, state.routes) != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, 2190, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) + != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + auto [sequence_rc, sequence] = context.simulator.replica_probe_log_last_sequence(); + if (sequence_rc != EXIT_SUCCESS + || context.simulator.replica_error( + state.deployment.target.backends(), 1205, + "simulated membership timeout") != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + auto [probe_rc, probe] = aurora_bgd_wait_for_replica_probe( + context.simulator, sequence, state.deployment.target.backends(), + Aurora_Replica_Probe_Kind::bgd_membership, kProbeTimeoutMs, + state.deployment.target.replica_set_id); + (void)probe; + bool retained = probe_rc == EXIT_SUCCESS + && runtime_status_is( + context.admin, 2190, "SWITCHOVER_IN_POST_PROCESSING") + && context.simulator.replica_update( + state.deployment.target.replica_set_id, + state.deployment.target.replica_rows(), + state.deployment.target.backends()) == EXIT_SUCCESS + && route_members(cl, context, state.deployment, state.routes, true); + ok(retained, + "generic membership errors retain active state and recover target routing"); + return EXIT_SUCCESS; +} + +int main() { + plan(2); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + if (test_generic_topology_error(context, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + test_generic_membership_error(cl, context, state); + +exit_cleanup: + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the topology-error fixture"); + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_worker_config_refresh-t.cpp b/test/tap/tests/test_aurora_bgd_worker_config_refresh-t.cpp new file mode 100644 index 0000000000..b51915066d --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_worker_config_refresh-t.cpp @@ -0,0 +1,104 @@ +/** + * @file test_aurora_bgd_worker_config_refresh-t.cpp + * @brief Aurora BGD active-worker refresh without FSM reset. + */ + +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +struct TestState { + Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_b_writer_only() }; + int writer_hostgroup { 2200 }; + int reader_hostgroup { 2201 }; + int green_writer_hostgroup { 2202 }; + int green_reader_hostgroup { 2203 }; + int refreshed_green_writer_hostgroup { 2204 }; + int refreshed_green_reader_hostgroup { 2205 }; + int route_hostgroup { 2206 }; +}; + +int test_active_worker_refresh( + CommandLine& cl, Context& context, TestState& state +) { + if (publish_available(context, state.deployment) != EXIT_SUCCESS + || configure( + context, state.deployment, state.writer_hostgroup, state.reader_hostgroup, + state.green_writer_hostgroup, state.green_reader_hostgroup, + false, 300) != EXIT_SUCCESS + || add_route( + context.admin, state.route_hostgroup, + state.deployment.production.members.front().endpoint.hostname) != EXIT_SUCCESS) { + diag("Error: failed to configure the worker-refresh fixture"); + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "worker-refresh scenario starts from AVAILABLE"); + + if (publish_status( + context, state.deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS + || aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_IN_POST_PROCESSING", + kWaitSeconds) != EXIT_SUCCESS + || set_default_hostgroup( + context.admin, state.route_hostgroup) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(route_to_backend( + cl, context, state.deployment.target.members.front().endpoint.backend()), + "POST_PROCESSING pins writer traffic before refresh"); + + if (aurora_bgd_execute_all(context.admin, { + "UPDATE mysql_servers SET weight=7,comment='Aurora BGD refreshed route' " + "WHERE hostgroup_id=2206", + "LOAD MYSQL SERVERS TO RUNTIME", + }) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_IN_POST_PROCESSING", + kWaitSeconds) == EXIT_SUCCESS + && route_to_backend( + cl, context, state.deployment.target.members.front().endpoint.backend()), + "an unrelated server reload preserves active state and the target pin"); + + if (aurora_bgd_execute_all(context.admin, { + "UPDATE mysql_aws_aurora_hostgroups SET green_writer_hostgroup=2204," + "green_reader_hostgroup=2205 WHERE writer_hostgroup=2200", + "LOAD MYSQL SERVERS TO RUNTIME", + }) != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + ok(wait_for_cond( + context.admin, + "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups " + "WHERE writer_hostgroup=2200 AND green_writer_hostgroup=2204 " + "AND green_reader_hostgroup=2205 " + "AND bgd_status='SWITCHOVER_IN_POST_PROCESSING'", + kWaitSeconds) == EXIT_SUCCESS + && route_to_backend( + cl, context, state.deployment.target.members.front().endpoint.backend()), + "green-hostgroup refresh preserves cached membership, active state, and the target pin"); + return EXIT_SUCCESS; +} + +int main() { + plan(4); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + test_active_worker_refresh(cl, context, state); + + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the worker-refresh fixture"); + } + return exit_status(); +} diff --git a/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp b/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp deleted file mode 100644 index 143e57d7c1..0000000000 --- a/test/tap/tests/test_aurora_bgd_worker_lifecycle-t.cpp +++ /dev/null @@ -1,375 +0,0 @@ -/** - * @file test_aurora_bgd_worker_lifecycle-t.cpp - * @brief Aurora BGD worker refresh, deactivation, and terminal rearming. - * - * Steps: - * - * 1. Refresh server, variable, and green-hostgroup configuration during POST_PROCESSING. - * 2. Deactivate the owning row and restore canonical production routing. - * 3. Refresh a completed worker without releasing its terminal latch. - * 4. Rearm the completed worker after a successful topology drain. - */ - -#include -#include -#include - -#include "aurora_bgd_tap.h" -#include "command_line.h" -#include "utils.h" - -using namespace std; - -const uint32_t kWaitSeconds = 5; - -struct TestState { - Aurora_BGD_Test_Deployment reload { aurora_bgd_deployment_a() }; - int reload_writer_hostgroup { 1620 }; - int reload_reader_hostgroup { 1621 }; - int reload_green_writer_hostgroup { 1622 }; - int reload_green_reader_hostgroup { 1623 }; - int reload_route_hostgroup { 1624 }; - int refreshed_green_writer_hostgroup { 1626 }; - int refreshed_green_reader_hostgroup { 1627 }; - Aurora_BGD_Test_Deployment terminal { aurora_bgd_deployment_b_writer_only() }; - int terminal_writer_hostgroup { 1630 }; - int terminal_reader_hostgroup { 1631 }; - int terminal_green_writer_hostgroup { 1632 }; - int terminal_green_reader_hostgroup { 1633 }; -}; - -int setup(CommandLine& cl, MYSQL*& admin, BGD_Simulator& sim) { - if (cl.getEnv()) { - diag("Error: failed to load TAP environment"); - return EXIT_FAILURE; - } - admin = init_mysql_conn(cl.admin_host, cl.admin_port, cl.admin_username, cl.admin_password); - if (admin == nullptr) { - diag("Error: failed to connect to ProxySQL Admin"); - return EXIT_FAILURE; - } - char username[] = "aurora1"; - char password[] = "pass1"; // NOSONAR: fixed simulator fixture credential. - if (sim.connect(cl.host, 3306, username, password) != EXIT_SUCCESS - || aurora_bgd_admin_cleanup(admin) != EXIT_SUCCESS - || sim.cleanup() != EXIT_SUCCESS) { - diag("Error: failed to initialize the shared AWS simulator"); - return EXIT_FAILURE; - } - return aurora_bgd_execute_all(admin, { - "DELETE FROM mysql_users WHERE username='testuser'", - "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " - "VALUES ('testuser','testuser',1,0,1)", - "LOAD MYSQL USERS TO RUNTIME", - }); -} - -int reset_scenario(MYSQL* admin, BGD_Simulator& sim) { - return aurora_bgd_admin_cleanup(admin) == EXIT_SUCCESS - && sim.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int cleanup(MYSQL* admin, BGD_Simulator& sim) { - int reset_rc = reset_scenario(admin, sim); - int user_rc = aurora_bgd_execute_all(admin, { - "DELETE FROM mysql_users WHERE username='testuser'", - "LOAD MYSQL USERS TO RUNTIME", - }); - mysql_close(admin); - return reset_rc == EXIT_SUCCESS && user_rc == EXIT_SUCCESS - ? EXIT_SUCCESS : EXIT_FAILURE; -} - -int publish_status( - BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment, const string& status -) { - return sim.topology_update( - aurora_bgd_topology_backends(deployment), - aurora_bgd_topology(deployment, status)); -} - -int set_default_hostgroup(MYSQL* admin, int hostgroup) { - return aurora_bgd_execute_all(admin, { - "UPDATE mysql_users SET default_hostgroup=" + to_string(hostgroup) + - " WHERE username='testuser'", - "LOAD MYSQL USERS TO RUNTIME", - }); -} - -int add_writer_route( - MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, int hostgroup -) { - return aurora_bgd_execute_all(admin, { - "INSERT INTO mysql_servers(hostgroup_id,hostname,port,status,use_ssl,comment) VALUES (" + - to_string(hostgroup) + "," + - aurora_bgd_sql_quote(deployment.production.members.front().endpoint.hostname) + - ",3306,'ONLINE',1,'Aurora BGD lifecycle route')", - "LOAD MYSQL SERVERS TO RUNTIME", - }); -} - -bool route_to_backend( - CommandLine& cl, BGD_Simulator& sim, const Endpoint& expected -) { - auto [sequence_rc, sequence] = sim.replica_probe_log_last_sequence(); - if (sequence_rc != EXIT_SUCCESS) { - return false; - } - MYSQL* client = init_mysql_conn(cl.host, cl.port, cl.username, cl.password); - if (client == nullptr) { - return false; - } - auto [query_rc, rows] = mysql_query_ext_rows(client, kAuroraBGDRouteProbeQuery); - (void)rows; - mysql_close(client); - return query_rc == EXIT_SUCCESS - && aurora_bgd_routing_probe_reached(sim, sequence, expected); -} - -bool route_writer( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, - Aurora_BGD_Test_Deployment& deployment, int route_hg, bool target -) { - if (set_default_hostgroup(admin, route_hg) != EXIT_SUCCESS) { - return false; - } - const Endpoint expected = target - ? deployment.target.members.front().endpoint.backend() - : deployment.production.members.front().endpoint.backend(); - return route_to_backend(cl, sim, expected); -} - -bool writer_placement( - MYSQL* admin, int writer_hg, int reader_hg, const string& hostname, bool demoted -) { - string query = - "SELECT ((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(writer_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + - ")=" + (demoted ? "0" : "1") + ") AND " - "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=" + - to_string(reader_hg) + " AND hostname=" + aurora_bgd_sql_quote(hostname) + - ")=" + (demoted ? "1" : "0") + ")"; - return wait_for_cond(admin, query, kWaitSeconds) == EXIT_SUCCESS; -} - -bool wait_for_inactive_none(MYSQL* admin, int writer_hg) { - return wait_for_cond( - admin, - "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups WHERE writer_hostgroup=" + - to_string(writer_hg) + " AND active=0 AND bgd_status='NONE'", - kWaitSeconds) == EXIT_SUCCESS; -} - -/** - * Refresh an active worker without resetting its FSM or traffic pins. - * - * - Reload unrelated server and monitor-variable configuration. - * - Refresh green staging hostgroups. - * - Preserve POST_PROCESSING status, cached membership, and target routing. - */ -int test_active_worker_refresh( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) -{ - Aurora_BGD_Test_Deployment& deployment = state.reload; - if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.reload_writer_hostgroup, - state.reload_reader_hostgroup, state.reload_green_writer_hostgroup, - state.reload_green_reader_hostgroup, false, 300) != EXIT_SUCCESS - || add_writer_route( - admin, deployment, state.reload_route_hostgroup) != EXIT_SUCCESS) { - diag("Error: failed to configure the active reload scenario"); - return EXIT_FAILURE; - } - - ok(aurora_bgd_wait_for_status( - admin, state.reload_writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, - "reload scenario starts from AVAILABLE"); - ok(publish_status(sim, deployment, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.reload_writer_hostgroup, - "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS - && route_writer( - cl, admin, sim, deployment, state.reload_route_hostgroup, true), - "active reload scenario pins writer traffic to the target"); - ok(aurora_bgd_execute_all( - admin, { "LOAD MYSQL SERVERS TO RUNTIME" }) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.reload_writer_hostgroup, - "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, - "an unrelated server reload preserves active FSM status"); - ok(route_writer( - cl, admin, sim, deployment, state.reload_route_hostgroup, true), - "an unrelated server reload preserves the applied traffic pin"); - ok(aurora_bgd_execute_all(admin, { - "SET mysql-aws_blue_green_deployment_auto_discovery='false'", - "LOAD MYSQL VARIABLES TO RUNTIME", - }) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.reload_writer_hostgroup, - "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) == EXIT_SUCCESS, - "disabling auto-discovery does not abort an admitted deployment"); - ok(route_writer( - cl, admin, sim, deployment, state.reload_route_hostgroup, true), - "a variable refresh preserves cached membership and target routing"); - ok(aurora_bgd_execute_all(admin, { - "UPDATE mysql_aws_aurora_hostgroups SET green_writer_hostgroup=" + - to_string(state.refreshed_green_writer_hostgroup) + - ",green_reader_hostgroup=" + - to_string(state.refreshed_green_reader_hostgroup) + - " WHERE writer_hostgroup=" + to_string(state.reload_writer_hostgroup), - "LOAD MYSQL SERVERS TO RUNTIME", - }) == EXIT_SUCCESS - && wait_for_cond( - admin, - "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups " - "WHERE writer_hostgroup=" + to_string(state.reload_writer_hostgroup) + - " AND green_writer_hostgroup=" + - to_string(state.refreshed_green_writer_hostgroup) + - " AND green_reader_hostgroup=" + - to_string(state.refreshed_green_reader_hostgroup) + - " AND bgd_status='SWITCHOVER_IN_POST_PROCESSING'", - kWaitSeconds) == EXIT_SUCCESS, - "green hostgroup changes refresh staging references without resetting the FSM"); - ok(route_writer( - cl, admin, sim, deployment, state.reload_route_hostgroup, true), - "green hostgroup refresh preserves applied pins and cached target IPs"); - return EXIT_SUCCESS; -} - -/** - * Deactivate an active worker. - * - * - Stop the worker after publishing NONE. - * - Restore canonical writer placement and production routing. - * - Preserve administrator-owned configuration and server rows. - */ -int test_active_worker_deactivation( - CommandLine& cl, MYSQL* admin, BGD_Simulator& sim, TestState& state) -{ - Aurora_BGD_Test_Deployment& deployment = state.reload; - ok(aurora_bgd_execute_all(admin, { - "UPDATE mysql_aws_aurora_hostgroups SET active=0 WHERE writer_hostgroup=" + - to_string(state.reload_writer_hostgroup), - "LOAD MYSQL SERVERS TO RUNTIME", - }) == EXIT_SUCCESS && wait_for_inactive_none(admin, state.reload_writer_hostgroup), - "deactivating the owning row stops the worker after publishing NONE"); - ok(writer_placement( - admin, state.reload_writer_hostgroup, state.reload_reader_hostgroup, - deployment.production.members.front().endpoint.hostname, false), - "worker teardown restores canonical writer placement"); - ok(route_writer( - cl, admin, sim, deployment, state.reload_route_hostgroup, false), - "worker teardown removes the applied traffic pin"); - ok(wait_for_cond( - admin, - "SELECT ((SELECT COUNT(*) FROM mysql_aws_aurora_hostgroups " - "WHERE writer_hostgroup=" + to_string(state.reload_writer_hostgroup) + - " AND active=0)=1) AND " - "((SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostname=" + - aurora_bgd_sql_quote(deployment.production.members.front().endpoint.hostname) + - ")>=1)", - kWaitSeconds) == EXIT_SUCCESS, - "teardown preserves user configuration and server rows"); - return EXIT_SUCCESS; -} - -/** - * Refresh and rearm a worker in the completed latch. - * - * - Change configured green hostgroups and monitor variables. - * - Preserve SWITCHOVER_COMPLETED across the refresh. - * - Release the latch only after a successful topology drain. - */ -int test_terminal_worker_refresh(MYSQL* admin, BGD_Simulator& sim, TestState& state) { - if (reset_scenario(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to reset before terminal reload scenario"); - return EXIT_FAILURE; - } - - Aurora_BGD_Test_Deployment& deployment = state.terminal; - if (aurora_bgd_publish(sim, deployment) != EXIT_SUCCESS - || aurora_bgd_admin_setup( - admin, deployment, state.terminal_writer_hostgroup, - state.terminal_reader_hostgroup, -1, -1, true, 300) != EXIT_SUCCESS) { - diag("Error: failed to configure the terminal reload scenario"); - return EXIT_FAILURE; - } - - ok(aurora_bgd_wait_for_status( - admin, state.terminal_writer_hostgroup, "AVAILABLE", kWaitSeconds) - == EXIT_SUCCESS, - "terminal reload scenario starts from AVAILABLE"); - ok(sim.topology_update( - aurora_bgd_topology_backends(deployment), - aurora_bgd_completed_topology(deployment)) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.terminal_writer_hostgroup, - "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS, - "direct completion enters the terminal latch"); - ok(aurora_bgd_execute_all(admin, { - "UPDATE mysql_aws_aurora_hostgroups SET green_writer_hostgroup=" + - to_string(state.terminal_green_writer_hostgroup) + - ",green_reader_hostgroup=" + - to_string(state.terminal_green_reader_hostgroup) + - " WHERE writer_hostgroup=" + to_string(state.terminal_writer_hostgroup), - "LOAD MYSQL SERVERS TO RUNTIME", - "SET mysql-aws_blue_green_deployment_auto_discovery='false'", - "LOAD MYSQL VARIABLES TO RUNTIME", - }) == EXIT_SUCCESS - && wait_for_cond( - admin, - "SELECT COUNT(*)=1 FROM runtime_mysql_aws_aurora_hostgroups " - "WHERE writer_hostgroup=" + to_string(state.terminal_writer_hostgroup) + - " AND green_writer_hostgroup=" + - to_string(state.terminal_green_writer_hostgroup) + - " AND green_reader_hostgroup=" + - to_string(state.terminal_green_reader_hostgroup) + - " AND bgd_status='SWITCHOVER_COMPLETED'", - kWaitSeconds) == EXIT_SUCCESS, - "configuration and variable refresh preserve the terminal latch"); - ok(sim.topology_delete(aurora_bgd_topology_backends(deployment)) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - admin, state.terminal_writer_hostgroup, "NONE", kWaitSeconds) == EXIT_SUCCESS, - "a successful topology drain rearms a refreshed terminal worker"); - return EXIT_SUCCESS; -} - -int main() { - plan(16); - - CommandLine cl {}; - MYSQL* admin = nullptr; - BGD_Simulator sim {}; - - if (setup(cl, admin, sim) != EXIT_SUCCESS) { - return exit_status(); - } - - TestState state {}; - - // ProxySQL: reload server, variable, and hostgroup configuration during POST_PROCESSING. - // Verify: the active FSM, cached target map, and applied pin remain intact. - if (test_active_worker_refresh(cl, admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // ProxySQL: deactivate the owning Aurora BGD row. - // Verify: the worker stops and restores production routing without deleting configuration. - if (test_active_worker_deactivation(cl, admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - - // ProxySQL: refresh configuration while a worker is latched at completion. - // Verify: the terminal state survives refresh and rearms after topology drain. - if (test_terminal_worker_refresh(admin, sim, state) != EXIT_SUCCESS) { - goto exit_cleanup; - } - -exit_cleanup: - if (cleanup(admin, sim) != EXIT_SUCCESS) { - diag("Error: failed to clean Aurora BGD worker-lifecycle test data"); - return EXIT_FAILURE; - } - return exit_status(); -} diff --git a/test/tap/tests/test_aurora_bgd_writer_switchover-t.cpp b/test/tap/tests/test_aurora_bgd_writer_switchover-t.cpp new file mode 100644 index 0000000000..920fddd41e --- /dev/null +++ b/test/tap/tests/test_aurora_bgd_writer_switchover-t.cpp @@ -0,0 +1,163 @@ +/** + * @file test_aurora_bgd_writer_switchover-t.cpp + * @brief Aurora BGD writer behavior across the active switchover phases. + */ + +#include +#include + +#include "aurora_bgd_scenario_tap.h" + +using namespace aurora_bgd_scenario; + +struct TestState { + Aurora_BGD_Test_Deployment deployment { aurora_bgd_deployment_a() }; + int writer_hostgroup { 2040 }; + int reader_hostgroup { 2041 }; + int green_writer_hostgroup { 2042 }; + int green_reader_hostgroup { 2043 }; + vector route_hostgroups { 2044, 2045, 2046 }; +}; + +int test_bgd_status_available(Context& context, TestState& state) { + if (publish_available(context, state.deployment) != EXIT_SUCCESS + || configure( + context, state.deployment, state.writer_hostgroup, state.reader_hostgroup, + state.green_writer_hostgroup, state.green_reader_hostgroup) != EXIT_SUCCESS + || add_member_routes( + context.admin, state.deployment, state.route_hostgroups) != EXIT_SUCCESS) { + diag("Error: failed to configure the writer-switchover fixture"); + return EXIT_FAILURE; + } + + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) == EXIT_SUCCESS, + "BGD status for wHG 2040 reports AVAILABLE"); + ok(writer_placement( + context.admin, state.writer_hostgroup, state.reader_hostgroup, + state.deployment.production.members.front().endpoint.hostname, false), + "AVAILABLE keeps the source writer in hostgroup 2040"); + return EXIT_SUCCESS; +} + +int test_switchover_initiated(Context& context, TestState& state) { + if (publish_status(context, state.deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_INITIATED"); + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_INITIATED", kWaitSeconds) + == EXIT_SUCCESS, + "BGD status for wHG 2040 reports SWITCHOVER_INITIATED"); + ok(writer_placement( + context.admin, state.writer_hostgroup, state.reader_hostgroup, + state.deployment.production.members.front().endpoint.hostname, false), + "SWITCHOVER_INITIATED leaves source placement unchanged"); + return EXIT_SUCCESS; +} + +int test_switchover_in_progress(Context& context, TestState& state) { + if (publish_status(context, state.deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_PROGRESS"); + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) + == EXIT_SUCCESS, + "BGD status for wHG 2040 reports SWITCHOVER_IN_PROGRESS"); + ok(writer_placement( + context.admin, state.writer_hostgroup, state.reader_hostgroup, + state.deployment.production.members.front().endpoint.hostname, true), + "SWITCHOVER_IN_PROGRESS demotes the source writer to hostgroup 2041"); + ok(server_count( + context.admin, "runtime_mysql_servers", state.reader_hostgroup, + state.deployment.production.members[1].endpoint.hostname, 1, "ONLINE") + && server_count( + context.admin, "runtime_mysql_servers", state.reader_hostgroup, + state.deployment.production.members[2].endpoint.hostname, 1, "ONLINE"), + "SWITCHOVER_IN_PROGRESS leaves source readers ONLINE in hostgroup 2041"); + return EXIT_SUCCESS; +} + +int test_switchover_post_processing( + CommandLine& cl, Context& context, TestState& state +) { + if (!route_members( + cl, context, state.deployment, state.route_hostgroups, false)) { + diag("Error: failed to create the source-backed pre-cutover pools"); + return EXIT_FAILURE; + } + for (int hostgroup : state.route_hostgroups) { + if (pool_connections(context.admin, hostgroup) < 1) { + diag("Error: route hostgroup %d has no pre-cutover pool", hostgroup); + return EXIT_FAILURE; + } + } + + if (publish_status( + context, state.deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS) { + diag("Error: failed to publish SWITCHOVER_IN_POST_PROCESSING"); + return EXIT_FAILURE; + } + ok(aurora_bgd_wait_for_status( + context.admin, state.writer_hostgroup, "SWITCHOVER_IN_POST_PROCESSING", + kWaitSeconds) == EXIT_SUCCESS, + "BGD status for wHG 2040 reports SWITCHOVER_IN_POST_PROCESSING"); + ok(writer_placement( + context.admin, state.writer_hostgroup, state.reader_hostgroup, + state.deployment.production.members.front().endpoint.hostname, false), + "POST_PROCESSING restores canonical writer placement"); + + bool pools_drained = true; + for (int hostgroup : state.route_hostgroups) { + pools_drained = pools_drained + && wait_for_pool_count(context.admin, hostgroup, "=0") == EXIT_SUCCESS; + } + ok(pools_drained, "POST_PROCESSING drains the pre-cutover member pools"); + ok(route_members( + cl, context, state.deployment, state.route_hostgroups, true), + "POST_PROCESSING routes every source hostname to its mapped target member"); + + vector pool_counts; + for (int hostgroup : state.route_hostgroups) { + pool_counts.push_back(pool_connections(context.admin, hostgroup)); + } + if (publish_status( + context, state.deployment, "SWITCHOVER_IN_POST_PROCESSING") != EXIT_SUCCESS) { + diag("Error: failed to repeat SWITCHOVER_IN_POST_PROCESSING"); + return EXIT_FAILURE; + } + usleep(300000); + bool pools_preserved = true; + for (size_t i = 0; i < state.route_hostgroups.size(); ++i) { + pools_preserved = pools_preserved + && pool_connections(context.admin, state.route_hostgroups[i]) >= pool_counts[i]; + } + ok(pools_preserved, + "repeated POST_PROCESSING does not replay completed pool retirement"); + return EXIT_SUCCESS; +} + +int main() { + plan(12); + + CommandLine cl {}; + Context context {}; + if (setup(cl, context) != EXIT_SUCCESS) { + return exit_status(); + } + + TestState state {}; + if (test_bgd_status_available(context, state) != EXIT_SUCCESS + || test_switchover_initiated(context, state) != EXIT_SUCCESS + || test_switchover_in_progress(context, state) != EXIT_SUCCESS) { + goto exit_cleanup; + } + test_switchover_post_processing(cl, context, state); + +exit_cleanup: + if (cleanup(context) != EXIT_SUCCESS) { + diag("Error: failed to clean the writer-switchover fixture"); + } + return exit_status(); +} From 02d654680744431c338abc62ec0f78b0ef7fd24d Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 20 Aug 2026 21:08:26 +0000 Subject: [PATCH 33/34] docs: document Aurora blue-green support --- include/DNS_Cache.hpp | 7 +- include/MySQL_HostGroups_Manager.h | 89 ++++++- include/MySQL_Monitor.hpp | 188 ++++++++++---- include/proxysql_admin.h | 14 +- lib/DNS_Cache.cpp | 6 + lib/MySQL_HostGroups_Manager.cpp | 108 +++++++- lib/MySQL_Monitor.cpp | 245 +++++++++++++++++- lib/ProxySQL_Admin.cpp | 32 +++ .../cluster_simulator/lib/aurora_utils.cpp | 14 + .../deps/cluster_simulator/lib/aurora_utils.h | 24 +- test/tap/tap/aurora_bgd_scenario_tap.h | 64 ++++- test/tap/tap/aurora_bgd_simulator.cpp | 25 ++ test/tap/tap/aurora_bgd_simulator.h | 77 ++++-- test/tap/tap/aurora_bgd_tap.h | 87 ++++++- test/tap/tap/bgd_simulator.cpp | 118 +++++++++ test/tap/tap/bgd_simulator.h | 146 +++++++++-- .../test_aurora_bgd_automatic_discovery-t.cpp | 8 + ...test_aurora_bgd_concurrent_isolation-t.cpp | 6 + ..._bgd_config_refresh_after_completion-t.cpp | 6 + ...aurora_bgd_disable_during_switchover-t.cpp | 7 + .../test_aurora_bgd_green_pool_cleanup-t.cpp | 7 + ...test_aurora_bgd_late_entry_completed-t.cpp | 6 + ..._aurora_bgd_late_entry_writer_phases-t.cpp | 7 + .../tap/tests/test_aurora_bgd_probe_tls-t.cpp | 4 + .../tests/test_aurora_bgd_reader_policy-t.cpp | 5 + ...aurora_bgd_reader_switchover_cleanup-t.cpp | 6 + ..._aurora_bgd_remove_during_switchover-t.cpp | 7 + .../test_aurora_bgd_repeated_deployment-t.cpp | 6 + test/tap/tests/test_aurora_bgd_rollback-t.cpp | 7 + test/tap/tests/test_aurora_bgd_smoke-t.cpp | 6 + ...est_aurora_bgd_topology_empty_absent-t.cpp | 6 + .../test_aurora_bgd_topology_errors-t.cpp | 6 + ...est_aurora_bgd_worker_config_refresh-t.cpp | 5 + .../test_aurora_bgd_writer_switchover-t.cpp | 8 + 34 files changed, 1218 insertions(+), 139 deletions(-) diff --git a/include/DNS_Cache.hpp b/include/DNS_Cache.hpp index b43ad1406e..0f790b89fa 100644 --- a/include/DNS_Cache.hpp +++ b/include/DNS_Cache.hpp @@ -81,7 +81,12 @@ class DNS_Cache { bool add(const std::string& hostname, std::vector&& ips); bool add_if_not_exist(const std::string& hostname, std::vector&& ips); void remove(const std::string& hostname); - // Clear ordinary resolved addresses while retaining explicit pins. + /** + * @brief Clear ordinary DNS resolutions while retaining explicit pins. + * + * @details Unpinned records are removed. Pinned records keep their fixed + * address while their ordinary resolution list and rotation counter are reset. + */ void clear(); bool empty() const; bool is_ip_valid(const std::string& hostname, const std::string& ip) const; diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index bd81cffc35..9384de901f 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -356,12 +356,15 @@ class Galera_Info { Galera_Info& operator=(const Galera_Info&) = delete; }; +/** + * @brief Runtime configuration retained for one Aurora writer hostgroup. + */ class AWS_Aurora_Info { public: - int writer_hostgroup; - int reader_hostgroup; - int green_writer_hostgroup; - int green_reader_hostgroup; + int writer_hostgroup; ///< Production writer hostgroup. + int reader_hostgroup; ///< Production reader hostgroup. + int green_writer_hostgroup; ///< Explicit green writer hostgroup; -1 means NULL. + int green_reader_hostgroup; ///< Explicit green reader hostgroup; -1 means NULL. int aurora_port; int max_lag_ms; int add_lag_ms; @@ -378,7 +381,49 @@ class AWS_Aurora_Info { char * comment; bool active; bool active_; + /** + * @brief Construct the runtime configuration for one Aurora cluster. + * + * @param w Writer hostgroup. + * @param r Reader hostgroup. + * @param gw Explicit green writer hostgroup, or -1 for NULL. + * @param gr Explicit green reader hostgroup, or -1 for NULL. + * @param _port Aurora backend port. + * @param _end_addr Aurora domain name. + * @param maxl Maximum accepted replica lag in milliseconds. + * @param al Lag threshold used when adding a reader. + * @param minl Minimum lag value used by Aurora monitoring. + * @param lnc Number of lag checks used by Aurora monitoring. + * @param ci Monitor check interval in milliseconds. + * @param ct Monitor check timeout in milliseconds. + * @param _a Whether the cluster is active. + * @param wiar Whether the writer also belongs to the reader hostgroup. + * @param nrw Weight assigned to newly discovered readers. + * @param amc Missing checks required before automatic removal. + * @param c Optional configuration comment. + */ AWS_Aurora_Info(int w, int r, int gw, int gr, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c); + /** + * @brief Apply a refreshed Aurora configuration to this runtime entry. + * + * @param r Reader hostgroup. + * @param gw Explicit green writer hostgroup, or -1 for NULL. + * @param gr Explicit green reader hostgroup, or -1 for NULL. + * @param _port Aurora backend port. + * @param _end_addr Aurora domain name. + * @param maxl Maximum accepted replica lag in milliseconds. + * @param al Lag threshold used when adding a reader. + * @param minl Minimum lag value used by Aurora monitoring. + * @param lnc Number of lag checks used by Aurora monitoring. + * @param ci Monitor check interval in milliseconds. + * @param ct Monitor check timeout in milliseconds. + * @param _a Whether the cluster is active. + * @param wiar Whether the writer also belongs to the reader hostgroup. + * @param nrw Weight assigned to newly discovered readers. + * @param amc Missing checks required before automatic removal. + * @param c Optional configuration comment. + * @return true when any retained configuration value changed. + */ bool update(int r, int gw, int gr, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c); ~AWS_Aurora_Info(); AWS_Aurora_Info(const AWS_Aurora_Info&) = delete; @@ -388,9 +433,13 @@ class AWS_Aurora_Info { /** * @brief Validate an Aurora hostgroup candidate and return its canonical configured projection. * - * The candidate must exactly match the configured Aurora Admin table projection. - * The returned result contains only valid rows in that canonical order. The - * caller owns the returned result. + * @details The candidate must exactly match the configured Aurora Admin table + * projection. The returned result contains only valid rows in canonical + * order. Rows that conflict by hostgroup role are rejected as one component. + * + * @param candidate Candidate Aurora configuration rows. + * @param errors Receives one message for each rejected projection or component. + * @return Heap-allocated canonical result owned by the caller. */ SQLite3_result* validate_and_filter_aws_aurora_hostgroups( const SQLite3_result* candidate, @@ -1127,9 +1176,12 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { /** * @brief Publish the node-local Aurora BGD state for one runtime row. * - * The caller owns validation of the state vocabulary. Writer hostgroups not - * present at runtime are ignored. Configuration reloads do not write this - * column. + * @details The caller owns validation of the state vocabulary. Writer + * hostgroups not present at runtime are ignored. Configuration reloads do + * not write this column. + * + * @param writer_hostgroup Writer hostgroup identifying the runtime row. + * @param bgd_status Validated Aurora BGD status string to publish. */ void update_aws_aurora_bgd_status(int writer_hostgroup, const std::string& bgd_status); /** @@ -1227,6 +1279,23 @@ class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { //void update_aws_aurora_set_reader(int _whid, int _rhid, char *_hostname, int _port); bool aws_aurora_replication_lag_action(int _whid, int _rhid, char *server_id, float current_replication_lag_ms, bool enable, bool is_writer, bool verbose=true); void update_aws_aurora_set_writer(int _whid, int _rhid, char *server_id, bool verbose=true); + /** + * @brief Place an Aurora member in its canonical writer hostgroup configuration. + * + * @details The explicit configuration values let BGD cleanup restore writer + * placement without rereading mutable Aurora configuration. The member is + * added to or removed from the reader hostgroup according to + * writer_is_also_reader. + * + * @param _whid Writer hostgroup. + * @param _rhid Reader hostgroup. + * @param server_id Aurora server identifier without the configured domain. + * @param verbose Whether to emit routing-change messages. + * @param domain_name Configured Aurora domain name. + * @param aurora_port Aurora backend port. + * @param writer_is_also_reader Whether the writer also belongs to the reader hostgroup. + * @param new_reader_weight Weight used when reader membership is created. + */ void update_aws_aurora_set_writer( int _whid, int _rhid, char *server_id, bool verbose, const char *domain_name, int aurora_port, int writer_is_also_reader, diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 4a2637f61a..d4e31a8667 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -566,18 +566,25 @@ struct AWS_RDS_BGD_Probe_Host { /** * @brief Aurora blue/green deployment phase published by the Aurora worker. + * + * @details Values after NONE mirror the statuses exposed by + * `mysql.rds_topology`. The worker publishes the current value in + * `runtime_mysql_aws_aurora_hostgroups.bgd_status`. */ enum class AWS_Aurora_BGD_Status { - NONE = 0, - AVAILABLE, - SWITCHOVER_INITIATED, - SWITCHOVER_IN_PROGRESS, - SWITCHOVER_IN_POST_PROCESSING, - SWITCHOVER_COMPLETED, + NONE = 0, ///< No active deployment has been observed. + AVAILABLE, ///< The deployment is available for switchover. + SWITCHOVER_INITIATED, ///< AWS has accepted the switchover request. + SWITCHOVER_IN_PROGRESS, ///< AWS is switching the writer endpoint. + SWITCHOVER_IN_POST_PROCESSING, ///< AWS is applying the final topology changes. + SWITCHOVER_COMPLETED, ///< Cleanup completed and the deployment is latched. }; /** * @brief Column positions in `AWS_Aurora_Hosts_resultset`. + * + * @details The resultset is the immutable configuration snapshot shared by + * the Aurora coordinator and its per-writer-hostgroup workers. */ enum AWS_Aurora_Hosts_Column { AWS_AURORA_WRITER_HOSTGROUP = 0, @@ -602,26 +609,45 @@ enum AWS_Aurora_Hosts_Column { /** * @brief Monitor worker owned by one Aurora writer hostgroup. + * + * @details The coordinator owns the object and joins its thread before + * destruction. Mutable control fields are atomic because the coordinator + * updates them while the worker is running. */ struct AWS_Aurora_BGD_Worker { - int writer_hg = 0; - pthread_t thread {}; - std::atomic_bool worker_stop {false}; - std::atomic current_checksum {0}; + int writer_hg = 0; ///< Writer hostgroup monitored by this worker. + pthread_t thread {}; ///< Worker thread handle owned by the coordinator. + std::atomic_bool worker_stop {false}; ///< Per-worker shutdown signal. + std::atomic current_checksum {0}; ///< Latest configuration checksum to apply. }; /** * @brief Stable identity of one Aurora blue/green deployment. + * + * @details AWS identifies a deployment through the target topology row. The + * identity remains latched after completion so repeated observations of the + * same completed deployment are idempotent. */ struct AWS_Aurora_BGD_Fingerprint { - std::string target_id; - std::string target_endpoint; - int target_port = 0; + std::string target_id; ///< Target identifier from `mysql.rds_topology`. + std::string target_endpoint; ///< Target cluster endpoint used for topology probes. + int target_port = 0; ///< Target endpoint port; zero means unset. + /** + * @brief Check whether the deployment identity is incomplete. + * + * @return true when any required target identity field is missing. + */ bool empty() const { return target_id.empty() || target_endpoint.empty() || target_port <= 0; } + /** + * @brief Compare two deployment identities. + * + * @param rhs Deployment identity to compare with this instance. + * @return true when the target identifier, endpoint, and port all match. + */ bool operator==(const AWS_Aurora_BGD_Fingerprint& rhs) const { return target_id == rhs.target_id && target_endpoint == rhs.target_endpoint @@ -631,57 +657,76 @@ struct AWS_Aurora_BGD_Fingerprint { /** * @brief One production or target Aurora member retained by the BGD worker. + * + * @details Production snapshots describe the configured cluster before the + * switchover. Target snapshots retain the matching production hostname and + * resolved target address needed to redirect traffic during post-processing. */ struct AWS_Aurora_BGD_Member { - std::string server_id; - std::string normalized_server_id; - std::string session_id; - std::string hostname; - std::string production_hostname; - std::string target_ip; - int port = 0; - int use_ssl = 0; - bool is_writer = false; - bool traffic_pin_applied = false; + std::string server_id; ///< Server identifier reported by Aurora. + std::string normalized_server_id; ///< Production identifier used to match a target member. + std::string session_id; ///< Aurora session identifier; MASTER_SESSION_ID marks the writer. + std::string hostname; ///< Hostname used to reach this observed member. + std::string production_hostname; ///< Configured hostname whose traffic is redirected. + std::string target_ip; ///< Resolved target address retained for DNS pinning. + int port = 0; ///< Backend port for the member. + int use_ssl = 0; ///< SSL setting inherited by target probes. + bool is_writer = false; ///< True when the member owns MASTER_SESSION_ID. + bool traffic_pin_applied = false; ///< True after production_hostname is pinned to target_ip. }; /** * @brief State carried by one existing per-writer Aurora monitor worker. + * + * @details The worker owns this state on its stack and mutates it across poll + * cycles without additional locking. Configuration-derived fields may be + * refreshed in place; deployment identity, membership snapshots, and action + * flags preserve the switchover FSM's progress. */ struct AWS_Aurora_BGD_State { - unsigned int writer_hg = 0; - unsigned int reader_hg = 0; - int green_writer_hg = -1; - int green_reader_hg = -1; - unsigned int max_lag_ms = 0; - unsigned int check_interval_ms = 0; - unsigned int check_timeout_ms = 0; - unsigned int add_lag_ms = 0; - unsigned int min_lag_ms = 0; - unsigned int lag_num_checks = 1; - unsigned int autopurge_missing_checks = 0; - int writer_is_also_reader = 0; - int new_reader_weight = 1; - int target_use_ssl = 0; - std::string domain_name; - - AWS_Aurora_BGD_Status status = AWS_Aurora_BGD_Status::NONE; - RDS_BGD_Topology_Monitor_State topology_state = TOPOLOGY_TABLE_CHECK; - AWS_Aurora_BGD_Fingerprint fingerprint; - std::vector production_probe_hosts; - std::vector production_members; - std::vector target_members; - - bool production_snapshot_frozen = false; - bool production_probe_suspended = false; - bool target_snapshot_complete = false; + unsigned int writer_hg = 0; ///< Production writer hostgroup. + unsigned int reader_hg = 0; ///< Production reader hostgroup. + int green_writer_hg = -1; ///< Configured green writer hostgroup; -1 means NULL. + int green_reader_hg = -1; ///< Configured green reader hostgroup; -1 means NULL. + unsigned int max_lag_ms = 0; ///< Maximum accepted Aurora replica lag. + unsigned int check_interval_ms = 0; ///< Baseline Aurora monitoring interval. + unsigned int check_timeout_ms = 0; ///< Timeout for Aurora and topology probes. + unsigned int add_lag_ms = 0; ///< Lag threshold used when adding a reader. + unsigned int min_lag_ms = 0; ///< Minimum lag value used by Aurora monitoring. + unsigned int lag_num_checks = 1; ///< Consecutive lag samples used by Aurora monitoring. + unsigned int autopurge_missing_checks = 0; ///< Missing samples required before automatic removal. + int writer_is_also_reader = 0; ///< Whether the writer also belongs to the reader hostgroup. + int new_reader_weight = 1; ///< Weight assigned to newly discovered readers. + int target_use_ssl = 0; ///< SSL setting used for target topology and membership probes. + std::string domain_name; ///< Domain appended to Aurora server identifiers. + + AWS_Aurora_BGD_Status status = AWS_Aurora_BGD_Status::NONE; ///< Current FSM phase. + RDS_BGD_Topology_Monitor_State topology_state = TOPOLOGY_TABLE_CHECK; ///< Next topology query form. + AWS_Aurora_BGD_Fingerprint fingerprint; ///< Active or completed deployment identity. + std::vector production_probe_hosts; ///< Configured production probe endpoints. + std::vector production_members; ///< Last complete production membership. + std::vector target_members; ///< Last complete target membership. + + bool production_snapshot_frozen = false; ///< Prevents active switchover membership from being replaced. + bool production_probe_suspended = false; ///< Stops ordinary Aurora probes while target traffic is active. + bool target_snapshot_complete = false; ///< True only when every production member has one target match. + /** + * @brief Check whether target membership is safe to apply. + * + * @return true when the retained target snapshot is complete and unambiguous. + */ bool has_complete_target_snapshot() const { return target_snapshot_complete; } }; -// Maps an Aurora switchover status enum to its runtime string. +/** + * @brief Map an Aurora BGD status to its runtime string. + * + * @param status Status to convert. + * @return Stable status string stored in the runtime Aurora hostgroup table. + */ const char* aws_aurora_bgd_status_str(AWS_Aurora_BGD_Status status); /** @@ -842,16 +887,40 @@ class MySQL_Monitor { void * monitor_group_replication(); void * monitor_group_replication_2(); void * monitor_galera(); + /** + * @brief Coordinate per-hostgroup Aurora monitor workers. + * + * @details Rebuilds the shared Aurora configuration snapshot when its + * checksum changes, starts one worker for each configured writer + * hostgroup, refreshes running workers, and joins workers removed from + * the active configuration during shutdown or reload. + * + * @return nullptr when monitoring stops. + */ void * monitor_aws_aurora(); /** * @brief Load the configuration rows for one Aurora worker. * + * @param writer_hg Writer hostgroup owned by the worker. + * @param current_checksum Per-hostgroup checksum published by the coordinator. + * @param candidate Receives a complete configuration snapshot on success. + * * @return true when the rows still match the coordinator-provided checksum. */ bool aws_aurora_bgd_load_worker_config( int writer_hg, uint64_t current_checksum, AWS_Aurora_BGD_State& candidate); /** * @brief Refresh only configuration-derived fields of a running Aurora worker. + * + * @details Preserves deployment identity, membership, and applied-action + * state while replacing the current configuration and forcing an + * immediate worker iteration. + * + * @param st Worker-owned Aurora BGD state to refresh. + * @param current_checksum Per-hostgroup checksum published by the coordinator. + * @param next_loop_at Next scheduled worker iteration; reset to zero on success. + * + * @return true when a matching configuration snapshot was applied. */ bool aws_aurora_bgd_refresh_worker_config( AWS_Aurora_BGD_State& st, uint64_t current_checksum, @@ -860,6 +929,10 @@ class MySQL_Monitor { * @brief Refresh the Aurora BGD worker's last complete production snapshot. * * @details Invalid or incomplete observations retain the previous snapshot. + * A changed production snapshot invalidates any retained target snapshot. + * + * @param st Worker-owned Aurora BGD state to update. + * @param result Successful ordinary Aurora membership observation. */ void aws_aurora_bgd_refresh_production_snapshot( AWS_Aurora_BGD_State& st, const AWS_Aurora_status_entry& result); @@ -869,6 +942,9 @@ class MySQL_Monitor { * @details INITIATED only suspends production probing. IN_PROGRESS demotes * the snapshotted writer on entry. POST_PROCESSING applies each complete * member mapping once and restores canonical writer placement. + * + * @param st Worker-owned state whose routing effects are applied. + * @param status_changed Whether the current status was entered in this cycle. */ void aws_aurora_bgd_apply_active_actions( AWS_Aurora_BGD_State& st, bool status_changed); @@ -878,6 +954,9 @@ class MySQL_Monitor { * @details Reconciles writer placement, removes mapped traffic pins, drains * eligible configured green pools, resumes production probing, and enters * the completed latch while retaining the deployment fingerprint. + * + * @param st Worker-owned state to clean up and latch. + * @param completed_fingerprint Identity reported by the completed target row. */ void aws_aurora_bgd_apply_completion( AWS_Aurora_BGD_State& st, @@ -888,12 +967,18 @@ class MySQL_Monitor { * @details Removes only applied traffic pins, drains affected production * pools, restores canonical writer placement, resumes ordinary probing, * and optionally clears the deployment identity. + * + * @param st Worker-owned state whose applied effects are reversed. + * @param next_status Validated status to enter after cleanup. + * @param clear_deployment Whether to discard the retained deployment identity. */ void aws_aurora_bgd_apply_rollback( AWS_Aurora_BGD_State& st, AWS_Aurora_BGD_Status next_status, bool clear_deployment); /** * @brief Release the completed latch after a successful topology drain. + * + * @param st Worker-owned state holding the completed deployment identity. */ void aws_aurora_bgd_release_completed_latch(AWS_Aurora_BGD_State& st); /** @@ -902,6 +987,9 @@ class MySQL_Monitor { * @details Discovery is serialized with the ordinary Aurora probe. This method * validates topology before publishing status and replaces target membership * only with a complete, unambiguous, fully resolved snapshot. + * + * @param st Worker-owned state advanced by the discovery cycle. + * @param worker_stop Per-worker shutdown signal checked around remote work. */ void aws_aurora_bgd_run_discovery_cycle( AWS_Aurora_BGD_State& st, std::atomic_bool& worker_stop); diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 750667fb1b..6141fc1f0f 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -335,8 +335,20 @@ bool materialize_mysql_aws_aurora_hostgroups( bool runtime ); -/** Copy the configured Aurora hostgroup projection between Admin and disk. */ +/** + * @brief Load the configured Aurora hostgroup projection from disk into Admin. + * + * @param db Admin database containing the main and disk schemas. + * @return true when the configured rows were copied successfully. + */ bool copy_mysql_aws_aurora_hostgroups_from_disk(SQLite3DB* db); + +/** + * @brief Save the configured Aurora hostgroup projection from Admin to disk. + * + * @param db Admin database containing the main and disk schemas. + * @return true when the configured rows were copied successfully. + */ bool copy_mysql_aws_aurora_hostgroups_to_disk(SQLite3DB* db); class ProxySQL_Admin { diff --git a/lib/DNS_Cache.cpp b/lib/DNS_Cache.cpp index b18c1483c5..5f4fea36f6 100644 --- a/lib/DNS_Cache.cpp +++ b/lib/DNS_Cache.cpp @@ -460,6 +460,12 @@ void DNS_Cache::remove(const std::string& hostname) { assert(rc == 0); } +/** + * @brief Clear ordinary DNS resolutions while retaining explicit pins. + * + * @details Unpinned records are removed. Pinned records keep their fixed + * address while their ordinary resolution list and rotation counter are reset. + */ void DNS_Cache::clear() { size_t records_removed = 0; int rc = pthread_rwlock_wrlock(&rwlock_); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 3e51d972e8..4a4e18402b 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -85,26 +85,49 @@ const char* const AWS_AURORA_CONFIG_COLUMNS[AWS_AURORA_CONFIG_COLUMN_COUNT] = { "comment" }; +/** + * @brief One hostgroup role extracted from an Aurora configuration row. + */ struct Aurora_Hostgroup_Role { - const char* field; - int value; + const char* field; ///< Configuration column naming the role. + int value; ///< Hostgroup identifier assigned to the role. }; +/** + * @brief Parsed Aurora row and the validation state of its hostgroup component. + */ struct Aurora_Config_Row { - std::vector fields; - std::vector roles; - std::vector errors; - int writer_hostgroup; - bool active; - bool locally_valid; + std::vector fields; ///< Canonical table projection. + std::vector roles; ///< Non-NULL hostgroups and their roles. + std::vector errors; ///< Validation failures for this row. + int writer_hostgroup; ///< Component identity used in diagnostics. + bool active; ///< Whether the row participates in role conflicts. + bool locally_valid; ///< Whether row-local validation succeeded. }; +/** + * @brief Convert an optional Aurora field to text for diagnostics. + * + * @param value Nullable SQLite result field. + * @return The field value, or the literal `NULL`. + */ std::string aws_aurora_nullable_value(const char* value) { return value ? value : "NULL"; } } // namespace +/** + * @brief Validate an Aurora hostgroup candidate and return its canonical configured projection. + * + * @details The candidate must exactly match the configured Aurora Admin table + * projection. The returned result contains only valid rows in canonical + * order. Rows that conflict by hostgroup role are rejected as one component. + * + * @param candidate Candidate Aurora configuration rows. + * @param errors Receives one message for each rejected projection or component. + * @return Heap-allocated canonical result owned by the caller. + */ SQLite3_result* validate_and_filter_aws_aurora_hostgroups( const SQLite3_result* candidate, std::vector& errors @@ -4206,6 +4229,16 @@ void MySQL_HostGroups_Manager::aws_rds_bgd_set_runtime_status(unsigned int write wrunlock(); } +/** + * @brief Publish the node-local Aurora BGD state for one runtime row. + * + * @details The caller owns validation of the state vocabulary. Writer + * hostgroups not present at runtime are ignored. Configuration reloads do + * not write this column. + * + * @param writer_hostgroup Writer hostgroup identifying the runtime row. + * @param bgd_status Validated Aurora BGD status string to publish. + */ void MySQL_HostGroups_Manager::update_aws_aurora_bgd_status(int writer_hostgroup, const std::string& bgd_status) { wrlock(); const char* query = "UPDATE mysql_aws_aurora_hostgroups SET bgd_status=?1 WHERE writer_hostgroup=?2"; @@ -6263,6 +6296,27 @@ SQLite3_result * MySQL_HostGroups_Manager::get_mysql_errors(bool reset) { return result; } +/** + * @brief Construct the runtime configuration for one Aurora cluster. + * + * @param w Writer hostgroup. + * @param r Reader hostgroup. + * @param gw Explicit green writer hostgroup, or -1 for NULL. + * @param gr Explicit green reader hostgroup, or -1 for NULL. + * @param _port Aurora backend port. + * @param _end_addr Aurora domain name. + * @param maxl Maximum accepted replica lag in milliseconds. + * @param al Lag threshold used when adding a reader. + * @param minl Minimum lag value used by Aurora monitoring. + * @param lnc Number of lag checks used by Aurora monitoring. + * @param ci Monitor check interval in milliseconds. + * @param ct Monitor check timeout in milliseconds. + * @param _a Whether the cluster is active. + * @param wiar Whether the writer also belongs to the reader hostgroup. + * @param nrw Weight assigned to newly discovered readers. + * @param amc Missing checks required before automatic removal. + * @param c Optional configuration comment. + */ AWS_Aurora_Info::AWS_Aurora_Info(int w, int r, int gw, int gr, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c) { comment=NULL; if (c) { @@ -6299,6 +6353,27 @@ AWS_Aurora_Info::~AWS_Aurora_Info() { } } +/** + * @brief Apply a refreshed Aurora configuration to this runtime entry. + * + * @param r Reader hostgroup. + * @param gw Explicit green writer hostgroup, or -1 for NULL. + * @param gr Explicit green reader hostgroup, or -1 for NULL. + * @param _port Aurora backend port. + * @param _end_addr Aurora domain name. + * @param maxl Maximum accepted replica lag in milliseconds. + * @param al Lag threshold used when adding a reader. + * @param minl Minimum lag value used by Aurora monitoring. + * @param lnc Number of lag checks used by Aurora monitoring. + * @param ci Monitor check interval in milliseconds. + * @param ct Monitor check timeout in milliseconds. + * @param _a Whether the cluster is active. + * @param wiar Whether the writer also belongs to the reader hostgroup. + * @param nrw Weight assigned to newly discovered readers. + * @param amc Missing checks required before automatic removal. + * @param c Optional configuration comment. + * @return true when any retained configuration value changed. + */ bool AWS_Aurora_Info::update(int r, int gw, int gr, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, int amc, char *c) { bool ret=false; active_=true; @@ -7201,6 +7276,23 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer(int _whid, int _rhid _whid, _rhid, _server_id, verbose, nullptr, -1, -1, -1); } +/** + * @brief Place an Aurora member in its canonical writer hostgroup configuration. + * + * @details The explicit configuration values let BGD cleanup restore writer + * placement without rereading mutable Aurora configuration. The member is + * added to or removed from the reader hostgroup according to + * writer_is_also_reader. + * + * @param _whid Writer hostgroup. + * @param _rhid Reader hostgroup. + * @param _server_id Aurora server identifier without the configured domain. + * @param verbose Whether to emit routing-change messages. + * @param domain_name_override Configured Aurora domain name. + * @param aurora_port_override Aurora backend port. + * @param writer_is_also_reader_override Whether the writer also belongs to the reader hostgroup. + * @param new_reader_weight_override Weight used when reader membership is created. + */ void MySQL_HostGroups_Manager::update_aws_aurora_set_writer( int _whid, int _rhid, char *_server_id, bool verbose, const char *domain_name_override, int aurora_port_override, diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 4977789d62..8c1370d091 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -6035,6 +6035,12 @@ bool AWS_Aurora_monitor_node::add_entry(AWS_Aurora_status_entry *ase) { } +/** + * @brief Map an Aurora BGD status to its runtime string. + * + * @param status Status to convert. + * @return Stable status string stored in the runtime Aurora hostgroup table. + */ const char* aws_aurora_bgd_status_str(AWS_Aurora_BGD_Status status) { switch (status) { case AWS_Aurora_BGD_Status::NONE: @@ -6055,19 +6061,32 @@ const char* aws_aurora_bgd_status_str(AWS_Aurora_BGD_Status status) { namespace { +/** + * @brief Result and connection state returned by one Aurora BGD probe. + */ struct AWS_Aurora_BGD_Query_Result { - unique_ptr mmsd; - int rc = 1; - unsigned int mysql_error = 0; + unique_ptr mmsd; ///< Probe connection, timing, and result owner. + int rc = 1; ///< Query status returned by the asynchronous monitor helper. + unsigned int mysql_error = 0; ///< MySQL error captured before the connection is released. }; +/** + * @brief Validated deployment state parsed from `mysql.rds_topology`. + */ struct AWS_Aurora_BGD_Topology_Observation { - bool valid = false; - bool completed = false; - AWS_Aurora_BGD_Status status = AWS_Aurora_BGD_Status::NONE; - AWS_Aurora_BGD_Fingerprint fingerprint; + bool valid = false; ///< True when all topology rows form an accepted snapshot. + bool completed = false; ///< True for the lone TARGET row completion shape. + AWS_Aurora_BGD_Status status = AWS_Aurora_BGD_Status::NONE; ///< Validated target status. + AWS_Aurora_BGD_Fingerprint fingerprint; ///< Stable identity of the target row. }; +/** + * @brief Build a member hostname from an Aurora server identifier and domain. + * + * @param server_id Aurora server identifier. + * @param domain_name Configured Aurora domain, with or without a leading dot. + * @return Fully qualified member hostname, or server_id when no domain is configured. + */ static std::string aws_aurora_bgd_member_hostname( const std::string& server_id, const std::string& domain_name ) { @@ -6079,6 +6098,13 @@ static std::string aws_aurora_bgd_member_hostname( : server_id + "." + domain_name; } +/** + * @brief Remove the configured Aurora domain from a member hostname. + * + * @param hostname Fully qualified configured member hostname. + * @param domain_name Configured Aurora domain, with or without a leading dot. + * @return Aurora server identifier, or an empty string when the domain does not match. + */ static std::string aws_aurora_bgd_server_id_from_hostname( const std::string& hostname, const std::string& domain_name ) { @@ -6094,6 +6120,13 @@ static std::string aws_aurora_bgd_server_id_from_hostname( return hostname.substr(0, hostname.size() - suffix.size()); } +/** + * @brief Parse one AWS topology status without accepting unknown values. + * + * @param raw_status Status text returned by `mysql.rds_topology`. + * @param status Receives the corresponding Aurora BGD status on success. + * @return true when raw_status belongs to the supported AWS vocabulary. + */ static bool aws_aurora_bgd_status_from_raw( const std::string& raw_status, AWS_Aurora_BGD_Status& status ) { @@ -6113,11 +6146,27 @@ static bool aws_aurora_bgd_status_from_raw( return true; } +/** + * @brief Check that an Aurora topology row contains every identity and state field. + * + * @param node Parsed topology row to inspect. + * @return true when the row has an id, endpoint, port, role, and status. + */ static bool aws_aurora_bgd_required_topology_fields(const AWS_RDS_Topology_Node& node) { return !node.id.empty() && !node.endpoint.empty() && node.port > 0 && !node.role.empty() && !node.status.empty(); } +/** + * @brief Validate an Aurora blue/green topology snapshot. + * + * @details Active phases require exactly one SOURCE and one TARGET row with + * matching statuses. Completion requires exactly one TARGET row. Any + * duplicate, incomplete, unknown, or inconsistent row rejects the snapshot. + * + * @param topology Parsed `mysql.rds_topology` result. + * @return Validated observation; valid remains false when the snapshot is rejected. + */ static AWS_Aurora_BGD_Topology_Observation aws_aurora_bgd_validate_topology( const AWS_RDS_Topology_Result& topology ) { @@ -6172,6 +6221,21 @@ static AWS_Aurora_BGD_Topology_Observation aws_aurora_bgd_validate_topology( return observation; } +/** + * @brief Execute one Aurora BGD query against a selected probe host. + * + * @details Reuses a monitor connection when possible, applies the Aurora + * timeout, and returns or destroys the connection according to the query + * outcome. + * + * @param host Backend endpoint and SSL setting to use. + * @param writer_hg Writer hostgroup used for monitor accounting. + * @param timeout_ms Query timeout in milliseconds. + * @param task_type Monitor task classification. + * @param query SQL text to execute. + * @param worker_stop Per-worker shutdown signal. + * @return Query result retaining the monitor state and captured MySQL error. + */ static AWS_Aurora_BGD_Query_Result aws_aurora_bgd_query( const AWS_RDS_BGD_Probe_Host& host, unsigned int writer_hg, @@ -6220,6 +6284,23 @@ static AWS_Aurora_BGD_Query_Result aws_aurora_bgd_query( return out; } +/** + * @brief Query eligible Aurora endpoints until one produces an accepted result. + * + * @details Candidate order starts at a random offset. Unpingable endpoints are + * skipped and counted; a missing topology table may be treated as a terminal + * observation when requested by the caller. + * + * @param hosts Candidate endpoints. + * @param writer_hg Writer hostgroup used for monitor accounting. + * @param timeout_ms Query timeout in milliseconds. + * @param task_type Monitor task classification. + * @param query SQL text to execute. + * @param worker_stop Per-worker shutdown signal. + * @param selected Optional destination for the endpoint that produced the result. + * @param stop_on_missing_table Whether MySQL error 1146 stops candidate traversal. + * @return Last query result, or the first accepted result. + */ static AWS_Aurora_BGD_Query_Result aws_aurora_bgd_query_candidates( const std::vector& hosts, unsigned int writer_hg, @@ -6262,6 +6343,12 @@ static AWS_Aurora_BGD_Query_Result aws_aurora_bgd_query_candidates( return last_result; } +/** + * @brief Publish a changed Aurora BGD state to the runtime hostgroup row. + * + * @param st Worker-owned state whose status is updated. + * @param status Validated status to publish. + */ static void aws_aurora_bgd_set_status( AWS_Aurora_BGD_State& st, AWS_Aurora_BGD_Status status ) { @@ -6276,6 +6363,13 @@ static void aws_aurora_bgd_set_status( MyHGM->update_aws_aurora_bgd_status(st.writer_hg, aws_aurora_bgd_status_str(status)); } +/** + * @brief Compare production membership snapshots independent of row order. + * + * @param lhs First membership snapshot. + * @param rhs Second membership snapshot. + * @return true when both snapshots contain the same identities, roles, and endpoints. + */ static bool aws_aurora_bgd_same_production_snapshot( const std::vector& lhs, const std::vector& rhs @@ -6298,6 +6392,17 @@ static bool aws_aurora_bgd_same_production_snapshot( return true; } +/** + * @brief Rebuild production membership from configured probe hosts and hostgroups. + * + * @details Used when discovery starts before an ordinary Aurora observation is + * available. The snapshot is accepted only when every configured member has + * a unique server identifier and exactly one member belongs to the writer + * hostgroup. + * + * @param st Worker-owned state receiving the rebuilt snapshot. + * @return true when a complete production snapshot was built. + */ static bool aws_aurora_bgd_rebuild_production_snapshot(AWS_Aurora_BGD_State& st) { std::unordered_set writer_endpoints; MyHGM->wrlock(); @@ -6351,6 +6456,19 @@ static bool aws_aurora_bgd_rebuild_production_snapshot(AWS_Aurora_BGD_State& st) return true; } +/** + * @brief Build a complete target membership snapshot from REPLICA_HOST_STATUS. + * + * @details Matches each current target member to one production member, + * validates writer and reader session identities, resolves target addresses, + * and preserves already-applied traffic actions only while their mapping is + * unchanged. + * + * @param st Worker-owned state containing production membership and deployment identity. + * @param result Target REPLICA_HOST_STATUS result. + * @param snapshot Receives the complete target snapshot on success. + * @return true when every production member has one valid target counterpart. + */ static bool aws_aurora_bgd_parse_target_membership( AWS_Aurora_BGD_State& st, MYSQL_RES* result, std::vector& snapshot @@ -6492,6 +6610,12 @@ static bool aws_aurora_bgd_parse_target_membership( return writers == 1 && snapshot.size() == st.production_members.size(); } +/** + * @brief Select target endpoints eligible for topology probes. + * + * @param st Worker-owned state containing target membership or deployment identity. + * @return Target member endpoints when complete, otherwise the target cluster endpoint. + */ static std::vector aws_aurora_bgd_target_probe_hosts( const AWS_Aurora_BGD_State& st ) { @@ -6511,6 +6635,15 @@ static std::vector aws_aurora_bgd_target_probe_hosts( return hosts; } +/** + * @brief Select endpoints eligible for target membership probes. + * + * @details Includes the target cluster endpoint in addition to retained member + * endpoints so membership discovery remains possible across endpoint changes. + * + * @param st Worker-owned state containing target membership and deployment identity. + * @return Deduplicated candidate endpoints for REPLICA_HOST_STATUS. + */ static std::vector aws_aurora_bgd_membership_probe_hosts( const AWS_Aurora_BGD_State& st ) { @@ -6533,6 +6666,15 @@ static std::vector aws_aurora_bgd_membership_probe_hosts } // namespace +/** + * @brief Refresh the Aurora BGD worker's last complete production snapshot. + * + * @details Invalid or incomplete observations retain the previous snapshot. + * A changed production snapshot invalidates any retained target snapshot. + * + * @param st Worker-owned Aurora BGD state to update. + * @param result Successful ordinary Aurora membership observation. + */ void MySQL_Monitor::aws_aurora_bgd_refresh_production_snapshot( AWS_Aurora_BGD_State& st, const AWS_Aurora_status_entry& result ) { @@ -6574,6 +6716,16 @@ void MySQL_Monitor::aws_aurora_bgd_refresh_production_snapshot( st.production_members = std::move(snapshot); } +/** + * @brief Apply the routing actions for an accepted active Aurora BGD state. + * + * @details INITIATED only suspends production probing. IN_PROGRESS demotes + * the snapshotted writer on entry. POST_PROCESSING applies each complete + * member mapping once and restores canonical writer placement. + * + * @param st Worker-owned state whose routing effects are applied. + * @param status_changed Whether the current status was entered in this cycle. + */ void MySQL_Monitor::aws_aurora_bgd_apply_active_actions( AWS_Aurora_BGD_State& st, bool status_changed ) { @@ -6647,6 +6799,16 @@ void MySQL_Monitor::aws_aurora_bgd_apply_active_actions( } } +/** + * @brief Run immediate effect-driven cleanup for TARGET completion. + * + * @details Reconciles writer placement, removes mapped traffic pins, drains + * eligible configured green pools, resumes production probing, and enters + * the completed latch while retaining the deployment fingerprint. + * + * @param st Worker-owned state to clean up and latch. + * @param completed_fingerprint Identity reported by the completed target row. + */ void MySQL_Monitor::aws_aurora_bgd_apply_completion( AWS_Aurora_BGD_State& st, const AWS_Aurora_BGD_Fingerprint& completed_fingerprint @@ -6719,6 +6881,17 @@ void MySQL_Monitor::aws_aurora_bgd_apply_completion( st.writer_hg, st.reader_hg); } +/** + * @brief Reverse applied pre-completion effects and enter a safe earlier state. + * + * @details Removes only applied traffic pins, drains affected production + * pools, restores canonical writer placement, resumes ordinary probing, + * and optionally clears the deployment identity. + * + * @param st Worker-owned state whose applied effects are reversed. + * @param next_status Validated status to enter after cleanup. + * @param clear_deployment Whether to discard the retained deployment identity. + */ void MySQL_Monitor::aws_aurora_bgd_apply_rollback( AWS_Aurora_BGD_State& st, AWS_Aurora_BGD_Status next_status, bool clear_deployment @@ -6764,6 +6937,11 @@ void MySQL_Monitor::aws_aurora_bgd_apply_rollback( st.writer_hg, st.reader_hg, aws_aurora_bgd_status_str(st.status)); } +/** + * @brief Release the completed latch after a successful topology drain. + * + * @param st Worker-owned state holding the completed deployment identity. + */ void MySQL_Monitor::aws_aurora_bgd_release_completed_latch(AWS_Aurora_BGD_State& st) { if (st.status != AWS_Aurora_BGD_Status::SWITCHOVER_COMPLETED) { return; @@ -6779,6 +6957,16 @@ void MySQL_Monitor::aws_aurora_bgd_release_completed_latch(AWS_Aurora_BGD_State& st.writer_hg, st.reader_hg); } +/** + * @brief Run the topology and target-membership probes owned by an Aurora worker. + * + * @details Discovery is serialized with the ordinary Aurora probe. This method + * validates topology before publishing status and replaces target membership + * only with a complete, unambiguous, fully resolved snapshot. + * + * @param st Worker-owned state advanced by the discovery cycle. + * @param worker_stop Per-worker shutdown signal checked around remote work. + */ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle( AWS_Aurora_BGD_State& st, std::atomic_bool& worker_stop ) { @@ -6915,6 +7103,15 @@ void MySQL_Monitor::aws_aurora_bgd_run_discovery_cycle( } } +/** + * @brief Load the configuration rows for one Aurora worker. + * + * @param writer_hg Writer hostgroup owned by the worker. + * @param current_checksum Per-hostgroup checksum published by the coordinator. + * @param candidate Receives a complete configuration snapshot on success. + * + * @return true when the rows still match the coordinator-provided checksum. + */ bool MySQL_Monitor::aws_aurora_bgd_load_worker_config( int writer_hg, uint64_t current_checksum, AWS_Aurora_BGD_State& candidate ) { @@ -6972,6 +7169,19 @@ bool MySQL_Monitor::aws_aurora_bgd_load_worker_config( return true; } +/** + * @brief Refresh only configuration-derived fields of a running Aurora worker. + * + * @details Preserves deployment identity, membership, and applied-action + * state while replacing the current configuration and forcing an immediate + * worker iteration. + * + * @param st Worker-owned Aurora BGD state to refresh. + * @param current_checksum Per-hostgroup checksum published by the coordinator. + * @param next_loop_at Next scheduled worker iteration; reset to zero on success. + * + * @return true when a matching configuration snapshot was applied. + */ bool MySQL_Monitor::aws_aurora_bgd_refresh_worker_config( AWS_Aurora_BGD_State& st, uint64_t current_checksum, unsigned long long& next_loop_at @@ -7007,6 +7217,17 @@ bool MySQL_Monitor::aws_aurora_bgd_refresh_worker_config( return true; } +/** + * @brief Run ordinary Aurora monitoring and BGD discovery for one writer hostgroup. + * + * @details The worker refreshes configuration in place, retains production + * membership from successful Aurora observations, and serializes BGD probes + * with the ordinary Aurora check loop. Applied BGD effects are rolled back + * before the worker exits. + * + * @param arg Pointer to the coordinator-owned AWS_Aurora_BGD_Worker. + * @return nullptr after shutdown or removal of the worker configuration. + */ void * monitor_AWS_Aurora_thread_HG(void *arg) { AWS_Aurora_BGD_Worker* worker = static_cast(arg); unsigned int wHG = worker->writer_hg; @@ -7430,6 +7651,16 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { } +/** + * @brief Coordinate per-hostgroup Aurora monitor workers. + * + * @details Rebuilds the shared Aurora configuration snapshot when its + * checksum changes, starts one worker for each configured writer + * hostgroup, refreshes running workers, and joins workers removed from + * the active configuration during shutdown or reload. + * + * @return nullptr when monitoring stops. + */ void * MySQL_Monitor::monitor_aws_aurora() { if (!wait_for_glo_mth()) return NULL; unsigned int MySQL_Monitor__thread_MySQL_Thread_Variables_version; diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 77f9cc320e..fac409ae39 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -207,6 +207,17 @@ static void BQE1(SQLite3DB *db, const vector& tbs, const string& p1, con } } +/** + * @brief Copy configured Aurora hostgroup columns between SQLite schemas. + * + * @details Runtime-only BGD status is intentionally excluded from the + * persisted projection. + * + * @param db Admin database containing both schemas. + * @param destination_schema Schema receiving the configured rows. + * @param source_schema Schema providing the configured rows. + * @return true when the copy statement succeeds. + */ static bool copy_mysql_aws_aurora_hostgroups( SQLite3DB* db, const string& destination_schema, @@ -220,10 +231,22 @@ static bool copy_mysql_aws_aurora_hostgroups( return db->execute(query.c_str()); } +/** + * @brief Load the configured Aurora hostgroup projection from disk into Admin. + * + * @param db Admin database containing the main and disk schemas. + * @return true when the configured rows were copied successfully. + */ bool copy_mysql_aws_aurora_hostgroups_from_disk(SQLite3DB* db) { return copy_mysql_aws_aurora_hostgroups(db, "main", "disk"); } +/** + * @brief Save the configured Aurora hostgroup projection from Admin to disk. + * + * @param db Admin database containing the main and disk schemas. + * @return true when the configured rows were copied successfully. + */ bool copy_mysql_aws_aurora_hostgroups_to_disk(SQLite3DB* db) { return copy_mysql_aws_aurora_hostgroups(db, "disk", "main"); } @@ -7396,6 +7419,15 @@ void ProxySQL_Admin::save_scheduler_runtime_to_database(bool _runtime) { free(args); } +/** + * @brief Replace an Admin Aurora hostgroup table with an HGM runtime snapshot. + * + * @param db Destination Admin database. + * @param resultset Aurora rows returned by MySQL_HostGroups_Manager. + * @param runtime Selects runtime_mysql_aws_aurora_hostgroups when true and + * mysql_aws_aurora_hostgroups when false. + * @return true when the destination table was replaced. + */ bool materialize_mysql_aws_aurora_hostgroups( SQLite3DB* db, const SQLite3_result* resultset, diff --git a/test/deps/cluster_simulator/lib/aurora_utils.cpp b/test/deps/cluster_simulator/lib/aurora_utils.cpp index fb7b1ef9ec..a555acf993 100644 --- a/test/deps/cluster_simulator/lib/aurora_utils.cpp +++ b/test/deps/cluster_simulator/lib/aurora_utils.cpp @@ -445,6 +445,20 @@ std::pair load_aurora_backend_addresses( } // namespace +/** + * @brief Publish ordinary Aurora state through backend-address replica sets. + * + * @details DOMAIN_NAME identifies the set, and CLUSTER_SIM_HOST_FILE resolves + * every SERVER_ID + DOMAIN_NAME member hostname to its simulated backend + * address. Publication is transactional. Snapshot replacement waits until + * every retained set has received an ordinary Aurora probe. + * + * @param proxysql_sqlite Connection to the simulator SQLite interface. + * @param servers Aurora members grouped by DOMAIN_NAME. + * @param mode State replacement policy for rows and backend controls. + * @return Pair containing EXIT_SUCCESS and an empty message, or EXIT_FAILURE + * and a diagnostic message. + */ std::pair prepare_aurora_cluster_state( MYSQL* proxysql_sqlite, const vector& servers, diff --git a/test/deps/cluster_simulator/lib/aurora_utils.h b/test/deps/cluster_simulator/lib/aurora_utils.h index 5006d50ccd..3bfa91d311 100644 --- a/test/deps/cluster_simulator/lib/aurora_utils.h +++ b/test/deps/cluster_simulator/lib/aurora_utils.h @@ -50,10 +50,13 @@ enum class aurora_state_id { new_state = 1 }; +/** + * @brief Select how a simulator publication replaces Aurora replica state. + */ enum class aurora_publication_mode { - replace_sets, - replace_snapshot_retaining_backends, - reset_scenario + replace_sets, ///< Replace only the replica sets present in the new state. + replace_snapshot_retaining_backends, ///< Replace all rows while retaining matching backend controls. + reset_scenario ///< Remove all Aurora rows and controls before publishing the new state. }; std::pair extract_aurora_servers_state( @@ -68,9 +71,18 @@ std::pair prepare_mysql_aurora_hostgroups( ); /** - * Publishes ordinary Aurora JSON state through backend-address replica sets. - * DOMAIN_NAME identifies the set, and CLUSTER_SIM_HOST_FILE resolves every - * SERVER_ID + DOMAIN_NAME member hostname to its simulated backend address. + * @brief Publish ordinary Aurora state through backend-address replica sets. + * + * @details DOMAIN_NAME identifies the set, and CLUSTER_SIM_HOST_FILE resolves + * every SERVER_ID + DOMAIN_NAME member hostname to its simulated backend + * address. Publication is transactional. Snapshot replacement waits until + * every retained set has received an ordinary Aurora probe. + * + * @param proxysql_sqlite Connection to the simulator SQLite interface. + * @param servers Aurora members grouped by DOMAIN_NAME. + * @param mode State replacement policy for rows and backend controls. + * @return Pair containing EXIT_SUCCESS and an empty message, or EXIT_FAILURE + * and a diagnostic message. */ std::pair prepare_aurora_cluster_state( MYSQL* proxysql_sqlite, diff --git a/test/tap/tap/aurora_bgd_scenario_tap.h b/test/tap/tap/aurora_bgd_scenario_tap.h index 821f4d8d1e..7961f0084c 100644 --- a/test/tap/tap/aurora_bgd_scenario_tap.h +++ b/test/tap/tap/aurora_bgd_scenario_tap.h @@ -19,9 +19,12 @@ using std::vector; const uint32_t kWaitSeconds = 5; const uint32_t kProbeTimeoutMs = 5000; +/** + * @brief Connections shared by one focused Aurora BGD TAP scenario. + */ struct Context { - MYSQL* admin { nullptr }; - BGD_Simulator simulator; + MYSQL* admin { nullptr }; ///< ProxySQL Admin connection owned by the scenario. + BGD_Simulator simulator; ///< Connected shared AWS simulator controller. }; inline bool scalar_is(MYSQL* admin, const string& query, const string& expected) { @@ -30,6 +33,12 @@ inline bool scalar_is(MYSQL* admin, const string& query, const string& expected) && rows.front().front() == expected; } +/** + * @brief Connect a scenario to ProxySQL and reset shared Aurora simulator state. + * @param cl TAP environment and connection options. + * @param context Receives the Admin and simulator connections. + * @return EXIT_SUCCESS when the isolated fixture is ready. + */ inline int setup(CommandLine& cl, Context& context) { if (cl.getEnv()) { diag("Error: failed to load TAP environment"); @@ -63,11 +72,21 @@ inline int setup(CommandLine& cl, Context& context) { }); } +/** + * @brief Remove runtime configuration and simulator state between scenarios. + * @param context Active scenario context. + * @return EXIT_SUCCESS when both stores were cleared. + */ inline int reset(Context& context) { return aurora_bgd_admin_cleanup(context.admin) == EXIT_SUCCESS && context.simulator.cleanup() == EXIT_SUCCESS ? EXIT_SUCCESS : EXIT_FAILURE; } +/** + * @brief Reset a scenario, remove its test user, and close the Admin connection. + * @param context Active scenario context. + * @return EXIT_SUCCESS when every cleanup operation succeeds. + */ inline int cleanup(Context& context) { int reset_rc = reset(context); int user_rc = context.admin == nullptr ? EXIT_FAILURE : aurora_bgd_execute_all(context.admin, { @@ -82,6 +101,20 @@ inline int cleanup(Context& context) { ? EXIT_SUCCESS : EXIT_FAILURE; } +/** + * @brief Configure one deployment through the common Aurora Admin fixture. + * @param context Active scenario context. + * @param deployment Deployment fixture to configure. + * @param writer_hostgroup Production writer hostgroup. + * @param reader_hostgroup Production reader hostgroup. + * @param green_writer_hostgroup Explicit green writer hostgroup, or -1 for NULL. + * @param green_reader_hostgroup Explicit green reader hostgroup, or -1 for NULL. + * @param automatic Whether BGD auto-discovery is enabled. + * @param check_interval_ms Aurora monitor interval. + * @param writer_is_also_reader Whether the writer also belongs to the reader hostgroup. + * @param use_ssl Whether configured production members use TLS. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ inline int configure( Context& context, Aurora_BGD_Test_Deployment& deployment, int writer_hostgroup, int reader_hostgroup, @@ -95,12 +128,25 @@ inline int configure( check_interval_ms, writer_is_also_reader, use_ssl); } +/** + * @brief Publish both membership sets and the AVAILABLE topology. + * @param context Active scenario context. + * @param deployment Deployment fixture to publish. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ inline int publish_available( Context& context, Aurora_BGD_Test_Deployment& deployment ) { return aurora_bgd_publish(context.simulator, deployment); } +/** + * @brief Replace the published topology phase for a deployment. + * @param context Active scenario context. + * @param deployment Deployment fixture whose backends serve the topology. + * @param status AWS topology status assigned to SOURCE and TARGET rows. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ inline int publish_status( Context& context, Aurora_BGD_Test_Deployment& deployment, const string& status ) { @@ -109,6 +155,13 @@ inline int publish_status( aurora_bgd_topology(deployment, status)); } +/** + * @brief Publish production and target membership with an initial topology phase. + * @param context Active scenario context. + * @param deployment Deployment fixture to publish. + * @param status Initial AWS topology status. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ inline int publish_initial( Context& context, Aurora_BGD_Test_Deployment& deployment, const string& status @@ -125,6 +178,13 @@ inline int publish_initial( ? EXIT_SUCCESS : EXIT_FAILURE; } +/** + * @brief Publish completion while retaining another deployment's serving backends. + * @param context Active scenario context. + * @param serving_deployment Fixture whose backends receive the topology response. + * @param completed_deployment Fixture providing the completed target identity. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ inline int publish_completed( Context& context, Aurora_BGD_Test_Deployment& serving_deployment, Aurora_BGD_Test_Deployment& completed_deployment diff --git a/test/tap/tap/aurora_bgd_simulator.cpp b/test/tap/tap/aurora_bgd_simulator.cpp index 1fb37f78c6..307d5e78b4 100644 --- a/test/tap/tap/aurora_bgd_simulator.cpp +++ b/test/tap/tap/aurora_bgd_simulator.cpp @@ -2,14 +2,29 @@ using namespace std; +/** + * @brief Return the address reached by the test network. + * + * @return Endpoint containing the fixed simulator IP and port. + */ Endpoint Aurora_BGD_Endpoint::backend() { return { ip, port }; } +/** + * @brief Return the hostname configured in ProxySQL. + * + * @return Endpoint containing the Aurora hostname and port. + */ Endpoint Aurora_BGD_Endpoint::host_endpoint() { return { hostname, port }; } +/** + * @brief Convert this fixture member to a simulator replica row. + * + * @return REPLICA_HOST_STATUS row containing the member fields. + */ Aurora_Replica_Row Aurora_BGD_Member::replica_row() { return { server_id, @@ -21,6 +36,11 @@ Aurora_Replica_Row Aurora_BGD_Member::replica_row() { }; } +/** + * @brief Convert every fixture member to a simulator replica row. + * + * @return REPLICA_HOST_STATUS rows in fixture order. + */ vector Aurora_BGD_Membership_Set::replica_rows() { vector rows {}; rows.reserve(members.size()); @@ -30,6 +50,11 @@ vector Aurora_BGD_Membership_Set::replica_rows() { return rows; } +/** + * @brief Convert serving endpoints to simulator backend addresses. + * + * @return Fixed simulator endpoints in fixture order. + */ vector Aurora_BGD_Membership_Set::backends() { vector endpoints {}; endpoints.reserve(serving_endpoints.size()); diff --git a/test/tap/tap/aurora_bgd_simulator.h b/test/tap/tap/aurora_bgd_simulator.h index 6df8f01db2..a24848f335 100644 --- a/test/tap/tap/aurora_bgd_simulator.h +++ b/test/tap/tap/aurora_bgd_simulator.h @@ -8,45 +8,80 @@ using namespace std; -/** An AWS-style Aurora hostname and its fixed simulator backend address. */ +/** + * @brief An AWS-style Aurora hostname and its fixed simulator backend address. + */ struct Aurora_BGD_Endpoint { - string hostname; - string ip; - int port; + string hostname; ///< Hostname presented to ProxySQL. + string ip; ///< Simulator backend address assigned to hostname. + int port; ///< Backend port used by both endpoint forms. + /** + * @brief Return the address reached by the test network. + * + * @return Endpoint containing the fixed simulator IP and port. + */ Endpoint backend(); + + /** + * @brief Return the hostname configured in ProxySQL. + * + * @return Endpoint containing the Aurora hostname and port. + */ Endpoint host_endpoint(); }; -/** One Aurora cluster member published through REPLICA_HOST_STATUS. */ +/** + * @brief One Aurora cluster member published through REPLICA_HOST_STATUS. + */ struct Aurora_BGD_Member { - string server_id; - string session_id; - Aurora_BGD_Endpoint endpoint; - double cpu { 0 }; - string last_update_timestamp; - double replica_lag_in_milliseconds { 0 }; - bool is_current { true }; + string server_id; ///< Aurora member identifier. + string session_id; ///< MASTER_SESSION_ID for the writer; stable id for a reader. + Aurora_BGD_Endpoint endpoint; ///< Member hostname and simulator backend mapping. + double cpu { 0 }; ///< CPU value returned by the simulated row. + string last_update_timestamp; ///< Timestamp returned by the simulated row. + double replica_lag_in_milliseconds { 0 }; ///< Replica lag returned by the simulated row. + bool is_current { true }; ///< Whether the row belongs to the current membership snapshot. + /** + * @brief Convert this fixture member to a simulator replica row. + * + * @return REPLICA_HOST_STATUS row containing the member fields. + */ Aurora_Replica_Row replica_row(); }; -/** A membership snapshot and every backend allowed to return it. */ +/** + * @brief A membership snapshot and every backend allowed to return it. + */ struct Aurora_BGD_Membership_Set { - string replica_set_id; - vector members; - vector serving_endpoints; + string replica_set_id; ///< Simulator identity for this cluster membership. + vector members; ///< Rows returned for the replica set. + vector serving_endpoints; ///< Backends allowed to serve the rows. + /** + * @brief Convert every fixture member to a simulator replica row. + * + * @return REPLICA_HOST_STATUS rows in fixture order. + */ vector replica_rows(); + + /** + * @brief Convert serving endpoints to simulator backend addresses. + * + * @return Fixed simulator endpoints in fixture order. + */ vector backends(); }; -/** Stable identity used to describe an AWS member rename during switchover. */ +/** + * @brief Stable identity used to describe an AWS member rename during switchover. + */ struct Aurora_BGD_Rename { - string production_server_id; - string target_server_id; - string session_id; - string cached_target_ip; + string production_server_id; ///< Member identifier before the AWS rename. + string target_server_id; ///< Member identifier in the target cluster. + string session_id; ///< Stable Aurora role or reader identity. + string cached_target_ip; ///< Address resolved for the target member before rename. }; #endif // TAP_AURORA_BGD_SIMULATOR_H diff --git a/test/tap/tap/aurora_bgd_tap.h b/test/tap/tap/aurora_bgd_tap.h index 07dda5dc3d..0cc1aa5a0c 100644 --- a/test/tap/tap/aurora_bgd_tap.h +++ b/test/tap/tap/aurora_bgd_tap.h @@ -17,18 +17,30 @@ using std::string; using std::to_string; using std::vector; +/** + * @brief Complete simulator fixture for one Aurora blue/green deployment. + */ struct Aurora_BGD_Test_Deployment { - string name; - string domain_name; - string blue_replica_set; - string target_replica_set; - Aurora_BGD_Endpoint target_cluster_endpoint; - Aurora_BGD_Membership_Set production; - Aurora_BGD_Membership_Set target; - string source_topology_id; - string target_topology_id; + string name; ///< Human-readable fixture name. + string domain_name; ///< Domain appended to member identifiers. + string blue_replica_set; ///< Production simulator membership identity. + string target_replica_set; ///< Target simulator membership identity. + Aurora_BGD_Endpoint target_cluster_endpoint; ///< Target cluster topology endpoint. + Aurora_BGD_Membership_Set production; ///< Production membership and serving backends. + Aurora_BGD_Membership_Set target; ///< Target membership and serving backends. + string source_topology_id; ///< SOURCE topology row identifier. + string target_topology_id; ///< TARGET topology row identifier. }; +/** + * @brief Build one Aurora member fixture with stable default metrics. + * + * @param server_id Aurora member identifier. + * @param session_id Writer or reader session identity. + * @param endpoint Hostname and fixed simulator address. + * @param current Whether the member belongs to current membership. + * @return Initialized member fixture. + */ inline Aurora_BGD_Member aurora_bgd_member( string server_id, string session_id, Aurora_BGD_Endpoint endpoint, bool current = true ) { @@ -41,6 +53,10 @@ inline Aurora_BGD_Member aurora_bgd_member( return member; } +/** + * @brief Build the primary three-member Aurora BGD fixture. + * @return Deployment A with one writer and two readers per environment. + */ inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_a() { Aurora_BGD_Test_Deployment deployment; deployment.name = "Aurora BGD deployment A"; @@ -85,6 +101,10 @@ inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_a() { return deployment; } +/** + * @brief Build an independent writer-only Aurora BGD fixture. + * @return Deployment B with one member per environment. + */ inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_b_writer_only() { Aurora_BGD_Test_Deployment deployment; deployment.name = "Aurora BGD deployment B writer-only"; @@ -117,6 +137,10 @@ inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_b_writer_only() { return deployment; } +/** + * @brief Build a writer-only fixture whose production identifier contains `green`. + * @return Deployment C with one member per environment. + */ inline Aurora_BGD_Test_Deployment aurora_bgd_deployment_c_writer_only() { Aurora_BGD_Test_Deployment deployment; deployment.name = "Aurora BGD deployment C writer-only"; @@ -183,6 +207,24 @@ inline int aurora_bgd_admin_cleanup(MYSQL* admin) { }); } +/** + * @brief Configure ProxySQL for one simulated Aurora deployment. + * + * @details Installs monitor credentials, the Aurora hostgroup row, production + * servers, and the test user's default route, then loads each module to runtime. + * + * @param admin ProxySQL Admin connection. + * @param deployment Deployment fixture to configure. + * @param writer_hg Production writer hostgroup. + * @param reader_hg Production reader hostgroup. + * @param green_writer_hg Explicit green writer hostgroup, or -1 for NULL. + * @param green_reader_hg Explicit green reader hostgroup, or -1 for NULL. + * @param auto_discovery Whether Aurora BGD auto-discovery is enabled. + * @param check_interval_ms Aurora monitor interval. + * @param writer_is_also_reader Whether the writer also belongs to the reader hostgroup. + * @param use_ssl Whether configured production members use TLS. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ inline int aurora_bgd_admin_setup( MYSQL* admin, Aurora_BGD_Test_Deployment& deployment, int writer_hg, int reader_hg, int green_writer_hg, int green_reader_hg, @@ -230,6 +272,11 @@ inline int aurora_bgd_admin_setup( return aurora_bgd_execute_all(admin, queries); } +/** + * @brief Build the two-row AVAILABLE topology for a deployment. + * @param deployment Deployment fixture. + * @return SOURCE and TARGET topology rows. + */ inline vector aurora_bgd_available_topology( Aurora_BGD_Test_Deployment& deployment ) { @@ -251,6 +298,12 @@ inline vector aurora_bgd_available_topology( }; } +/** + * @brief Build an active two-row topology with one shared status. + * @param deployment Deployment fixture. + * @param status AWS switchover status assigned to both rows. + * @return SOURCE and TARGET topology rows. + */ inline vector aurora_bgd_topology( Aurora_BGD_Test_Deployment& deployment, const string& status ) { @@ -261,6 +314,11 @@ inline vector aurora_bgd_topology( return rows; } +/** + * @brief Build the lone TARGET row used for completed Aurora switchovers. + * @param deployment Deployment fixture. + * @return One SWITCHOVER_COMPLETED target row. + */ inline vector aurora_bgd_completed_topology( Aurora_BGD_Test_Deployment& deployment ) { @@ -275,6 +333,11 @@ inline vector aurora_bgd_completed_topology( }; } +/** + * @brief Collect every production and target backend eligible for topology probes. + * @param deployment Deployment fixture. + * @return Simulator backend endpoints without altering fixture order. + */ inline vector aurora_bgd_topology_backends(Aurora_BGD_Test_Deployment& deployment) { vector backends = deployment.production.backends(); vector target_backends = deployment.target.backends(); @@ -282,6 +345,12 @@ inline vector aurora_bgd_topology_backends(Aurora_BGD_Test_Deployment& return backends; } +/** + * @brief Publish both membership sets and the AVAILABLE topology. + * @param sim Connected shared AWS simulator. + * @param deployment Deployment fixture to publish. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ inline int aurora_bgd_publish( BGD_Simulator& sim, Aurora_BGD_Test_Deployment& deployment ) { diff --git a/test/tap/tap/bgd_simulator.cpp b/test/tap/tap/bgd_simulator.cpp index 8c0641fa24..269f8f1465 100644 --- a/test/tap/tap/bgd_simulator.cpp +++ b/test/tap/tap/bgd_simulator.cpp @@ -11,10 +11,20 @@ using namespace std; namespace { +/** + * @brief Convert a topology probe kind to its simulator log value. + * @param kind Probe kind to convert. + * @return Stable simulator log string. + */ const char* probe_kind_string(BGD_Probe_Kind kind) { return kind == BGD_Probe_Kind::table_check ? "table_check" : "metadata"; } +/** + * @brief Parse a topology probe kind from a simulator log row. + * @param value Stored probe kind. + * @return Result code and parsed kind. + */ rc_t parse_probe_kind(string value) { if (value == "table_check") { return { EXIT_SUCCESS, BGD_Probe_Kind::table_check }; @@ -25,10 +35,20 @@ rc_t parse_probe_kind(string value) { return { EXIT_FAILURE, BGD_Probe_Kind::table_check }; } +/** + * @brief Convert an Aurora probe kind to its simulator log value. + * @param kind Probe kind to convert. + * @return Stable simulator log string. + */ const char* replica_probe_kind_string(Aurora_Replica_Probe_Kind kind) { return kind == Aurora_Replica_Probe_Kind::ordinary ? "ordinary" : "bgd_membership"; } +/** + * @brief Parse an Aurora probe kind from a simulator log row. + * @param value Stored probe kind. + * @return Result code and parsed kind. + */ rc_t parse_replica_probe_kind(string value) { if (value == "ordinary") { return { EXIT_SUCCESS, Aurora_Replica_Probe_Kind::ordinary }; @@ -41,6 +61,12 @@ rc_t parse_replica_probe_kind(string value) { } // namespace +/** + * @brief Replace the topology rows served by a set of backends. + * @param backends Backends that should serve the topology. + * @param rows Topology rows returned by those backends. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int BGD_Simulator::topology_update(vector backends, vector rows) { if (backends.empty()) { return EXIT_FAILURE; @@ -70,6 +96,11 @@ int BGD_Simulator::topology_update(vector backends, vector backends) { if (backends.empty()) { return EXIT_FAILURE; @@ -87,10 +118,22 @@ int BGD_Simulator::topology_delete(vector backends) { return execute_transaction(statements); } +/** + * @brief Simulate an absent topology table on a set of backends. + * @param backends Backends that should report the table as absent. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int BGD_Simulator::topology_drop(vector backends) { return topology_error(backends, 1146, "Table 'mysql.rds_topology' doesn't exist"); } +/** + * @brief Make topology probes return a selected MySQL error. + * @param backends Backends that should return the error. + * @param error_code Nonzero MySQL error code. + * @param error_msg MySQL error text. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int BGD_Simulator::topology_error(vector backends, int error_code, string error_msg) { if (backends.empty() || error_code == 0) { return EXIT_FAILURE; @@ -113,6 +156,13 @@ int BGD_Simulator::topology_error(vector backends, int error_code, str return execute_transaction(statements); } +/** + * @brief Replace one Aurora membership set and its serving backends. + * @param replica_set_id Stable simulator membership identity. + * @param rows Replica rows returned for the set. + * @param backends Backends that should serve the set. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int BGD_Simulator::replica_update( string replica_set_id, vector rows, @@ -150,6 +200,11 @@ int BGD_Simulator::replica_update( return execute_transaction(statements); } +/** + * @brief Remove one Aurora membership set and its backend controls. + * @param replica_set_id Stable simulator membership identity. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int BGD_Simulator::replica_delete(string replica_set_id) { if (replica_set_id.empty()) { return EXIT_FAILURE; @@ -163,12 +218,24 @@ int BGD_Simulator::replica_delete(string replica_set_id) { return execute_transaction(statements); } +/** + * @brief Simulate an absent REPLICA_HOST_STATUS table on selected backends. + * @param backends Backends that should report the table as absent. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int BGD_Simulator::replica_drop(vector backends) { return replica_error( backends, 1146, "Table 'information_schema.REPLICA_HOST_STATUS' doesn't exist"); } +/** + * @brief Make Aurora membership probes return a selected MySQL error. + * @param backends Backends that should return the error. + * @param error_code Nonzero MySQL error code. + * @param error_msg MySQL error text. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int BGD_Simulator::replica_error( vector backends, int error_code, string error_msg) { @@ -191,6 +258,10 @@ int BGD_Simulator::replica_error( return execute_transaction(statements); } +/** + * @brief Remove all shared topology, replica, control, and probe-log state. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int BGD_Simulator::cleanup() { vector statements { "DELETE FROM READONLY_STATUS", @@ -219,6 +290,10 @@ int BGD_Simulator::cleanup() { return execute_transaction(statements); } +/** + * @brief Read the last topology-probe sequence. + * @return Result code and the current sequence number. + */ rc_t BGD_Simulator::probe_log_last_sequence() { if (connection() == nullptr) { return { EXIT_FAILURE, 0 }; @@ -236,6 +311,11 @@ rc_t BGD_Simulator::probe_log_last_sequence() { }; } +/** + * @brief Read topology probes recorded after a sequence. + * @param sequence_id Exclusive lower sequence bound. + * @return Result code and ordered probe records. + */ rc_t> BGD_Simulator::probe_log_since(uint64_t sequence_id) { if (connection() == nullptr) { return { EXIT_FAILURE, {} }; @@ -271,6 +351,15 @@ rc_t> BGD_Simulator::probe_log_since(uint64_t sequence_id) return { EXIT_SUCCESS, move(logs) }; } +/** + * @brief Wait for a matching topology probe. + * @param sequence_id Exclusive lower sequence bound. + * @param backend Expected backend. + * @param probe_kind Expected topology query form. + * @param timeout_ms Maximum wait in milliseconds. + * @param encrypted Expected TLS state, or -1 to accept either. + * @return Result code and the matching probe record. + */ rc_t BGD_Simulator::wait_for_probe_log( uint64_t sequence_id, Endpoint backend, @@ -310,6 +399,10 @@ rc_t BGD_Simulator::wait_for_probe_log( return { ETIMEDOUT, {} }; } +/** + * @brief Read the last Aurora replica-probe sequence. + * @return Result code and the current sequence number. + */ rc_t BGD_Simulator::replica_probe_log_last_sequence() { if (connection() == nullptr) { return { EXIT_FAILURE, 0 }; @@ -326,6 +419,11 @@ rc_t BGD_Simulator::replica_probe_log_last_sequence() { }; } +/** + * @brief Read Aurora replica probes recorded after a sequence. + * @param sequence_id Exclusive lower sequence bound. + * @return Result code and ordered probe records. + */ rc_t> BGD_Simulator::replica_probe_log_since( uint64_t sequence_id) { @@ -363,6 +461,16 @@ rc_t> BGD_Simulator::replica_probe_log_since( return { EXIT_SUCCESS, move(logs) }; } +/** + * @brief Wait for a matching Aurora replica probe. + * @param sequence_id Exclusive lower sequence bound. + * @param backend Expected backend. + * @param probe_kind Expected Aurora query path. + * @param timeout_ms Maximum wait in milliseconds. + * @param encrypted Expected TLS state, or -1 to accept either. + * @param replica_set_id Expected set identity, or empty to accept any set. + * @return Result code and the matching probe record. + */ rc_t BGD_Simulator::wait_for_replica_probe_log( uint64_t sequence_id, Endpoint backend, @@ -406,6 +514,11 @@ rc_t BGD_Simulator::wait_for_replica_probe_log( return { ETIMEDOUT, {} }; } +/** + * @brief Execute simulator state changes as one transaction. + * @param statements SQL statements to execute in order. + * @return EXIT_SUCCESS after commit, otherwise EXIT_FAILURE after rollback. + */ int BGD_Simulator::execute_transaction(vector& statements) { if (execute("START TRANSACTION") != EXIT_SUCCESS) { return EXIT_FAILURE; @@ -423,6 +536,11 @@ int BGD_Simulator::execute_transaction(vector& statements) { return EXIT_SUCCESS; } +/** + * @brief Build a SQL predicate identifying one simulator backend. + * @param backend Backend address and port. + * @return SQL predicate for simulator control tables. + */ string BGD_Simulator::backend_predicate(Endpoint backend) { return "backend_ip=" + sql_quote(backend.host) + " AND backend_port=" + to_string(backend.port); diff --git a/test/tap/tap/bgd_simulator.h b/test/tap/tap/bgd_simulator.h index b4bc70e028..cdd86c7709 100644 --- a/test/tap/tap/bgd_simulator.h +++ b/test/tap/tap/bgd_simulator.h @@ -13,71 +13,140 @@ using std::vector; /** One row returned by the simulated mysql.rds_topology table. */ struct BGD_Topology_Row { - string id; - string endpoint; - int port; - string role; - string status; + string id; ///< Topology deployment member identifier. + string endpoint; ///< Endpoint reported by the topology row. + int port; ///< Endpoint port. + string role; ///< SOURCE or TARGET role reported by AWS. + string status; ///< Switchover status reported by AWS. }; /** Identifies a topology query recorded by the shared AWS BGD simulator. */ enum class BGD_Probe_Kind { - table_check, - metadata, + table_check, ///< Probe checking whether the topology table contains rows. + metadata, ///< Probe fetching the complete topology metadata. }; /** One topology query observed by the shared AWS BGD simulator. */ struct BGD_Probe_Log { - uint64_t sequence_id; - Endpoint backend; - BGD_Probe_Kind probe_kind; - bool encrypted; + uint64_t sequence_id; ///< Monotonic simulator log sequence. + Endpoint backend; ///< Backend that received the query. + BGD_Probe_Kind probe_kind; ///< Form of topology query observed. + bool encrypted; ///< Whether the simulated connection used TLS. }; /** One row returned by the simulated Aurora replica-status service. */ struct Aurora_Replica_Row { - string server_id; - string session_id; - double cpu; - string last_update_timestamp; - double replica_lag_in_milliseconds; - bool is_current; + string server_id; ///< Aurora member identifier. + string session_id; ///< Aurora writer or reader session identity. + double cpu; ///< CPU value returned by the simulator. + string last_update_timestamp; ///< Row timestamp returned by the simulator. + double replica_lag_in_milliseconds; ///< Replica lag returned by the simulator. + bool is_current; ///< Whether the row belongs to current membership. }; /** Identifies which Aurora replica-status query reached the simulator. */ enum class Aurora_Replica_Probe_Kind { - ordinary, - bgd_membership, + ordinary, ///< Ordinary Aurora monitor membership query. + bgd_membership, ///< BGD target-membership query. }; /** One Aurora replica-status query observed by the shared AWS simulator. */ struct Aurora_Replica_Probe_Log { - uint64_t sequence_id; - Endpoint backend; - Aurora_Replica_Probe_Kind probe_kind; - string replica_set_id; - bool encrypted; + uint64_t sequence_id; ///< Monotonic simulator log sequence. + Endpoint backend; ///< Backend that received the query. + Aurora_Replica_Probe_Kind probe_kind; ///< Aurora query path observed. + string replica_set_id; ///< Membership set served by the backend. + bool encrypted; ///< Whether the simulated connection used TLS. }; /** Controls shared AWS BGD topology and Aurora replica simulator responses. */ class BGD_Simulator : public Cluster_Simulator { public: + /** + * @brief Replace the topology rows served by a set of backends. + * @param backends Backends that should serve the topology. + * @param rows Topology rows returned by those backends. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int topology_update(vector backends, vector rows); + /** + * @brief Publish an empty topology table on a set of backends. + * @param backends Backends that should return an empty table. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int topology_delete(vector backends); + /** + * @brief Simulate an absent topology table on a set of backends. + * @param backends Backends that should report the table as absent. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int topology_drop(vector backends); + /** + * @brief Make topology probes return a selected MySQL error. + * @param backends Backends that should return the error. + * @param error_code Nonzero MySQL error code. + * @param error_msg MySQL error text. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int topology_error(vector backends, int error_code, string error_msg); + /** + * @brief Replace one Aurora membership set and its serving backends. + * @param replica_set_id Stable simulator membership identity. + * @param rows Replica rows returned for the set. + * @param backends Backends that should serve the set. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int replica_update( string replica_set_id, vector rows, vector backends ); + /** + * @brief Remove one Aurora membership set and its backend controls. + * @param replica_set_id Stable simulator membership identity. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int replica_delete(string replica_set_id); + /** + * @brief Simulate an absent REPLICA_HOST_STATUS table on selected backends. + * @param backends Backends that should report the table as absent. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int replica_drop(vector backends); + /** + * @brief Make Aurora membership probes return a selected MySQL error. + * @param backends Backends that should return the error. + * @param error_code Nonzero MySQL error code. + * @param error_msg MySQL error text. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int replica_error(vector backends, int error_code, string error_msg); + /** + * @brief Remove all shared topology, replica, control, and probe-log state. + * @return EXIT_SUCCESS on success, otherwise EXIT_FAILURE. + */ int cleanup(); + /** + * @brief Read the last topology-probe sequence. + * @return Result code and the current sequence number. + */ rc_t probe_log_last_sequence(); + /** + * @brief Read topology probes recorded after a sequence. + * @param sequence_id Exclusive lower sequence bound. + * @return Result code and ordered probe records. + */ rc_t> probe_log_since(uint64_t sequence_id); + /** + * @brief Wait for a matching topology probe. + * @param sequence_id Exclusive lower sequence bound. + * @param backend Expected backend. + * @param probe_kind Expected topology query form. + * @param timeout_ms Maximum wait in milliseconds. + * @param encrypted Expected TLS state, or -1 to accept either. + * @return Result code and the matching probe record. + */ rc_t wait_for_probe_log( uint64_t sequence_id, Endpoint backend, @@ -85,8 +154,27 @@ class BGD_Simulator : public Cluster_Simulator { uint32_t timeout_ms, int encrypted = -1 ); + /** + * @brief Read the last Aurora replica-probe sequence. + * @return Result code and the current sequence number. + */ rc_t replica_probe_log_last_sequence(); + /** + * @brief Read Aurora replica probes recorded after a sequence. + * @param sequence_id Exclusive lower sequence bound. + * @return Result code and ordered probe records. + */ rc_t> replica_probe_log_since(uint64_t sequence_id); + /** + * @brief Wait for a matching Aurora replica probe. + * @param sequence_id Exclusive lower sequence bound. + * @param backend Expected backend. + * @param probe_kind Expected Aurora query path. + * @param timeout_ms Maximum wait in milliseconds. + * @param encrypted Expected TLS state, or -1 to accept either. + * @param replica_set_id Expected set identity, or empty to accept any set. + * @return Result code and the matching probe record. + */ rc_t wait_for_replica_probe_log( uint64_t sequence_id, Endpoint backend, @@ -97,7 +185,17 @@ class BGD_Simulator : public Cluster_Simulator { ); protected: + /** + * @brief Build a SQL predicate identifying one simulator backend. + * @param backend Backend address and port. + * @return SQL predicate for simulator control tables. + */ static string backend_predicate(Endpoint backend); + /** + * @brief Execute simulator state changes as one transaction. + * @param statements SQL statements to execute in order. + * @return EXIT_SUCCESS after commit, otherwise EXIT_FAILURE after rollback. + */ int execute_transaction(vector& statements); }; diff --git a/test/tap/tests/test_aurora_bgd_automatic_discovery-t.cpp b/test/tap/tests/test_aurora_bgd_automatic_discovery-t.cpp index c5b5f979de..c5649d4a14 100644 --- a/test/tap/tests/test_aurora_bgd_automatic_discovery-t.cpp +++ b/test/tap/tests/test_aurora_bgd_automatic_discovery-t.cpp @@ -1,6 +1,11 @@ /** * @file test_aurora_bgd_automatic_discovery-t.cpp * @brief Aurora BGD runtime derivation for NULL green hostgroups. + * + * @details Verifies that automatic discovery: + * 1. Preserves NULL green hostgroups in configured and runtime rows. + * 2. Moves from absent topology to AVAILABLE without rewriting configuration. + * 3. Remains idempotent across repeated AVAILABLE observations. */ #include @@ -40,6 +45,7 @@ bool persistent_row_stays_null(Context& context, int writer_hostgroup) { "1"); } +/** @brief Verify discovery preserves NULL green hostgroups in both table projections. */ int test_null_green_hostgroups(Context& context, TestState& state) { if (publish_available(context, state.first) != EXIT_SUCCESS || configure( @@ -61,6 +67,7 @@ int test_null_green_hostgroups(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Verify an absent topology can later be discovered as AVAILABLE. */ int test_absent_then_available(Context& context, TestState& state) { if (context.simulator.replica_update( state.second.production.replica_set_id, @@ -98,6 +105,7 @@ int test_absent_then_available(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Verify repeated discovery leaves the runtime and configured rows unchanged. */ int test_repeated_discovery(Context& context, TestState& state) { usleep(300000); ok(runtime_row_stays_null(context, 2024, 2025, "AVAILABLE") diff --git a/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp b/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp index 57fd351546..8939b5677d 100644 --- a/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp +++ b/test/tap/tests/test_aurora_bgd_concurrent_isolation-t.cpp @@ -1,6 +1,10 @@ /** * @file test_aurora_bgd_concurrent_isolation-t.cpp * @brief Independent Aurora BGD state for three writer hostgroups. + * + * @details Verifies that three workers can reach AVAILABLE independently and + * that advancing one worker does not change another worker's phase, routing, + * or probe policy. */ #include @@ -45,6 +49,7 @@ bool worker_matches( worker.deployment.production.members.front().endpoint.hostname, demoted); } +/** @brief Verify three configured deployments independently reach AVAILABLE. */ int test_three_workers_available(Context& context, TestState& state) { if (configure_available(context, state.first, false) != EXIT_SUCCESS || configure_available(context, state.second, true) != EXIT_SUCCESS @@ -67,6 +72,7 @@ int test_three_workers_available(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Advance selected workers and verify the remaining worker is unaffected. */ int test_independent_phase_changes( CommandLine& cl, Context& context, TestState& state ) { diff --git a/test/tap/tests/test_aurora_bgd_config_refresh_after_completion-t.cpp b/test/tap/tests/test_aurora_bgd_config_refresh_after_completion-t.cpp index cc18f8e0a1..2cdecc8467 100644 --- a/test/tap/tests/test_aurora_bgd_config_refresh_after_completion-t.cpp +++ b/test/tap/tests/test_aurora_bgd_config_refresh_after_completion-t.cpp @@ -1,6 +1,10 @@ /** * @file test_aurora_bgd_config_refresh_after_completion-t.cpp * @brief Aurora BGD configuration refresh while completion is latched. + * + * @details Completes a deployment from IN_PROGRESS, then changes the worker's + * green hostgroups and confirms the completed fingerprint remains latched + * while the refreshed configuration becomes visible. */ #include @@ -15,6 +19,7 @@ struct TestState { int reader_hostgroup { 2211 }; }; +/** @brief Enter the completed latch and verify canonical writer placement. */ int test_completed_latch(Context& context, TestState& state) { if (publish_available(context, state.deployment) != EXIT_SUCCESS || configure( @@ -44,6 +49,7 @@ int test_completed_latch(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Verify a completed worker applies refreshed green hostgroups in place. */ int test_config_refresh_after_completion(Context& context, TestState& state) { if (aurora_bgd_execute_all(context.admin, { "UPDATE mysql_aws_aurora_hostgroups SET check_timeout_ms=950 " diff --git a/test/tap/tests/test_aurora_bgd_disable_during_switchover-t.cpp b/test/tap/tests/test_aurora_bgd_disable_during_switchover-t.cpp index 869c88348d..db80da2939 100644 --- a/test/tap/tests/test_aurora_bgd_disable_during_switchover-t.cpp +++ b/test/tap/tests/test_aurora_bgd_disable_during_switchover-t.cpp @@ -1,6 +1,10 @@ /** * @file test_aurora_bgd_disable_during_switchover-t.cpp * @brief Aurora BGD cleanup when the owning row is deactivated. + * + * @details Moves one deployment from AVAILABLE to IN_PROGRESS, deactivates + * its Aurora row, and verifies worker cleanup restores the source writer and + * removes the runtime deployment state. */ #include @@ -17,6 +21,7 @@ struct TestState { int green_reader_hostgroup { 2223 }; }; +/** @brief Establish the deployment's AVAILABLE baseline. */ int test_bgd_status_available(Context& context, TestState& state) { if (publish_available(context, state.deployment) != EXIT_SUCCESS || configure( @@ -30,6 +35,7 @@ int test_bgd_status_available(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Enter IN_PROGRESS and verify source-writer demotion. */ int test_writer_switchover_in_progress(Context& context, TestState& state) { if (publish_status( context, state.deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS) { @@ -46,6 +52,7 @@ int test_writer_switchover_in_progress(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Deactivate the row and verify the worker reverses its routing effect. */ int test_disable_during_switchover(Context& context, TestState& state) { if (aurora_bgd_execute_all(context.admin, { "UPDATE mysql_aws_aurora_hostgroups SET active=0 WHERE writer_hostgroup=2220", diff --git a/test/tap/tests/test_aurora_bgd_green_pool_cleanup-t.cpp b/test/tap/tests/test_aurora_bgd_green_pool_cleanup-t.cpp index 82c35da7cc..86e510ee38 100644 --- a/test/tap/tests/test_aurora_bgd_green_pool_cleanup-t.cpp +++ b/test/tap/tests/test_aurora_bgd_green_pool_cleanup-t.cpp @@ -1,6 +1,10 @@ /** * @file test_aurora_bgd_green_pool_cleanup-t.cpp * @brief Aurora BGD rollback and completion behavior for green status pools. + * + * @details Builds green pools containing ONLINE, SHUNNED, OFFLINE_SOFT, and + * OFFLINE_HARD servers. It verifies rollback preserves the pools, successful + * completion drains eligible pools, and offline pools remain untouched. */ #include @@ -82,6 +86,7 @@ bool configured_rows_preserved(Context& context, TestState& state) { return true; } +/** @brief Verify rollback does not drain configured green connection pools. */ int test_rollback_preserves_green_pools( CommandLine& cl, Context& context, TestState& state ) { @@ -118,6 +123,7 @@ int test_rollback_preserves_green_pools( return EXIT_SUCCESS; } +/** @brief Verify completion drains ONLINE and SHUNNED green connection pools. */ int test_successful_cleanup_drains_non_offline( Context& context, TestState& state ) { @@ -145,6 +151,7 @@ int test_successful_cleanup_drains_non_offline( return EXIT_SUCCESS; } +/** @brief Verify completion leaves OFFLINE_SOFT and OFFLINE_HARD pools unchanged. */ int test_cleanup_preserves_offline_pools(Context& context, TestState& state) { state.pools_after.clear(); for (GreenServer& server : state.servers) { diff --git a/test/tap/tests/test_aurora_bgd_late_entry_completed-t.cpp b/test/tap/tests/test_aurora_bgd_late_entry_completed-t.cpp index 42dc34d8e4..e91ae26c78 100644 --- a/test/tap/tests/test_aurora_bgd_late_entry_completed-t.cpp +++ b/test/tap/tests/test_aurora_bgd_late_entry_completed-t.cpp @@ -1,6 +1,10 @@ /** * @file test_aurora_bgd_late_entry_completed-t.cpp * @brief First observation of Aurora BGD completion and later rearm. + * + * @details Starts a worker on a lone completed TARGET row, verifies the + * resulting cleanup is a routing no-op, then drains the topology and checks + * that discovery rearms for a later deployment. */ #include @@ -16,6 +20,7 @@ struct TestState { int route_hostgroup { 2122 }; }; +/** @brief Verify direct entry at completion preserves source routing and latches identity. */ int test_first_completed( CommandLine& cl, Context& context, TestState& state ) { @@ -67,6 +72,7 @@ int test_first_completed( return EXIT_SUCCESS; } +/** @brief Verify an empty topology releases the completed latch back to NONE. */ int test_completed_empty_topology(Context& context, TestState& state) { if (context.simulator.topology_delete( aurora_bgd_topology_backends(state.deployment)) != EXIT_SUCCESS) { diff --git a/test/tap/tests/test_aurora_bgd_late_entry_writer_phases-t.cpp b/test/tap/tests/test_aurora_bgd_late_entry_writer_phases-t.cpp index f064fde453..bbee0decf7 100644 --- a/test/tap/tests/test_aurora_bgd_late_entry_writer_phases-t.cpp +++ b/test/tap/tests/test_aurora_bgd_late_entry_writer_phases-t.cpp @@ -1,6 +1,10 @@ /** * @file test_aurora_bgd_late_entry_writer_phases-t.cpp * @brief First observation of each active Aurora BGD writer phase. + * + * @details Starts independent workers directly in INITIATED, IN_PROGRESS, and + * POST_PROCESSING to verify phase-specific routing actions and the switch + * from ordinary production probes to target-membership probes. */ #include @@ -37,6 +41,7 @@ bool active_probe_policy( return membership_seen; } +/** @brief Verify direct INITIATED entry suspends ordinary production probing. */ int test_first_initiated(Context& context, TestState& state) { if (publish_initial( context, state.initiated, "SWITCHOVER_INITIATED") != EXIT_SUCCESS @@ -59,6 +64,7 @@ int test_first_initiated(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Verify direct IN_PROGRESS entry demotes the source writer. */ int test_first_in_progress(Context& context, TestState& state) { if (reset(context) != EXIT_SUCCESS || publish_initial( @@ -78,6 +84,7 @@ int test_first_in_progress(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Verify direct POST_PROCESSING entry redirects every mapped member. */ int test_first_post_processing( CommandLine& cl, Context& context, TestState& state ) { diff --git a/test/tap/tests/test_aurora_bgd_probe_tls-t.cpp b/test/tap/tests/test_aurora_bgd_probe_tls-t.cpp index bba132d235..8611d1648f 100644 --- a/test/tap/tests/test_aurora_bgd_probe_tls-t.cpp +++ b/test/tap/tests/test_aurora_bgd_probe_tls-t.cpp @@ -1,6 +1,9 @@ /** * @file test_aurora_bgd_probe_tls-t.cpp * @brief Aurora BGD topology and target-membership probe TLS policy. + * + * @details Configures TLS on the production Aurora row and verifies both + * topology discovery and target-membership discovery inherit that setting. */ #include @@ -17,6 +20,7 @@ struct TestState { int green_reader_hostgroup { 2033 }; }; +/** @brief Verify topology and membership probes both use the configured TLS policy. */ int test_tls_probe_policy(Context& context, TestState& state) { if (publish_available(context, state.deployment) != EXIT_SUCCESS || configure( diff --git a/test/tap/tests/test_aurora_bgd_reader_policy-t.cpp b/test/tap/tests/test_aurora_bgd_reader_policy-t.cpp index 1703ea6086..95b3786ecd 100644 --- a/test/tap/tests/test_aurora_bgd_reader_policy-t.cpp +++ b/test/tap/tests/test_aurora_bgd_reader_policy-t.cpp @@ -1,6 +1,10 @@ /** * @file test_aurora_bgd_reader_policy-t.cpp * @brief Aurora BGD mapped-reader eligibility and routing. + * + * @details Publishes a complete target membership, enters POST_PROCESSING, + * and verifies each configured production reader remains eligible while its + * hostname routes to the matched target reader. */ #include @@ -19,6 +23,7 @@ struct TestState { vector reader_routes { 2054, 2055 }; }; +/** @brief Verify mapped readers stay ONLINE and route to their target counterparts. */ int test_mapped_readers(CommandLine& cl, Context& context, TestState& state) { if (publish_available(context, state.deployment) != EXIT_SUCCESS || configure( diff --git a/test/tap/tests/test_aurora_bgd_reader_switchover_cleanup-t.cpp b/test/tap/tests/test_aurora_bgd_reader_switchover_cleanup-t.cpp index af245a02d0..d120784bd2 100644 --- a/test/tap/tests/test_aurora_bgd_reader_switchover_cleanup-t.cpp +++ b/test/tap/tests/test_aurora_bgd_reader_switchover_cleanup-t.cpp @@ -1,6 +1,10 @@ /** * @file test_aurora_bgd_reader_switchover_cleanup-t.cpp * @brief Aurora BGD immediate completion cleanup and topology-drain rearm. + * + * @details Redirects all mapped members in POST_PROCESSING, publishes the lone + * completed TARGET row, and verifies immediate routing cleanup. A later empty + * topology must release the completed latch. */ #include @@ -20,6 +24,7 @@ struct TestState { vector source_routes { 2077, 2078, 2079 }; }; +/** @brief Verify completion removes target pins and restores production routing. */ int test_completion_cleanup( CommandLine& cl, Context& context, TestState& state ) { @@ -82,6 +87,7 @@ int test_completion_cleanup( return EXIT_SUCCESS; } +/** @brief Verify an empty topology releases the completed latch to NONE. */ int test_topology_drain(Context& context, TestState& state) { if (context.simulator.topology_delete( aurora_bgd_topology_backends(state.deployment)) != EXIT_SUCCESS) { diff --git a/test/tap/tests/test_aurora_bgd_remove_during_switchover-t.cpp b/test/tap/tests/test_aurora_bgd_remove_during_switchover-t.cpp index c175df1340..e27e043638 100644 --- a/test/tap/tests/test_aurora_bgd_remove_during_switchover-t.cpp +++ b/test/tap/tests/test_aurora_bgd_remove_during_switchover-t.cpp @@ -1,6 +1,10 @@ /** * @file test_aurora_bgd_remove_during_switchover-t.cpp * @brief Aurora BGD cleanup when the owning row is deleted. + * + * @details Moves one deployment from AVAILABLE to IN_PROGRESS, deletes its + * Aurora configuration row, and verifies worker removal restores the source + * writer and removes runtime deployment state. */ #include @@ -17,6 +21,7 @@ struct TestState { int green_reader_hostgroup { 2233 }; }; +/** @brief Establish the deployment's AVAILABLE baseline. */ int test_bgd_status_available(Context& context, TestState& state) { if (publish_available(context, state.deployment) != EXIT_SUCCESS || configure( @@ -30,6 +35,7 @@ int test_bgd_status_available(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Enter IN_PROGRESS and verify source-writer demotion. */ int test_writer_switchover_in_progress(Context& context, TestState& state) { if (publish_status( context, state.deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS) { @@ -46,6 +52,7 @@ int test_writer_switchover_in_progress(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Delete the owning row and verify worker teardown reverses routing effects. */ int test_remove_during_switchover(Context& context, TestState& state) { if (aurora_bgd_execute_all(context.admin, { "DELETE FROM mysql_aws_aurora_hostgroups WHERE writer_hostgroup=2230", diff --git a/test/tap/tests/test_aurora_bgd_repeated_deployment-t.cpp b/test/tap/tests/test_aurora_bgd_repeated_deployment-t.cpp index 461b00efc3..1f3dd9c8a7 100644 --- a/test/tap/tests/test_aurora_bgd_repeated_deployment-t.cpp +++ b/test/tap/tests/test_aurora_bgd_repeated_deployment-t.cpp @@ -1,6 +1,10 @@ /** * @file test_aurora_bgd_repeated_deployment-t.cpp * @brief Reuse one Aurora worker for a later deployment fingerprint. + * + * @details Completes one deployment, drains its topology, then publishes a + * different target fingerprint on the same worker and verifies the new + * deployment can independently reach POST_PROCESSING. */ #include @@ -61,6 +65,7 @@ int complete_deployment(Context& context, Aurora_BGD_Test_Deployment& deployment ? EXIT_SUCCESS : EXIT_FAILURE; } +/** @brief Complete the first deployment and release its completed latch. */ int test_deployment_a(Context& context, TestState& state) { if (publish_available(context, state.first) != EXIT_SUCCESS || configure( @@ -80,6 +85,7 @@ int test_deployment_a(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Verify the rearmed worker accepts and redirects the second deployment. */ int test_deployment_b( CommandLine& cl, Context& context, TestState& state ) { diff --git a/test/tap/tests/test_aurora_bgd_rollback-t.cpp b/test/tap/tests/test_aurora_bgd_rollback-t.cpp index 998d8e04c3..a5a066afdd 100644 --- a/test/tap/tests/test_aurora_bgd_rollback-t.cpp +++ b/test/tap/tests/test_aurora_bgd_rollback-t.cpp @@ -1,6 +1,10 @@ /** * @file test_aurora_bgd_rollback-t.cpp * @brief Aurora BGD rollback from each active writer phase. + * + * @details Runs independent deployments through INITIATED, IN_PROGRESS, and + * POST_PROCESSING before publishing AVAILABLE again. Each case verifies only + * the routing and status effects that its phase must reverse. */ #include @@ -28,6 +32,7 @@ int prepare_available( ? EXIT_SUCCESS : EXIT_FAILURE; } +/** @brief Verify INITIATED can return to AVAILABLE without routing changes. */ int test_initiated_rollback(Context& context, TestState& state) { if (prepare_available(context, state, 2130, 2131) != EXIT_SUCCESS || publish_status( @@ -48,6 +53,7 @@ int test_initiated_rollback(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Verify IN_PROGRESS rollback restores canonical writer placement. */ int test_in_progress_rollback(Context& context, TestState& state) { if (reset(context) != EXIT_SUCCESS || prepare_available(context, state, 2140, 2141) != EXIT_SUCCESS @@ -78,6 +84,7 @@ int test_in_progress_rollback(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Verify POST_PROCESSING rollback removes every applied traffic pin. */ int test_post_processing_rollback( CommandLine& cl, Context& context, TestState& state ) { diff --git a/test/tap/tests/test_aurora_bgd_smoke-t.cpp b/test/tap/tests/test_aurora_bgd_smoke-t.cpp index 231f26649e..72b73b0053 100644 --- a/test/tap/tests/test_aurora_bgd_smoke-t.cpp +++ b/test/tap/tests/test_aurora_bgd_smoke-t.cpp @@ -1,6 +1,10 @@ /** * @file test_aurora_bgd_smoke-t.cpp * @brief Basic Aurora BGD AVAILABLE discovery. + * + * @details Publishes one valid deployment and verifies AVAILABLE state, + * continued ordinary production monitoring, target topology probing, and + * target-membership discovery. */ #include @@ -17,6 +21,7 @@ struct TestState { int green_reader_hostgroup { 2013 }; }; +/** @brief Verify AVAILABLE discovery and its production/topology probe policy. */ int test_available_topology(Context& context, TestState& state) { if (publish_available(context, state.deployment) != EXIT_SUCCESS || configure( @@ -40,6 +45,7 @@ int test_available_topology(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Verify AVAILABLE discovery probes the target membership set. */ int test_target_membership(Context& context, TestState& state) { ok(membership_probe_reached(context, 0, state.deployment), "AVAILABLE discovery probes the target replica set"); diff --git a/test/tap/tests/test_aurora_bgd_topology_empty_absent-t.cpp b/test/tap/tests/test_aurora_bgd_topology_empty_absent-t.cpp index e5d3fbf451..e82fa6ceed 100644 --- a/test/tap/tests/test_aurora_bgd_topology_empty_absent-t.cpp +++ b/test/tap/tests/test_aurora_bgd_topology_empty_absent-t.cpp @@ -1,6 +1,10 @@ /** * @file test_aurora_bgd_topology_empty_absent-t.cpp * @brief Aurora BGD cancellation on successful empty or absent topology. + * + * @details Starts independent deployments in IN_PROGRESS, then verifies both + * a successful empty result and a missing topology table cancel the active + * deployment, restore the writer, and resume ordinary Aurora probing. */ #include @@ -30,6 +34,7 @@ int prepare_in_progress( ? EXIT_SUCCESS : EXIT_FAILURE; } +/** @brief Verify an empty topology cancels an active deployment safely. */ int test_empty_before_completion(Context& context, TestState& state) { if (prepare_in_progress(context, state, 2160, 2161) != EXIT_SUCCESS) { return EXIT_FAILURE; @@ -51,6 +56,7 @@ int test_empty_before_completion(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Verify a missing topology table cancels an active deployment safely. */ int test_absent_before_completion(Context& context, TestState& state) { if (reset(context) != EXIT_SUCCESS || prepare_in_progress(context, state, 2170, 2171) != EXIT_SUCCESS) { diff --git a/test/tap/tests/test_aurora_bgd_topology_errors-t.cpp b/test/tap/tests/test_aurora_bgd_topology_errors-t.cpp index 353ccb5170..959b3b3ffe 100644 --- a/test/tap/tests/test_aurora_bgd_topology_errors-t.cpp +++ b/test/tap/tests/test_aurora_bgd_topology_errors-t.cpp @@ -1,6 +1,10 @@ /** * @file test_aurora_bgd_topology_errors-t.cpp * @brief Aurora BGD state retention across generic metadata errors. + * + * @details Injects non-table-missing errors into topology and target-membership + * probes. Each case verifies the active phase and its already-applied routing + * effects remain unchanged until a valid observation arrives. */ #include @@ -15,6 +19,7 @@ struct TestState { vector routes { 2194, 2195, 2196 }; }; +/** @brief Verify a generic topology error retains active state and writer placement. */ int test_generic_topology_error(Context& context, TestState& state) { if (publish_initial( context, state.deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS @@ -45,6 +50,7 @@ int test_generic_topology_error(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Verify a generic membership error retains post-processing traffic pins. */ int test_generic_membership_error( CommandLine& cl, Context& context, TestState& state ) { diff --git a/test/tap/tests/test_aurora_bgd_worker_config_refresh-t.cpp b/test/tap/tests/test_aurora_bgd_worker_config_refresh-t.cpp index b51915066d..7272ac587f 100644 --- a/test/tap/tests/test_aurora_bgd_worker_config_refresh-t.cpp +++ b/test/tap/tests/test_aurora_bgd_worker_config_refresh-t.cpp @@ -1,6 +1,10 @@ /** * @file test_aurora_bgd_worker_config_refresh-t.cpp * @brief Aurora BGD active-worker refresh without FSM reset. + * + * @details Enters POST_PROCESSING, changes the explicit green hostgroups, and + * verifies the existing worker retains its deployment phase and traffic pins + * while applying the refreshed configuration in place. */ #include @@ -20,6 +24,7 @@ struct TestState { int route_hostgroup { 2206 }; }; +/** @brief Verify an active worker refreshes configuration without resetting its FSM. */ int test_active_worker_refresh( CommandLine& cl, Context& context, TestState& state ) { diff --git a/test/tap/tests/test_aurora_bgd_writer_switchover-t.cpp b/test/tap/tests/test_aurora_bgd_writer_switchover-t.cpp index 920fddd41e..946d9a3256 100644 --- a/test/tap/tests/test_aurora_bgd_writer_switchover-t.cpp +++ b/test/tap/tests/test_aurora_bgd_writer_switchover-t.cpp @@ -1,6 +1,10 @@ /** * @file test_aurora_bgd_writer_switchover-t.cpp * @brief Aurora BGD writer behavior across the active switchover phases. + * + * @details Advances one deployment through AVAILABLE, INITIATED, IN_PROGRESS, + * and POST_PROCESSING. Each phase verifies its distinct status, writer + * placement, probe policy, or target-routing effect. */ #include @@ -19,6 +23,7 @@ struct TestState { vector route_hostgroups { 2044, 2045, 2046 }; }; +/** @brief Verify AVAILABLE preserves canonical source-writer placement. */ int test_bgd_status_available(Context& context, TestState& state) { if (publish_available(context, state.deployment) != EXIT_SUCCESS || configure( @@ -40,6 +45,7 @@ int test_bgd_status_available(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Verify INITIATED publishes status without moving the source writer. */ int test_switchover_initiated(Context& context, TestState& state) { if (publish_status(context, state.deployment, "SWITCHOVER_INITIATED") != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_INITIATED"); @@ -56,6 +62,7 @@ int test_switchover_initiated(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Verify IN_PROGRESS demotes the source writer exactly once. */ int test_switchover_in_progress(Context& context, TestState& state) { if (publish_status(context, state.deployment, "SWITCHOVER_IN_PROGRESS") != EXIT_SUCCESS) { diag("Error: failed to publish SWITCHOVER_IN_PROGRESS"); @@ -79,6 +86,7 @@ int test_switchover_in_progress(Context& context, TestState& state) { return EXIT_SUCCESS; } +/** @brief Verify POST_PROCESSING restores writer placement and redirects all members. */ int test_switchover_post_processing( CommandLine& cl, Context& context, TestState& state ) { From ad59272792d1658bfd38a1a00b479d0d14d10a9a Mon Sep 17 00:00:00 2001 From: Wazir Ahmed Date: Thu, 20 Aug 2026 21:36:07 +0000 Subject: [PATCH 34/34] test: tighten Aurora BGD scenario helpers --- test/tap/tap/aurora_bgd_scenario_tap.h | 11 +++++-- .../test_aurora_bgd_repeated_deployment-t.cpp | 29 +++++++++++-------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/test/tap/tap/aurora_bgd_scenario_tap.h b/test/tap/tap/aurora_bgd_scenario_tap.h index 7961f0084c..591a9c5938 100644 --- a/test/tap/tap/aurora_bgd_scenario_tap.h +++ b/test/tap/tap/aurora_bgd_scenario_tap.h @@ -60,16 +60,23 @@ inline int setup(CommandLine& cl, Context& context) { return EXIT_FAILURE; } if (aurora_bgd_admin_cleanup(context.admin) != EXIT_SUCCESS - || context.simulator.cleanup() != EXIT_SUCCESS) { + || context.simulator.cleanup() != EXIT_SUCCESS) { diag("Error: failed to clear prior Aurora BGD state"); + mysql_close(context.admin); + context.admin = nullptr; return EXIT_FAILURE; } - return aurora_bgd_execute_all(context.admin, { + int user_rc = aurora_bgd_execute_all(context.admin, { "DELETE FROM mysql_users WHERE username='testuser'", "INSERT INTO mysql_users(username,password,active,default_hostgroup,transaction_persistent) " "VALUES ('testuser','testuser',1,0,1)", "LOAD MYSQL USERS TO RUNTIME", }); + if (user_rc != EXIT_SUCCESS) { + mysql_close(context.admin); + context.admin = nullptr; + } + return user_rc; } /** diff --git a/test/tap/tests/test_aurora_bgd_repeated_deployment-t.cpp b/test/tap/tests/test_aurora_bgd_repeated_deployment-t.cpp index 1f3dd9c8a7..718037c337 100644 --- a/test/tap/tests/test_aurora_bgd_repeated_deployment-t.cpp +++ b/test/tap/tests/test_aurora_bgd_repeated_deployment-t.cpp @@ -50,19 +50,23 @@ struct TestState { vector completion_routes { 2247, 2248, 2249 }; }; -int complete_deployment(Context& context, Aurora_BGD_Test_Deployment& deployment) { +int complete_deployment( + Context& context, Aurora_BGD_Test_Deployment& deployment, int writer_hostgroup +) { return publish_status(context, deployment, "SWITCHOVER_IN_PROGRESS") == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + context.admin, writer_hostgroup, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) + == EXIT_SUCCESS + && publish_status( + context, deployment, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS && aurora_bgd_wait_for_status( - context.admin, 2240, "SWITCHOVER_IN_PROGRESS", kWaitSeconds) == EXIT_SUCCESS - && publish_status( - context, deployment, "SWITCHOVER_IN_POST_PROCESSING") == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - context.admin, 2240, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) - == EXIT_SUCCESS - && publish_completed(context, deployment, deployment) == EXIT_SUCCESS - && aurora_bgd_wait_for_status( - context.admin, 2240, "SWITCHOVER_COMPLETED", kWaitSeconds) == EXIT_SUCCESS - ? EXIT_SUCCESS : EXIT_FAILURE; + context.admin, writer_hostgroup, "SWITCHOVER_IN_POST_PROCESSING", kWaitSeconds) + == EXIT_SUCCESS + && publish_completed(context, deployment, deployment) == EXIT_SUCCESS + && aurora_bgd_wait_for_status( + context.admin, writer_hostgroup, "SWITCHOVER_COMPLETED", kWaitSeconds) + == EXIT_SUCCESS + ? EXIT_SUCCESS : EXIT_FAILURE; } /** @brief Complete the first deployment and release its completed latch. */ @@ -74,7 +78,8 @@ int test_deployment_a(Context& context, TestState& state) { || aurora_bgd_wait_for_status( context.admin, state.writer_hostgroup, "AVAILABLE", kWaitSeconds) != EXIT_SUCCESS - || complete_deployment(context, state.first) != EXIT_SUCCESS) { + || complete_deployment( + context, state.first, state.writer_hostgroup) != EXIT_SUCCESS) { diag("Error: failed to complete deployment A"); return EXIT_FAILURE; }