diff --git a/README.md b/README.md index b6efb73..c60adcc 100644 --- a/README.md +++ b/README.md @@ -181,9 +181,9 @@ Check [benchmark](benchmark/README.md) ## TODO * SSL support -* Performance improvement: - granular locking on client/server list - active wait list processing +* Session pooling +* Enrich Statistics & Output +* Sharding? ## License diff --git a/benchmark/README.md b/benchmark/README.md index 9f85838..eb30d7b 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -1,6 +1,6 @@ # PostgreSQL Connection Pool Benchmark Suite -Comprehensive benchmarking suite for comparing **pgpool** vs **pgbouncer** vs **direct PostgreSQL** connections using the industry-standard `pgbench` tool. +Comprehensive benchmarking suite for comparing **pgpool** vs **pgcat** vs **pgbouncer** vs **direct PostgreSQL** connections using the industry-standard `pgbench` tool. ## Overview @@ -14,14 +14,18 @@ This benchmark suite tests: ``` benchmark/ -├── config/ # Configuration files -│ ├── pgpool.ini # pgpool configuration +├── config/ # Configuration files +│ ├── pgpool.ini # pgpool configuration │ ├── pgbouncer.ini # pgbouncer configuration +│ ├── pgcat.toml # pgcat configuration │ └── userlist.txt # pgbouncer auth file ├── scripts/ # Benchmark scripts │ ├── setup.sh # Environment setup │ ├── run_benchmark.sh # Main benchmark runner -│ ├── generate_report.sh # Report generator +│ ├── generate_report.sh # Report gener + +4. **Connection Time** - Lower is better + - Overhead of establishing conator │ └── cleanup.sh # Cleanup script ├── results/ # Benchmark results (auto-generated) │ └── YYYYMMDD_HHMMSS/ # Timestamped results @@ -31,11 +35,11 @@ benchmark/ ## Prerequisites -1. **PostgreSQL** (tested with PostgreSQL 15+) +1. **PostgreSQL** (tested with PostgreSQL 16+) 2. **pgbench** (included with PostgreSQL client tools) 3. **pgpool** (built from this repository) -4. **pgbouncer** (optional, for comparison) -5. **Go 1.22+** (to build pgpool) +4. **pgcat** ([built from git repository](https://github.com/postgresml/pgcat)) +5. **pgbouncer** (1.24.0) ### Installation @@ -91,18 +95,29 @@ max_clients = 200 Edit `config/pgbouncer.ini` and `config/userlist.txt` with your credentials. + +#### pgcat Configuration + +Edit `pgcat.toml` with test db and credentials. + ### 3. Start Poolers #### Start pgpool ```bash # In terminal 1 cd /path/to/pgpool -./pgpool -conf benchmark/config/pgpool.ini -debug +./pgpool -conf benchmark/config/pgpool.ini ``` -#### Start pgbouncer (optional) +#### Start pgbouncer ```bash # In terminal 2 +cargo run --release +``` + +#### Start pgbouncer +```bash +# In terminal 3 cd benchmark/config pgbouncer -d pgbouncer.ini ``` @@ -139,7 +154,7 @@ export DB_NAME=postgres # Pool ports export PGPOOL_PORT=5433 -export PGBOUNCER_PORT=6432 +export PGBOUNCER_PORT=6433 # Benchmark configuration export DURATION=60 # seconds per test @@ -171,12 +186,7 @@ The TPC-B workload is automatically initialized by pgbench with the specified sc - Mean response time - Overall user experience -3. **P95/P99 Latency** - Lower is better - - Tail latency - - Worst-case performance - - Critical for user experience - -4. **Connection Time** - Lower is better +3. **Connection Time** - Lower is better - Overhead of establishing connections - Important for connection churn scenarios diff --git a/benchmark/config/pgbouncer.ini b/benchmark/config/pgbouncer.ini index f7773d3..10532a7 100644 --- a/benchmark/config/pgbouncer.ini +++ b/benchmark/config/pgbouncer.ini @@ -3,7 +3,7 @@ postgres = host=127.0.0.1 port=5432 dbname=postgres [pgbouncer] listen_addr = 127.0.0.1 -listen_port = 6432 +listen_port = 6433 auth_type = scram-sha-256 auth_file = userlist.txt diff --git a/benchmark/config/pgcat.toml b/benchmark/config/pgcat.toml new file mode 100644 index 0000000..b745bbc --- /dev/null +++ b/benchmark/config/pgcat.toml @@ -0,0 +1,350 @@ +# +# PgCat config example. +# + +# +# General pooler settings +[general] +# What IP to run on, 0.0.0.0 means accessible from everywhere. +host = "0.0.0.0" + +# Port to run on, same as PgBouncer used in this example. +port = 6432 + +# Whether to enable prometheus exporter or not. +enable_prometheus_exporter = true + +# Port at which prometheus exporter listens on. +prometheus_exporter_port = 9930 + +# How long to wait before aborting a server connection (ms). +connect_timeout = 5000 # milliseconds + +# How long an idle connection with a server is left open (ms). +idle_timeout = 30000 # milliseconds + +# Max connection lifetime before it's closed, even if actively used. +server_lifetime = 86400000 # 24 hours + +# How long a client is allowed to be idle while in a transaction (ms). +idle_client_in_transaction_timeout = 0 # milliseconds + +# How much time to give the health check query to return with a result (ms). +healthcheck_timeout = 1000 # milliseconds + +# How long to keep connection available for immediate re-use, without running a healthcheck query on it +healthcheck_delay = 30000 # milliseconds + +# How much time to give clients during shutdown before forcibly killing client connections (ms). +shutdown_timeout = 60000 # milliseconds + +# How long to ban a server if it fails a health check (seconds). +ban_time = 60 # seconds + +# If we should log client connections +log_client_connections = false + +# If we should log client disconnections +log_client_disconnections = false + +# When set to true, PgCat reloads configs if it detects a change in the config file. +autoreload = 15000 + +# Number of worker threads the Runtime will use (4 by default). +worker_threads = 5 + +# Number of seconds of connection idleness to wait before sending a keepalive packet to the server. +tcp_keepalives_idle = 5 +# Number of unacknowledged keepalive packets allowed before giving up and closing the connection. +tcp_keepalives_count = 5 +# Number of seconds between keepalive packets. +tcp_keepalives_interval = 5 + +# Path to TLS Certificate file to use for TLS connections +# tls_certificate = ".circleci/server.cert" +# Path to TLS private key file to use for TLS connections +# tls_private_key = ".circleci/server.key" + +# Enable/disable server TLS +server_tls = false + +# Verify server certificate is completely authentic. +verify_server_certificate = false + +# User name to access the virtual administrative database (pgbouncer or pgcat) +# Connecting to that database allows running commands like `SHOW POOLS`, `SHOW DATABASES`, etc.. +admin_username = "admin_user" +# Password to access the virtual administrative database +admin_password = "admin_pass" + +# Default plugins that are configured on all pools. +[plugins] + +# Prewarmer plugin that runs queries on server startup, before giving the connection +# to the client. +[plugins.prewarmer] +enabled = false +queries = [ + "SELECT pg_prewarm('pgbench_accounts')", +] + +# Log all queries to stdout. +[plugins.query_logger] +enabled = false + +# Block access to tables that Postgres does not allow us to control. +[plugins.table_access] +enabled = false +tables = [ + "pg_user", + "pg_roles", + "pg_database", +] + +# Intercept user queries and give a fake reply. +[plugins.intercept] +enabled = true + +[plugins.intercept.queries.0] + +query = "select current_database() as a, current_schemas(false) as b" +schema = [ + ["a", "text"], + ["b", "text"], +] +result = [ + ["${DATABASE}", "{public}"], +] + +[plugins.intercept.queries.1] + +query = "select current_database(), current_schema(), current_user" +schema = [ + ["current_database", "text"], + ["current_schema", "text"], + ["current_user", "text"], +] +result = [ + ["${DATABASE}", "public", "${USER}"], +] + + +# pool configs are structured as pool. +# the pool_name is what clients use as database name when connecting. +# For a pool named `sharded_db`, clients access that pool using connection string like +# `postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded_db` +[pools.sharded_db] +# Pool mode (see PgBouncer docs for more). +# `session` one server connection per connected client +# `transaction` one server connection per client transaction +pool_mode = "transaction" + +# Load balancing mode +# `random` selects the server at random +# `loc` selects the server with the least outstanding busy conncetions +load_balancing_mode = "random" + +# If the client doesn't specify, PgCat routes traffic to this role by default. +# `any` round-robin between primary and replicas, +# `replica` round-robin between replicas only without touching the primary, +# `primary` all queries go to the primary unless otherwise specified. +default_role = "any" + +# Prepared statements cache size. +# TODO: update documentation +prepared_statements_cache_size = 500 + +# If Query Parser is enabled, we'll attempt to parse +# every incoming query to determine if it's a read or a write. +# If it's a read query, we'll direct it to a replica. Otherwise, if it's a write, +# we'll direct it to the primary. +query_parser_enabled = true + +# If the query parser is enabled and this setting is enabled, we'll attempt to +# infer the role from the query itself. +query_parser_read_write_splitting = true + +# If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for +# load balancing of read queries. Otherwise, the primary will only be used for write +# queries. The primary can always be explicitly selected with our custom protocol. +primary_reads_enabled = true + +# Allow sharding commands to be passed as statement comments instead of +# separate commands. If these are unset this functionality is disabled. +# sharding_key_regex = '/\* sharding_key: (\d+) \*/' +# shard_id_regex = '/\* shard_id: (\d+) \*/' +# regex_search_limit = 1000 # only look at the first 1000 characters of SQL statements + +# Defines the behavior when no shard is selected in a sharded system. +# `random`: picks a shard at random +# `random_healthy`: picks a shard at random favoring shards with the least number of recent errors +# `shard_`: e.g. shard_0, shard_4, etc. picks a specific shard, everytime +# default_shard = "shard_0" + +# So what if you wanted to implement a different hashing function, +# or you've already built one and you want this pooler to use it? +# Current options: +# `pg_bigint_hash`: PARTITION BY HASH (Postgres hashing function) +# `sha1`: A hashing function based on SHA1 +sharding_function = "pg_bigint_hash" + +# Query to be sent to servers to obtain the hash used for md5 authentication. The connection will be +# established using the database configured in the pool. This parameter is inherited by every pool +# and can be redefined in pool configuration. +# auth_query="SELECT usename, passwd FROM pg_shadow WHERE usename='$1'" + +# User to be used for connecting to servers to obtain the hash used for md5 authentication by sending the query +# specified in `auth_query_user`. The connection will be established using the database configured in the pool. +# This parameter is inherited by every pool and can be redefined in pool configuration. +# auth_query_user = "sharding_user" + +# Password to be used for connecting to servers to obtain the hash used for md5 authentication by sending the query +# specified in `auth_query_user`. The connection will be established using the database configured in the pool. +# This parameter is inherited by every pool and can be redefined in pool configuration. +# auth_query_password = "sharding_user" + +# Automatically parse this from queries and route queries to the right shard! +# automatic_sharding_key = "data.id" + +# Idle timeout can be overwritten in the pool +idle_timeout = 40000 + +# Connect timeout can be overwritten in the pool +connect_timeout = 3000 + +# When enabled, ip resolutions for server connections specified using hostnames will be cached +# and checked for changes every `dns_max_ttl` seconds. If a change in the host resolution is found +# old ip connections are closed (gracefully) and new connections will start using new ip. +# dns_cache_enabled = false + +# Specifies how often (in seconds) cached ip addresses for servers are rechecked (see `dns_cache_enabled`). +# dns_max_ttl = 30 + +# Plugins can be configured on a pool-per-pool basis. This overrides the global plugins setting, +# so all plugins have to be configured here again. +[pool.sharded_db.plugins] + +[pools.sharded_db.plugins.prewarmer] +enabled = true +queries = [ + "SELECT pg_prewarm('pgbench_accounts')", +] + +[pools.sharded_db.plugins.query_logger] +enabled = false + +[pools.sharded_db.plugins.table_access] +enabled = false +tables = [ + "pg_user", + "pg_roles", + "pg_database", +] + +[pools.sharded_db.plugins.intercept] +enabled = true + +[pools.sharded_db.plugins.intercept.queries.0] + +query = "select current_database() as a, current_schemas(false) as b" +schema = [ + ["a", "text"], + ["b", "text"], +] +result = [ + ["${DATABASE}", "{public}"], +] + +[pools.sharded_db.plugins.intercept.queries.1] + +query = "select current_database(), current_schema(), current_user" +schema = [ + ["current_database", "text"], + ["current_schema", "text"], + ["current_user", "text"], +] +result = [ + ["${DATABASE}", "public", "${USER}"], +] + +# User configs are structured as pool..users. +# This section holds the credentials for users that may connect to this cluster +[pools.sharded_db.users.0] +# PostgreSQL username used to authenticate the user and connect to the server +# if `server_username` is not set. +username = "pgtest" + +# PostgreSQL password used to authenticate the user and connect to the server +# if `server_password` is not set. +password = "test123" + +pool_mode = "transaction" + +# PostgreSQL username used to connect to the server. +# server_username = "another_user" + +# PostgreSQL password used to connect to the server. +# server_password = "another_password" + +# Maximum number of server connections that can be established for this user +# The maximum number of connection from a single Pgcat process to any database in the cluster +# is the sum of pool_size across all users. +pool_size = 20 + + +# Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way. +# 0 means it is disabled. +statement_timeout = 0 + +[pools.sharded_db.users.1] +username = "other_user" +password = "other_user" +pool_size = 21 +statement_timeout = 15000 +connect_timeout = 1000 +idle_timeout = 1000 + +# Shard configs are structured as pool..shards. +# Each shard config contains a list of servers that make up the shard +# and the database name to use. +[pools.sharded_db.shards.0] +# Array of servers in the shard, each server entry is an array of `[host, port, role]` +servers = [["127.0.0.1", 5432, "primary"], ["localhost", 5432, "replica"]] + +# Array of mirrors for the shard, each mirror entry is an array of `[host, port, index of server in servers array]` +# Traffic hitting the server identified by the index will be sent to the mirror. +# mirrors = [["1.2.3.4", 5432, 0], ["1.2.3.4", 5432, 1]] + +# Database name (e.g. "postgres") +database = "shard0" + +[pools.sharded_db.shards.1] +servers = [["127.0.0.1", 5432, "primary"], ["localhost", 5432, "replica"]] +database = "shard1" + +[pools.sharded_db.shards.2] +servers = [["127.0.0.1", 5432, "primary" ], ["localhost", 5432, "replica" ]] +database = "shard2" + + +[pools.postgres] +pool_mode = "transaction" +default_role = "primary" +query_parser_enabled = true +primary_reads_enabled = true +sharding_function = "pg_bigint_hash" + +[pools.postgres.users.0] +username = "pgtest" +password = "test123" +pool_size = 20 +min_pool_size = 3 +server_lifetime = 60000 +statement_timeout = 0 + +[pools.postgres.shards.0] +servers = [ + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5432, "replica" ] +] +database = "postgres" diff --git a/benchmark/report.md b/benchmark/report.md index 957db1c..f883734 100644 --- a/benchmark/report.md +++ b/benchmark/report.md @@ -25,10 +25,10 @@ Memory ``` Benchmark Configuration ======================= -Timestamp: 20251119_123527 -Duration: 30s +Timestamp: 20251208_094817 +Duration: 40s Scale Factor: 1 -Connection Counts: 10 30 50 +Connection Counts: 10 50 Workload: TPC-B-like (pgbench default) Database Configuration @@ -41,7 +41,8 @@ DB User: pgtest Pool Configuration ========== pgpool: localhost 5433 -pgbouncer: localhost 6432 +pgcat: localhost 6432 +pgbouncer: localhost 6433 ``` ## Workload: tpcb @@ -50,38 +51,35 @@ pgbouncer: localhost 6432 #### Throughput (TPS - Higher is Better) -| Connections | direct | pgpool | pgbouncer | -|-------------|-----------|-----------|-----------| -| 10 | 585.238364 | 586.118595 | 604.660117 | -| 30 | 582.777928 | 532.676985 | 714.917133 | -| 50 | 589.293650 | 398.812499 | 783.499747 | +| Connections | direct | pgpool | pgcat | pgbouncer | +|-------------|-----------|-----------|-----------|-----------| +| 10 | 540.992544 | 629.573937 | 400.953194 | 508.470016 | +| 50 | 520.948243 | 513.098964 | 444.584482 | 719.135004 | #### Average Latency (ms - Lower is Better) -| Connections | direct | pgpool | pgbouncer | -|-------------|-----------|-----------|-----------| -| 10 | 17.087 | 17.061 | 16.538 | -| 30 | 51.478 | 56.319 | 41.963 | -| 50 | 84.847 | 125.372 | 63.816 | +| Connections | direct | pgpool | pgcat | pgbouncer | +|-------------|-----------|-----------|-----------|-----------| +| 10 | 18.485 | 15.884 | 24.941 | 19.667 | +| 50 | 95.979 | 97.447 | 112.465 | 69.528 | +--- ### Protocol: prepared #### Throughput (TPS - Higher is Better) -| Connections | direct | pgpool | pgbouncer | -|-------------|-----------|-----------|-----------| -| 10 | 657.164027 | 693.167688 | 700.220246 | -| 30 | 638.176354 | 616.605597 | 818.683994 | -| 50 | 648.179381 | 590.052991 | 841.249706 | +| Connections | direct | pgpool | pgcat | pgbouncer | +|-------------|-----------|-----------|-----------|-----------| +| 10 | 603.806235 | 671.575983 | NA | 484.158180 | +| 50 | 571.118586 | 819.022499 | NA | 774.326083 | #### Average Latency (ms - Lower is Better) -| Connections | direct | pgpool | pgbouncer | -|-------------|-----------|-----------|-----------| -| 10 | 15.217 | 14.427 | 14.281 | -| 30 | 47.009 | 48.653 | 36.644 | -| 50 | 77.139 | 84.738 | 59.435 | +| Connections | direct | pgpool | pgcat | pgbouncer | +|-------------|-----------|-----------|-----------|-----------| +| 10 | 16.562 | 14.890 | NA | 20.654 | +| 50 | 87.547 | 61.048 | NA | 64.572 | --- @@ -89,18 +87,16 @@ pgbouncer: localhost 6432 #### Throughput (TPS - Higher is Better) -| Connections | direct | pgpool | pgbouncer | -|-------------|-----------|-----------|-----------| -| 10 | 619.224861 | 753.693232 | 663.680040 | -| 30 | 608.975762 | 828.391829 | 746.985918 | -| 50 | 613.787577 | 802.184268 | 749.947154 | +| Connections | direct | pgpool | pgcat | pgbouncer | +|-------------|-----------|-----------|-----------|-----------| +| 10 | 572.214114 | 701.134502 | 416.828321 | 548.480541 | +| 50 | 442.619273 | 852.513829 | 573.369829 | 734.414236 | #### Average Latency (ms - Lower is Better) -| Connections | direct | pgpool | pgbouncer | -|-------------|-----------|-----------|-----------| -| 10 | 16.149 | 13.268 | 15.068 | -| 30 | 49.263 | 36.215 | 40.161 | -| 50 | 81.461 | 62.330 | 66.671 | +| Connections | direct | pgpool | pgcat | pgbouncer | +|-------------|-----------|-----------|-----------|-----------| +| 10 | 17.476 | 14.263 | 23.991 | 18.232 | +| 50 | 112.964 | 58.650 | 87.204 | 68.081 | --- diff --git a/benchmark/scripts/generate_report.sh b/benchmark/scripts/generate_report.sh index 87db921..19d6753 100755 --- a/benchmark/scripts/generate_report.sh +++ b/benchmark/scripts/generate_report.sh @@ -85,6 +85,7 @@ echo "Target,Workload,Connections,Protocol,TPS,Latency_Avg_ms,Latency_Stddev_ms, TARGETS="" [ -d "${RESULTS_DIR}/direct" ] && TARGETS="${TARGETS} direct" [ -d "${RESULTS_DIR}/pgpool" ] && TARGETS="${TARGETS} pgpool" +[ -d "${RESULTS_DIR}/pgcat" ] && TARGETS="${TARGETS} pgcat" [ -d "${RESULTS_DIR}/pgbouncer" ] && TARGETS="${TARGETS} pgbouncer" # Detect workloads and connection counts from files diff --git a/benchmark/scripts/run_benchmark.sh b/benchmark/scripts/run_benchmark.sh index efee8e7..19738fb 100755 --- a/benchmark/scripts/run_benchmark.sh +++ b/benchmark/scripts/run_benchmark.sh @@ -25,12 +25,14 @@ DB_NAME="${DB_NAME:-postgres}" PGPOOL_HOST="${PGPOOL_HOST:-localhost}" PGPOOL_PORT="${PGPOOL_PORT:-5433}" PGBOUNCER_HOST="${PGBOUNCER_HOST:-localhost}" -PGBOUNCER_PORT="${PGBOUNCER_PORT:-6432}" +PGBOUNCER_PORT="${PGBOUNCER_PORT:-6433}" +PGCAT_HOST="${PGCAT_HOST:-localhost}" +PGCAT_PORT="${PGCAT_PORT:-6432}" # Benchmark configuration -DURATION="${DURATION:-30}" +DURATION="${DURATION:-40}" SCALE_FACTOR="${SCALE_FACTOR:-1}" -CONNECTION_COUNTS="${CONNECTION_COUNTS:-10 50 100}" +CONNECTION_COUNTS="${CONNECTION_COUNTS:-10 50}" PROTOCOLS=("simple" "extended" "prepared") # Results directory @@ -43,6 +45,7 @@ echo -e "${BLUE}=== PostgreSQL Connection Pool Benchmark ===${NC}\n" # Create results directories mkdir -p "${RUN_DIR}/pgpool" mkdir -p "${RUN_DIR}/pgbouncer" +mkdir -p "${RUN_DIR}/pgcat" mkdir -p "${RUN_DIR}/direct" mkdir -p "${RUN_DIR}/logs" @@ -66,6 +69,7 @@ DB User: ${DB_USER} Pool Configuration ========== pgpool: ${PGPOOL_HOST} ${PGPOOL_PORT} +pgcat: ${PGCAT_HOST} ${PGCAT_PORT} pgbouncer: ${PGBOUNCER_HOST} ${PGBOUNCER_PORT} EOF @@ -140,6 +144,9 @@ DIRECT_AVAILABLE=$? check_service "pgpool" "${PGPOOL_HOST}" "${PGPOOL_PORT}" PGPOOL_AVAILABLE=$? +check_service "pgcat" "${PGCAT_HOST}" "${PGCAT_PORT}" +PGCAT_AVAILABLE=$? + check_service "pgbouncer" "${PGBOUNCER_HOST}" "${PGBOUNCER_PORT}" PGBOUNCER_AVAILABLE=$? @@ -154,6 +161,7 @@ fi TARGETS=() [ $DIRECT_AVAILABLE -eq 0 ] && TARGETS+=("direct:${DB_HOST}:${DB_PORT}") [ $PGPOOL_AVAILABLE -eq 0 ] && TARGETS+=("pgpool:${PGPOOL_HOST}:${PGPOOL_PORT}") +[ $PGCAT_AVAILABLE -eq 0 ] && TARGETS+=("pgcat:${PGCAT_HOST}:${PGCAT_PORT}") [ $PGBOUNCER_AVAILABLE -eq 0 ] && TARGETS+=("pgbouncer:${PGBOUNCER_HOST}:${PGBOUNCER_PORT}") echo -e "${BLUE}=== Initializing pgbench tables ===${NC}\n" diff --git a/pool/client.go b/pool/client.go index 9478144..81cc972 100644 --- a/pool/client.go +++ b/pool/client.go @@ -70,25 +70,46 @@ type clientAuth struct { } type ClientConn struct { - node Node - msgReader *proto.MessageReader - Conn *net.TCPConn - ErrCount int - ProcID uint32 - Secret uint32 - auth clientAuth - PrepStmts map[string]*Stmt - Server *ServConn - Wait chan WaitType - State ClientState + node Node + msgReader *proto.MessageReader + Conn *net.TCPConn + ErrCount int + ProcID uint32 + Secret uint32 + auth clientAuth + PrepStmts map[string]*Stmt + Server *ServConn + serverWBuf []byte + Wait chan WaitType + State ClientState } func (client *ClientConn) ID() string { return client.Conn.RemoteAddr().String() } +func (client *ClientConn) WriteToServer(msg proto.Message) error { + slog.Debug("buffer", "client", client.ID(), "msg", msg.Type().String()) + + // buffer client messages for bulk writing + client.serverWBuf = append(client.serverWBuf, msg.Encode()...) + if client.State != ClientSyncWait && len(client.serverWBuf) < 4096 { + return nil + } + + slog.Debug("batch send", "client", client.ID(), "message", msg.Type().String()) + + if err := client.Server.Write(&proto.RawMsg{Data: client.serverWBuf}); err != nil { + return err + } + + client.serverWBuf = []byte{} + + return nil +} + func (client *ClientConn) Write(msg proto.Message) error { - slog.Debug("send message", "client", client.ID(), "msg", msg.Type().String()) + slog.Debug("-->>", "client", client.ID(), "msg", msg.Type().String()) if _, err := client.Conn.Write(msg.Encode()); err != nil { slog.Error("client write", "error", err) @@ -106,11 +127,12 @@ func (pls *Pools) NewClient(conn *net.TCPConn) { var pool *Pool client := &ClientConn{ - Conn: conn, - msgReader: proto.NewReader(conn, true), - State: ClientInit, - PrepStmts: map[string]*Stmt{}, - Wait: make(chan WaitType), + Conn: conn, + msgReader: proto.NewReader(conn, true), + State: ClientInit, + PrepStmts: map[string]*Stmt{}, + Wait: make(chan WaitType), + serverWBuf: make([]byte, 0, 4096), } slog.Info("new client:", "addr", conn.RemoteAddr().String()) @@ -123,7 +145,7 @@ func (pls *Pools) NewClient(conn *net.TCPConn) { break } - slog.Debug("client:", "addr", client.ID(), "message", msg.Type().String) + slog.Debug("recv:", "client", client.ID(), "msg", msg.Type().String) switch msg.Type() { case proto.StartupMsg: @@ -132,6 +154,8 @@ func (pls *Pools) NewClient(conn *net.TCPConn) { if err == nil { pool.handleClient(client) } + client.State = ClientTerminate + case proto.SSLRequest: // TODO: support SSL _ = client.Write(&proto.RawMsg{Data: []byte{'N'}}) @@ -147,12 +171,12 @@ func (pls *Pools) NewClient(conn *net.TCPConn) { } } - slog.Info("close client:", "addr", client.ID()) - _ = client.Conn.Close() - if pool != nil { pool.remove(client) } + + slog.Info("close client:", "addr", client.ID()) + _ = client.Conn.Close() } func (pls *Pools) clientAuth(client *ClientConn, db, user string) (*Pool, error) { @@ -345,7 +369,7 @@ func (p *Pool) handleClient(client *ClientConn) { break } - slog.Debug("client:", "addr", client.ID(), "message", fmt.Sprintf("%#v", msg)) + slog.Debug("recv:", "client", client.ID(), "msg", fmt.Sprintf("%#v", msg)) switch msg.Type() { case proto.Terminate: @@ -397,10 +421,7 @@ func (p *Pool) handleClient(client *ClientConn) { client.doBind(msg.(*proto.BindMsg)) continue case proto.Sync, proto.CopyFail, proto.CopyDone: - if p.WaitLst.Len() > 0 { - client.State = ClientSyncWait - } - + client.State = ClientSyncWait default: } @@ -409,9 +430,7 @@ func (p *Pool) handleClient(client *ClientConn) { continue } - slog.Debug("forward:", "client", client.ID(), "server", client.Server.ID(), "message", msg.Type().String()) - - _ = client.Server.Write(msg) + _ = client.WriteToServer(msg) // we must wait/sync with server message processing, otherwise // the message processing for query session between client/server thread will be messed up @@ -426,7 +445,12 @@ func (p *Pool) handleClient(client *ClientConn) { } server := client.Server - if p.WaitLst.Len() > 0 { + if p.WaitLen.Load() == 0 { + server.Wait <- ServerCopyResp + continue + } + + if p.WaitLen.Load() > 0 { p.free(client.Server) slog.Debug("Unpair", "server", server.ID(), "client", client.ID()) } diff --git a/pool/deque.go b/pool/deque.go index 3f553c5..5b6a371 100644 --- a/pool/deque.go +++ b/pool/deque.go @@ -2,6 +2,7 @@ package pool import ( "sync" + "sync/atomic" ) type DequeNode[T any] struct { @@ -13,42 +14,23 @@ type DequeNode[T any] struct { type Deque[T any] struct { head *DequeNode[T] tail *DequeNode[T] - size int - lock sync.RWMutex + size atomic.Int64 + lock sync.Mutex } func NewDeque[T any]() *Deque[T] { - return &Deque[T]{ + d := &Deque[T]{ head: nil, tail: nil, - size: 0, } -} - -func (d *Deque[T]) PushFront(data T) { - d.lock.Lock() - defer d.lock.Unlock() - - node := &DequeNode[T]{data: data} - - if d.head == nil { - d.head = node - d.tail = node - } else { - node.next = d.head - d.head.prev = node - d.head = node - } - - d.size++ + d.size.Store(0) + return d } func (d *Deque[T]) PushBack(data T) { - d.lock.Lock() - defer d.lock.Unlock() - node := &DequeNode[T]{data: data} + d.lock.Lock() if d.tail == nil { d.head = node d.tail = node @@ -57,8 +39,9 @@ func (d *Deque[T]) PushBack(data T) { d.tail.next = node d.tail = node } + d.lock.Unlock() - d.size++ + d.size.Add(1) } func (d *Deque[T]) PopFront() (T, bool) { @@ -80,76 +63,19 @@ func (d *Deque[T]) PopFront() (T, bool) { d.head.prev = nil } - d.size-- + d.size.Add(-1) return data, true } -func (d *Deque[T]) PopBack() (T, bool) { - d.lock.Lock() - defer d.lock.Unlock() - - var zero T - if d.tail == nil { - return zero, false - } - - data := d.tail.data - - if d.head == d.tail { - d.head = nil - d.tail = nil - } else { - d.tail = d.tail.prev - d.tail.next = nil - } - - d.size-- - return data, true -} - -func (d *Deque[T]) PeekFront() (T, bool) { - d.lock.RLock() - defer d.lock.RUnlock() - - var zero T - if d.head == nil { - return zero, false - } - - return d.head.data, true -} - -func (d *Deque[T]) PeekBack() (T, bool) { - d.lock.RLock() - defer d.lock.RUnlock() - - var zero T - if d.tail == nil { - return zero, false - } - - return d.tail.data, true -} - func (d *Deque[T]) Len() int { - d.lock.RLock() - defer d.lock.RUnlock() - - return d.size -} - -func (d *Deque[T]) IsEmpty() bool { - d.lock.RLock() - defer d.lock.RUnlock() - - return d.size == 0 + // Lock-free size check - this is the main benefit + return int(d.size.Load()) } func (d *Deque[T]) Clear() { d.lock.Lock() - defer d.lock.Unlock() - d.head = nil d.tail = nil - d.size = 0 -} \ No newline at end of file + d.size.Store(0) + d.lock.Unlock() +} diff --git a/pool/deque_test.go b/pool/deque_test.go index 77c03e4..2996f32 100644 --- a/pool/deque_test.go +++ b/pool/deque_test.go @@ -13,40 +13,9 @@ func TestDeque_NewDeque(t *testing.T) { if deque.Len() != 0 { t.Errorf("Expected empty deque to have length 0, got %d", deque.Len()) } - if !deque.IsEmpty() { - t.Error("Expected new deque to be empty") - } } -func TestDeque_PushFront(t *testing.T) { - deque := pool.NewDeque[int]() - - deque.PushFront(1) - if deque.Len() != 1 { - t.Errorf("Expected length 1, got %d", deque.Len()) - } - - front, ok := deque.PeekFront() - if !ok || front != 1 { - t.Errorf("Expected front to be 1, got %d", front) - } - - back, ok := deque.PeekBack() - if !ok || back != 1 { - t.Errorf("Expected back to be 1, got %d", back) - } - - deque.PushFront(2) - front, ok = deque.PeekFront() - if !ok || front != 2 { - t.Errorf("Expected front to be 2, got %d", front) - } - - back, ok = deque.PeekBack() - if !ok || back != 1 { - t.Errorf("Expected back to be 1, got %d", back) - } -} +// TestDeque_PushFront removed - PushFront method not implemented func TestDeque_PushBack(t *testing.T) { deque := pool.NewDeque[int]() @@ -56,25 +25,9 @@ func TestDeque_PushBack(t *testing.T) { t.Errorf("Expected length 1, got %d", deque.Len()) } - front, ok := deque.PeekFront() - if !ok || front != 1 { - t.Errorf("Expected front to be 1, got %d", front) - } - - back, ok := deque.PeekBack() - if !ok || back != 1 { - t.Errorf("Expected back to be 1, got %d", back) - } - deque.PushBack(2) - front, ok = deque.PeekFront() - if !ok || front != 1 { - t.Errorf("Expected front to be 1, got %d", front) - } - - back, ok = deque.PeekBack() - if !ok || back != 2 { - t.Errorf("Expected back to be 2, got %d", back) + if deque.Len() != 2 { + t.Errorf("Expected length 2, got %d", deque.Len()) } } @@ -98,39 +51,9 @@ func TestDeque_PopFront(t *testing.T) { if deque.Len() != 2 { t.Errorf("Expected length 2, got %d", deque.Len()) } - - front, ok := deque.PeekFront() - if !ok || front != 2 { - t.Errorf("Expected front to be 2, got %d", front) - } } -func TestDeque_PopBack(t *testing.T) { - deque := pool.NewDeque[int]() - - _, ok := deque.PopBack() - if ok { - t.Error("Expected PopBack on empty deque to return false") - } - - deque.PushBack(1) - deque.PushBack(2) - deque.PushBack(3) - - val, ok := deque.PopBack() - if !ok || val != 3 { - t.Errorf("Expected to pop 3, got %d", val) - } - - if deque.Len() != 2 { - t.Errorf("Expected length 2, got %d", deque.Len()) - } - - back, ok := deque.PeekBack() - if !ok || back != 2 { - t.Errorf("Expected back to be 2, got %d", back) - } -} +// TestDeque_PopBack removed - PopBack method not implemented func TestDeque_GenericPointer(t *testing.T) { type TestStruct struct { @@ -142,19 +65,9 @@ func TestDeque_GenericPointer(t *testing.T) { obj1 := &TestStruct{value: 42} obj2 := &TestStruct{value: 84} - deque.PushFront(obj1) + deque.PushBack(obj1) deque.PushBack(obj2) - front, ok := deque.PeekFront() - if !ok || front.value != 42 { - t.Errorf("Expected front value to be 42, got %d", front.value) - } - - back, ok := deque.PeekBack() - if !ok || back.value != 84 { - t.Errorf("Expected back value to be 84, got %d", back.value) - } - popped, ok := deque.PopFront() if !ok || popped.value != 42 { t.Errorf("Expected popped value to be 42, got %d", popped.value) @@ -177,53 +90,26 @@ func TestDeque_Clear(t *testing.T) { if deque.Len() != 0 { t.Errorf("Expected length 0 after clear, got %d", deque.Len()) } - - if !deque.IsEmpty() { - t.Error("Expected deque to be empty after clear") - } } func TestDeque_SingleElement(t *testing.T) { deque := pool.NewDeque[int]() - deque.PushFront(42) + deque.PushBack(42) - val, ok := deque.PopBack() + val, ok := deque.PopFront() if !ok || val != 42 { t.Errorf("Expected to pop 42, got %d", val) } - if !deque.IsEmpty() { + if deque.Len() != 0 { t.Error("Expected deque to be empty after popping single element") } } // Concurrent Tests -func TestDeque_ConcurrentPushFront(t *testing.T) { - deque := pool.NewDeque[int]() - numRoutines := 100 - itemsPerRoutine := 100 - - var wg sync.WaitGroup - wg.Add(numRoutines) - - for i := range numRoutines { - go func(start int) { - defer wg.Done() - for j := range itemsPerRoutine { - deque.PushFront(start*itemsPerRoutine + j) - } - }(i) - } - - wg.Wait() - - expectedLen := numRoutines * itemsPerRoutine - if deque.Len() != expectedLen { - t.Errorf("Expected length %d, got %d", expectedLen, deque.Len()) - } -} +// TestDeque_ConcurrentPushFront removed - PushFront method not implemented func TestDeque_ConcurrentPushBack(t *testing.T) { deque := pool.NewDeque[int]() @@ -283,48 +169,12 @@ func TestDeque_ConcurrentPopFront(t *testing.T) { t.Errorf("Expected to pop %d items, got %d", numItems, successCount.Load()) } - if !deque.IsEmpty() { + if deque.Len() != 0 { t.Errorf("Expected deque to be empty, but has %d items", deque.Len()) } } -func TestDeque_ConcurrentPopBack(t *testing.T) { - deque := pool.NewDeque[int]() - numItems := 10000 - - // Pre-populate the deque - for i := range numItems { - deque.PushBack(i) - } - - numGoroutines := 100 - var wg sync.WaitGroup - var successCount atomic.Int32 - wg.Add(numGoroutines) - - for range numGoroutines { - go func() { - defer wg.Done() - for { - _, ok := deque.PopBack() - if !ok { - break - } - successCount.Add(1) - } - }() - } - - wg.Wait() - - if int(successCount.Load()) != numItems { - t.Errorf("Expected to pop %d items, got %d", numItems, successCount.Load()) - } - - if !deque.IsEmpty() { - t.Errorf("Expected deque to be empty, but has %d items", deque.Len()) - } -} +// TestDeque_ConcurrentPopBack removed - PopBack method not implemented func TestDeque_ConcurrentMixedOperations(t *testing.T) { deque := pool.NewDeque[int]() @@ -336,17 +186,8 @@ func TestDeque_ConcurrentMixedOperations(t *testing.T) { var popCount atomic.Int32 // Concurrent push operations - wg.Add(numGoroutines * 2) + wg.Add(numGoroutines) for i := range numGoroutines { - // PushFront goroutines - go func(start int) { - defer wg.Done() - for j := range operationsPerGoroutine { - deque.PushFront(start*operationsPerGoroutine + j) - pushCount.Add(1) - } - }(i) - // PushBack goroutines go func(start int) { defer wg.Done() @@ -354,19 +195,19 @@ func TestDeque_ConcurrentMixedOperations(t *testing.T) { deque.PushBack(start*operationsPerGoroutine + j) pushCount.Add(1) } - }(i + numGoroutines) + }(i) } wg.Wait() // Verify all items were pushed - expectedLen := numGoroutines * 2 * operationsPerGoroutine + expectedLen := numGoroutines * operationsPerGoroutine if deque.Len() != expectedLen { t.Errorf("Expected length %d after pushes, got %d", expectedLen, deque.Len()) } // Concurrent pop operations - wg.Add(numGoroutines * 2) + wg.Add(numGoroutines) for range numGoroutines { // PopFront goroutines go func() { @@ -379,18 +220,6 @@ func TestDeque_ConcurrentMixedOperations(t *testing.T) { popCount.Add(1) } }() - - // PopBack goroutines - go func() { - defer wg.Done() - for { - _, ok := deque.PopBack() - if !ok { - break - } - popCount.Add(1) - } - }() } wg.Wait() @@ -399,7 +228,7 @@ func TestDeque_ConcurrentMixedOperations(t *testing.T) { t.Errorf("Expected to pop %d items, got %d", expectedLen, popCount.Load()) } - if !deque.IsEmpty() { + if deque.Len() != 0 { t.Errorf("Expected deque to be empty, but has %d items", deque.Len()) } } @@ -416,35 +245,7 @@ func TestDeque_ConcurrentReaders(t *testing.T) { numGoroutines := 100 iterations := 1000 var wg sync.WaitGroup - wg.Add(numGoroutines * 3) - - // Concurrent PeekFront operations - for range numGoroutines { - go func() { - defer wg.Done() - for range iterations { - val, ok := deque.PeekFront() - if !ok || val != 0 { - t.Errorf("PeekFront failed or returned wrong value") - return - } - } - }() - } - - // Concurrent PeekBack operations - for range numGoroutines { - go func() { - defer wg.Done() - for range iterations { - val, ok := deque.PeekBack() - if !ok || val != numItems-1 { - t.Errorf("PeekBack failed or returned wrong value") - return - } - } - }() - } + wg.Add(numGoroutines) // Concurrent Len operations for range numGoroutines { @@ -514,7 +315,7 @@ func TestDeque_ConcurrentPushAndPop(t *testing.T) { t.Errorf("Produced %d items but consumed %d", produced.Load(), consumed.Load()) } - if !deque.IsEmpty() { + if deque.Len() != 0 { t.Errorf("Expected deque to be empty, but has %d items", deque.Len()) } } @@ -548,7 +349,7 @@ func TestDeque_ConcurrentClear(t *testing.T) { // After all operations, clear one final time and verify deque.Clear() - if !deque.IsEmpty() { + if deque.Len() != 0 { t.Errorf("Expected deque to be empty after clear, but has %d items", deque.Len()) } } @@ -559,25 +360,17 @@ func TestDeque_ConcurrentStressTest(t *testing.T) { operationsPerGoroutine := 500 var wg sync.WaitGroup - wg.Add(numGoroutines * 4) + wg.Add(numGoroutines * 2) // Mix of all operations for i := range numGoroutines { - // PushFront - go func(id int) { - defer wg.Done() - for j := range operationsPerGoroutine { - deque.PushFront(id*operationsPerGoroutine + j) - } - }(i) - // PushBack go func(id int) { defer wg.Done() for j := range operationsPerGoroutine { deque.PushBack(id*operationsPerGoroutine + j) } - }(i + numGoroutines) + }(i) // PopFront go func() { @@ -586,20 +379,10 @@ func TestDeque_ConcurrentStressTest(t *testing.T) { deque.PopFront() } }() - - // PopBack - go func() { - defer wg.Done() - for range operationsPerGoroutine { - deque.PopBack() - } - }() } wg.Wait() // Just verify the deque is in a valid state (no panics occurred) _ = deque.Len() - _, _ = deque.PeekFront() - _, _ = deque.PeekBack() } diff --git a/pool/list.go b/pool/list.go index 59171a1..274581e 100644 --- a/pool/list.go +++ b/pool/list.go @@ -11,7 +11,6 @@ type Node struct { // DList represents a generic doubly linked list type DList struct { - head *Node tail *Node size int lock sync.RWMutex @@ -19,41 +18,15 @@ type DList struct { func NewDList() *DList { return &DList{ - head: nil, tail: nil, size: 0, } } -func (dl *DList) Head() *Node { - dl.lock.RLock() - defer dl.lock.RUnlock() - return dl.head -} - func (dl *DList) Tail() *Node { - dl.lock.RLock() - defer dl.lock.RUnlock() return dl.tail } -func (dl *DList) PushFront(node *Node) { - dl.lock.Lock() - defer dl.lock.Unlock() - - node.prev = nil - node.next = dl.head - - if dl.head == nil { - dl.tail = node - } else { - dl.head.prev = node - } - - dl.head = node - dl.size++ -} - func (dl *DList) PushBack(node *Node) { dl.lock.Lock() defer dl.lock.Unlock() @@ -61,9 +34,7 @@ func (dl *DList) PushBack(node *Node) { node.prev = dl.tail node.next = nil - if dl.tail == nil { - dl.head = node - } else { + if dl.tail != nil { dl.tail.next = node } @@ -71,64 +42,10 @@ func (dl *DList) PushBack(node *Node) { dl.size++ } -func (dl *DList) PopFront() (*Node, bool) { - dl.lock.Lock() - defer dl.lock.Unlock() - - if dl.head == nil { - return nil, false - } - - node := dl.head - - if dl.head == dl.tail { - dl.head = nil - dl.tail = nil - } else { - dl.head = dl.head.next - dl.head.prev = nil - } - - node.next = nil - node.prev = nil - - dl.size-- - return node, true -} - -func (dl *DList) PopBack() (*Node, bool) { - dl.lock.Lock() - defer dl.lock.Unlock() - - if dl.tail == nil { - return nil, false - } - - node := dl.tail - - if dl.head == dl.tail { - dl.head = nil - dl.tail = nil - } else { - dl.tail = dl.tail.prev - dl.tail.next = nil - } - - node.prev = nil - node.next = nil - - dl.size-- - return node, true -} - func (dl *DList) Delete(node *Node) bool { dl.lock.Lock() defer dl.lock.Unlock() - if dl.head == node { - dl.head = node.next - } - if dl.tail == node { dl.tail = node.prev } @@ -155,18 +72,10 @@ func (dl *DList) Len() int { return dl.size } -func (dl *DList) IsEmpty() bool { - dl.lock.RLock() - defer dl.lock.RUnlock() - - return dl.size == 0 -} - func (dl *DList) Clear() { dl.lock.Lock() defer dl.lock.Unlock() - dl.head = nil dl.tail = nil dl.size = 0 } @@ -175,9 +84,9 @@ func (dl *DList) ForEach(fn func(*Node)) { dl.lock.RLock() defer dl.lock.RUnlock() - current := dl.head + current := dl.tail for current != nil { fn(current) - current = current.next + current = current.prev } } diff --git a/pool/list_test.go b/pool/list_test.go index 988db21..2d132ab 100644 --- a/pool/list_test.go +++ b/pool/list_test.go @@ -12,9 +12,6 @@ func TestNewDList(t *testing.T) { if dl == nil { t.Fatal("NewDList returned nil") } - if dl.Head() != nil { - t.Error("New list should have nil Head") - } if dl.Tail() != nil { t.Error("New list should have nil Tail") } @@ -23,38 +20,6 @@ func TestNewDList(t *testing.T) { } } -func TestDList_PushFront(t *testing.T) { - dl := pool.NewDList() - - // Test pushing to empty list - node1 := &pool.Node{} - dl.PushFront(node1) - - if dl.Head() != node1 { - t.Error("Head should be node1") - } - if dl.Tail() != node1 { - t.Error("Tail should be node1") - } - if dl.Len() != 1 { - t.Errorf("Size should be 1, got %d", dl.Len()) - } - - // Test pushing second node - node2 := &pool.Node{} - dl.PushFront(node2) - - if dl.Head() != node2 { - t.Error("Head should be node2") - } - if dl.Tail() != node1 { - t.Error("Tail should still be node1") - } - if dl.Len() != 2 { - t.Errorf("Size should be 2, got %d", dl.Len()) - } -} - func TestDList_PushBack(t *testing.T) { dl := pool.NewDList() @@ -62,9 +27,6 @@ func TestDList_PushBack(t *testing.T) { node1 := &pool.Node{} dl.PushBack(node1) - if dl.Head() != node1 { - t.Error("Head should be node1") - } if dl.Tail() != node1 { t.Error("Tail should be node1") } @@ -76,9 +38,6 @@ func TestDList_PushBack(t *testing.T) { node2 := &pool.Node{} dl.PushBack(node2) - if dl.Head() != node1 { - t.Error("Head should still be node1") - } if dl.Tail() != node2 { t.Error("Tail should be node2") } @@ -87,120 +46,6 @@ func TestDList_PushBack(t *testing.T) { } } -func TestDList_PopFront(t *testing.T) { - dl := pool.NewDList() - - // Test popping from empty list - node, ok := dl.PopFront() - if ok { - t.Error("PopFront should return false for empty list") - } - if node != nil { - t.Error("PopFront should return nil node for empty list") - } - - // Test popping single element - node1 := &pool.Node{} - dl.PushFront(node1) - poppedNode, ok := dl.PopFront() - - if !ok { - t.Error("PopFront should return true") - } - if poppedNode != node1 { - t.Error("Should return the pushed node") - } - if dl.Head() != nil { - t.Error("Head should be nil after popping last element") - } - if dl.Tail() != nil { - t.Error("Tail should be nil after popping last element") - } - if dl.Len() != 0 { - t.Errorf("Size should be 0, got %d", dl.Len()) - } - - // Test popping from multiple elements - node2 := &pool.Node{} - node3 := &pool.Node{} - dl.PushFront(node2) - dl.PushFront(node3) - - poppedNode, ok = dl.PopFront() - if !ok { - t.Error("PopFront should return true") - } - if poppedNode != node3 { - t.Error("Should return node3") - } - if dl.Head() != node2 { - t.Error("Head should be node2") - } - if dl.Tail() != node2 { - t.Error("Tail should be node2") - } - if dl.Len() != 1 { - t.Errorf("Size should be 1, got %d", dl.Len()) - } -} - -func TestPopBack(t *testing.T) { - dl := pool.NewDList() - - // Test popping from empty list - node, ok := dl.PopBack() - if ok { - t.Error("PopBack should return false for empty list") - } - if node != nil { - t.Error("PopBack should return nil node for empty list") - } - - // Test popping single element - node1 := &pool.Node{} - dl.PushBack(node1) - poppedNode, ok := dl.PopBack() - - if !ok { - t.Error("PopBack should return true") - } - if poppedNode != node1 { - t.Error("Should return the pushed node") - } - if dl.Head() != nil { - t.Error("Head should be nil after popping last element") - } - if dl.Tail() != nil { - t.Error("Tail should be nil after popping last element") - } - if dl.Len() != 0 { - t.Errorf("Size should be 0, got %d", dl.Len()) - } - - // Test popping from multiple elements - node2 := &pool.Node{} - node3 := &pool.Node{} - dl.PushBack(node2) - dl.PushBack(node3) - - poppedNode, ok = dl.PopBack() - if !ok { - t.Error("PopBack should return true") - } - if poppedNode != node3 { - t.Error("Should return node3") - } - if dl.Head() != node2 { - t.Error("Head should be node2") - } - if dl.Tail() != node2 { - t.Error("Tail should be node2") - } - if dl.Len() != 1 { - t.Errorf("Size should be 1, got %d", dl.Len()) - } -} - func TestDList_Delete(t *testing.T) { dl := pool.NewDList() @@ -221,26 +66,23 @@ func TestDList_Delete(t *testing.T) { t.Errorf("Size should be 2, got %d", dl.Len()) } - // Test deleting head + // Test deleting first node (node1) ok = dl.Delete(node1) if !ok { t.Error("Delete should return true") } - if dl.Head() != node3 { - t.Error("Head should be node3") - } if dl.Len() != 1 { t.Errorf("Size should be 1, got %d", dl.Len()) } + if dl.Tail() != node3 { + t.Error("Tail should be node3") + } - // Test deleting tail (last remaining node) + // Test deleting last remaining node ok = dl.Delete(node3) if !ok { t.Error("Delete should return true") } - if dl.Head() != nil { - t.Error("Head should be nil") - } if dl.Tail() != nil { t.Error("Tail should be nil") } @@ -256,57 +98,27 @@ func TestDList_Len(t *testing.T) { t.Errorf("Len should be 0, got %d", dl.Len()) } - dl.PushFront(&pool.Node{}) + dl.PushBack(&pool.Node{}) if dl.Len() != 1 { t.Errorf("Len should be 1, got %d", dl.Len()) } - - dl.PushBack(&pool.Node{}) - if dl.Len() != 2 { - t.Errorf("Len should be 2, got %d", dl.Len()) - } -} - -func TestDList_IsEmpty(t *testing.T) { - dl := pool.NewDList() - - if !dl.IsEmpty() { - t.Error("New list should be empty") - } - - dl.PushFront(&pool.Node{}) - if dl.IsEmpty() { - t.Error("List with elements should not be empty") - } - - dl.PopFront() - if !dl.IsEmpty() { - t.Error("List should be empty after removing all elements") - } } func TestDList_Clear(t *testing.T) { dl := pool.NewDList() // Add some elements - dl.PushFront(&pool.Node{}) dl.PushBack(&pool.Node{}) dl.PushBack(&pool.Node{}) dl.Clear() - if dl.Head() != nil { - t.Error("Head should be nil after Clear") - } if dl.Tail() != nil { t.Error("Tail should be nil after Clear") } if dl.Len() != 0 { t.Errorf("Size should be 0 after Clear, got %d", dl.Len()) } - if !dl.IsEmpty() { - t.Error("List should be empty after Clear") - } } func TestDList_ForEach(t *testing.T) { @@ -336,9 +148,11 @@ func TestDList_ForEach(t *testing.T) { t.Errorf("Expected 3 nodes visited, got %d", len(visitedNodes)) } + // ForEach traverses from tail to head (reverse order) for i, node := range visitedNodes { - if node != nodes[i] { - t.Errorf("Node at index %d doesn't match expected node", i) + expectedIndex := len(nodes) - 1 - i + if node != nodes[expectedIndex] { + t.Errorf("Node at index %d doesn't match expected node at index %d", i, expectedIndex) } } } @@ -349,18 +163,20 @@ func TestDList_ConcurrentAccess(t *testing.T) { const numOperations = 100 var wg sync.WaitGroup + var nodeMutex sync.Mutex + nodes := make([]*pool.Node, 0, numGoroutines*numOperations) // Test concurrent push operations for range numGoroutines { wg.Add(1) go func() { defer wg.Done() - for j := range numOperations { - if j%2 == 0 { - dl.PushFront(&pool.Node{}) - } else { - dl.PushBack(&pool.Node{}) - } + for range numOperations { + node := &pool.Node{} + dl.PushBack(node) + nodeMutex.Lock() + nodes = append(nodes, node) + nodeMutex.Unlock() } }() } @@ -372,102 +188,22 @@ func TestDList_ConcurrentAccess(t *testing.T) { t.Errorf("Expected size %d after concurrent pushes, got %d", expectedSize, dl.Len()) } - // Test concurrent pop operations - for range numGoroutines { + // Test concurrent delete operations + for i := range numGoroutines { wg.Add(1) - go func() { + go func(goroutineID int) { defer wg.Done() - for j := range numOperations { - if j%2 == 0 { - dl.PopFront() - } else { - dl.PopBack() - } + start := goroutineID * numOperations + end := start + numOperations + for j := start; j < end; j++ { + dl.Delete(nodes[j]) } - }() + }(i) } wg.Wait() if dl.Len() != 0 { - t.Errorf("Expected size 0 after concurrent pops, got %d", dl.Len()) - } - if !dl.IsEmpty() { - t.Error("List should be empty after all pops") - } -} - -func TestDList_MixedOperations(t *testing.T) { - dl := pool.NewDList() - - // Test sequence of mixed operations - node1 := &pool.Node{} - node2 := &pool.Node{} - node3 := &pool.Node{} - - dl.PushFront(node1) - dl.PushBack(node2) - dl.PushFront(node3) - - if dl.Len() != 3 { - t.Errorf("Expected length 3, got %d", dl.Len()) - } - - // List should be: node3 -> node1 -> node2 - if dl.Head() != node3 { - t.Error("Head should be node3") - } - if dl.Tail() != node2 { - t.Error("Tail should be node2") - } - - node, ok := dl.PopBack() - if !ok || node != node2 { - t.Error("Should pop node2") - } - - dl.Delete(node1) - - if dl.Len() != 1 { - t.Errorf("Expected length 1, got %d", dl.Len()) - } - if dl.Head() != node3 || dl.Tail() != node3 { - t.Error("Only node3 should remain") - } - - dl.Clear() - if !dl.IsEmpty() { - t.Error("List should be empty after Clear") - } - - const numGoroutines = 10 - const numOperations = 100 - - totalSize := numGoroutines * numOperations - var wg sync.WaitGroup - - for range totalSize / 5 { - dl.PushBack(&pool.Node{}) - } - - for range numGoroutines { - wg.Add(1) - go func() { - defer wg.Done() - for j := range numOperations { - if j%5 == 0 { - dl.PopFront() - } else { - dl.PushFront(&pool.Node{}) - } - } - }() - } - - wg.Wait() - - expectedSize := totalSize * 4 / 5 - if dl.Len() != expectedSize { - t.Errorf("Expected size %d after concurrent pushes and pop, got %d", expectedSize, dl.Len()) + t.Errorf("Expected size 0 after concurrent deletes, got %d", dl.Len()) } } diff --git a/pool/pool.go b/pool/pool.go index 77cbe8c..2ec8646 100644 --- a/pool/pool.go +++ b/pool/pool.go @@ -4,6 +4,7 @@ import ( "fmt" "log/slog" "sync" + "sync/atomic" "time" "unsafe" ) @@ -29,7 +30,8 @@ type Pool struct { User User CancelServer *ServConn // dedicated to send cancel request to server FreeLst *Deque[*ServConn] - WaitLst *Deque[*ClientConn] + WaitClient chan *ClientConn + WaitLen atomic.Int32 Servers *DList Clients *DList } @@ -46,10 +48,10 @@ func (pls *Pools) Get(db, user string) (*Pool, error) { } pool := &Pool{ - Servers: NewDList(), - Clients: NewDList(), - FreeLst: NewDeque[*ServConn](), - WaitLst: NewDeque[*ClientConn](), + Servers: NewDList(), + Clients: NewDList(), + FreeLst: NewDeque[*ServConn](), + WaitClient: make(chan *ClientConn), } for _, d := range pls.Cfg.Dbs { @@ -95,8 +97,8 @@ func (p *Pool) stats() { slog.Debug("Stats", "pool", p.Db.Name+":"+p.User.Name, "servers", p.Servers.Len(), "clients", p.Clients.Len(), - "free servers", p.FreeLst.Len(), - "wait", p.WaitLst.Len()) + "wait", p.WaitLen.Load(), + "free servers", p.FreeLst.Len()) time.Sleep(time.Second * 10) } @@ -105,27 +107,18 @@ func (p *Pool) stats() { // asynchrous routine to process WaitLst in the pool func (p *Pool) do() { for { - if p.WaitLst.IsEmpty() { - time.Sleep(time.Second) - continue - } - - client, ok := p.WaitLst.PopFront() - if !ok { - continue - } - + client := <-p.WaitClient if client.State != ClientWait { continue } + p.WaitLen.Add(-1) var server *ServConn popserver: srv, ok := p.FreeLst.PopFront() if !ok { if srv, err := p.NewServConn(); err != nil { - p.WaitLst.PushFront(client) - continue + goto popserver } else { server = srv } @@ -145,9 +138,15 @@ func (p *Pool) do() { } func (p *Pool) wait(client *ClientConn) { + if srv, ok := p.FreeLst.PopFront(); ok { + p.use(srv, client) + return + } + slog.Debug("Wait Start", "client", client.ID()) client.State = ClientWait - p.WaitLst.PushBack(client) + p.WaitLen.Add(1) + p.WaitClient <- client <-client.Wait slog.Debug("Wait End", "client", client.ID()) } @@ -206,8 +205,6 @@ func (p *Pool) CloseAll() { _ = client.Conn.Close() }) - // passively rely on schedule routine to remove outdated client in wait list - p.WaitLst.Clear() p.Clients.Clear() p.Servers.ForEach(func(node *Node) { diff --git a/pool/server.go b/pool/server.go index 7b39e40..41e36e6 100644 --- a/pool/server.go +++ b/pool/server.go @@ -29,18 +29,19 @@ const ( // Server Connection type ServConn struct { - node Node - Conn *net.TCPConn - msgReader *proto.MessageReader - scram scramInfo - Client *ClientConn - Wait chan WaitType - Params map[string]string - PrepStmts map[string]*Stmt - ProcID uint32 - Secret uint32 - State ServerState - lock sync.RWMutex + node Node + Conn *net.TCPConn + msgReader *proto.MessageReader + scram scramInfo + Client *ClientConn + clientWBuf []byte + Wait chan WaitType + Params map[string]string + PrepStmts map[string]*Stmt + ProcID uint32 + Secret uint32 + State ServerState + lock sync.RWMutex } func (sc *ServConn) ID() string { @@ -48,17 +49,44 @@ func (sc *ServConn) ID() string { } func (sc *ServConn) Write(msg proto.Message) error { - slog.Debug("send", "server", sc.ID(), "message", msg.Type().String()) + data := msg.Encode() + slog.Debug("-->>", "server", sc.ID(), "data", string(data)) - if _, err := sc.Conn.Write(msg.Encode()); err != nil { + if _, err := sc.Conn.Write(data); err != nil { slog.Error("server write", "error", err) return err } return nil } +func (sc *ServConn) WriteToClient(msg proto.Message) error { + slog.Debug("buffer", "server", sc.ID(), "message", msg.Type().String()) + + sc.clientWBuf = append(sc.clientWBuf, msg.Encode()...) + + switch msg.Type() { + case proto.ReadyForQuery, proto.ErrorResp, proto.CopyFail: + break + default: + if len(sc.clientWBuf) < 4096 { + return nil + } + } + + slog.Debug("batch send", "from server", sc.ID(), "on message", msg.Type().String()) + + err := sc.Client.Write(&proto.RawMsg{Data: sc.clientWBuf}) + if err != nil { + slog.Error("server write", "error", err) + } + + sc.clientWBuf = []byte{} + + return err +} + func (p *Pool) EnsureServer() (*ServConn, error) { - if node := p.Servers.Head(); node != nil { + if node := p.Servers.Tail(); node != nil { return (*ServConn)(unsafe.Pointer(node)), nil } @@ -81,13 +109,12 @@ func (p *Pool) handleServer(sc *ServConn) { break } - slog.Debug("server message:", "addr", sc.ID(), "msg", fmt.Sprintf("%#v", m)) + slog.Debug("recv:", "server", sc.ID(), "msg", fmt.Sprintf("%#v", m)) switch m.Type() { case proto.ReadyForQuery: - // TODO: client can be removed asynchronously on error if sc.Client != nil { - _ = sc.Client.Write(m) + _ = sc.WriteToClient(m) msg := m.(*proto.ReadyQuery) if sc.Client.State == ClientSyncWait { if msg.State == 'T' { @@ -118,7 +145,7 @@ func (p *Pool) handleServer(sc *ServConn) { slog.Error("server without client:ignore message:", "type", m.Type().String()) continue } - _ = sc.Client.Write(m) + _ = sc.WriteToClient(m) } } @@ -146,12 +173,13 @@ func (p *Pool) NewServConn() (*ServConn, error) { _ = conn.SetKeepAlive(true) srvConn := &ServConn{ - State: ServerInit, - Conn: conn, - Wait: make(chan WaitType), - msgReader: proto.NewReader(conn, false), - Params: map[string]string{}, - PrepStmts: map[string]*Stmt{}, + State: ServerInit, + Conn: conn, + Wait: make(chan WaitType), + msgReader: proto.NewReader(conn, false), + Params: map[string]string{}, + PrepStmts: map[string]*Stmt{}, + clientWBuf: make([]byte, 0, 4096), } if err = p.serverAuth(srvConn); err != nil { return nil, err