test: make TEST_TAP_TIMEOUT actually fire, and enable it by default

test/infra/control/env-isolated.bash

@@ -104,7 +104,14 @@ export TEST_PY_TAP_REPEAT="${TEST_PY_TAP_REPEAT:-1}"
 export TEST_PY_TAP_SHUFFLE_LIMIT="${TEST_PY_TAP_SHUFFLE_LIMIT:-0}"
 export TEST_PY_TAP_DUMP_RUNTIME="${TEST_PY_TAP_DUMP_RUNTIME:-1}"
 export TEST_PY_TAP_DUMP_STATS="${TEST_PY_TAP_DUMP_STATS:-1}"
-export TEST_TAP_TIMEOUT="${TEST_TAP_TIMEOUT:-0}"
+# Per-test wall-clock budget, in seconds. 0 disables it entirely, which was
+# the previous default: a hung TAP test then ran until the CI job itself was
+# killed. 1800 is ~2.4x the slowest single test measured across 47 groups
+# (reg_test_3765_ssl_pollout-t, 12.5 min; then test_cluster_sync-t 10.5,
+# set_testing-240-t 7.8, test_auth_methods-t 7.7 -- only 4 tests exceed 5
+# minutes at all), so it cannot fire on a merely slow test while still
+# catching a hang long before the 90-minute step budget.
+export TEST_TAP_TIMEOUT="${TEST_TAP_TIMEOUT:-1800}"
 
 # Cluster sync test support — expose first cluster node admin port for replica validation
 if [ "${NUM_CLUSTER_NODES}" -gt 0 ]; then

test/scripts/bin/proxysql-tester.py

@@ -5,6 +5,7 @@
 import os
 import pymysql
 import sys
+import select
 import subprocess
 import random
 import time
@@ -842,27 +843,58 @@ def disk_usage():
                         sys.exit(1)
                     continue
 
-                # Run test with timeout if specified
+                # Run test with timeout if specified.
+                #
+                # The read has to be non-blocking for tap_timeout to mean
+                # anything. readline() blocks until a full line arrives, so on
+                # a test that hangs while producing no output -- precisely the
+                # case this timeout exists to catch -- the deadline check below
+                # it was simply never reached, and the test ran until CI killed
+                # the job. select() bounds the wait so the check always runs,
+                # and reading raw chunks rather than lines means a test that
+                # stops mid-line cannot wedge us either.
                 try:
                     start_time = time.time()
+                    buf = b''
                     while True:
-                        line = fop.stdout.readline()
-                        if not line and fop.poll() is not None:
-                            break
-                        if line:
-                            log.debug(f"msg: {line.decode('utf-8').strip()}")
-
                         if tap_timeout > 0 and (time.time() - start_time) > tap_timeout:
                             raise subprocess.TimeoutExpired(fop.args, tap_timeout)
-
+
+                        wait = 1.0
+                        if tap_timeout > 0:
+                            wait = max(0.0, min(1.0, tap_timeout - (time.time() - start_time)))
+                        ready, _, _ = select.select([fop.stdout], [], [], wait)
+
+                        if ready:
+                            chunk = os.read(fop.stdout.fileno(), 65536)
+                            if not chunk:
+                                break                     # EOF: test finished
+                            buf += chunk
+                            while b'\n' in buf:
+                                line, buf = buf.split(b'\n', 1)
+                                log.debug(f"msg: {line.decode('utf-8', 'replace').strip()}")
+                        elif fop.poll() is not None:
+                            break
+
+                    if buf:                               # trailing partial line
+                        log.debug(f"msg: {buf.decode('utf-8', 'replace').strip()}")
+
                     fop.wait()
                 except subprocess.TimeoutExpired:
                     fop.kill()
                     log.critical(f"TAP test {fo_num+1}/{len(tap_tests)} '{os.path.basename(fo_cmd)}' timed out after {tap_timeout} seconds")
                     self.padmin_command(f"LOGENTRY '{TAP} test {fo_num+1}/{len(tap_tests)} \'{os.path.basename(fo_cmd)}\' timed out after {tap_timeout} seconds'")
                     # Drain any remaining output
                     for line in fop.stdout:
-                        log.debug(f"msg: {line.decode('utf-8').strip()}")
+                        log.debug(f"msg: {line.decode('utf-8', 'replace').strip()}")
+                    # Reap the child. kill() signals but does not wait, so
+                    # returncode stays None until we do -- and the shared exit
+                    # path below evaluates abs(int(fop.returncode)), which
+                    # raises TypeError on None. That path was unreachable while
+                    # the timeout could never fire; now that it can, a timed-out
+                    # test would abort the whole group instead of failing one
+                    # test. wait() yields -SIGKILL, so the test scores non-zero.
+                    fop.wait()
                 except Exception as e:
                     log.critical(f"TAP test {fo_num+1}/{len(tap_tests)} '{os.path.basename(fo_cmd)}' - test threw an exception !!!: {e}")
                     self.padmin_command(f"LOGENTRY '{TAP} test {fo_num+1}/{len(tap_tests)} \'{os.path.basename(fo_cmd)}\' - test threw an exception !!!'")