diff --git a/.github/workflows/hotpath.yml b/.github/workflows/hotpath.yml new file mode 100644 index 0000000..d00da4b --- /dev/null +++ b/.github/workflows/hotpath.yml @@ -0,0 +1,58 @@ +name: Hotpath profiling coverage + +on: + workflow_dispatch: + workflow_call: + +permissions: + contents: read + +jobs: + profile: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + feature: [hotpath-profile, hotpath-alloc-profile] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: '3.14' + - uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly-2026-09-25 + - uses: Swatinem/rust-cache@v2 + - uses: astral-sh/setup-uv@v6 + - name: Install build and test dependencies + run: uv sync --locked --no-install-project + - name: Audit hooks and test the inventory parser + run: | + cargo test --locked --manifest-path tools/hotpath-coverage/Cargo.toml + cargo run --locked --manifest-path tools/hotpath-coverage/Cargo.toml + - name: Build the instrumented extension + run: uv run --no-sync maturin develop --release --features ${{ matrix.feature }} + - name: Generate TLS fixtures + run: uv run --no-sync python scripts/generate_test_tls_certs.py + - name: Test profile lifecycle and tooling + run: | + uv run --no-sync pytest -q tests/test_hotpath_profile.py + uv run --no-sync pytest -q -m tooling tests/tooling/test_profile_with_hotpath.py + - name: Collect every workload in fresh processes + run: >- + uv run --no-sync python scripts/profile_with_hotpath.py all + --output target/hotpath-ci + --callbacks 1000 --tasks 200 --task-batch-size 50 + --tcp-roundtrips 30 --timers 200 --timer-batch-size 50 + --connections 5 --probe-iterations 3 --concurrency 2 + --requests-per-connection 3 --bulk-bytes 131072 + --idle-connections 4 --idle-cycles 2 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ matrix.feature }}-${{ github.run_attempt }} + path: target/hotpath-ci/ + retention-days: 14 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a367419..19ad1c5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -64,6 +64,9 @@ jobs: vibeio-native: uses: ./.github/workflows/vibeio-native.yml + hotpath-profile: + uses: ./.github/workflows/hotpath.yml + clippy: name: clippy (risk-focused) runs-on: ubuntu-latest diff --git a/Cargo.toml b/Cargo.toml index 3484955..c844312 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,8 @@ windows-sys = { version = "0.61", features = ["Wdk_Foundation", "Wdk_Storage_Fil [features] default = [] hotpath-profile = ["dep:hotpath", "hotpath/hotpath"] +# Separate diagnostic build: Rust allocations, not CPython's allocator. +hotpath-alloc-profile = ["hotpath-profile", "hotpath/hotpath-alloc"] # Opt-in: faster scheduler turns, but measured TCP workloads can regress. scheduler-batch-cache = [] blocking-default = ["dep:rusty_pool"] diff --git a/benches/profile_workloads.py b/benches/profile_workloads.py new file mode 100644 index 0000000..3ff56ae --- /dev/null +++ b/benches/profile_workloads.py @@ -0,0 +1,190 @@ +"""Small correctness-checked probes for runtime paths outside TCP benchmarks.""" + +from __future__ import annotations + +import asyncio +import contextvars +import os +import signal +import socket +import sys +import tempfile +import threading +from pathlib import Path + + +async def threadsafe(iterations: int) -> None: + loop = asyncio.get_running_loop() + done = loop.create_future() + seen = [] + value = contextvars.ContextVar("profile_value", default=-1) + + def receive(i): + seen.append((i, value.get())) + if len(seen) == iterations: + done.set_result(None) + + def producer(): + for i in range(iterations): + value.set(i) + loop.call_soon_threadsafe(receive, i) + + worker = threading.Thread(target=producer, name="profile-producer") + worker.start() + try: + await done + finally: + worker.join() + assert seen == [(i, i) for i in range(iterations)] + + +async def executor_dns(iterations: int) -> None: + loop = asyncio.get_running_loop() + for i in range(iterations): + assert await loop.run_in_executor(None, sum, range(i + 1)) == i * (i + 1) // 2 + addresses = await loop.getaddrinfo("localhost", 80, type=socket.SOCK_STREAM) + assert addresses + host, port = await loop.getnameinfo( + ("127.0.0.1", 80), socket.NI_NUMERICHOST | socket.NI_NUMERICSERV + ) + assert (host, port) == ("127.0.0.1", "80") + + +async def udp(iterations: int) -> None: + loop = asyncio.get_running_loop() + with ( + socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as left, + socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as right, + ): + left.bind(("127.0.0.1", 0)) + right.bind(("127.0.0.1", 0)) + left.setblocking(False) + right.setblocking(False) + for i in range(iterations): + payload = i.to_bytes(8, "little") + await loop.sock_sendto(left, payload, right.getsockname()) + data, addr = await loop.sock_recvfrom(right, 64) + assert data == payload + await loop.sock_sendto(right, data, addr) + buffer = bytearray(64) + size, _ = await loop.sock_recvfrom_into(left, buffer) + assert buffer[:size] == payload + + +async def socket_ops(iterations: int) -> None: + loop = asyncio.get_running_loop() + with socket.socket() as listener, socket.socket() as client: + listener.bind(("127.0.0.1", 0)) + listener.listen() + listener.setblocking(False) + client.setblocking(False) + connecting = asyncio.ensure_future( + loop.sock_connect(client, listener.getsockname()) + ) + server, _ = await loop.sock_accept(listener) + await connecting + with server: + server.setblocking(False) + for _ in range(iterations): + await loop.sock_sendall(client, b"socket probe") + data = await loop.sock_recv(server, 64) + assert data == b"socket probe" + await loop.sock_sendall(server, data) + buffer = bytearray(64) + size = await loop.sock_recv_into(client, buffer) + assert buffer[:size] == data + + +async def subprocess_pipes(iterations: int) -> None: + for _ in range(iterations): + process = await asyncio.create_subprocess_exec( + sys.executable, + "-c", + "import sys; sys.stdout.buffer.write(sys.stdin.buffer.read()); sys.stderr.write('err')", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + out, err = await process.communicate(b"pipe probe" * 1024) + assert out == b"pipe probe" * 1024 and err == b"err" + assert process.returncode == 0 + finally: + if process.returncode is None: + process.kill() + await process.wait() + + +async def unix_streams(iterations: int) -> None: + async def echo(reader, writer): + try: + while data := await reader.read(4096): + writer.write(data) + await writer.drain() + finally: + writer.close() + await writer.wait_closed() + + with tempfile.TemporaryDirectory(prefix="rsloop-profile-") as directory: + path = str(Path(directory) / "sock") + server = await asyncio.start_unix_server(echo, path) + try: + reader, writer = await asyncio.open_unix_connection(path) + try: + for _ in range(iterations): + writer.write(b"unix") + await writer.drain() + assert await reader.readexactly(4) == b"unix" + finally: + writer.close() + await writer.wait_closed() + finally: + server.close() + await server.wait_closed() + + +async def signals(iterations: int) -> None: + loop = asyncio.get_running_loop() + received = asyncio.Event() + # Registration is dispatched to rsloop's runtime thread. SIGWINCH is + # harmless before installation; retry delivery until that thread is ready. + loop.add_signal_handler(signal.SIGWINCH, received.set) + try: + for _ in range(iterations): + received.clear() + while not received.is_set(): + os.kill(os.getpid(), signal.SIGWINCH) + try: + await asyncio.wait_for(received.wait(), 0.05) + except asyncio.TimeoutError: + pass + finally: + loop.remove_signal_handler(signal.SIGWINCH) + + +async def cancellation(iterations: int) -> None: + async def waiting(): + await asyncio.sleep(60) + + for _ in range(iterations): + task = asyncio.create_task(waiting()) + await asyncio.sleep(0) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + else: + raise AssertionError("cancellation was lost") + + +RUNNERS = { + "threadsafe": threadsafe, + "executor_dns": executor_dns, + "udp": udp, + "socket_ops": socket_ops, + "subprocess_pipes": subprocess_pipes, + "unix_streams": unix_streams, + "signals": signals, + "cancellation": cancellation, +} diff --git a/benches/scheduler_workloads.py b/benches/scheduler_workloads.py index b4c81a3..af83c33 100644 --- a/benches/scheduler_workloads.py +++ b/benches/scheduler_workloads.py @@ -15,8 +15,14 @@ async def bench_timers( """Include scheduling, expiration/cancellation, and handle release.""" if iterations <= 0 or batch_size <= 0: raise ValueError("iterations and batch_size must be positive") - if mode not in {"timers_retained", "timers_discarded", "timers_cancelled"}: + if mode not in { + "timers_retained", + "timers_discarded", + "timers_cancelled", + "timers_mixed", + }: raise ValueError(f"unknown timer mode: {mode}") + cancelled = mode in {"timers_cancelled", "timers_mixed"} loop = asyncio.get_running_loop() started = time.perf_counter() outstanding = iterations @@ -32,12 +38,20 @@ def fire(completion: asyncio.Future[None] = done) -> None: completion.set_result(None) handles = [] - for _ in range(count): + for index in range(count): if mode == "timers_discarded": loop.call_later(0, fire) else: - handles.append(loop.call_later(0, fire)) - if mode == "timers_cancelled": + # Deterministic scattered future deadlines exercise the heap + # independently of the zero-delay FIFO. Cancel before expiry. + delay = ( + 60 + ((index * 15319) % 2048) / 1024 + if mode == "timers_mixed" + else 0 + ) + handles.append(loop.call_later(delay, fire)) + if cancelled: + handle = None for handle in handles: handle.cancel() del handle @@ -45,7 +59,7 @@ def fire(completion: asyncio.Future[None] = done) -> None: # cancelled deadlines to be processed as well. loop.call_later(0, done.set_result, None) await done - assert remaining == (count if mode == "timers_cancelled" else 0) + assert remaining == (count if cancelled else 0) handles.clear() outstanding -= count return ChildResult( diff --git a/docs/development.md b/docs/development.md index 279d6f5..be1f38c 100644 --- a/docs/development.md +++ b/docs/development.md @@ -19,6 +19,10 @@ Quick Rust check: cargo check ``` +For function timing, async poll data, allocation profiles, and a workload +coverage inventory, see [Hotpath Profiling](hotpath-rs-profile.md). Profiling +hooks are opt-in and absent from normal builds. + The nightly pin supplies the allocator API merged in [rust-lang/rust#156882](https://github.com/rust-lang/rust/pull/156882). The opt-in `scheduler-batch-cache` Cargo feature uses the stabilized `Allocator` and diff --git a/docs/hotpath-lab.md b/docs/hotpath-lab.md index 27d115c..eb67afa 100644 --- a/docs/hotpath-lab.md +++ b/docs/hotpath-lab.md @@ -101,13 +101,18 @@ This harness does not apply model output or create commits automatically. workloads; holdout is the default comparison suite. Additional opt-in scheduler probes are available through `--workloads`: -`timers_retained`, `timers_discarded`, `timers_cancelled`, and +`timers_retained`, `timers_discarded`, `timers_cancelled`, `timers_mixed`, and `tcp_connect_churn`. Timer probes schedule zero-delay timers in batches and include dispatch or cancellation and handle release. Retained handles remain live until their batch drains; discarded handles can reuse a freelist while callbacks remain pending. Training uses 1,000,000 timers in batches of 10,000; holdout uses 1,200,000 in batches of 777. +`timers_mixed` instead schedules deterministic, scattered deadlines 60–62 seconds +ahead, cancels them, and waits for their queue entries to drain. It checks heap +costs independently of the zero-delay timer path; none of these callbacks should +execute. Include it when changing timer data structures. + The connection probe uses explicit stdlib stream protocols, so native stream wrappers cannot bypass generic TCP-reader startup. It creates a connection, echoes one payload, and closes it before the next connection. Training uses diff --git a/docs/hotpath-rs-profile.md b/docs/hotpath-rs-profile.md index 16cd64e..e03696d 100644 --- a/docs/hotpath-rs-profile.md +++ b/docs/hotpath-rs-profile.md @@ -1,25 +1,470 @@ -# hotpath-rs profiling experiment +# Profiling rsloop with hotpath-rs -This branch adds an opt-in `hotpath-profile` Cargo feature. Normal builds have no -profiler hooks. The feature instruments callback scheduling and invocation, -context operations, dispatcher work, and selected TCP stream paths. A pair of -private `_loop.hotpath_start(path)` / `_loop.hotpath_stop()` functions bounds the -profile around Python-driven workloads; `scripts/profile_with_hotpath.py` runs -the existing `compare_event_loops.py` workloads with a short warmup. +The opt-in `hotpath-profile` feature instruments rsloop's Rust function bodies +across the Python bindings, callback/context/timer machinery, dispatcher, +transports, TLS, subprocesses, socket operations, and embedded vibeio runtime. +This includes the platform drivers and optional runtime modules. Normal builds +compile these hooks out and do not link hotpath. -## Reproduce +Reports default to JSON with **unlimited rows** for functions, futures, and +threads. Function entries include source locations, call/sample counts, and +p50/p95/p99 durations. Async functions use `future = true`, adding poll counts +and poll durations. A separate `hotpath-alloc-profile` build counts Rust +allocations through hotpath's global allocator. + +## Collect a coverage suite + +Use the project interpreter (`.venv/bin/python` on Unix, +`.venv/Scripts/python.exe` on Windows). Build first, then profile; do not compile, +run tests, or run another benchmark while collecting profiles. ```bash +.venv/bin/python scripts/generate_test_tls_certs.py tests/fixtures/tls .venv/bin/maturin develop --release --features hotpath-profile -.venv/bin/python scripts/profile_with_hotpath.py callbacks --output target/hotpath-rs/callbacks.txt -.venv/bin/python scripts/profile_with_hotpath.py tasks --output target/hotpath-rs/tasks.txt -.venv/bin/python scripts/profile_with_hotpath.py tcp_streams --output target/hotpath-rs/tcp_streams.txt +.venv/bin/python scripts/profile_with_hotpath.py all --output target/hotpath-rs/timing +``` + +`all` runs each of these workloads in its own fresh process: + +| Area | Workloads | +| --- | --- | +| Scheduling | callbacks, tasks, task options, retained/discarded/cancelled timers, mixed future deadlines | +| TCP and application protocols | native echo, generic protocol connection churn, HTTP, TLS HTTP, WebSocket framing with/without TLS, mixed messages, bulk transfer, idle connections | +| Other runtime paths | thread-safe/context callbacks, executor/DNS, UDP sockets, raw TCP sockets, subprocess pipes, Unix streams, POSIX signals, cancellation | + +The probes check their results. Unix/signal probes are explicitly skipped on +unsupported platforms. TLS needs the generated local certificates. A failed or +timed-out child makes the overall command fail, while preserving other reports +and failure logs. Each child has a timeout. Existing outputs are not overwritten. + +Each output directory contains: + +- `.json`: the full upstream hotpath report, without a top-50 cutoff. +- `.metadata.json`: the actual extension path/hash, build features, + interpreter/platform, workload parameters, effective profiler environment, + outcome and instrumented duration including report flushing. +- `.log`: child output/errors, including failed runs. +- `manifest.json`: commands, statuses, checkout revision, dirty state, and source + hashes. These hashes describe the checkout, **not proof that the installed + extension was built from it**. Rebuild after editing Rust; the binary hash + identifies what actually ran. +- `coverage.json`: the source inventory with an `observed` flag for every + definition, exclusion reasons, and any observed names the inventory cannot map. +- `summary.md`: per-workload counts and links to the reports. + +Profiles include cold loop setup, execution, and teardown. There is no in-process +warmup: hotpath's future/thread registries are process-global, so warming up in +the same process can contaminate their statistics. The private native API allows +one start/stop session per process and rejects attempts to restart it. + +The collector clears inherited `HOTPATH_*` settings before importing rsloop so +old filters, row limits, or output paths cannot silently change the result. +Effective settings are recorded. `--focus` explicitly filters function timing; +future/thread rows can still describe other work. `--time-sampling-rate` is in +`(0, 1]` and defaults to 1. A sampled report retains call counts and sampled-call +counts; estimates are not equivalent to timing every call. + +## Focus on a path or run your application + +```bash +.venv/bin/python scripts/profile_with_hotpath.py tcp_streams \ + --output target/hotpath-rs/tcp.json --tcp-roundtrips 5000 +.venv/bin/python scripts/profile_with_hotpath.py tls_http \ + --output target/hotpath-rs/tls.json --focus tls --time-sampling-rate 0.1 +.venv/bin/python scripts/profile_with_hotpath.py module \ + --output target/hotpath-rs/tls-tests.json --module pytest \ + --module-args -q tests/test_tls.py +.venv/bin/python scripts/profile_with_hotpath.py module \ + --output target/hotpath-rs/application.json --module your_application \ + --module-args --your-option value +``` + +Put `--module-args` last. Applications must select rsloop themselves. Modules +run inside the same profiling session, and their failures propagate after the +report is flushed. Use `--format table` for a single human-readable report; +`all` requires JSON for coverage analysis. Parameters such as `--timers`, +`--connections`, `--concurrency`, `--bulk-bytes`, and `--probe-iterations` let you +extend short probes into representative experiments. + +For allocations, use a separate build and fresh directory: + +```bash +.venv/bin/maturin develop --release --features hotpath-alloc-profile +.venv/bin/python scripts/profile_with_hotpath.py all --output target/hotpath-rs/allocations .venv/bin/maturin develop --release -.venv/bin/python benches/compare_event_loops.py --loops rsloop \ - --workloads callbacks,tasks,tcp_streams --repeat 5 --warmups 2 \ - --json-output target/hotpath-rs/uninstrumented.json ``` +Allocation mode changes the function report section from timing to allocation +statistics. With the collector's default environment, these are **exclusive +allocation bytes**, excluding nested instrumented calls; check the report's +`description` field. They are not inclusive like function timing. It does not +account for CPython's allocator, native libraries that +allocate outside Rust's global allocator, or explicit `System` allocations. +It measures allocation traffic, not retained heap size or peak RSS. Use the +normal benchmark harness for peak RSS. Always rebuild without profiling before +comparing performance. + +Instrumentation can also introduce allocations: hotpath 0.28.3 creates an +`Arc` for each measured async invocation in allocation mode. +Those allocations can appear in an enclosing rsloop function. Check the locked +profiler implementation and repeat with focused instrumentation before treating +small per-await allocations as production costs. + +## Keep coverage from drifting + +```bash +cargo run --locked --manifest-path tools/hotpath-coverage/Cargo.toml +cargo run --locked --manifest-path tools/hotpath-coverage/Cargo.toml -- --json +# Add attributes for newly introduced functions, then review and format: +cargo run --locked --manifest-path tools/hotpath-coverage/Cargo.toml -- --instrument +cargo fmt --all +``` + +The audit parses Rust syntax, including inactive platform/feature branches, +and rejects missing function hooks or async functions without future tracking. +It excludes tests, verification/mock helpers, const functions, profiler control, +ABI callbacks, and known signal-safe/post-fork code. It also rejects hooks added +to these excluded function definitions. Signal-handler call chains require +manual safety review when changed: a syntax audit is not call-graph analysis. +The standalone vibeio harness accepts the profiling cfg but keeps hooks disabled. +`scripts/run_rust_tests.py` gives profiling-feature test threads a 32 MiB default +stack: unoptimized nested async wrappers can otherwise exceed libtest's 2 MiB +stack when moving owned I/O arrays. It preserves an explicit `RUST_MIN_STACK`. +For direct debug `cargo test` with profiling, set `RUST_MIN_STACK=33554432` too. +Use release builds for profile collection. + +The inventory covers explicit functions/methods, not macro-generated code or +anonymous closures/async blocks as independent rows. Their executed work is +included in enclosing spans or instrumented callees; instrument a specific +boundary when finer attribution is needed. External dependencies, PyO3-generated +wrappers, Python frames, and kernel internals are not individually instrumented. + +**Unobserved does not mean cheap or unused.** The inventory includes Windows, +macOS, Linux, optional Cargo features, error branches and APIs a particular +workload never calls. Run representative applications, targeted test modules, +and native-platform builds to fill those gaps. Do not describe a Linux smoke +suite as 100% execution coverage. + +Function times are inclusive wall durations: nested calls overlap, blocking +socket waits are not CPU work, and parallel workers can sum beyond wall time. +Future poll durations help distinguish execution from suspension; thread CPU +samples are coarse and short-lived threads may not be sampled. Do not add nested +rows or call them self time. Broad instrumentation substantially perturbs tiny +functions. Use the profiles to choose a hypothesis, then validate it on normal +release builds with [paired holdouts](hotpath-lab.md). + +The implementation follows hotpath's [function/future instrumentation](https://hotpath.rs/functions) +and [report configuration](https://hotpath.rs/configuration). The locked hotpath +version and report format are recorded in the raw report. + +## Coverage check on October 4, 2026 + +On Linux x86-64, CPython 3.14.7, Rust 1.100.0-nightly and hotpath 0.28.3, +all 23 default workloads completed in both timing and allocation builds. +The syntax audit found 1,995 instrumented definitions with 1,959 distinct names +across all platform/feature branches, plus 27 intentionally excluded definitions. +Timing workloads observed 789 names; allocation workloads observed 788. Every +observed rsloop function mapped back to the inventory, and reports were untruncated. + +| Timing workload | Function rows | Future rows | +| --- | ---: | ---: | +| Callbacks | 170 | 0 | +| Native TCP streams | 494 | 9 | +| TLS HTTP | 538 | 3 | +| Subprocess pipes | 350 | 0 | + +Function row counts include the session wrapper. Counts can vary with scheduling +and workload sizes. These are coverage observations, not evidence of a speedup. +The local ignored artifacts are under `target/hotpath-rs/full-timing/` and +`target/hotpath-rs/full-allocations/`, each with a summary, provenance, and raw +reports. They describe the working-tree implementation, not an immutable release. +The `Hotpath profiling coverage` workflow repeats a smaller suite in both modes +and uploads the reports; the main test workflow invokes it too. + +## Performance investigation at `0e0c8d0` + +On October 4, 2026, a fresh investigation used the clean committed source on +the same Linux/CPython/toolchain platform above. All **23 workloads passed in +both timing and allocation modes**, observing 800 distinct rsloop function +names in each mode, with no unmapped names or truncated reports. No production +code was changed in this investigation. + +Raw reports, binary hashes, commands, and coverage are retained locally under +`target/hotpath-rs/performance-0e0c8d0/`. The `timing/` and `allocations/` +directories contain the full suites; `focused/` contains three independent +timing processes per selected path, and `allocations-focused/` contains a +profiler-overhead check. These ignored artifacts are not shipped with the repo. + +For each profiling build, the full-suite command was: + +```bash +.venv/bin/python scripts/profile_with_hotpath.py all \ + --output target/hotpath-rs/performance-0e0c8d0/timing \ + --callbacks 200000 --tasks 50000 --tcp-roundtrips 5000 \ + --timers 50000 --connections 500 --probe-iterations 200 \ + --concurrency 8 --requests-per-connection 200 --bulk-bytes 16777216 \ + --idle-connections 100 --idle-cycles 50 +``` + +Use a fresh output directory and change `timing` to `allocations` after building +the allocation feature. Task/timer batches remained at 500 and TCP payloads +at 1 KiB. Bulk transfer received 16 MiB on each of eight connections. + +### Focused timing findings + +Each focused run used `--focus` with the function name below and the same +relevant workload parameters. Values are median function totals across three +fresh processes; shares use the corresponding profiled session duration. + +| Workload | Focus | Median total (range) | Median session share | +| --- | --- | ---: | ---: | +| 200,000 callbacks | `schedule_callback_args` | 36.95 ms (36.87–37.42) | 33.60% | +| 50,000 retained timers | `LocalTimers::collect` | 12.60 ms (12.19–12.69) | 24.58% | +| 50,000 cancelled timers | `LocalTimers::collect` | 12.68 ms (12.60–12.73) | 27.07% | +| 128 MiB bulk receive | `ExactReadAccumulator::fill_from` | 38.46 ms (37.88–38.98) | 50.86% | +| 5,000 TCP roundtrips | `try_direct_tasked_write` | 46.22 ms (45.56–48.80) | 36.35% | + +These remain instrumented wall times, not production self time or achievable +speedups. Filtering leaves instrumentation checks and future tracking in the +binary. In the broad timer profiles, collection took about 51 ms; filtering +comparison hooks reduced that to about 12.6 ms. The broad profiles counted +436,400 heap comparisons for retained timers and 440,100 for cancelled timers. +The normal *whole timer workload* took only about 17–18 ms, so interpreting +the broad profile literally would substantially overstate this cost. + +### Recommended experiments, in order + +1. **Size initial pooled writes to the request.** In 500 generic TCP + connection/echo/close cycles, read pools allocated 31.2 MiB and write pools + 15.6 MiB, about 46.9 MiB combined, for 1 KiB messages. The write-pool row + records 1,000 acquisitions averaging 16 KiB. + `WriteBufferPoolState::acquire` in `src/transport/stream/buffers.rs` floors + allocations at `WRITE_BUFFER_BLOCK_SIZE` (16 KiB). Test a smaller initial + capacity that grows when needed while preserving the existing pool-slot + limit and write-buffer accounting. Reducing those 1,000 initial allocations + from 16 KiB to 1 KiB would remove about 14.6 MiB of allocation traffic in + this probe; that is a capacity estimate, not a measured speed or RSS gain. + Validate short connections, HTTP/TLS, mixed message sizes, bulk transfer, + partial writes, backpressure, and close ordering. Change read-buffer sizing + separately because smaller reads can increase syscall frequency. +2. **Reduce large exact-read copying if a safe design pays off.** + `ExactReadAccumulator::fill_from` in `src/transport/stream/fast.rs` remains + expensive with focused instrumentation. It already copies directly into a + single final Python bytes allocation, so removing a temporary accumulator + would not help. Explore receiving directly into the private result buffer + for large pending exact reads, or an internal owned-buffer handoff that + avoids the current copy. This is a substantial ownership/lifetime change: + preserve cancellation, partial EOF, transport shutdown, Python attachment, + and the rule that uninitialized bytes must never be exposed. Compare + several payload sizes and concurrency levels; the measured span includes + memory writes and possible page faults, not just instruction overhead. +3. **Reduce timer allocation and heap work.** `schedule_timer` accounted for + about 3.5 MiB of exclusive Rust allocations per 50,000 timers. The source + creates an `Arc` per timer; cancellation still leaves heap + entries to pop and release through the ready queue. Candidate designs + include reusable timer storage with generation-checked handles, or + cancellation-aware heap rebuilding. Keep equal-deadline order, finalizer + reentrancy, retained handles, and `when()` semantics. These probes use + zero-delay timers; add mixed future deadlines and cancellation densities + before selecting a data structure. +4. **Investigate callback object/queue costs at finer boundaries.** Scheduling + remains material, but `schedule_callback_args` includes context capture, + handle creation, and queue insertion. The 200,000-callback burst records + 12.0 MiB of allocation traffic in `try_enqueue_local_ready`; p50/p95/p99 + are all zero, consistent with occasional `VecDeque` growth, not a Rust + allocation per callback. The 50,000-task workload with bounded batches + records only 29.9 KiB there. Separate large-burst capacity growth from + steady-state scheduling, and add targeted spans around Python handle + creation before choosing a layout change. Rust allocation reports do not + measure CPython object allocations. The previous handle-freelist experiment + already rejected that approach for large bursts. + +The first experiment has the most concrete, limited change to test. The others +need more design work. Every candidate still requires an uninstrumented paired +comparison using [the holdout workflow](hotpath-lab.md). + +### Costs that should not drive a change yet + +- The TCP reader future reported **640,128 bytes in 20,004 allocations**. + There were 10,002 calls each to `acquire_read_buffer_async` and + `wait_until_async_readable`; each measured async call creates a 32-byte + hotpath allocation bridge on this platform. Repeating with + `--focus '/reader_task::run_tcp_socket_reader_task|::buffers::|::core_events::/'` + reduced the reader-task allocation row and its poll allocation count to + **zero**, while the read-pool row stayed at 96 KiB. Do not optimize away + rsloop awaits to fix these profiler allocations. +- In bulk transfer, `ReadBufferPool::wait_async` spans totalled 429.07 ms + across connections, but its future polls totalled only **0.324 ms**. The + span primarily describes suspension/backpressure, not expensive CPU work. + TLS socket polling similarly overlaps across worker threads. +- `try_direct_tasked_write` includes a mutex and the socket write. Its timing + does not establish mutex contention. Existing write coalescing reduced + 4,800 application writes to 3,200 direct-write attempts in HTTP; additional + batching must be justified against request latency. +- Runtime batch vectors still allocate 2 KiB per entry, but enabling the + existing cache is not an established fix: the + [earlier cache experiment](allocator-batch-results.md) confirmed a TCP + regression. Revisit only with a different hypothesis and fresh comparisons. + +### Normal release baseline + +After restoring `.venv/bin/maturin develop --release`, nine workloads ran in +five fresh measured processes each, with one discarded process per workload +beforehand. Workload order was shuffled within each measured block with seed +41026; GC was disabled. The local `normal_baseline.py` calls the same existing +benchmark functions as the collector. `normal-baseline/` retains its plan, +source hashes, extension hash/features, raw samples, CPU time, and peak RSS. +The binary was verified to have both profiling features disabled and no +`hotpath_start` entry point. No compilation or competing benchmark ran during +collection; CPU affinity was unrestricted. + +| Workload | Median workload time | Observed range | +| --- | ---: | ---: | +| Callbacks | 44.01 ms | 43.32–44.24 ms | +| Tasks | 85.70 ms | 84.21–86.26 ms | +| Retained timers | 17.81 ms | 17.57–18.87 ms | +| Cancelled timers | 17.11 ms | 16.91–17.17 ms | +| TCP streams | 79.31 ms | 78.31–79.92 ms | +| TCP connection churn | 83.23 ms | 82.39–84.23 ms | +| HTTP keep-alive | 31.60 ms | 31.53–31.85 ms | +| TLS HTTP | 55.29 ms | 53.27–55.49 ms | +| Bulk transfer | 61.65 ms | 60.65–62.54 ms | + +These are diagnostic baselines, not candidate comparisons. They use each +benchmark's own timed region, whereas hotpath sessions also include cold +loop setup/teardown and other wrapper work. Five short samples are not the +paired holdout gate and do not establish any optimization speedup. + +## Profile-guided implementation + +The implementation on `perf/profile-guided-optimizations`, based on `0e0c8d0`, +keeps two production changes: + +- Pooled write buffers start at the requested capacity with a 256-byte minimum + for tiny headers, instead of a 16 KiB minimum. Growth, pool-slot limits, + oversized-buffer rejection, and backpressure accounting are unchanged. +- Loop-thread zero-delay timers use an ordered FIFO alongside the timer heap. + Future deadlines and out-of-order insertions use the heap. The next timer is + selected by the same deadline/sequence ordering across both containers. + Pending timers outside the running loop retain their original heap, and + stopping a loop transfers outstanding entries back to it. Callback release + still happens outside timer borrows/locks. Per-timer `Arc` allocation and + Python handle/cancellation semantics are unchanged. + +The new `timers_mixed` workload schedules scattered future deadlines and cancels +them before expiry. It is available in both profiling and holdout tools, and +brings the profiling collector's default suite to 24 workloads. This probe +exposed extra work in an initial design that tried FIFO insertion for all +deadlines; the retained fast path is specifically for zero-delay timers. + +No bulk-read implementation change is retained. One trial fed fragments directly +into the private result bytes and returned their buffers to the pool earlier; +the combined candidate's bulk-transfer time increased 2.36% (interval ++0.26% to +3.96%). Another preserved pool recycling and deferred Future-reference +cloning until completion, but did not establish a bulk gain; its combined +candidate also regressed TCP time. These were combined experiments, not isolated +attribution of each slowdown to the read code. Both trials were reverted. +Regression tests for fragmented exact reads, overflow, partial EOF, exceptions, +cancellation, and a real large transfer with trailing bytes remain. + +Correctness validation passed 411 Rust tests with all features, 230 Python tests +(three skipped on the normal build), and 94 tooling tests. New Rust tests compare +random insertion, removal, and stop/start heap transfers against `BinaryHeap`, +and check that small pooled writes grow without losing data and reuse storage. + +Local benchmark artifacts are under `target/profile-guided-optimizations/`. +The `baseline/` and `final/` directories preserve the exact normal-release +packages and binary hashes; `final/source.patch` records the Rust change from +the base commit. Each comparison archives its runner/workloads, predeclared +parameters/order/seed, raw fresh-process samples, and paired bootstrap results. +`paired-12/` and `confirmation-12/` preserve the rejected intermediate trials; +`final-12/` measures the retained implementation. No profiling, builds, or tests +ran concurrently with these comparisons. CPU affinity was unrestricted. + +### Retained implementation: paired measurements + +The final comparison used 12 balanced, randomized baseline/candidate process +pairs per workload (seed 41029), with one full warmup and one measured workload +per process, GC disabled, and uninstrumented release binaries. Every measured +workload exceeded 0.25 seconds. Parameters were 1.5 million callbacks, 250,000 +tasks, 1.2 million timers, task/timer batches of 777, 25,000 TCP roundtrips with +8 KiB payloads, 2,500 connection cycles, eight concurrent connections with +4,000 requests each, and 128 MiB per bulk connection. + +Changes below are geometric means of **paired** time ratios, not ratios of the +displayed medians. The intervals use 20,000 bootstrap resamples and a 99.79% +confidence target, adjusting across 12 timing and 12 RSS metrics. + +| Workload | Baseline median (ms) | Candidate median (ms) | Paired time change (interval) | +| --- | ---: | ---: | ---: | +| Callbacks | 323.79 | 328.44 | +0.21% [-3.98%, +2.39%] | +| Tasks | 417.06 | 420.27 | +0.51% [-0.69%, +1.64%] | +| Retained zero-delay timers | 424.57 | 394.28 | -7.30% [-7.87%, -6.62%] | +| Discarded zero-delay timers | 389.26 | 353.88 | -8.37% [-9.98%, -6.86%] | +| Cancelled zero-delay timers | 412.40 | 377.56 | -8.74% [-9.98%, -7.68%] | +| Mixed future deadlines | 513.34 | 516.53 | +1.05% [+0.17%, +2.12%] | +| TCP streams | 484.69 | 483.39 | -0.10% [-1.88%, +1.83%] | +| TCP connection churn | 428.56 | 434.17 | -0.06% [-2.97%, +2.23%] | +| HTTP keep-alive | 578.02 | 594.49 | +2.55% [+1.54%, +3.89%] | +| TLS HTTP | 331.98 | 337.70 | +0.83% [-2.33%, +4.21%] | +| Mixed streams | 684.33 | 678.48 | -0.34% [-3.25%, +2.46%] | +| Bulk transfer | 444.10 | 445.81 | +0.30% [-1.62%, +2.06%] | + +Connection-churn peak RSS decreased **9.48%**, with an interval of +[-9.55%, -9.41%]. Timer RSS was unchanged at the process peak measurement's +resolution. The mixed-deadline probe showed a small timing increase within the +3% budget; this change is a zero-delay optimization, not an improvement to every +timer distribution. + +The broad balanced gate is **inconclusive**: HTTP/TLS timing and some network +RSS upper bounds exceed the 3% regression budget. A separately predeclared +32-pair follow-up for HTTP, TLS, and bulk transfer (seed 41030, same sizes, +99.17% intervals across its six metrics) produced: + +| Workload | Time change (interval) | Peak RSS change (interval) | +| --- | ---: | ---: | +| HTTP keep-alive | +1.17% [-1.07%, +4.23%] | +0.31% [-0.97%, +1.62%] | +| TLS HTTP | +0.35% [-1.70%, +2.34%] | +0.13% [-0.95%, +1.30%] | +| Bulk transfer | -0.61% [-1.36%, +0.03%] | +0.92% [-1.49%, +3.53%] | + +The follow-up resolves the TLS bounds but leaves HTTP timing and bulk RSS +uncertain against that budget. It does not turn the original gate into a pass. +The supported gains are faster zero-delay timers and lower connection-churn +memory use; a general network speedup or absence of all regressions is not +established. Follow-up artifacts are in `network-32/`. The final baseline and +candidate extension hashes are respectively +`be2ada747687c4a05bcee520e03a5640b74225d1ab64e7596f4d178c6ef78c29` +and `05a273296b2a2bd959ad84f53ec949aa4a70fa488fd29fcd347af484e959b7bc`. + +### Hotpath verification of the retained changes + +Fresh profiles of the final source are retained in `timing/` and `allocations/` +under the implementation artifact directory. The syntax audit covers 2,029 +definitions (2,002 instrumented and 27 excluded), with no missing hooks. The +profiling lifecycle Python test also passed on the allocation build. + +For 50,000 timers in batches of 500, the measured `TimerEntry::partial_cmp` +call counts changed as follows relative to the earlier `0e0c8d0` profiles: + +| Workload | Before | After | Reduction | +| --- | ---: | ---: | ---: | +| Retained zero-delay timers | 436,400 | 150,000 | 65.6% | +| Cancelled zero-delay timers | 440,100 | 150,300 | 65.8% | + +The 500-connection, 1 KiB echo probe still made 1,000 pooled write acquisitions, +but their allocation traffic fell from 16,384,000 to 1,024,000 bytes: **93.75% +less**, or about 14.65 MiB saved. Read-pool traffic remained about 31.2 MiB. +These are Rust allocation requests, not resident-memory measurements; the RSS +result above comes from separate uninstrumented processes with 8 KiB messages. +Timing profiles for retained, cancelled, and mixed-deadline timers, and +allocation profiles for connection churn, TCP streams, and mixed timers all +completed successfully. The normal release binary was restored afterward. + +## Historical selective experiment + +The figures below predate the broad instrumentation and current collector. +They are retained as historical observations, not measurements of this build. + The observations below are from commit `aa40fec` plus the profiling changes, on an Intel Core i9-9900K, Linux x86-64, CPython 3.14.7, Rust 1.100.0-nightly, and hotpath 0.28.3. Each workload was run in a fresh process. @@ -51,7 +496,7 @@ processes each, after two warmups). The instrumented single runs took 139.4, 170.7, and 91.9 ms respectively. These are sequential, unpaired observations, but show that the profiler especially distorts the tiny callback paths. -## Next experiments +### Experiments proposed from the historical profile 1. For callback-heavy workloads, break down `schedule_callback_args` further, particularly `Py::new` handle creation and reference ownership. That path diff --git a/mkdocs.yml b/mkdocs.yml index 6abb529..284ff6b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -12,6 +12,7 @@ nav: - Rust Extensions: rust-extensions.md - Examples: examples.md - Benchmarks: benchmarks.md + - Hotpath Profiling: hotpath-rs-profile.md - How It Works: how-it-works.md - Project Structure: project-structure.md - Development: development.md diff --git a/scripts/hotpath_lab.py b/scripts/hotpath_lab.py index cc47934..c546c66 100644 --- a/scripts/hotpath_lab.py +++ b/scripts/hotpath_lab.py @@ -41,6 +41,7 @@ "timers_retained", "timers_discarded", "timers_cancelled", + "timers_mixed", "tcp_connect_churn", ) DEFAULT_WORKLOADS = ",".join(WORKLOADS[:6]) diff --git a/scripts/profile_with_hotpath.py b/scripts/profile_with_hotpath.py index 4f540d9..42f93f3 100644 --- a/scripts/profile_with_hotpath.py +++ b/scripts/profile_with_hotpath.py @@ -1,81 +1,467 @@ -"""Profile one existing rsloop benchmark in an instrumented extension. +"""Collect complete hotpath reports, one fresh process per workload. -Build first with ``maturin develop --release --features hotpath-profile``. -Run each workload in its own process so each report has a clear scope. +Build with ``maturin develop --release --features hotpath-profile`` first. +Use ``all --output target/hotpath-rs/run`` for the coverage suite, or profile +an application/test module with ``module --module NAME --module-args ...``. """ from __future__ import annotations import argparse import gc -import importlib.util +import hashlib +import json +import math +import os +import platform +import runpy +import subprocess import sys +import time +from dataclasses import asdict from pathlib import Path +from typing import Any, cast ROOT = Path(__file__).resolve().parents[1] -BENCHMARK = ROOT / "benches" / "compare_event_loops.py" +BASIC = ("callbacks", "tasks", "task_options", "tcp_streams") +TIMERS = ("timers_retained", "timers_discarded", "timers_cancelled", "timers_mixed") +MATRIX = ( + "http_keepalive", + "tls_http", + "websocket_messages", + "websocket_tls", + "mixed_streams", + "bulk_transfer", + "idle_connections", +) +PROBES = ( + "threadsafe", + "executor_dns", + "udp", + "socket_ops", + "subprocess_pipes", + "unix_streams", + "signals", + "cancellation", +) +WORKLOADS = (*BASIC, *TIMERS, "tcp_connect_churn", *MATRIX, *PROBES) -def load_benchmark(): - spec = importlib.util.spec_from_file_location("compare_event_loops", BENCHMARK) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module +def positive(value: str) -> int: + number = int(value) + if number <= 0: + raise argparse.ArgumentTypeError("must be positive") + return number -def main() -> None: +def parse_args(argv=None): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("workload", choices=("callbacks", "tasks", "tcp_streams")) + parser.add_argument("workload", choices=(*WORKLOADS, "all", "module")) parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--callbacks", type=int, default=200_000) - parser.add_argument("--tasks", type=int, default=50_000) - parser.add_argument("--task-batch-size", type=int, default=5_000) - parser.add_argument("--tcp-roundtrips", type=int, default=5_000) - parser.add_argument("--payload-size", type=int, default=1024) - args = parser.parse_args() + parser.add_argument( + "--format", choices=("json", "json-pretty", "table"), default="json" + ) + parser.add_argument("--focus", help="hotpath function name substring (or /regex/)") + parser.add_argument("--time-sampling-rate", type=float, default=1.0) + parser.add_argument("--timeout", type=float, default=120.0) + parser.add_argument("--callbacks", type=positive, default=20_000) + parser.add_argument("--tasks", type=positive, default=5_000) + parser.add_argument("--task-batch-size", type=positive, default=500) + parser.add_argument("--tcp-roundtrips", type=positive, default=500) + parser.add_argument("--payload-size", type=positive, default=1024) + parser.add_argument("--timers", type=positive, default=5_000) + parser.add_argument("--timer-batch-size", type=positive, default=500) + parser.add_argument("--connections", type=positive, default=50) + parser.add_argument("--probe-iterations", type=positive, default=20) + parser.add_argument("--concurrency", type=positive, default=4) + parser.add_argument("--requests-per-connection", type=positive, default=20) + parser.add_argument("--bulk-bytes", type=positive, default=1024 * 1024) + parser.add_argument("--idle-connections", type=positive, default=20) + parser.add_argument("--idle-cycles", type=positive, default=10) + parser.add_argument("--tls-dir", type=Path, default=ROOT / "tests/fixtures/tls") + parser.add_argument("--module", help="Python module to run inside the profile") + parser.add_argument("--child", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--module-args", nargs=argparse.REMAINDER, default=[]) + args = parser.parse_args(argv) + if ( + not math.isfinite(args.time_sampling_rate) + or not 0 < args.time_sampling_rate <= 1 + ): + parser.error("--time-sampling-rate must be in (0, 1]") + if not math.isfinite(args.timeout) or args.timeout <= 0: + parser.error("--timeout must be finite and positive") + if args.workload == "all" and args.format == "table": + parser.error("all requires JSON reports for coverage analysis") + if (args.workload == "module") != bool(args.module): + parser.error( + "module workload requires --module; other workloads do not accept it" + ) + return args + + +def profile_environment(args): + # hotpath env settings override the builder and some are cached at import. + # Clear stale filters, row limits, output paths and sampling configuration. + env = {k: v for k, v in os.environ.items() if not k.startswith("HOTPATH_")} + env.update(HOTPATH_LIMIT="0", HOTPATH_THREADS_INTERVAL_MS="10", PYTHONHASHSEED="0") + if args.focus: + env["HOTPATH_FOCUS"] = args.focus + return env + + +def write_json(path, data): + path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n") + + +def sha256(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def inventory(): + result = subprocess.run( + [ + "cargo", + "run", + "--quiet", + "--locked", + "--manifest-path", + str(ROOT / "tools/hotpath-coverage/Cargo.toml"), + "--", + "--json", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + return json.loads(result.stdout) + + +def report_names(report): + for section, values in report.items(): + if ( + isinstance(values, dict) + and "total_count" in values + and values["total_count"] != values.get("included_count") + ): + raise ValueError(f"{section} is truncated; remove hotpath row limits") + names = set() + for section in ("functions_timing", "functions_alloc"): + values = report.get(section) or {} + names.update(row["name"] for row in values.get("data", [])) + if not names: + raise ValueError("hotpath report contains no measured functions") + return names + + +def coverage(entries, reports): + observed = {name for report in reports for name in report_names(report)} + rows = [] + for entry in entries: + row = dict(entry) + row["observed"] = row["name"] in observed + rows.append(row) + eligible = {r["name"] for r in rows if r["status"] == "instrumented"} + return { + "instrumented_names": len(eligible), + "observed_names": len(eligible & observed), + "unmapped_observed_names": sorted(observed - eligible - {"rsloop"}), + "note": "Static inventory includes inactive platforms/features. Unobserved does not mean unused or cheap. Times are inclusive, not CPU self time.", + "functions": rows, + } + + +def summary(manifest, coverage_data, reports): + lines = [ + "# rsloop hotpath profile", + "", + ( + "Diagnostic instrumentation changes runtime cost. Function times overlap; " + "they are inclusive wall durations, not CPU self time. Compare performance " + "using a separate uninstrumented build." + ), + "", + ( + f"Observed {coverage_data['observed_names']} of " + f"{coverage_data['instrumented_names']} instrumented names across all source " + "platforms/features. See coverage.json for unobserved paths and exclusions." + ), + "", + "| Workload | Status | Function rows | Future rows | Thread rows |", + "| --- | --- | ---: | ---: | ---: |", + ] + successful = iter(reports) + for run in manifest["runs"]: + if run["status"] != "ok": + lines.append(f"| {run['workload']} | {run['status']} | — | — | — |") + continue + report = next(successful) + functions = report.get("functions_timing", report.get("functions_alloc", {})) + lines.append( + f"| [{run['workload']}]({Path(run['report']).name}) | ok | " + f"{functions.get('included_count', 0)} | " + f"{len(report.get('futures', {}).get('data', []))} | " + f"{report.get('threads', {}).get('included_count', 0)} |" + ) + return "\n".join(lines) + "\n" + +def run_workload(args): + import asyncio + + import rsloop + + sys.path.insert(0, str(ROOT / "benches")) + import compare_event_loops as small + import profile_workloads as probes + import scheduler_workloads as scheduler + import workload_matrix as matrix + + name = args.workload + if name == "module": + sys.argv = [args.module, *args.module_args] + try: + runpy.run_module(args.module, run_name="__main__", alter_sys=True) + except SystemExit as exc: + if exc.code not in (None, 0): + raise + return {"module": args.module, "arguments": args.module_args} + if name == "callbacks": + coro = small.bench_callbacks("rsloop", args.callbacks) + elif name in ("tasks", "task_options"): + runner = small.bench_tasks if name == "tasks" else small.bench_task_options + coro = runner("rsloop", args.tasks, args.task_batch_size) + elif name == "tcp_streams": + coro = small.bench_tcp_streams("rsloop", args.tcp_roundtrips, args.payload_size) + elif name in TIMERS: + coro = scheduler.bench_timers( + "rsloop", args.timers, args.timer_batch_size, name + ) + elif name == "tcp_connect_churn": + coro = scheduler.bench_tcp_connect_churn( + "rsloop", args.connections, args.payload_size + ) + elif name in PROBES: + coro = probes.RUNNERS[name](args.probe_iterations) + else: + old_argv = sys.argv + sys.argv = ["matrix"] + try: + matrix_args = matrix.parse_args() + finally: + sys.argv = old_argv + for key in ( + "concurrency", + "requests_per_connection", + "bulk_bytes", + "idle_connections", + "idle_cycles", + "tls_dir", + ): + setattr(matrix_args, key, getattr(args, key)) + matrix.validate_args(matrix_args) + coro = matrix.SCENARIO_RUNNERS[name]("rsloop", matrix_args) + # Include loop construction, shutdown, worker joins, and cancellation. + result = rsloop.run(asyncio.wait_for(coro, args.timeout)) + return ( + asdict(result) if result is not None else {"iterations": args.probe_iterations} + ) + + +def child(args): + import rsloop from rsloop import _loop - if not hasattr(_loop, "hotpath_start"): - raise SystemExit("build rsloop with --features hotpath-profile first") - - bench = load_benchmark() - - def run(count: int): - if args.workload == "callbacks": - coro = bench.bench_callbacks("rsloop", count) - elif args.workload == "tasks": - coro = bench.bench_tasks("rsloop", count, args.task_batch_size) - else: - coro = bench.bench_tcp_streams("rsloop", count, args.payload_size) - return bench.run_with_loop("rsloop", coro) - - warmup = {"callbacks": 1_000, "tasks": 500, "tcp_streams": 50} - count = { - "callbacks": args.callbacks, - "tasks": args.tasks, - "tcp_streams": args.tcp_roundtrips, - }[args.workload] - run(warmup[args.workload]) - - args.output.parent.mkdir(parents=True, exist_ok=True) + native = cast(Any, _loop) + if not hasattr(native, "hotpath_start"): + raise RuntimeError("build rsloop with --features hotpath-profile first") + extension = Path(native.__file__).resolve() + metadata = { + "workload": args.workload, + "pid": os.getpid(), + "python": sys.version, + "platform": platform.platform(), + "extension": str(extension), + "extension_sha256": sha256(extension), + "build_info": rsloop.build_info(), + "settings": { + k: str(v) if isinstance(v, Path) else v for k, v in vars(args).items() + }, + "environment": { + k: v for k, v in os.environ.items() if k.startswith(("HOTPATH_", "RSLOOP_")) + }, + "scope": "cold workload, including loop setup/teardown; no in-process warmup", + } gc.collect() + gc_enabled = gc.isenabled() gc.disable() + started = time.perf_counter() try: - _loop.hotpath_start(str(args.output)) + native.hotpath_start( + str(args.output), + format=args.format, + time_sampling_rate=args.time_sampling_rate, + ) try: - result = run(count) + metadata["result"] = run_workload(args) + metadata["status"] = "ok" finally: - _loop.hotpath_stop() + native.hotpath_stop() + except BaseException as exc: + metadata["status"] = "failed" + metadata["error"] = repr(exc) + raise finally: - gc.enable() + metadata["instrumented_seconds_with_report_flush"] = ( + time.perf_counter() - started + ) + if gc_enabled: + gc.enable() + write_json(args.output.with_suffix(".metadata.json"), metadata) - print( - f"{result.workload}: {result.operations:,} operations in {result.seconds:.4f}s" + +def child_command(args, name, output): + command = [ + sys.executable, + str(Path(__file__).resolve()), + name, + "--child", + "--output", + str(output), + ] + for key, value in vars(args).items(): + if key in {"workload", "child", "output", "module_args"} or value is None: + continue + command.extend(["--" + key.replace("_", "-"), str(value)]) + if args.module_args: + command.extend(["--module-args", *args.module_args]) + return command + + +def unsupported(name): + if name == "unix_streams" and sys.platform == "win32": + return "Unix stream APIs are unavailable on Windows" + if name == "signals" and os.name != "posix": + return "POSIX signal probe" + return None + + +def main(argv=None): + args = parse_args(argv) + if args.child: + child(args) + return + args.output = args.output.expanduser().resolve() + output = args.output + sidecars = (".metadata.json", ".manifest.json", ".coverage.json", ".log") + if output.exists() or any(output.with_suffix(s).exists() for s in sidecars): + raise SystemExit(f"refusing to overwrite {output}") + entries = inventory() + if args.workload == "all": + output.mkdir(parents=True) + names = WORKLOADS + else: + output.parent.mkdir(parents=True, exist_ok=True) + names = (args.workload,) + source_hashes = { + str(p.relative_to(ROOT)): sha256(p) for p in sorted(ROOT.glob("src/**/*.rs")) + } + source_hashes.update( + { + str(p.relative_to(ROOT)): sha256(p) + for directory in ("benches", "scripts", "python/rsloop") + for p in sorted((ROOT / directory).glob("*.py")) + } + ) + source_hashes.update( + { + name: sha256(ROOT / name) + for name in ("Cargo.toml", "Cargo.lock", "rust-toolchain.toml", "build.rs") + } ) - print(f"hotpath report: {args.output}") + manifest = { + "revision": subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True + ).strip(), + "git_status": subprocess.check_output( + ["git", "status", "--porcelain"], cwd=ROOT, text=True + ), + "source_sha256": source_hashes, + "note": "Source hashes describe the checkout at collection time; extension SHA-256 identifies the actual binary. Rebuild after source edits.", + "runs": [], + } + manifest_path = ( + output / "manifest.json" + if args.workload == "all" + else output.with_suffix(".manifest.json") + ) + reports = [] + failures = [] + write_json(manifest_path, manifest) + for name in names: + reason = unsupported(name) + if reason: + manifest["runs"].append( + {"workload": name, "status": "skipped", "reason": reason} + ) + write_json(manifest_path, manifest) + continue + report = output / f"{name}.json" if args.workload == "all" else output + command = child_command(args, name, report) + record = {"workload": name, "command": command, "report": str(report)} + try: + result = subprocess.run( + command, + cwd=ROOT, + env=profile_environment(args), + capture_output=True, + text=True, + timeout=args.timeout + 30, + check=False, + ) + report.with_suffix(".log").write_text(result.stdout + result.stderr) + if result.returncode: + raise RuntimeError( + f"child exited {result.returncode}; see {report.with_suffix('.log')}" + ) + if args.format != "table": + data = json.loads(report.read_text()) + report_names(data) + reports.append(data) + elif not report.is_file(): + raise RuntimeError("hotpath did not create a report") + record["status"] = "ok" + print(f"{name}: {report}", flush=True) + except (OSError, ValueError, RuntimeError, subprocess.TimeoutExpired) as exc: + if isinstance(exc, subprocess.TimeoutExpired): + parts = [ + p.decode(errors="replace") if isinstance(p, bytes) else (p or "") + for p in (exc.stdout, exc.stderr) + ] + report.with_suffix(".log").write_text( + "".join(parts) + "\nProfile child timed out.\n" + ) + record.update(status="failed", error=str(exc)) + failures.append(name) + print(f"{name}: failed: {exc}", file=sys.stderr, flush=True) + manifest["runs"].append(record) + write_json(manifest_path, manifest) + if reports: + destination = ( + output / "coverage.json" + if args.workload == "all" + else output.with_suffix(".coverage.json") + ) + coverage_data = coverage(entries, reports) + write_json(destination, coverage_data) + if args.workload == "all": + (output / "summary.md").write_text( + summary(manifest, coverage_data, reports) + ) + if failures: + raise SystemExit( + f"profiling failed: {', '.join(failures)} (partial results retained)" + ) if __name__ == "__main__": diff --git a/scripts/run_rust_tests.py b/scripts/run_rust_tests.py index 2e5eb29..0267511 100644 --- a/scripts/run_rust_tests.py +++ b/scripts/run_rust_tests.py @@ -51,6 +51,14 @@ def main() -> int: python_home, libdir, python_abi = python_link_config(interpreter) env = os.environ.copy() + if args.all_features or any( + name in (args.features or "").split(",") + for name in ("hotpath-profile", "hotpath-alloc-profile") + ): + # hotpath wraps nested async bodies in additional futures. In debug + # builds, moving owned I/O arrays through those wrappers can exceed + # libtest's 2 MiB stack before polling. Keep normal builds unchanged. + env.setdefault("RUST_MIN_STACK", str(32 * 1024 * 1024)) env["PYO3_PYTHON"] = str(interpreter) env["PYTHONHOME"] = python_home if os.name == "nt": diff --git a/src/async_event.rs b/src/async_event.rs index 401b7ce..4c83eea 100644 --- a/src/async_event.rs +++ b/src/async_event.rs @@ -10,12 +10,20 @@ pub struct AsyncEvent { } impl AsyncEvent { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "AsyncEvent") + )] pub fn new() -> Self { Self { waiters: Mutex::new(Vec::with_capacity(4)), } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "AsyncEvent") + )] pub fn listen(&self) -> oneshot::Receiver<()> { let (tx, rx) = oneshot::channel(); self.waiters @@ -25,6 +33,10 @@ impl AsyncEvent { rx } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "AsyncEvent") + )] pub fn notify_all(&self) { let mut waiters = self.waiters.lock().expect("poisoned async event waiters"); let drained = waiters.drain(..); diff --git a/src/bindings/loop_api.rs b/src/bindings/loop_api.rs index e4f45be..979b570 100644 --- a/src/bindings/loop_api.rs +++ b/src/bindings/loop_api.rs @@ -59,16 +59,19 @@ pub struct PyLoop { } impl PyLoop { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[inline] fn as_py_any(py: Python<'_>, slf: &Py) -> Py { slf.clone_ref(py).into_any() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[inline] fn task_locals(py: Python<'_>, slf: &Py) -> PyResult { TaskLocals::new(Self::as_py_any(py, slf).into_bound(py)).copy_context(py) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn schedule_now( &self, py: Python<'_>, @@ -83,16 +86,19 @@ impl PyLoop { Ok(handle.into_any()) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[allow(dead_code)] fn not_implemented(feature: &str) -> PyErr { PyNotImplementedError::new_err(format!("{feature} is not implemented in rust-impl yet")) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn map_loop_error(err: LoopCoreError) -> PyErr { PyRuntimeError::new_err(err.to_string()) } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[pyfunction] /// Creates a new Python-visible rsloop event loop. /// diff --git a/src/bindings/loop_api/asyncgens.rs b/src/bindings/loop_api/asyncgens.rs index f586880..e4e8a0a 100644 --- a/src/bindings/loop_api/asyncgens.rs +++ b/src/bindings/loop_api/asyncgens.rs @@ -22,6 +22,10 @@ pub(super) struct AsyncgenHooksGuard { impl AsyncgenHooksGuard { // Install loop-specific async-generator hooks temporarily; `Drop` restores // the process-wide hooks even when `run_forever` exits with an error. + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "AsyncgenHooksGuard") + )] pub(super) fn install( py: Python<'_>, loop_obj: &Py, @@ -61,6 +65,10 @@ impl AsyncgenHooksGuard { } impl Drop for AsyncgenHooksGuard { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { Python::attach(|py| { let sys = match py.import("sys") { @@ -75,6 +83,7 @@ impl Drop for AsyncgenHooksGuard { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn new_asyncgens_set(py: Python<'_>) -> PyResult> { // A strong set retains every completed async context manager until loop // shutdown and prevents abandoned generators from reaching their finalizer. @@ -83,6 +92,7 @@ fn new_asyncgens_set(py: Python<'_>) -> PyResult> { Ok(PySet::empty(py)?.unbind()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn active_asyncgens_set(py: Python<'_>, core: &Arc) -> PyResult> { let mut state = core.state.lock().expect("poisoned loop state"); if let Some(active) = state.active_asyncgens.as_ref() { @@ -93,6 +103,7 @@ fn active_asyncgens_set(py: Python<'_>, core: &Arc) -> PyResult( slf: Py, py: Python<'py>, @@ -154,6 +165,7 @@ pub(super) fn shutdown_asyncgens<'py>( }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[pyfunction] /// Registers an asynchronous generator with its owning loop on first iteration. /// @@ -198,6 +210,7 @@ pub fn asyncgen_firstiter_hook( Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[pyfunction] /// Unregisters and schedules finalization of an asynchronous generator. /// diff --git a/src/bindings/loop_api/asyncio_cache.rs b/src/bindings/loop_api/asyncio_cache.rs index 40779f0..9961a78 100644 --- a/src/bindings/loop_api/asyncio_cache.rs +++ b/src/bindings/loop_api/asyncio_cache.rs @@ -64,18 +64,21 @@ pub(super) struct TaskKwargSupport { pub(super) eager_start: bool, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn asyncio_task_cls(py: Python<'_>) -> PyResult<&Py> { PYTHON_API_CACHES .asyncio_task_cls .get_or_try_init(py, || Ok(py.import("asyncio")?.getattr("Task")?.unbind())) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn asyncio_future_cls(py: Python<'_>) -> PyResult<&Py> { PYTHON_API_CACHES .asyncio_future_cls .get_or_try_init(py, || Ok(py.import("asyncio")?.getattr("Future")?.unbind())) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn asyncio_get_running_loop_fn(py: Python<'_>) -> PyResult<&Py> { PYTHON_API_CACHES .asyncio_get_running_loop_fn @@ -87,12 +90,14 @@ pub(super) fn asyncio_get_running_loop_fn(py: Python<'_>) -> PyResult<&Py }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn asyncio_task_kwarg_support(py: Python<'_>) -> PyResult<&'static TaskKwargSupport> { PYTHON_API_CACHES .asyncio_task_kwarg_support .get_or_try_init(py, || detect_asyncio_task_kwarg_support(py)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn detect_asyncio_task_kwarg_support(py: Python<'_>) -> PyResult { let inspect = py.import("inspect")?; let Some(signature) = asyncio_task_signature(py, &inspect)? else { @@ -119,6 +124,7 @@ fn detect_asyncio_task_kwarg_support(py: Python<'_>) -> PyResult( py: Python<'py>, inspect: &Bound<'py, PyModule>, @@ -132,6 +138,7 @@ fn asyncio_task_signature<'py>( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn has_keyword_only_parameter( parameters: &Bound<'_, PyAny>, keyword_only: &Bound<'_, PyAny>, @@ -143,6 +150,7 @@ fn has_keyword_only_parameter( parameter.getattr("kind")?.eq(keyword_only) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn mark_task_kwarg_supported(support: &mut TaskKwargSupport, kwarg_name: &str) { match kwarg_name { "name" => support.name = true, @@ -152,11 +160,13 @@ fn mark_task_kwarg_supported(support: &mut TaskKwargSupport, kwarg_name: &str) { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub(super) fn call_callable_noargs(py: Python<'_>, callable: &Py) -> PyResult> { ffi_helpers::call_noargs(py, callable) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub(super) fn call_callable_onearg( py: Python<'_>, @@ -166,6 +176,7 @@ pub(super) fn call_callable_onearg( ffi_helpers::call_onearg(py, callable, arg) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(any(Py_3_12, all(Py_3_11, not(Py_LIMITED_API))))] fn keyword_tuple( slot: &'static PyOnceLock>, @@ -175,6 +186,7 @@ fn keyword_tuple( slot.get_or_try_init(py, || Ok(PyTuple::new(py, names)?.unbind())) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(any(Py_3_12, all(Py_3_11, not(Py_LIMITED_API))))] pub(super) fn asyncio_future_loop_kwnames(py: Python<'_>) -> PyResult<&Py> { keyword_tuple( @@ -184,6 +196,7 @@ pub(super) fn asyncio_future_loop_kwnames(py: Python<'_>) -> PyResult<&Py, diff --git a/src/bindings/loop_api/connections.rs b/src/bindings/loop_api/connections.rs index b85ede2..78ab2cd 100644 --- a/src/bindings/loop_api/connections.rs +++ b/src/bindings/loop_api/connections.rs @@ -32,6 +32,7 @@ pub(super) struct CreateUnixConnectionParams { pub(super) tls: TlsParams, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn create_connection<'py>( slf: Py, py: Python<'py>, @@ -69,6 +70,7 @@ pub(super) fn create_connection<'py>( }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Walks the resolved addresses in order, returning the first socket that /// connects and reporting the last failure when none do. async fn connect_first_reachable_address( @@ -111,6 +113,7 @@ async fn connect_first_reachable_address( }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Wraps a connected socket in a transport and returns the `(transport, /// protocol)` pair. The TLS handshake runs on a blocking worker because it can /// block on peer I/O. @@ -138,6 +141,7 @@ async fn finish_client_connection( Python::attach(|py| transport_protocol_pair(py, transport.into_any(), protocol)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn create_connection_transport( slf: Py, py: Python<'_>, @@ -160,6 +164,7 @@ pub(super) fn create_connection_transport( transport_protocol_pair(py, transport.into_any(), &protocol) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn create_unix_connection<'py>( slf: Py, py: Python<'py>, @@ -205,6 +210,7 @@ pub(super) fn create_unix_connection<'py>( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn connect_accepted_socket<'py>( slf: Py, py: Python<'py>, @@ -239,6 +245,7 @@ pub(super) fn connect_accepted_socket<'py>( }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn start_tls<'py>( slf: Py, py: Python<'py>, diff --git a/src/bindings/loop_api/executor.rs b/src/bindings/loop_api/executor.rs index 44d58ea..0f057c4 100644 --- a/src/bindings/loop_api/executor.rs +++ b/src/bindings/loop_api/executor.rs @@ -23,6 +23,7 @@ pub(super) struct AddrInfoRequest { pub(super) flags: i32, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn warn_default_executor_timeout(py: Python<'_>, timeout: f64) -> PyResult<()> { let warnings = py.import("warnings")?; let builtins = py.import("builtins")?; @@ -41,6 +42,7 @@ fn warn_default_executor_timeout(py: Python<'_>, timeout: f64) -> PyResult<()> { Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn run_in_executor<'py>( slf: Py, py: Python<'py>, @@ -87,6 +89,7 @@ pub(super) fn run_in_executor<'py>( }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn getaddrinfo<'py>( slf: Py, py: Python<'py>, @@ -115,6 +118,7 @@ pub(super) fn getaddrinfo<'py>( .map(|awaitable| awaitable.into_bound(py)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Construct the single unambiguous TCP/UDP result for an IP literal and an /// integer port. Leave resolver flags, services, scoped addresses, unspecified /// socket types and invalid combinations to the system resolver. Custom loop @@ -186,6 +190,7 @@ fn numeric_addrinfo<'py>( Ok(Some(future.into_bound(py))) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn getnameinfo<'py>( slf: Py, py: Python<'py>, @@ -206,6 +211,7 @@ pub(super) fn getnameinfo<'py>( .map(|awaitable| awaitable.into_bound(py)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn shutdown_default_executor<'py>( slf: Py, py: Python<'py>, @@ -250,6 +256,7 @@ pub(super) fn shutdown_default_executor<'py>( }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Shuts the executor down on a helper thread so the wait can time out; on /// timeout it warns and falls back to a non-waiting `shutdown(False)`. async fn shutdown_executor_with_timeout( diff --git a/src/bindings/loop_api/fast_callbacks.rs b/src/bindings/loop_api/fast_callbacks.rs index ddaeced..a9fc084 100644 --- a/src/bindings/loop_api/fast_callbacks.rs +++ b/src/bindings/loop_api/fast_callbacks.rs @@ -11,6 +11,7 @@ use pyo3::{ffi, get_trampoline_function}; use super::PyLoop; use crate::engine::{CallbackArgs, CallbackKind}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] unsafe fn schedule( py: Python<'_>, slf: *mut ffi::PyObject, @@ -131,6 +132,7 @@ unsafe fn schedule( Ok(handle.into_ptr()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] unsafe fn soon( py: Python<'_>, slf: *mut ffi::PyObject, @@ -142,6 +144,7 @@ unsafe fn soon( unsafe { schedule(py, slf, args, nargs, names, CallbackKind::Soon) } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] unsafe fn threadsafe( py: Python<'_>, slf: *mut ffi::PyObject, @@ -153,6 +156,7 @@ unsafe fn threadsafe( unsafe { schedule(py, slf, args, nargs, names, CallbackKind::Threadsafe) } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(crate) fn install_fast_callbacks(py: Python<'_>) -> PyResult<()> { // Descriptors borrow their method definition forever. These contain only // static strings/function pointers, no interpreter-owned objects. Allocate diff --git a/src/bindings/loop_api/ffi_helpers.rs b/src/bindings/loop_api/ffi_helpers.rs index 400f64b..d015b32 100644 --- a/src/bindings/loop_api/ffi_helpers.rs +++ b/src/bindings/loop_api/ffi_helpers.rs @@ -5,10 +5,12 @@ use pyo3::prelude::*; #[cfg(any(Py_3_12, all(Py_3_11, not(Py_LIMITED_API))))] use pyo3::ffi; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn call_noargs(py: Python<'_>, callable: &Py) -> PyResult> { Ok(callable.bind(py).call0()?.unbind()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn call_onearg( py: Python<'_>, callable: &Py, @@ -17,6 +19,7 @@ pub(super) fn call_onearg( Ok(callable.bind(py).call1((arg,))?.unbind()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(any(Py_3_12, all(Py_3_11, not(Py_LIMITED_API))))] pub(super) fn vectorcall( py: Python<'_>, diff --git a/src/bindings/loop_api/lifecycle.rs b/src/bindings/loop_api/lifecycle.rs index c510d46..2786d1a 100644 --- a/src/bindings/loop_api/lifecycle.rs +++ b/src/bindings/loop_api/lifecycle.rs @@ -8,6 +8,7 @@ use super::PyLoop; use super::asyncgens::AsyncgenHooksGuard; use crate::engine::LoopCoreError; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn close(loop_ref: &PyLoop, py: Python<'_>) -> PyResult<()> { let executor = { let mut state = loop_ref.core.state.lock().expect("poisoned loop state"); @@ -31,6 +32,7 @@ pub(super) fn close(loop_ref: &PyLoop, py: Python<'_>) -> PyResult<()> { Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn run_forever(slf: Py, py: Python<'_>) -> PyResult<()> { let loop_obj = PyLoop::as_py_any(py, &slf); let core = slf.borrow(py).core.clone(); @@ -38,6 +40,7 @@ pub(super) fn run_forever(slf: Py, py: Python<'_>) -> PyResult<()> { core.run_forever(py, loop_obj) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn run_until_complete( slf: Py, py: Python<'_>, @@ -88,6 +91,7 @@ pub(super) fn run_until_complete( Ok(wrapped.call_method0("result")?.unbind()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// `add_done_callback` target installed by `run_until_complete`: stop the loop /// once the awaited future finishes, but let `SystemExit` and /// `KeyboardInterrupt` propagate out of `run_forever` instead. diff --git a/src/bindings/loop_api/methods.rs b/src/bindings/loop_api/methods.rs index 5451990..05c1faa 100644 --- a/src/bindings/loop_api/methods.rs +++ b/src/bindings/loop_api/methods.rs @@ -29,6 +29,7 @@ use crate::engine::{CallbackKind, LoopCore, PyTimerHandle}; #[pymethods] impl PyLoop { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[new] pub(super) fn new() -> Self { Self { @@ -36,6 +37,7 @@ impl PyLoop { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(callback, *args, context=None))] fn call_soon( &self, @@ -53,6 +55,7 @@ impl PyLoop { ) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(callback, *args, context=None))] fn call_soon_threadsafe( &self, @@ -70,6 +73,7 @@ impl PyLoop { ) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(delay, callback, *args, context=None))] fn call_later( &self, @@ -96,6 +100,7 @@ impl PyLoop { Py::new(py, PyTimerHandle::new(ready.id(), when, &ready)) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(when, callback, *args, context=None))] fn call_at( &self, @@ -109,46 +114,57 @@ impl PyLoop { self.call_later(py, when - self.time(), callback, args, context) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn time(&self) -> f64 { self.core.time() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn stop(&self) -> PyResult<()> { self.core.schedule_stop().map_err(Self::map_loop_error) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn close(&self, py: Python<'_>) -> PyResult<()> { lifecycle::close(self, py) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn is_running(&self) -> bool { self.core.is_running() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn is_closed(&self) -> bool { self.core.is_closed() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn get_debug(&self) -> bool { self.core.get_debug() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn set_debug(&self, enabled: bool) { self.core.set_debug(enabled); } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn run_forever(slf: Py, py: Python<'_>) -> PyResult<()> { lifecycle::run_forever(slf, py) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn run_until_complete(slf: Py, py: Python<'_>, future: Py) -> PyResult> { lifecycle::run_until_complete(slf, py, future) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn create_future(slf: Py, py: Python<'_>) -> PyResult> { tasks::create_future(slf, py) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(coro, *, name=None, context=None, eager_start=None, **kwargs))] fn create_task( slf: Py, @@ -172,6 +188,7 @@ impl PyLoop { ) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn set_task_factory(&self, factory: Option>) { let installed = factory.is_some(); self.core @@ -182,6 +199,7 @@ impl PyLoop { self.core.set_task_factory_installed(installed); } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn get_task_factory(&self, py: Python<'_>) -> Option> { self.core .state @@ -192,10 +210,12 @@ impl PyLoop { .map(|factory| factory.clone_ref(py)) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn default_exception_handler(&self, py: Python<'_>, context: Py) -> PyResult<()> { self.core.default_exception_handler(py, context) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn get_exception_handler(&self, py: Python<'_>) -> Option> { self.core .state @@ -206,6 +226,7 @@ impl PyLoop { .map(|handler| handler.clone_ref(py)) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn set_exception_handler(&self, handler: Option>) { self.core .state @@ -214,12 +235,14 @@ impl PyLoop { .exception_handler = handler; } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn call_exception_handler(slf: Py, py: Python<'_>, context: Py) -> PyResult<()> { slf.borrow(py) .core .call_exception_handler(py, Some(&Self::as_py_any(py, &slf)), context) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn set_default_executor(&self, executor: Option>) { self.core .state @@ -228,6 +251,7 @@ impl PyLoop { .default_executor = executor; } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[getter] fn slow_callback_duration(&self) -> f64 { self.core @@ -237,6 +261,7 @@ impl PyLoop { .slow_callback_duration } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[setter(slow_callback_duration)] fn set_slow_callback_duration(&self, value: f64) { self.core @@ -246,6 +271,7 @@ impl PyLoop { .slow_callback_duration = value; } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn __repr__(&self) -> String { format!( "", @@ -255,6 +281,7 @@ impl PyLoop { ) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(protocol_factory, host=None, port=None, *, family=0, flags=1, sock=None, backlog=100, ssl=None, reuse_address=None, reuse_port=None, keep_alive=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, start_serving=true))] #[expect( clippy::too_many_arguments, @@ -300,6 +327,7 @@ impl PyLoop { ) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(protocol_factory, host=None, port=None, *, ssl=None, family=0, proto=0, flags=0, sock=None, local_addr=None, server_hostname=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, happy_eyeballs_delay=None, interleave=None, all_errors=false))] #[expect( clippy::too_many_arguments, @@ -347,6 +375,7 @@ impl PyLoop { ) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(protocol_factory, sock, *, ssl=None, server_hostname=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None))] #[allow(clippy::too_many_arguments)] fn _create_connection_transport( @@ -373,6 +402,7 @@ impl PyLoop { ) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(protocol_factory, path=None, *, sock=None, backlog=100, ssl=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, start_serving=true, cleanup_socket=true))] #[expect( clippy::too_many_arguments, @@ -406,6 +436,7 @@ impl PyLoop { ) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(protocol_factory, path=None, *, ssl=None, sock=None, server_hostname=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None))] #[expect( clippy::too_many_arguments, @@ -439,6 +470,7 @@ impl PyLoop { ) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(protocol_factory, sock, *, ssl=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None))] fn connect_accepted_socket( slf: Py, @@ -458,6 +490,7 @@ impl PyLoop { ) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(transport, protocol, sslcontext, *, server_side=false, server_hostname=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None))] #[expect( clippy::too_many_arguments, @@ -489,6 +522,7 @@ impl PyLoop { ) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(fd, callback, *args))] fn add_reader( &self, @@ -500,10 +534,12 @@ impl PyLoop { watchers::add_reader(self, py, fd, callback, args) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn remove_reader(&self, py: Python<'_>, fd: &Bound<'_, PyAny>) -> PyResult { watchers::remove_reader(self, py, fd) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(fd, callback, *args))] fn add_writer( &self, @@ -515,10 +551,12 @@ impl PyLoop { watchers::add_writer(self, py, fd, callback, args) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn remove_writer(&self, py: Python<'_>, fd: &Bound<'_, PyAny>) -> PyResult { watchers::remove_writer(self, py, fd) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn sock_recv( slf: Py, py: Python<'_>, @@ -528,6 +566,7 @@ impl PyLoop { sock_ops::sock_recv(slf, py, sock, nbytes) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn sock_recv_into( slf: Py, py: Python<'_>, @@ -537,6 +576,7 @@ impl PyLoop { sock_ops::sock_recv_into(slf, py, sock, buf) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn sock_sendall( slf: Py, py: Python<'_>, @@ -546,10 +586,12 @@ impl PyLoop { sock_ops::sock_sendall(slf, py, sock, data) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn sock_accept(slf: Py, py: Python<'_>, sock: Py) -> PyResult> { sock_ops::sock_accept(slf, py, sock) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn sock_connect( slf: Py, py: Python<'_>, @@ -559,6 +601,7 @@ impl PyLoop { sock_ops::sock_connect(slf, py, sock, address) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn _sock_connect_fast<'py>( slf: Py, py: Python<'py>, @@ -568,6 +611,7 @@ impl PyLoop { sock_ops::sock_connect_fast(slf, py, sock, address) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(host, port, *, family=0, r#type=0, proto=0, flags=0))] #[expect( clippy::too_many_arguments, @@ -597,6 +641,7 @@ impl PyLoop { ) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(sockaddr, flags=0))] fn getnameinfo( slf: Py, @@ -607,6 +652,7 @@ impl PyLoop { executor::getnameinfo(slf, py, sockaddr, flags) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(executor, func, *args))] fn run_in_executor<'py>( slf: Py, @@ -618,6 +664,7 @@ impl PyLoop { super::executor::run_in_executor(slf, py, executor, func, args) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(protocol_factory, cmd, *, stdin=default_stdio_pipe(), stdout=default_stdio_pipe(), stderr=default_stdio_pipe(), universal_newlines=false, shell=true, bufsize=0, encoding=None, errors=None, text=None, **kwargs))] #[expect( clippy::too_many_arguments, @@ -660,6 +707,7 @@ impl PyLoop { process_spawn::spawn_subprocess(&slf, py, params, move |py| shell_command(py, &cmd)) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(protocol_factory, program, *args, stdin=default_stdio_pipe(), stdout=default_stdio_pipe(), stderr=default_stdio_pipe(), universal_newlines=false, shell=false, bufsize=0, encoding=None, errors=None, text=None, **kwargs))] #[expect( clippy::too_many_arguments, @@ -704,6 +752,7 @@ impl PyLoop { }) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn connect_read_pipe( slf: Py, py: Python<'_>, @@ -713,6 +762,7 @@ impl PyLoop { pipes::connect_read_pipe(slf, py, protocol_factory, pipe) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn connect_write_pipe( slf: Py, py: Python<'_>, @@ -722,6 +772,7 @@ impl PyLoop { pipes::connect_write_pipe(slf, py, protocol_factory, pipe) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(sig, callback, *args))] fn add_signal_handler( slf: Py, @@ -733,14 +784,17 @@ impl PyLoop { signals::add_signal_handler(slf, py, sig, callback, args) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn remove_signal_handler(slf: Py, py: Python<'_>, sig: i32) -> PyResult { signals::remove_signal_handler(slf, py, sig) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] fn shutdown_asyncgens(slf: Py, py: Python<'_>) -> PyResult> { asyncgens::shutdown_asyncgens(slf, py) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyLoop"))] #[pyo3(signature=(timeout=None))] fn shutdown_default_executor( slf: Py, diff --git a/src/bindings/loop_api/pipes.rs b/src/bindings/loop_api/pipes.rs index 4aa5de9..a1d659e 100644 --- a/src/bindings/loop_api/pipes.rs +++ b/src/bindings/loop_api/pipes.rs @@ -6,6 +6,7 @@ use super::PyLoop; use super::spawn_env::{LoopSpawnEnv, transport_protocol_pair}; use crate::transport::stream::{spawn_read_pipe_transport, spawn_write_pipe_transport}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn connect_read_pipe<'py>( slf: Py, py: Python<'py>, @@ -25,6 +26,7 @@ pub(super) fn connect_read_pipe<'py>( }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn connect_write_pipe<'py>( slf: Py, py: Python<'py>, diff --git a/src/bindings/loop_api/process_handles.rs b/src/bindings/loop_api/process_handles.rs index a9c3559..64dcb24 100644 --- a/src/bindings/loop_api/process_handles.rs +++ b/src/bindings/loop_api/process_handles.rs @@ -7,6 +7,7 @@ use pyo3::prelude::*; use crate::fd_ops; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn file_from_fd(fd: fd_ops::RawFd) -> PyResult { #[cfg(windows)] { @@ -25,6 +26,7 @@ pub(super) fn file_from_fd(fd: fd_ops::RawFd) -> PyResult { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn new_pipe() -> PyResult<(File, File)> { #[cfg(windows)] { @@ -42,6 +44,7 @@ pub(super) fn new_pipe() -> PyResult<(File, File)> { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn file_from_owned_handle(handle: windows_sys::Win32::Foundation::HANDLE) -> File { use std::os::windows::io::FromRawHandle; @@ -51,6 +54,7 @@ fn file_from_owned_handle(handle: windows_sys::Win32::Foundation::HANDLE) -> Fil unsafe { File::from_raw_handle(handle.cast()) } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] fn file_from_owned_fd(fd: i32) -> File { use std::os::fd::FromRawFd; @@ -60,6 +64,7 @@ fn file_from_owned_fd(fd: i32) -> File { unsafe { File::from_raw_fd(fd) } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn create_pipe_handles() -> PyResult<( windows_sys::Win32::Foundation::HANDLE, @@ -81,6 +86,7 @@ fn create_pipe_handles() -> PyResult<( Ok((read_end, write_end)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] fn create_pipe_fds() -> PyResult<[i32; 2]> { let mut fds = [0_i32; 2]; diff --git a/src/bindings/loop_api/process_spawn.rs b/src/bindings/loop_api/process_spawn.rs index c958636..86608fc 100644 --- a/src/bindings/loop_api/process_spawn.rs +++ b/src/bindings/loop_api/process_spawn.rs @@ -42,6 +42,7 @@ enum ProcessTextMode { Conflict, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn process_text_mode( universal_newlines: bool, text: Option, @@ -64,6 +65,7 @@ enum NormalizedUmask { Invalid, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn normalize_process_umask(mask: i64) -> NormalizedUmask { if mask == -1 { NormalizedUmask::Unchanged @@ -95,6 +97,10 @@ struct ProcessSpawnConfig { } impl Default for UnixPreExecConfig { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn default() -> Self { Self { restore_signals: true, @@ -120,6 +126,7 @@ pub(super) struct SubprocessParams { pub(super) api_name: &'static str, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Shared tail of both subprocess methods: build the protocol, spawn the child, /// and wrap it in a transport. Only the `Command` construction differs, so it /// arrives as a closure that runs under the GIL on the spawning thread. @@ -180,6 +187,7 @@ where }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// `/bin/sh -c ` on Unix, `%COMSPEC% /c ""` on Windows. pub(super) fn shell_command(py: Python<'_>, cmd: &Py) -> PyResult { let shell_cmd = cmd.bind(py).extract::()?; @@ -199,6 +207,7 @@ pub(super) fn shell_command(py: Python<'_>, cmd: &Py) -> PyResult, program: &Py, @@ -211,6 +220,7 @@ pub(super) fn exec_command( Ok(command) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn resolve_numeric_id( py: Python<'_>, value: &Bound<'_, PyAny>, @@ -232,6 +242,7 @@ fn resolve_numeric_id( ))) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn resolve_extra_groups(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult> { let mut groups = Vec::new(); for item in value.try_iter()? { @@ -248,6 +259,7 @@ fn resolve_extra_groups(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult, universal_newlines: bool, @@ -289,6 +301,7 @@ pub(super) fn parse_process_text_config( })) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn apply_process_basic_kw( py: Python<'_>, command: &mut Command, @@ -303,6 +316,7 @@ fn apply_process_basic_kw( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// `Popen` keywords that are platform-specific or already implied by the way /// rsloop spawns. Wrappers around `loop.subprocess_exec()` — `AnyIO` is the one /// that caught this — forward every one of them unconditionally at its @@ -351,6 +365,7 @@ fn apply_platform_process_kw( Ok(true) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn windows_only_process_kw_error(key: &str) -> PyErr { PyNotImplementedError::new_err(format!( @@ -358,12 +373,14 @@ fn windows_only_process_kw_error(key: &str) -> PyErr { )) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(not(windows))] fn windows_only_process_kw_error(key: &str) -> PyErr { // Same wording as subprocess.Popen so callers recognise it. PyValueError::new_err(format!("{key} is only supported on Windows platforms")) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn apply_process_creation_flags(command: &mut Command, flags: i64) -> PyResult<()> { let flags = u32::try_from(flags).map_err(|_| { @@ -375,11 +392,13 @@ fn apply_process_creation_flags(command: &mut Command, flags: i64) -> PyResult<( Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(not(windows))] fn apply_process_creation_flags(_command: &mut Command, _flags: i64) -> PyResult<()> { Err(windows_only_process_kw_error("creationflags")) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn process_fspath(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult { py.import("os")? .getattr("fspath")? @@ -387,6 +406,7 @@ fn process_fspath(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult .extract::() } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn apply_process_cwd( py: Python<'_>, command: &mut Command, @@ -398,6 +418,7 @@ fn apply_process_cwd( Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn apply_process_env(command: &mut Command, value: &Bound<'_, PyAny>) -> PyResult<()> { if !value.is_none() { for (env_key, env_value) in value.cast::()?.iter() { @@ -407,6 +428,7 @@ fn apply_process_env(command: &mut Command, value: &Bound<'_, PyAny>) -> PyResul Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn apply_process_executable( py: Python<'_>, command: &mut Command, @@ -434,6 +456,7 @@ struct UnixProcessKw<'a, 'py> { value: &'a Bound<'py, PyAny>, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn apply_unix_process_kw( py: Python<'_>, unix: &mut UnixPreExecConfig, @@ -456,6 +479,7 @@ fn apply_unix_process_kw( apply_unix_misc_process_kw(&mut kw) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn is_known_unix_process_kw(key: &str) -> bool { matches!( key, @@ -463,6 +487,7 @@ fn is_known_unix_process_kw(key: &str) -> bool { ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn apply_unix_fd_process_kw(kw: &mut UnixProcessKw<'_, '_>) -> PyResult { match kw.key { "process_group" => kw.unix.process_group = Some(kw.value.extract::()?), @@ -478,6 +503,7 @@ fn apply_unix_fd_process_kw(kw: &mut UnixProcessKw<'_, '_>) -> PyResult { Ok(true) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn apply_unix_identity_process_kw( py: Python<'_>, kw: &mut UnixProcessKw<'_, '_>, @@ -501,6 +527,7 @@ fn apply_unix_identity_process_kw( Ok(true) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn apply_unix_misc_process_kw(kw: &mut UnixProcessKw<'_, '_>) -> PyResult { match kw.key { "umask" => { @@ -523,6 +550,7 @@ fn apply_unix_misc_process_kw(kw: &mut UnixProcessKw<'_, '_>) -> PyResult Ok(true) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn apply_unix_bool_process_kw(kw: &mut UnixProcessKw<'_, '_>) -> PyResult { match kw.key { "restore_signals" => kw.unix.restore_signals = kw.value.is_truthy()?, @@ -532,6 +560,7 @@ fn apply_unix_bool_process_kw(kw: &mut UnixProcessKw<'_, '_>) -> PyResult Ok(true) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn apply_common_process_kwargs( py: Python<'_>, command: &mut Command, diff --git a/src/bindings/loop_api/process_stdio.rs b/src/bindings/loop_api/process_stdio.rs index fb92a1a..bab648e 100644 --- a/src/bindings/loop_api/process_stdio.rs +++ b/src/bindings/loop_api/process_stdio.rs @@ -45,6 +45,7 @@ enum ProcessStdioPlanError { StdoutRedirect, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn process_stdio_plan( stdin: ProcessStdioSpec, stdout: ProcessStdioSpec, @@ -67,6 +68,10 @@ fn process_stdio_plan( } impl ProcessStdioSpecs { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessStdioSpecs") + )] pub(super) fn parse( py: Python<'_>, stdin: &Py, @@ -83,6 +88,7 @@ impl ProcessStdioSpecs { static PIPE_CELL: PyOnceLock> = PyOnceLock::new(); +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Default for an omitted `stdin`/`stdout`/`stderr`: `subprocess.PIPE` (== -1), /// matching `CPython`'s loop methods. An explicit `None` arrives as `Option::None` /// instead and is honored as "inherit the parent's fd" by `parse_process_stdio`. @@ -99,6 +105,7 @@ pub(super) fn default_stdio_pipe() -> Py { }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn parse_process_stdio( py: Python<'_>, value: &Py, @@ -114,6 +121,7 @@ fn parse_process_stdio( ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn parse_subprocess_stdio_marker( py: Python<'_>, value: &Bound<'_, PyAny>, @@ -134,10 +142,12 @@ fn parse_subprocess_stdio_marker( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn stdio_from_fd(fd: fd_ops::RawFd) -> PyResult { process_handles::file_from_fd(fd).map(std::process::Stdio::from) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn apply_stdio( command: &mut Command, specs: ProcessStdioSpecs, @@ -175,6 +185,7 @@ pub(super) fn apply_stdio( Ok((stdout_override, stderr_override)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn output_stdio(spec: ProcessStdioSpec) -> PyResult { use std::process::Stdio; @@ -189,6 +200,7 @@ fn output_stdio(spec: ProcessStdioSpec) -> PyResult { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn stderr_stdio( stderr: ProcessStdioSpec, stdout: ProcessStdioSpec, @@ -199,6 +211,7 @@ fn stderr_stdio( output_stdio(stderr) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn stderr_stdout_stdio(stdout: ProcessStdioSpec) -> PyResult { use std::process::Stdio; diff --git a/src/bindings/loop_api/servers.rs b/src/bindings/loop_api/servers.rs index 6db7326..58cfeb6 100644 --- a/src/bindings/loop_api/servers.rs +++ b/src/bindings/loop_api/servers.rs @@ -37,6 +37,7 @@ pub(super) struct CreateUnixServerParams { pub(super) cleanup_socket: bool, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn create_server<'py>( slf: Py, py: Python<'py>, @@ -93,6 +94,7 @@ pub(super) fn create_server<'py>( }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn create_unix_server<'py>( slf: Py, py: Python<'py>, diff --git a/src/bindings/loop_api/signals.rs b/src/bindings/loop_api/signals.rs index f104b8e..a4d120e 100644 --- a/src/bindings/loop_api/signals.rs +++ b/src/bindings/loop_api/signals.rs @@ -19,6 +19,7 @@ use crate::context::capture_context; use crate::engine::SignalHandlerTemplate; use crate::engine::{LoopCommand, LoopSignalCommand}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn add_signal_handler( slf: Py, py: Python<'_>, @@ -76,6 +77,7 @@ pub(super) fn add_signal_handler( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn remove_signal_handler(slf: Py, py: Python<'_>, sig: i32) -> PyResult { let loop_ref = slf.borrow(py); let core = loop_ref.core.clone(); @@ -96,6 +98,7 @@ pub(super) fn remove_signal_handler(slf: Py, py: Python<'_>, sig: i32) - Ok(removed) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Called by the signal watcher, not from a signal handler: re-enters the loop /// through `call_soon_threadsafe` so the user callback runs on the loop thread /// in the context captured at registration. diff --git a/src/bindings/loop_api/sock_ops.rs b/src/bindings/loop_api/sock_ops.rs index 73c7cb7..37c9386 100644 --- a/src/bindings/loop_api/sock_ops.rs +++ b/src/bindings/loop_api/sock_ops.rs @@ -9,6 +9,7 @@ use pyo3::prelude::*; use super::PyLoop; use super::socket_connect::connect_socket_to_address; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn sock_recv<'py>( slf: Py, py: Python<'py>, @@ -23,6 +24,7 @@ pub(super) fn sock_recv<'py>( ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn sock_recv_into<'py>( slf: Py, py: Python<'py>, @@ -37,6 +39,7 @@ pub(super) fn sock_recv_into<'py>( ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn sock_sendall<'py>( slf: Py, py: Python<'py>, @@ -64,6 +67,7 @@ pub(super) fn sock_sendall<'py>( ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn sock_accept<'py>( slf: Py, py: Python<'py>, @@ -72,6 +76,7 @@ pub(super) fn sock_accept<'py>( super::socket_operation::start(slf, py, sock, super::socket_operation::SocketAction::Accept) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn sock_connect<'py>( slf: Py, py: Python<'py>, @@ -85,6 +90,7 @@ pub(super) fn sock_connect<'py>( }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Connects an INET/INET6 stream socket, returning a loop-native Future /// (not a coroutine — awaited directly, never `create_task`ed). On Unix the /// writability wait runs on the vibeio reactor and its completion is diff --git a/src/bindings/loop_api/socket_connect.rs b/src/bindings/loop_api/socket_connect.rs index 0509e7e..44319db 100644 --- a/src/bindings/loop_api/socket_connect.rs +++ b/src/bindings/loop_api/socket_connect.rs @@ -21,6 +21,7 @@ use crate::fd_ops; const WSAEISCONN: i32 = 10056; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] pub(super) async fn connect_socket_to_address(sock: Py, address: Py) -> PyResult<()> { // Initiate the connect and look up the descriptor in a single GIL // acquisition. Every `Python::attach` here runs on the async runtime's @@ -65,6 +66,7 @@ pub(super) async fn connect_socket_to_address(sock: Py, address: Py) -> PyResult { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn socket_so_error(py: Python<'_>, sock: &Py) -> PyResult { let socket_mod = py.import("socket")?; @@ -112,6 +115,7 @@ fn socket_so_error(py: Python<'_>, sock: &Py) -> PyResult { .extract(py) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn socket_os_error(py: Python<'_>, errno: i32) -> PyResult<()> { let builtins = py.import("builtins")?; let oserror = builtins.getattr("OSError")?; @@ -121,6 +125,7 @@ fn socket_os_error(py: Python<'_>, errno: i32) -> PyResult<()> { ))?)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn is_already_connected_socket_error(py: Python<'_>, err: &PyErr) -> PyResult { let builtins = py.import("builtins")?; let oserror = builtins.getattr("OSError")?; @@ -135,16 +140,19 @@ fn is_already_connected_socket_error(py: Python<'_>, err: &PyErr) -> PyResult bool { errno == libc::EISCONN || errno == WSAEISCONN } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn is_connect_in_progress_errno(errno: i32) -> bool { errno == libc::EINPROGRESS || errno == libc::EALREADY || errno == libc::EWOULDBLOCK } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Attempts to initiate the connect via a direct `libc::connect` for a numeric /// address, skipping Python's `socket.connect` (its dispatch, address parsing, /// and — the expensive part — raising a `BlockingIOError` for EINPROGRESS on @@ -182,6 +190,7 @@ fn libc_connect_numeric(fd: fd_ops::RawFd, address: &Bound<'_, PyAny>) -> PyResu } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Initiates a non-blocking connect on the loop thread and, when it does not /// complete synchronously, hands the writability wait to the vibeio reactor on /// this loop's own runtime. Returns the loop Future the caller awaits. diff --git a/src/bindings/loop_api/socket_operation.rs b/src/bindings/loop_api/socket_operation.rs index 799a6b8..4faf41d 100644 --- a/src/bindings/loop_api/socket_operation.rs +++ b/src/bindings/loop_api/socket_operation.rs @@ -32,10 +32,18 @@ pub(super) enum SocketAction { } impl SocketAction { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "SocketAction") + )] fn writable(&self) -> bool { matches!(self, Self::SendAll { .. }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "SocketAction") + )] fn attempt(&mut self, py: Python<'_>, socket: &Py) -> PyResult>> { let result = match self { Self::Recv(count) => socket.call_method1(py, "recv", (*count,)), @@ -104,6 +112,7 @@ impl SocketAction { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn retry_or_error(py: Python<'_>, error: PyErr) -> PyResult>> { if fd_ops::is_retryable_socket_error(py, &error)? { Ok(None) @@ -124,6 +133,10 @@ struct CancelWait(Arc); #[pymethods] impl CancelWait { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "CancelWait") + )] fn __call__(&self, _future: &Bound<'_, PyAny>) { self.0.done.store(true, Ordering::Release); self.0.waker.wake(); @@ -142,11 +155,16 @@ struct SocketOperation { #[pymethods] impl SocketOperation { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "SocketOperation") + )] fn __call__(slf: Py, py: Python<'_>) -> PyResult<()> { poll_operation(py, slf) } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn start<'py>( slf: Py, py: Python<'py>, @@ -209,6 +227,7 @@ pub(super) fn start<'py>( Ok(future.into_bound(py)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn poll_operation(py: Python<'_>, operation: Py) -> PyResult<()> { let mut op = operation.borrow_mut(py); if op.future.call_method0(py, "done")?.extract::(py)? { @@ -242,6 +261,7 @@ fn poll_operation(py: Python<'_>, operation: Py) -> PyResult<() Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn duplicate_socket(py: Python<'_>, socket: &Py) -> PyResult { let fd = fd_ops::fileobj_to_fd(py, socket.bind(py))?; #[cfg(unix)] @@ -262,6 +282,7 @@ fn duplicate_socket(py: Python<'_>, socket: &Py) -> PyResult, operation: Py) -> PyResult<()> { let op = operation.borrow(py); let duplicate = match duplicate_socket(py, &op.socket) { @@ -296,6 +317,7 @@ fn arm_wait(py: Python<'_>, operation: Py) -> PyResult<()> { Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] async fn wait_socket( core: Arc, socket: socket2::Socket, diff --git a/src/bindings/loop_api/sockets.rs b/src/bindings/loop_api/sockets.rs index 340c481..e9507b6 100644 --- a/src/bindings/loop_api/sockets.rs +++ b/src/bindings/loop_api/sockets.rs @@ -32,6 +32,7 @@ pub(super) struct TcpServerSocketOptions { pub(super) keep_alive: Option, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn resolve_stream_addrinfos( py: Python<'_>, host: Option>, @@ -68,6 +69,7 @@ struct AddrInfoQuery { flags: i32, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn call_getaddrinfo<'py>( py: Python<'py>, socket_mod: &Bound<'py, PyModule>, @@ -86,6 +88,7 @@ fn call_getaddrinfo<'py>( .call((host, port), Some(&kwargs)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn parse_stream_addrinfo(entry: Bound<'_, PyAny>) -> PyResult { let tuple = entry.cast::()?; Ok(( @@ -96,6 +99,7 @@ fn parse_stream_addrinfo(entry: Bound<'_, PyAny>) -> PyResult, family: i32, @@ -110,6 +114,7 @@ pub(super) fn build_stream_socket( Ok(sock.unbind()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] fn set_socket_bool_option_unix( py: Python<'_>, @@ -137,6 +142,7 @@ fn set_socket_bool_option_unix( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn listener_sources_from_sockets( py: Python<'_>, sockets: &[Py], @@ -167,6 +173,7 @@ pub(super) fn listener_sources_from_sockets( Ok(listeners) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn build_tcp_server_sockets( py: Python<'_>, host: Option>, @@ -222,6 +229,7 @@ struct TcpSocketOptionRefs<'py, 'a> { so_keepalive: &'a Bound<'py, PyAny>, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn apply_tcp_server_socket_options( py: Python<'_>, sock: &Py, @@ -255,11 +263,13 @@ fn apply_tcp_server_socket_options( Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] fn set_tcp_keepalive_option(py: Python<'_>, sock: &Py, keep_alive: bool) -> PyResult<()> { set_socket_bool_option_unix(py, sock, libc::SOL_SOCKET, libc::SO_KEEPALIVE, keep_alive) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(not(unix))] fn set_tcp_keepalive_option( py: Python<'_>, @@ -279,6 +289,7 @@ fn set_tcp_keepalive_option( Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub(super) fn build_unix_server_socket( py: Python<'_>, @@ -304,6 +315,7 @@ pub(super) fn build_unix_server_socket( Ok(sock.unbind()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Builds the unnamed `AF_UNIX` socket used to dial a Unix server. #[cfg(unix)] pub(super) fn build_unix_client_socket(py: Python<'_>) -> PyResult> { diff --git a/src/bindings/loop_api/spawn_env.rs b/src/bindings/loop_api/spawn_env.rs index b73a5d4..1acaeb5 100644 --- a/src/bindings/loop_api/spawn_env.rs +++ b/src/bindings/loop_api/spawn_env.rs @@ -25,6 +25,10 @@ pub(super) struct LoopSpawnEnv { } impl LoopSpawnEnv { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "LoopSpawnEnv") + )] /// Snapshots the loop and the caller's context. Must run on the calling /// thread, before the transport future is constructed. pub(super) fn capture(py: Python<'_>, slf: &Py) -> PyResult { @@ -39,6 +43,10 @@ impl LoopSpawnEnv { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "LoopSpawnEnv") + )] /// Builds the spawn context handed to the transport constructors. `protocol` /// is the protocol instance, or the protocol factory for server creation. pub(super) fn spawn_context( @@ -56,6 +64,10 @@ impl LoopSpawnEnv { ) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "LoopSpawnEnv") + )] /// Instantiates the protocol in the caller's context, rejecting the call if /// the loop is no longer running. pub(super) fn call_protocol_factory( @@ -75,6 +87,7 @@ impl LoopSpawnEnv { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// The `(transport, protocol)` tuple every `asyncio` transport-creating method /// resolves to. pub(super) fn transport_protocol_pair( @@ -86,6 +99,7 @@ pub(super) fn transport_protocol_pair( Ok(result.unbind().into_any()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn is_asyncio_subprocess_stream_protocol( py: Python<'_>, protocol: &Py, diff --git a/src/bindings/loop_api/tasks.rs b/src/bindings/loop_api/tasks.rs index 4d6f0e6..17ef3a8 100644 --- a/src/bindings/loop_api/tasks.rs +++ b/src/bindings/loop_api/tasks.rs @@ -31,6 +31,7 @@ pub(super) struct TaskOptions { pub(super) kwargs: Option>, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn is_current_running_loop(py: Python<'_>, loop_obj: &Py) -> PyResult { let current = asyncio_get_running_loop_fn(py)?.call0(py)?; if current.is_none(py) { @@ -39,6 +40,7 @@ fn is_current_running_loop(py: Python<'_>, loop_obj: &Py) -> PyResult, loop_obj: &Py) -> PyResult> { #[cfg(any(Py_3_12, all(Py_3_11, not(Py_LIMITED_API))))] { @@ -56,10 +58,12 @@ fn create_asyncio_future_for_loop(py: Python<'_>, loop_obj: &Py) -> PyRes } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn create_asyncio_future_for_running_loop(py: Python<'_>) -> PyResult> { call_callable_noargs(py, asyncio_future_cls(py)?) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Fast-path future creation for internal callers that hold a loop object: /// when `loop_obj` is exactly a `PyLoop` running on this thread, skip the /// Python-level `create_future` method dispatch. Returns `Ok(None)` when the @@ -77,6 +81,7 @@ pub(crate) fn try_fast_create_future( create_asyncio_future_for_running_loop(py).map(Some) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(crate) fn try_fast_create_task( py: Python<'_>, loop_obj: &Py, @@ -92,6 +97,7 @@ pub(crate) fn try_fast_create_task( create_asyncio_task_for_running_loop(py, loop_obj.bind(py), coro).map(Some) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn create_asyncio_task_for_loop( py: Python<'_>, loop_obj: &Py, @@ -139,6 +145,7 @@ fn create_asyncio_task_for_loop( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn create_asyncio_task_for_running_loop( py: Python<'_>, @@ -149,6 +156,7 @@ fn create_asyncio_task_for_running_loop( call_callable_onearg(py, task_cls, coro.bind(py)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn create_asyncio_task_with_kwargs( py: Python<'_>, loop_obj: Option<&Py>, @@ -162,6 +170,7 @@ fn create_asyncio_task_with_kwargs( asyncio_task_cls(py)?.call(py, (coro,), Some(&task_kwargs)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn trim_task_source_traceback(py: Python<'_>, task: &Py) -> PyResult<()> { let Ok(source_traceback) = task.getattr(py, "_source_traceback") else { return Ok(()); @@ -178,6 +187,7 @@ fn trim_task_source_traceback(py: Python<'_>, task: &Py) -> PyResult<()> source_traceback.del_item(source_traceback.len()? - 1) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn create_future(slf: Py, py: Python<'_>) -> PyResult> { if slf.get().core.on_runtime_thread() { return create_asyncio_future_for_running_loop(py); @@ -191,6 +201,7 @@ pub(super) fn create_future(slf: Py, py: Python<'_>) -> PyResult, py: Python<'_>, diff --git a/src/bindings/loop_api/tls_params.rs b/src/bindings/loop_api/tls_params.rs index c68fd79..b8d7c99 100644 --- a/src/bindings/loop_api/tls_params.rs +++ b/src/bindings/loop_api/tls_params.rs @@ -30,6 +30,7 @@ struct TlsValidationInputs { has_shutdown_timeout: bool, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn tls_validation_error(inputs: TlsValidationInputs) -> Option { if inputs.has_ssl { None @@ -53,6 +54,7 @@ pub(super) struct TlsParams { } impl TlsParams { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TlsParams"))] /// Options for a method that has no `server_hostname` parameter. pub(super) fn without_hostname( ssl: Option>, @@ -67,10 +69,12 @@ impl TlsParams { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TlsParams"))] pub(super) fn is_enabled(&self) -> bool { self.ssl.is_some() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TlsParams"))] /// Rejects TLS-only keywords passed without `ssl`. The check order matches /// what the individual methods used to do, so the reported error for a call /// that misuses several keywords at once does not change. @@ -95,6 +99,7 @@ impl TlsParams { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TlsParams"))] /// Client-side settings, or `None` when the caller passed no `ssl`. pub(super) fn client_settings(&self, py: Python<'_>) -> PyResult> { let Some(ssl) = self.ssl.as_ref() else { @@ -110,6 +115,7 @@ impl TlsParams { .map(Some) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TlsParams"))] /// Server-side settings, or `None` when the caller passed no `ssl`. pub(super) fn server_settings(&self, py: Python<'_>) -> PyResult> { let Some(ssl) = self.ssl.as_ref() else { @@ -124,6 +130,7 @@ impl TlsParams { .map(Some) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TlsParams"))] /// Server settings shared with the accept tasks that outlive this call. pub(super) fn shared_server_settings( &self, diff --git a/src/bindings/loop_api/watchers.rs b/src/bindings/loop_api/watchers.rs index 789abca..be06911 100644 --- a/src/bindings/loop_api/watchers.rs +++ b/src/bindings/loop_api/watchers.rs @@ -16,6 +16,7 @@ use crate::context::capture_context; use crate::engine::{CallbackKind, FdWatch, LoopCommand, LoopIoCommand, ReadyCallback}; use crate::fd_ops; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn add_reader( loop_ref: &PyLoop, py: Python<'_>, @@ -62,6 +63,7 @@ pub(super) fn add_reader( .map_err(PyLoop::map_loop_error) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn remove_reader( loop_ref: &PyLoop, py: Python<'_>, @@ -104,6 +106,7 @@ pub(super) fn remove_reader( Ok(removed) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn add_writer( loop_ref: &PyLoop, py: Python<'_>, @@ -147,6 +150,7 @@ pub(super) fn add_writer( .map_err(PyLoop::map_loop_error) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn remove_writer( loop_ref: &PyLoop, py: Python<'_>, diff --git a/src/blocking.rs b/src/blocking.rs index 364e767..66525ec 100644 --- a/src/blocking.rs +++ b/src/blocking.rs @@ -4,6 +4,7 @@ use std::thread; use futures::channel::oneshot; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Runs one blocking closure on a named worker and returns its result asynchronously. pub async fn run(name: impl Into, task: F) -> Result where diff --git a/src/build_metadata.rs b/src/build_metadata.rs index 060b120..7b632dd 100644 --- a/src/build_metadata.rs +++ b/src/build_metadata.rs @@ -12,6 +12,7 @@ const REACTOR: &str = "kqueue"; #[cfg(not(any(windows, target_os = "linux", target_os = "macos")))] const REACTOR: &str = "mio"; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Return stable diagnostics that help identify the installed native build. #[pyfunction] pub(crate) fn build_info(py: Python<'_>) -> PyResult> { @@ -43,5 +44,10 @@ pub(crate) fn build_info(py: Python<'_>) -> PyResult> { }, )?; info.set_item("tls_backend", "rustls")?; + info.set_item("hotpath_profile", cfg!(feature = "hotpath-profile"))?; + info.set_item( + "hotpath_alloc_profile", + cfg!(feature = "hotpath-alloc-profile"), + )?; Ok(info.unbind()) } diff --git a/src/context.rs b/src/context.rs index 5b65d29..5cfed21 100644 --- a/src/context.rs +++ b/src/context.rs @@ -7,6 +7,7 @@ use pyo3::types::PyTuple; static SET_RUNNING_LOOP_FN: PyOnceLock> = PyOnceLock::new(); +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn set_running_loop_fn(py: Python<'_>) -> PyResult<&Py> { SET_RUNNING_LOOP_FN.get_or_try_init(py, || { @@ -31,6 +32,7 @@ pub fn capture_context(py: Python<'_>, explicit: Option>) -> PyResult< Ok((context, true)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub fn is_nested_context_error(py: Python<'_>, err: &PyErr) -> bool { err.is_instance_of::(py) @@ -68,11 +70,13 @@ pub fn exit_context(py: Python<'_>, context: &Py) -> PyResult<()> { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn call_noargs(py: Python<'_>, callback: &Py) -> PyResult> { Ok(callback.bind(py).call0()?.unbind()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn call_onearg( py: Python<'_>, @@ -82,6 +86,7 @@ fn call_onearg( Ok(callback.bind(py).call1((arg,))?.unbind()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn run_in_context( py: Python<'_>, context: &Py, @@ -112,6 +117,7 @@ pub fn run_in_context( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub fn run_in_context_noargs( py: Python<'_>, @@ -140,6 +146,7 @@ pub fn run_in_context_noargs( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub fn run_in_context_onearg( py: Python<'_>, @@ -169,12 +176,14 @@ pub fn run_in_context_onearg( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub fn ensure_running_loop(py: Python<'_>, loop_obj: &Py) -> PyResult<()> { set_running_loop_fn(py)?.call1(py, (loop_obj.clone_ref(py),))?; Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub fn clear_running_loop(py: Python<'_>) -> PyResult<()> { set_running_loop_fn(py)?.call1(py, (py.None(),))?; diff --git a/src/engine/callbacks.rs b/src/engine/callbacks.rs index a2e7209..ac08541 100644 --- a/src/engine/callbacks.rs +++ b/src/engine/callbacks.rs @@ -74,6 +74,10 @@ pub struct ReadyCallback { } impl ReadyCallback { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadyCallback") + )] #[inline] /// Builds a ready callback and selects a zero-, one-, or many-argument fast path. /// @@ -102,6 +106,10 @@ impl ReadyCallback { Self::from_args(id, kind, callback, args, context, context_needs_run) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadyCallback") + )] pub(crate) fn from_args( id: CallbackId, kind: CallbackKind, @@ -130,12 +138,20 @@ impl ReadyCallback { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadyCallback") + )] #[inline] /// Returns the loop-unique identifier assigned to this callback. pub fn id(&self) -> CallbackId { self.id } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadyCallback") + )] #[inline] /// Returns the scheduling source used for diagnostics and re-arming I/O. pub fn kind(&self) -> CallbackKind { @@ -150,18 +166,30 @@ impl ReadyCallback { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadyCallback") + )] #[inline] /// Borrows the underlying Python callable. pub fn callback(&self) -> &Py { &self.callback } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadyCallback") + )] #[inline] /// Borrows the captured Python `contextvars.Context`. pub fn context(&self) -> &Py { &self.context } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadyCallback") + )] #[inline] /// Reports whether invocation needs to enter the captured context. pub fn context_needs_run(&self) -> bool { @@ -210,12 +238,20 @@ impl ReadyCallback { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadyCallback") + )] #[inline] /// Reports whether this callback has been cancelled. pub fn cancelled(&self) -> bool { self.cancelled.load(Ordering::Relaxed) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadyCallback") + )] #[inline] /// Marks the callback as cancelled. /// @@ -239,12 +275,14 @@ pub struct PyHandle { } impl PyHandle { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyHandle"))] #[inline] /// Wraps a callback in a Python-visible handle. pub fn new(callback: ReadyCallback) -> Self { Self { callback } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyHandle"))] #[inline] /// Borrows the callback controlled by this handle. pub fn ready(&self) -> &ReadyCallback { @@ -254,15 +292,18 @@ impl PyHandle { #[pymethods] impl PyHandle { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyHandle"))] fn cancel(&self) -> PyResult<()> { self.callback.cancel(); Ok(()) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyHandle"))] fn cancelled(&self) -> bool { self.callback.cancelled() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyHandle"))] fn __repr__(&self) -> String { format!( "", @@ -291,6 +332,10 @@ pub struct PyTimerHandle { } impl PyTimerHandle { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyTimerHandle") + )] #[inline] /// Creates a timer handle for a callback scheduled at loop time `when`. pub fn new(callback_id: CallbackId, when: f64, callback: &Arc) -> Self { @@ -479,6 +524,10 @@ mod tests { #[pymethods] impl PyTimerHandle { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyTimerHandle") + )] fn cancel(&self) -> PyResult<()> { self.cancelled.store(true, Ordering::Relaxed); if let Some(callback) = self.callback.upgrade() { @@ -487,14 +536,26 @@ impl PyTimerHandle { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyTimerHandle") + )] fn cancelled(&self) -> bool { self.cancelled.load(Ordering::Relaxed) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyTimerHandle") + )] fn when(&self) -> f64 { self.when } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyTimerHandle") + )] fn __repr__(&self) -> String { format!( "", diff --git a/src/engine/dispatcher.rs b/src/engine/dispatcher.rs index b5aef8b..958be82 100644 --- a/src/engine/dispatcher.rs +++ b/src/engine/dispatcher.rs @@ -61,6 +61,7 @@ enum WatchTask { } impl WatchTask { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "WatchTask"))] fn spawn_thread(name: String, task: impl FnOnce(Arc) + Send + 'static) -> Self { let stop = Arc::new(AtomicBool::new(false)); let thread_stop = Arc::clone(&stop); @@ -71,6 +72,7 @@ impl WatchTask { Self::Thread { stop, join } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "WatchTask"))] fn abort(self) { match self { Self::Thread { stop, join } => { @@ -81,6 +83,7 @@ impl WatchTask { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "WatchTask"))] fn cancel(self) { match self { Self::Thread { stop, .. } => { @@ -91,16 +94,19 @@ impl WatchTask { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn abort_watch_task(task: WatchTask) { task.abort(); } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn cancel_watch_task(task: WatchTask) { task.cancel(); } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn run_runtime_thread(core: Arc, command_rx: Receiver) { let runtime = crate::vibeio::RuntimeBuilder::new() .rsloop_profile() @@ -128,6 +134,10 @@ pub fn run_runtime_thread(core: Arc, command_rx: Receiver impl Future for RuntimeDispatcher { type Output = (); + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { // Register before inspecting the command channel so a concurrent send // either wakes this poll or remains visible when the channel is @@ -157,6 +167,10 @@ impl Future for RuntimeDispatcher { } impl RuntimeDispatcher { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeDispatcher") + )] #[inline] fn has_ready(&self) -> bool { !self.ready_batch.is_empty() @@ -180,6 +194,10 @@ impl RuntimeDispatcher { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeDispatcher") + )] fn handle_command(&mut self, command: LoopCommand) -> bool { match command { LoopCommand::ScheduleReady(callback) => { @@ -475,6 +493,10 @@ impl RuntimeDispatcher { self.core.signal_ready(); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeDispatcher") + )] fn finish_run(&mut self) { let Some(active_run) = self.active_run.take() else { return; @@ -491,6 +513,10 @@ impl RuntimeDispatcher { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeDispatcher") + )] fn cleanup_watchers(&mut self) { #[cfg(unix)] for (_, watcher) in self.signal_tasks.drain() { diff --git a/src/engine/loop_core.rs b/src/engine/loop_core.rs index 14ce15c..2357e28 100644 --- a/src/engine/loop_core.rs +++ b/src/engine/loop_core.rs @@ -23,7 +23,7 @@ use super::commands::{ LoopCommand, LoopFutureCommand, LoopIoCommand, LoopRunCommand, LoopTransportCommand, ReadyItem, }; use super::dispatcher::run_runtime_thread; -use super::timer_entry::TimerEntry; +use super::timer_entry::{TimerEntry, TimerQueue}; use crate::context::{capture_context, clear_running_loop, ensure_running_loop}; use crate::errors::handle_callback_error; use crate::fd_ops::RawFd; @@ -68,6 +68,7 @@ pub struct LoopWake { } impl LoopWake { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopWake"))] fn new() -> Self { Self { ready_pending: AtomicBool::new(false), @@ -75,6 +76,7 @@ impl LoopWake { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopWake"))] /// Marks the ready queue non-empty and wakes the parked loop thread. Cheap /// and idempotent while a wake is already pending. #[inline] @@ -102,6 +104,10 @@ impl WaitForWake { Self::until(wake, Instant::now() + timeout) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WaitForWake") + )] fn until(wake: Arc, deadline: Instant) -> Self { Self { wake, @@ -110,6 +116,7 @@ impl WaitForWake { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Bounded busy-wait before parking in `block_on`. Cross-thread wakeups (reader /// worker threads, the transitional runtime thread) otherwise pay the full /// `driver.wait` park + interrupt round-trip, which dominates request/response @@ -140,6 +147,10 @@ const SPIN_MISS_COOLDOWN_PARKS: u32 = 8; impl Future for WaitForWake { type Output = (); + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { if self.wake.ready_pending.load(Ordering::Acquire) { @@ -171,7 +182,7 @@ struct ActiveLoopTls { core: Cell<*const LoopCore>, ready_queue: Cell<*mut VecDeque>, drain_active: Cell, - timers: Cell<*mut BinaryHeap>, + timers: Cell<*mut TimerQueue>, } thread_local! { @@ -190,23 +201,27 @@ thread_local! { /// The guard returns outstanding timers when a run ends, including unwinding. struct LocalTimers<'a> { core: &'a LoopCore, - #[allow( - clippy::box_collection, - reason = "TLS points to the heap object, whose address must survive moving this guard" - )] - heap: Box>, + heap: Box, } impl<'a> LocalTimers<'a> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "LocalTimers") + )] fn new(core: &'a LoopCore) -> Self { - let mut heap = Box::new(BinaryHeap::new()); + let mut heap = Box::new(TimerQueue::new()); ACTIVE_LOOP_TLS.with(|tls| tls.timers.set(&mut *heap)); Self { core, heap } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "LocalTimers") + )] fn collect(&mut self, ready: &mut VecDeque) { if self.core.pending_timers_dirty.swap(false, Ordering::AcqRel) { - self.heap.append( + self.heap.append_heap( &mut self .core .pending_timers @@ -227,6 +242,10 @@ impl<'a> LocalTimers<'a> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "LocalTimers") + )] fn deadline(&self) -> Instant { let signal_deadline = Instant::now() + SIGNAL_POLL_INTERVAL; self.heap @@ -236,14 +255,20 @@ impl<'a> LocalTimers<'a> { } impl Drop for LocalTimers<'_> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { ACTIVE_LOOP_TLS.with(|tls| tls.timers.set(std::ptr::null_mut())); if !self.heap.is_empty() { - self.core - .pending_timers - .lock() - .expect("poisoned pending timers") - .append(&mut self.heap); + self.heap.drain_into( + &mut self + .core + .pending_timers + .lock() + .expect("poisoned pending timers"), + ); self.core .pending_timers_dirty .store(true, Ordering::Release); @@ -261,6 +286,10 @@ pub enum LoopCoreError { } impl fmt::Display for LoopCoreError { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Closed => write!(f, "event loop is closed"), @@ -314,6 +343,7 @@ pub struct LoopState { } impl LoopState { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopState"))] fn new() -> Self { Self { closed: false, @@ -356,6 +386,7 @@ pub struct LoopCore { } impl LoopCore { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Creates a loop core and starts its command-dispatcher thread. /// /// Python callbacks are not run on that thread; the dispatcher only @@ -390,6 +421,7 @@ impl LoopCore { core } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Submits a control command to the loop dispatcher. /// /// Commands originating on the active loop thread may be handled locally to @@ -403,6 +435,7 @@ impl LoopCore { self.send_remote_command(command) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] fn send_remote_command(&self, command: LoopCommand) -> Result<(), LoopCoreError> { self.command_tx .send(command) @@ -443,32 +476,38 @@ impl LoopCore { self.send_remote_command(LoopCommand::ScheduleReadyHandle(handle)) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Reports whether a run session is currently active. pub fn is_running(&self) -> bool { self.state.lock().expect("poisoned loop state").running } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Reports whether this loop has been permanently closed. pub fn is_closed(&self) -> bool { self.state.lock().expect("poisoned loop state").closed } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Enables or disables asyncio debug diagnostics. pub fn set_debug(&self, enabled: bool) { self.debug_enabled.store(enabled, Ordering::SeqCst); } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Returns whether asyncio debug diagnostics are enabled. pub fn get_debug(&self) -> bool { self.debug_enabled.load(Ordering::SeqCst) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] #[inline] /// Reports whether a custom Python task factory is installed. pub fn has_task_factory(&self) -> bool { self.task_factory_installed.load(Ordering::Relaxed) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] #[inline] /// Updates the fast-path flag for custom task-factory installation. /// @@ -478,11 +517,13 @@ impl LoopCore { .store(installed, Ordering::Relaxed); } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Allocates a loop-unique callback identifier. pub fn next_callback_id(&self) -> CallbackId { self.next_callback_id.fetch_add(1, Ordering::Relaxed) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Returns elapsed monotonic seconds on this loop's clock. pub fn time(&self) -> f64 { self.start.elapsed().as_secs_f64() @@ -546,6 +587,7 @@ impl LoopCore { Ok(handle) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Captures context and schedules a callback after `delay`. /// /// Returns the shared callback and its absolute value on [`LoopCore::time`], @@ -582,7 +624,7 @@ impl LoopCore { if std::ptr::eq(tls.core.get(), Arc::as_ptr(self)) && !tls.timers.get().is_null() { // SAFETY: LocalTimers owns this stable heap on the current // thread. No heap borrow crosses callback execution. - unsafe { (*tls.timers.get()).push(entry) }; + unsafe { (*tls.timers.get()).push(entry, delay.is_zero()) }; None } else { Some(entry) @@ -606,6 +648,7 @@ impl LoopCore { Ok((ready, when)) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Runs callbacks and I/O until [`LoopCore::schedule_stop`] is processed. /// /// The caller is the Python loop thread. While parked, the GIL is released @@ -1006,12 +1049,14 @@ impl LoopCore { } impl LoopCore { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] fn reset_run_state_after_finish_error(&self) { let mut state = self.state.lock().expect("poisoned loop state"); state.running = false; state.stopping = false; } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Requests a graceful stop of the active run session. /// /// Already-ready callbacks are still processed according to asyncio's @@ -1020,6 +1065,7 @@ impl LoopCore { self.send_command(LoopCommand::RequestStop) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Permanently closes the loop and joins its dispatcher thread. /// /// Closing is idempotent, but an actively running loop returns @@ -1079,6 +1125,7 @@ impl LoopCore { } impl LoopCore { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Dispatches an asyncio error-context dictionary to the configured handler. /// /// Falls back to [`LoopCore::default_exception_handler`] when no custom @@ -1109,6 +1156,7 @@ impl LoopCore { self.default_exception_handler(py, context) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Writes an unhandled callback error and traceback to Python's `sys.stderr`. pub fn default_exception_handler(&self, py: Python<'_>, context: Py) -> PyResult<()> { let sys = py.import("sys")?; @@ -1158,6 +1206,7 @@ impl LoopCore { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] fn rearm_fd_watch_if_needed(&self, ready: &ReadyCallback) { // Readiness callbacks are one-shot at the runtime layer. Re-arm them // only after the current ready batch has drained so callbacks queued @@ -1203,15 +1252,18 @@ impl LoopCore { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] #[inline] pub(crate) fn mark_runtime_thread(&self) { ACTIVE_LOOP_TLS.with(|tls| tls.core.set(self as *const Self)); } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] pub(crate) fn set_runtime_waker(&self, waker: Option) { *self.runtime_waker.lock().expect("poisoned runtime waker") = waker; } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Marks the ready queue non-empty and wakes the parked loop thread. Used by /// cross-thread ready producers (the transitional runtime thread, signal and /// transport workers). @@ -1220,6 +1272,7 @@ impl LoopCore { self.wake.signal(); } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Spawns a detached I/O task on this loop's on-thread vibeio runtime. Must /// be called on the loop thread (asyncio contract). The task begins running /// the next time the loop parks in `block_on`; its completions push ready @@ -1243,6 +1296,7 @@ impl LoopCore { }) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Spawns a cancellable I/O task (accept loop / socket reader) on this loop's /// runtime, tracked by `fd` so `stop_io_task` can cancel it. Any existing /// task registered for `fd` is cancelled first. Must run on the loop thread. @@ -1272,6 +1326,7 @@ impl LoopCore { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Cancels the tracked I/O task registered for `fd`, if any. Must run on the /// loop thread. /// @@ -1291,11 +1346,13 @@ impl LoopCore { }) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] #[inline] pub(crate) fn install_local_ready_queue(&self, ready: *mut VecDeque) { ACTIVE_LOOP_TLS.with(|tls| tls.ready_queue.set(ready)); } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] #[inline] pub(crate) fn clear_runtime_thread(&self) { ACTIVE_LOOP_TLS.with(|tls| { @@ -1307,16 +1364,19 @@ impl LoopCore { }); } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] #[inline] pub(crate) fn set_ready_drain_active(&self, active: bool) { ACTIVE_LOOP_TLS.with(|tls| tls.drain_active.set(active)); } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] #[inline] pub(crate) fn on_runtime_thread(&self) -> bool { ACTIVE_LOOP_TLS.with(|tls| std::ptr::eq(tls.core.get(), self)) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] /// Resolves a TCP connect whose writability wait finished on the vibeio /// reactor. Runs on the loop thread so the `SO_ERROR` check and the /// `set_result` / `set_exception` happen with the GIL already held for the @@ -1369,6 +1429,7 @@ impl LoopCore { Ok(()) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] #[inline] fn try_handle_local_command(&self, command: LoopCommand) -> Result<(), LoopCommand> { match command { @@ -1551,6 +1612,7 @@ impl LoopCore { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] #[inline] fn try_enqueue_local_ready(&self, item: ReadyItem) -> Result<(), ReadyItem> { ACTIVE_LOOP_TLS.with(|tls| { @@ -1577,6 +1639,7 @@ impl LoopCore { }) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "LoopCore"))] #[inline] fn try_enqueue_active_ready(&self, item: ReadyItem) -> Result<(), ReadyItem> { let active_dispatch = self diff --git a/src/engine/timer_entry.rs b/src/engine/timer_entry.rs index b27c4f6..5ff7d98 100644 --- a/src/engine/timer_entry.rs +++ b/src/engine/timer_entry.rs @@ -1,9 +1,11 @@ //! Timer-heap entry with stable ordering for equal deadlines. use std::cmp::Ordering; +use std::collections::{BinaryHeap, VecDeque}; use std::sync::Arc; use std::time::Instant; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn compare_timer_parts( left_when: &T, left_seq: u64, @@ -23,7 +25,94 @@ pub(super) struct TimerEntry { pub(super) callback: Arc, } +/// Immediately due timers normally arrive in deadline order and use a FIFO. +/// Positive delays and out-of-order arrivals use the ordinary heap. Keep the +/// two paths separate so scattered future deadlines pay no FIFO comparisons. +/// Neither insertion nor removal drops callbacks while the queue is borrowed. +#[derive(Default)] +pub(super) struct TimerQueue { + ordered: VecDeque, + unordered: BinaryHeap, +} + +impl TimerQueue { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TimerQueue") + )] + pub(super) fn new() -> Self { + Self::default() + } + + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TimerQueue") + )] + pub(super) fn is_empty(&self) -> bool { + self.ordered.is_empty() && self.unordered.is_empty() + } + + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TimerQueue") + )] + pub(super) fn push(&mut self, entry: TimerEntry, immediate: bool) { + // TimerEntry's comparison is reversed for the min-deadline heap. + // Remote scheduling/merging can reorder even zero-delay timers. + if immediate && self.ordered.back().is_none_or(|last| entry <= *last) { + self.ordered.push_back(entry); + } else { + self.unordered.push(entry); + } + } + + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TimerQueue") + )] + pub(super) fn peek(&self) -> Option<&TimerEntry> { + match (self.ordered.front(), self.unordered.peek()) { + (Some(front), Some(top)) if top > front => Some(top), + (Some(front), _) => Some(front), + (None, top) => top, + } + } + + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TimerQueue") + )] + pub(super) fn pop(&mut self) -> Option { + match (self.ordered.front(), self.unordered.peek()) { + (Some(front), Some(top)) if top > front => self.unordered.pop(), + (Some(_), _) => self.ordered.pop_front(), + (None, _) => self.unordered.pop(), + } + } + + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TimerQueue") + )] + pub(super) fn append_heap(&mut self, other: &mut BinaryHeap) { + self.unordered.append(other); + } + + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TimerQueue") + )] + pub(super) fn drain_into(&mut self, other: &mut BinaryHeap) { + other.append(&mut self.unordered); + other.extend(self.ordered.drain(..)); + } +} + impl PartialEq for TimerEntry { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn eq(&self, other: &Self) -> bool { self.when == other.when && self.seq == other.seq } @@ -32,12 +121,20 @@ impl PartialEq for TimerEntry { impl Eq for TimerEntry {} impl PartialOrd for TimerEntry { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } impl Ord for TimerEntry { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn cmp(&self, other: &Self) -> Ordering { compare_timer_parts(&self.when, self.seq, &other.when, other.seq) } @@ -123,7 +220,7 @@ mod tests { use pyo3::prelude::*; use pyo3::types::PyTuple; - use super::TimerEntry; + use super::{TimerEntry, TimerQueue}; use crate::engine::callbacks::{CallbackKind, ReadyCallback}; fn timer_entry(py: Python<'_>, when: Instant, seq: u64) -> TimerEntry { @@ -166,4 +263,54 @@ mod tests { assert!(heap.is_empty()); }); } + + proptest::proptest! { + #[test] + fn timer_queue_matches_heap_through_insert_pop_and_merge( + operations in proptest::collection::vec((0_u8..6, 0_u64..100), 1..300) + ) { + crate::initialize_python_for_tests(); + Python::attach(|py| { + let now = Instant::now(); + let mut queues = [TimerQueue::new(), TimerQueue::new()]; + let mut heaps = [BinaryHeap::new(), BinaryHeap::new()]; + for (seq, (op, delay)) in operations.into_iter().enumerate() { + let index = usize::from(op % 2); + match op { + 0 | 1 => { + let when = now + Duration::from_millis(delay); + queues[index].push(timer_entry(py, when, seq as u64), seq % 3 == 0); + heaps[index].push(timer_entry(py, when, seq as u64)); + } + 2 | 3 => { + let actual = queues[index].pop().map(|e| (e.when, e.seq)); + let expected = heaps[index].pop().map(|e| (e.when, e.seq)); + assert_eq!(actual, expected); + } + _ => { + let mut pending = BinaryHeap::new(); + queues[1 - index].drain_into(&mut pending); + assert!(queues[1 - index].is_empty()); + queues[index].append_heap(&mut pending); + assert!(pending.is_empty()); + let mut other = std::mem::take(&mut heaps[1 - index]); + heaps[index].append(&mut other); + } + } + for i in 0..2 { + assert_eq!(queues[i].is_empty(), heaps[i].is_empty()); + assert_eq!(queues[i].peek().map(|e| (e.when, e.seq)), + heaps[i].peek().map(|e| (e.when, e.seq))); + } + } + for i in 0..2 { + while let Some(expected) = heaps[i].pop() { + let actual = queues[i].pop().expect("remaining timer"); + assert_eq!((actual.when, actual.seq), (expected.when, expected.seq)); + } + assert!(queues[i].pop().is_none()); + } + }); + } + } } diff --git a/src/errors.rs b/src/errors.rs index f500612..2833f7a 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -6,6 +6,7 @@ use pyo3::types::PyDict; use crate::engine::LoopCore; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn handle_callback_error( py: Python<'_>, loop_core: &LoopCore, diff --git a/src/lib.rs b/src/lib.rs index 3bb6277..28a0abb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -83,6 +83,7 @@ pub(crate) fn initialize_python_for_tests() { // `PyOnceLock`, and `#[pyclass]` interior mutability uses PyO3's atomic borrow // flags, so importing under a free-threaded interpreter no longer needs to // re-enable the GIL. +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[pymodule(gil_used = false)] fn _loop(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add("__version__", env!("CARGO_PKG_VERSION"))?; diff --git a/src/module_init.rs b/src/module_init.rs index 7d94140..ebe6fc2 100644 --- a/src/module_init.rs +++ b/src/module_init.rs @@ -14,6 +14,7 @@ use crate::transport::stream::{ reset_transport_stats, start_server, transport_stats, }; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(crate) fn add_module_contents(m: &Bound<'_, PyModule>) -> PyResult<()> { add_module_classes(m)?; add_event_loop_functions(m)?; @@ -23,6 +24,7 @@ pub(crate) fn add_module_contents(m: &Bound<'_, PyModule>) -> PyResult<()> { Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn add_module_classes(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; crate::bindings::install_fast_callbacks(m.py())?; @@ -37,6 +39,7 @@ fn add_module_classes(m: &Bound<'_, PyModule>) -> PyResult<()> { Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn add_event_loop_functions(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(new_event_loop, m)?)?; m.add_function(wrap_pyfunction!(asyncgen_firstiter_hook, m)?)?; @@ -46,12 +49,14 @@ fn add_event_loop_functions(m: &Bound<'_, PyModule>) -> PyResult<()> { Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn add_stream_functions(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(open_connection, m)?)?; m.add_function(wrap_pyfunction!(start_server, m)?)?; Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn add_diagnostic_functions(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(build_info, m)?)?; m.add_function(wrap_pyfunction!(transport_stats, m)?)?; @@ -64,6 +69,7 @@ fn add_diagnostic_functions(m: &Bound<'_, PyModule>) -> PyResult<()> { Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn add_module_compat_aliases(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add("_future_done_stop", m.getattr("future_done_stop")?)?; m.add( diff --git a/src/platform/fd/mod.rs b/src/platform/fd/mod.rs index 6d35be1..43f211a 100644 --- a/src/platform/fd/mod.rs +++ b/src/platform/fd/mod.rs @@ -14,6 +14,7 @@ pub type RawFd = i64; #[cfg(windows)] const FD_POLL_INTERVAL_MS: i32 = 50; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn fileobj_to_fd(_py: Python<'_>, fileobj: &Bound<'_, PyAny>) -> PyResult { if let Ok(fd) = fileobj.extract::() { return Ok(fd); @@ -22,10 +23,12 @@ pub fn fileobj_to_fd(_py: Python<'_>, fileobj: &Bound<'_, PyAny>) -> PyResult() } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn fileobj_keepalive(fileobj: &Bound<'_, PyAny>) -> Py { fileobj.clone().unbind().into_any() } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn dup_raw_fd(fd: RawFd) -> io::Result { #[cfg(unix)] { @@ -52,6 +55,7 @@ enum PollReadiness { Ready { read: bool, write: bool }, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] fn decode_poll_revents(read: bool, write: bool, revents: i32) -> PollReadiness { if revents & i32::from(libc::POLLNVAL) != 0 { @@ -65,6 +69,7 @@ fn decode_poll_revents(read: bool, write: bool, revents: i32) -> PollReadiness { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub fn poll_fd(fd: RawFd, read: bool, write: bool, timeout_ms: i32) -> io::Result<(bool, bool)> { if !read && !write { @@ -110,10 +115,12 @@ pub fn poll_fd(fd: RawFd, read: bool, write: bool, timeout_ms: i32) -> io::Resul } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] pub async fn wait_writable(fd: RawFd) -> PyResult<()> { wait_for_interest(fd, false, true).await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] async fn wait_for_interest(fd: RawFd, read: bool, write: bool) -> PyResult<()> { #[cfg(windows)] { @@ -162,6 +169,7 @@ async fn wait_for_interest(fd: RawFd, read: bool, write: bool) -> PyResult<()> { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn is_retryable_socket_error(py: Python<'_>, err: &PyErr) -> PyResult { let builtins = py.import("builtins")?; let blocking = builtins.getattr("BlockingIOError")?; @@ -169,10 +177,12 @@ pub fn is_retryable_socket_error(py: Python<'_>, err: &PyErr) -> PyResult Ok(err.is_instance(py, &blocking) || err.is_instance(py, &interrupted)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn raw_fd_fits_c_int(fd: RawFd) -> bool { fd >= RawFd::from(libc::c_int::MIN) && fd <= RawFd::from(libc::c_int::MAX) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn raw_fd_to_c_int(fd: RawFd) -> io::Result { if raw_fd_fits_c_int(fd) { Ok(libc::c_int::try_from(fd) @@ -185,6 +195,7 @@ fn raw_fd_to_c_int(fd: RawFd) -> io::Result { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// A `connect()` attempt that is still completing in the background. #[cfg(unix)] #[inline] @@ -192,6 +203,7 @@ pub fn is_connect_in_progress_errno(errno: i32) -> bool { errno == libc::EINPROGRESS || errno == libc::EALREADY || errno == libc::EWOULDBLOCK } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// The socket is already connected (a benign outcome for `connect()`). #[cfg(unix)] #[inline] @@ -199,6 +211,7 @@ pub fn is_already_connected_errno(errno: i32) -> bool { errno == libc::EISCONN } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Reads the pending `SO_ERROR` for a socket via a direct `getsockopt`, so the /// connect-completion path resolves without acquiring the GIL. #[cfg(unix)] diff --git a/src/platform/fd/windows.rs b/src/platform/fd/windows.rs index f49e68b..bc5357a 100644 --- a/src/platform/fd/windows.rs +++ b/src/platform/fd/windows.rs @@ -15,6 +15,7 @@ use windows_sys::Win32::System::Threading::GetCurrentProcess; use super::{RawFd, raw_fd_to_c_int}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn dup_raw_fd(fd: RawFd) -> io::Result { match duplicate_socket(fd) { Ok(duped) => Ok(duped), @@ -32,6 +33,7 @@ pub(super) fn dup_raw_fd(fd: RawFd) -> io::Result { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn duplicate_socket(fd: RawFd) -> io::Result { let socket = raw_fd_to_socket(fd)?; let socket = socket_from_raw(socket); @@ -41,12 +43,14 @@ pub(super) fn duplicate_socket(fd: RawFd) -> io::Result { Ok(raw.cast_signed()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn socket_from_raw(socket: SOCKET) -> Socket { // SAFETY: The caller provides a raw socket handle that should be temporarily owned by // `Socket`; callers must prevent unintended closure when they only borrow the source handle. unsafe { Socket::from_raw_socket(socket as _) } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn duplicate_handle_raw( process: HANDLE, handle: HANDLE, @@ -66,6 +70,7 @@ fn duplicate_handle_raw( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn raw_fd_to_handle(fd: RawFd) -> io::Result { let fd = raw_fd_to_c_int(fd)?; // SAFETY: `_get_osfhandle` only reads the C runtime fd table for this validated fd and returns @@ -77,6 +82,7 @@ pub fn raw_fd_to_handle(fd: RawFd) -> io::Result { Ok(handle as HANDLE) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn duplicate_handle(handle: HANDLE) -> io::Result { if handle.is_null() || handle == INVALID_HANDLE_VALUE { return Err(io::Error::new( @@ -96,10 +102,12 @@ pub fn duplicate_handle(handle: HANDLE) -> io::Result { Ok(duplicated) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn duplicate_handle_from_fd(fd: RawFd) -> io::Result { duplicate_handle(raw_fd_to_handle(fd)?) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn poll_fd(fd: RawFd, read: bool, write: bool, timeout_ms: i32) -> io::Result<(bool, bool)> { if !read && !write { return Ok((false, false)); @@ -139,11 +147,13 @@ pub fn poll_fd(fd: RawFd, read: bool, write: bool, timeout_ms: i32) -> io::Resul )) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn raw_fd_to_socket(fd: RawFd) -> io::Result { fd.try_into() .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "socket handle out of range")) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn new_fd_set(socket: SOCKET, enabled: bool) -> FD_SET { let mut set = FD_SET { fd_count: 0, diff --git a/src/profile.rs b/src/profile.rs index 87f412d..7082028 100644 --- a/src/profile.rs +++ b/src/profile.rs @@ -2,23 +2,75 @@ use std::sync::Mutex; -use hotpath::{HotpathGuard, HotpathGuardBuilder}; -use pyo3::exceptions::PyRuntimeError; +use hotpath::{Format, HotpathGuard, HotpathGuardBuilder}; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; -static PROFILE: Mutex> = Mutex::new(None); +// hotpath has process-global registries. A fresh process per session keeps +// function, future, and thread statistics from leaking between workloads. +#[allow( + clippy::large_enum_variant, + reason = "A single static session stores the guard inline" +)] +enum Session { + New, + Active(HotpathGuard), + Finished, +} +static PROFILE: Mutex = Mutex::new(Session::New); + +#[cfg(feature = "hotpath-alloc-profile")] +#[global_allocator] +static ALLOCATOR: hotpath::CountingAllocator = + hotpath::CountingAllocator::new(); #[pyfunction] -pub(crate) fn hotpath_start(output_path: String) -> PyResult<()> { - let mut profile = PROFILE.lock().expect("poisoned hotpath profile"); - if profile.is_some() { - return Err(PyRuntimeError::new_err( - "hotpath profiling is already active", +#[pyo3(signature = (output_path, *, format="json", functions_limit=0, time_sampling_rate=1.0))] +pub(crate) fn hotpath_start( + output_path: String, + format: &str, + functions_limit: usize, + time_sampling_rate: f64, +) -> PyResult<()> { + let format = match format { + "json" => Format::Json, + "json-pretty" => Format::JsonPretty, + "table" => Format::Table, + _ => { + return Err(PyValueError::new_err( + "format must be json, json-pretty, or table", + )); + } + }; + if !time_sampling_rate.is_finite() + || !(0.0..=1.0).contains(&time_sampling_rate) + || time_sampling_rate == 0.0 + { + return Err(PyValueError::new_err( + "time_sampling_rate must be in (0, 1]", )); } - *profile = Some( + let mut profile = PROFILE.lock().expect("poisoned hotpath profile"); + match &*profile { + Session::New => {} + Session::Active(_) => { + return Err(PyRuntimeError::new_err( + "hotpath profiling is already active", + )); + } + Session::Finished => { + return Err(PyRuntimeError::new_err( + "start a fresh process for each hotpath profile", + )); + } + } + *profile = Session::Active( HotpathGuardBuilder::new("rsloop") - .functions_limit(50) + .limit(0) + .functions_limit(functions_limit) + .percentiles(&[50.0, 95.0, 99.0]) + .time_sampling_rate(time_sampling_rate) + .format(format) .output_path(output_path) .build(), ); @@ -26,11 +78,16 @@ pub(crate) fn hotpath_start(output_path: String) -> PyResult<()> { } #[pyfunction] -pub(crate) fn hotpath_stop() -> PyResult<()> { - let guard = PROFILE.lock().expect("poisoned hotpath profile").take(); - let Some(guard) = guard else { +pub(crate) fn hotpath_stop(py: Python<'_>) -> PyResult<()> { + let mut profile = PROFILE.lock().expect("poisoned hotpath profile"); + if !matches!(*profile, Session::Active(_)) { return Err(PyRuntimeError::new_err("hotpath profiling is not active")); + } + let Session::Active(guard) = std::mem::replace(&mut *profile, Session::Finished) else { + unreachable!() }; - drop(guard); + drop(profile); + // Report flushing can block; let other Python-attached workers finish. + py.detach(|| drop(guard)); Ok(()) } diff --git a/src/python_names.rs b/src/python_names.rs index bf6b789..7edcaf7 100644 --- a/src/python_names.rs +++ b/src/python_names.rs @@ -3,62 +3,76 @@ use pyo3::prelude::*; use pyo3::types::PyString; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(crate) fn cancelled<'py>(py: Python<'py>) -> &'py Bound<'py, PyString> { pyo3::intern!(py, "cancelled") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(any(Py_3_12, all(Py_3_11, not(Py_LIMITED_API))))] pub(crate) fn context_kw<'py>(py: Python<'py>) -> &'py Bound<'py, PyString> { pyo3::intern!(py, "context") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(crate) fn create_future<'py>(py: Python<'py>) -> &'py Bound<'py, PyString> { pyo3::intern!(py, "create_future") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub(crate) fn errno<'py>(py: Python<'py>) -> &'py Bound<'py, PyString> { pyo3::intern!(py, "errno") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(crate) fn done<'py>(py: Python<'py>) -> &'py Bound<'py, PyString> { pyo3::intern!(py, "done") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(any(Py_3_12, all(Py_3_11, not(Py_LIMITED_API))))] pub(crate) fn loop_kw<'py>(py: Python<'py>) -> &'py Bound<'py, PyString> { pyo3::intern!(py, "loop") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(any(Py_3_12, all(Py_3_11, not(Py_LIMITED_API))))] pub(crate) fn name_kw<'py>(py: Python<'py>) -> &'py Bound<'py, PyString> { pyo3::intern!(py, "name") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(crate) fn pause_reading<'py>(py: Python<'py>) -> &'py Bound<'py, PyString> { pyo3::intern!(py, "pause_reading") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(crate) fn pause_writing<'py>(py: Python<'py>) -> &'py Bound<'py, PyString> { pyo3::intern!(py, "pause_writing") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(crate) fn resume_reading<'py>(py: Python<'py>) -> &'py Bound<'py, PyString> { pyo3::intern!(py, "resume_reading") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(crate) fn resume_writing<'py>(py: Python<'py>) -> &'py Bound<'py, PyString> { pyo3::intern!(py, "resume_writing") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(crate) fn set_exception<'py>(py: Python<'py>) -> &'py Bound<'py, PyString> { pyo3::intern!(py, "set_exception") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(crate) fn set_result<'py>(py: Python<'py>) -> &'py Bound<'py, PyString> { pyo3::intern!(py, "set_result") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub(crate) fn call_method0( _py: Python<'_>, @@ -68,6 +82,7 @@ pub(crate) fn call_method0( Ok(obj.call_method0(method)?.unbind()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub(crate) fn call_method1( _py: Python<'_>, diff --git a/src/rust_async.rs b/src/rust_async.rs index 4c0fc08..dfb4ada 100644 --- a/src/rust_async.rs +++ b/src/rust_async.rs @@ -6,6 +6,7 @@ use pyo3::prelude::*; pub use pyo3_async_runtimes::{TaskLocals, into_future_with_locals}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Capture the current Python event loop and contextvars so a Rust future can /// be attached to the active `rsloop` task. #[inline] @@ -13,6 +14,7 @@ pub fn get_current_locals(py: Python<'_>) -> PyResult { pyo3_async_runtimes::async_std::get_current_locals(py) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Convert a `Send` Rust future into a Python awaitable bound to the currently /// running Python loop. pub fn future_into_py(py: Python<'_>, fut: F) -> PyResult> @@ -23,6 +25,7 @@ where future_into_py_with_locals(py, get_current_locals(py)?, fut) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Convert a `Send` Rust future into a Python awaitable using explicit task /// locals captured earlier. pub fn future_into_py_with_locals( @@ -37,6 +40,7 @@ where pyo3_async_runtimes::async_std::future_into_py_with_locals(py, locals, fut) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Convert a `!Send` Rust future into a Python awaitable bound to the current /// Python loop. pub fn local_future_into_py(py: Python<'_>, fut: F) -> PyResult> @@ -47,6 +51,7 @@ where local_future_into_py_with_locals(py, get_current_locals(py)?, fut) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Convert a `!Send` Rust future into a Python awaitable using explicit task /// locals captured earlier. #[allow(deprecated)] diff --git a/src/transport/process/core_events.rs b/src/transport/process/core_events.rs index de5786e..36c6848 100644 --- a/src/transport/process/core_events.rs +++ b/src/transport/process/core_events.rs @@ -30,6 +30,7 @@ enum ProcessEventKind { Lost, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn process_event_kind(event: &PendingProcessEvent) -> ProcessEventKind { match event { PendingProcessEvent::PipeDataReceived { .. } => ProcessEventKind::PipeData, @@ -39,10 +40,15 @@ fn process_event_kind(event: &PendingProcessEvent) -> ProcessEventKind { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn process_event_stops_drain(kind: ProcessEventKind) -> bool { kind == ProcessEventKind::Lost } impl ProcessTransportCore { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn enqueue_pending_event(self: &Arc, event: PendingProcessEvent) { self.pending_events .lock() @@ -61,6 +67,10 @@ impl ProcessTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(crate) fn drain_pending_events_with_py(self: &Arc, py: Python<'_>) -> PyResult<()> { let mut drained = VecDeque::new(); loop { @@ -120,6 +130,10 @@ impl ProcessTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn call_protocol_with_tuple( &self, py: Python<'_>, @@ -139,6 +153,10 @@ impl ProcessTransportCore { run_in_context(py, &context, context_needs_run, &callback, &tuple) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn call_in_loop_context( &self, f: impl for<'py> FnOnce(Python<'py>) -> PyResult, @@ -149,6 +167,10 @@ impl ProcessTransportCore { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn call_protocol_method0( &self, py: Python<'_>, @@ -158,6 +180,10 @@ impl ProcessTransportCore { self.call_protocol_with_tuple(py, method, &args) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn call_protocol_method1( &self, py: Python<'_>, @@ -168,6 +194,10 @@ impl ProcessTransportCore { self.call_protocol_with_tuple(py, method, &args) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn call_protocol_method2( &self, py: Python<'_>, @@ -179,6 +209,10 @@ impl ProcessTransportCore { self.call_protocol_with_tuple(py, method, &args) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn report_error(&self, err: PyErr, message: &str) { let _ = Python::attach(|py| -> PyResult<()> { let context = PyDict::new(py); diff --git a/src/transport/process/core_protocol.rs b/src/transport/process/core_protocol.rs index 93efe93..ca81c0a 100644 --- a/src/transport/process/core_protocol.rs +++ b/src/transport/process/core_protocol.rs @@ -22,10 +22,12 @@ enum ProcessExitDecision { First { should_finish: bool }, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn process_connection_lost_eligible(exited: bool, open_pipes_empty: bool) -> bool { exited && open_pipes_empty } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn record_process_exit( exited: &mut bool, returncode: &mut Option, @@ -43,6 +45,10 @@ fn record_process_exit( } impl ProcessTransportCore { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn connection_made(&self, transport: Py) -> PyResult<()> { self.call_in_loop_context(|py| { self.call_protocol_method1(py, "connection_made", transport.into_any())?; @@ -50,6 +56,10 @@ impl ProcessTransportCore { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn pipe_data_received_with_py( &self, py: Python<'_>, @@ -80,6 +90,10 @@ impl ProcessTransportCore { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn pipe_data_received(self: &Arc, fd: i32, data: &[u8]) -> PyResult<()> { if !self.loop_core.on_runtime_thread() { self.enqueue_pending_event(PendingProcessEvent::PipeDataReceived { @@ -92,6 +106,10 @@ impl ProcessTransportCore { self.call_in_loop_context(|py| self.pipe_data_received_with_py(py, fd, data)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn pipe_connection_lost_value_with_py( &self, py: Python<'_>, @@ -108,6 +126,10 @@ impl ProcessTransportCore { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn pipe_connection_lost_message( self: &Arc, fd: i32, @@ -153,6 +175,10 @@ impl ProcessTransportCore { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn pipe_connection_lost( self: &Arc, fd: i32, @@ -162,12 +188,20 @@ impl ProcessTransportCore { self.pipe_connection_lost_message(fd, exc) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn process_exited_with_py(&self, py: Python<'_>, returncode: i32) -> PyResult<()> { let _ = returncode; self.call_protocol_method0(py, "process_exited")?; Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn process_exited(self: &Arc, returncode: i32) -> PyResult<()> { let should_finish = { let mut state = self.state.lock().expect("poisoned process state"); @@ -205,6 +239,10 @@ impl ProcessTransportCore { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn connection_lost_with_py( &self, py: Python<'_>, @@ -217,6 +255,10 @@ impl ProcessTransportCore { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn connection_lost_message(self: &Arc, exc: Option) -> PyResult<()> { { let mut state = self.state.lock().expect("poisoned process state"); @@ -237,6 +279,10 @@ impl ProcessTransportCore { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn connection_lost(self: &Arc, exc: Option) -> PyResult<()> { let exc = exc.map(|err| Python::attach(|py| err.value(py).to_string())); self.connection_lost_message(exc) diff --git a/src/transport/process/core_state.rs b/src/transport/process/core_state.rs index e16bf9a..afc031e 100644 --- a/src/transport/process/core_state.rs +++ b/src/transport/process/core_state.rs @@ -9,6 +9,10 @@ use pyo3::prelude::*; use super::ProcessTransportCore; impl ProcessTransportCore { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] #[inline] pub(super) fn get_returncode(&self) -> Option { self.state @@ -17,11 +21,19 @@ impl ProcessTransportCore { .returncode } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] #[inline] pub(super) fn is_closing(&self) -> bool { self.state.lock().expect("poisoned process state").closing } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn pipe_transport(&self, py: Python<'_>, fd: i32) -> Option> { self.state .lock() @@ -31,6 +43,10 @@ impl ProcessTransportCore { .map(|transport| transport.clone_ref(py)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn has_open_pipe(&self, fd: i32) -> bool { self.state .lock() @@ -39,6 +55,10 @@ impl ProcessTransportCore { .contains(&fd) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportCore") + )] pub(super) fn register_pipe_transports(&self, transports: Vec<(i32, Py)>) { if transports.is_empty() { return; diff --git a/src/transport/process/params.rs b/src/transport/process/params.rs index af63e4a..2acf701 100644 --- a/src/transport/process/params.rs +++ b/src/transport/process/params.rs @@ -41,6 +41,10 @@ pub struct ProcessTransportParams { } impl ProcessTransportParams { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportParams") + )] pub fn new(spawn_context: TransportSpawnContext, child: Child) -> Self { let TransportSpawnContext { loop_core, @@ -63,11 +67,19 @@ impl ProcessTransportParams { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportParams") + )] pub fn with_text_config(mut self, text_config: Option) -> Self { self.text_config = text_config; self } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessTransportParams") + )] pub fn with_stdio_overrides( mut self, stdout_override: Option, diff --git a/src/transport/process/py_pipes.rs b/src/transport/process/py_pipes.rs index 9cc7162..a030c31 100644 --- a/src/transport/process/py_pipes.rs +++ b/src/transport/process/py_pipes.rs @@ -16,22 +16,46 @@ use super::{PyProcessPipeTransport, PyProcessStdinProtocol}; #[pymethods] impl PyProcessPipeTransport { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessPipeTransport") + )] fn close(&self) { self.core.closing.store(true, Ordering::SeqCst); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessPipeTransport") + )] fn is_closing(&self) -> bool { self.core.closing.load(Ordering::SeqCst) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessPipeTransport") + )] fn get_extra_info(&self, py: Python<'_>, _name: &str, default: Option>) -> Py { default.unwrap_or_else(|| py.None()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessPipeTransport") + )] fn pause_reading(&self) {} + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessPipeTransport") + )] fn resume_reading(&self) {} + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessPipeTransport") + )] fn __repr__(&self) -> String { format!( "", @@ -43,12 +67,28 @@ impl PyProcessPipeTransport { #[pymethods] impl PyProcessStdinProtocol { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessStdinProtocol") + )] fn connection_made(&self, _transport: Py) {} + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessStdinProtocol") + )] fn pause_writing(&self) {} + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessStdinProtocol") + )] fn resume_writing(&self) {} + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessStdinProtocol") + )] #[pyo3(signature=(_exc=None))] fn connection_lost(&self, _exc: Option>) -> PyResult<()> { if !self.core.has_open_pipe(0) { diff --git a/src/transport/process/py_transport.rs b/src/transport/process/py_transport.rs index 80f371e..023a4f0 100644 --- a/src/transport/process/py_transport.rs +++ b/src/transport/process/py_transport.rs @@ -12,23 +12,43 @@ use super::{ProcessCommand, PyProcessTransport}; #[pymethods] impl PyProcessTransport { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessTransport") + )] fn get_pid(&self) -> u32 { self.core.state.lock().expect("poisoned process state").pid } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessTransport") + )] #[inline] fn get_returncode(&self) -> Option { self.core.get_returncode() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessTransport") + )] fn is_closing(&self) -> bool { self.core.is_closing() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessTransport") + )] fn get_pipe_transport(&self, py: Python<'_>, fd: i32) -> Option> { self.core.pipe_transport(py, fd) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessTransport") + )] fn send_signal(&self, sig: i32) -> PyResult<()> { if self.core.get_returncode().is_some() { return Err(PyProcessLookupError::new_err("process is not running")); @@ -39,6 +59,10 @@ impl PyProcessTransport { .map_err(|_| PyProcessLookupError::new_err("process is not running")) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessTransport") + )] fn terminate(&self) -> PyResult<()> { if self.core.get_returncode().is_some() { return Err(PyProcessLookupError::new_err("process is not running")); @@ -49,6 +73,10 @@ impl PyProcessTransport { .map_err(|_| PyProcessLookupError::new_err("process is not running")) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessTransport") + )] fn kill(&self) -> PyResult<()> { if self.core.get_returncode().is_some() { return Err(PyProcessLookupError::new_err("process is not running")); @@ -59,6 +87,10 @@ impl PyProcessTransport { .map_err(|_| PyProcessLookupError::new_err("process is not running")) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessTransport") + )] fn close(&self, py: Python<'_>) -> PyResult<()> { { let mut state = self.core.state.lock().expect("poisoned process state"); @@ -71,6 +103,10 @@ impl PyProcessTransport { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessTransport") + )] fn __repr__(&self) -> String { format!( "", @@ -80,6 +116,10 @@ impl PyProcessTransport { ) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyProcessTransport") + )] fn _wait<'py>(&self, py: Python<'py>) -> PyResult> { let locals = crate::transport::stream::task_locals_for_loop(py, &self.core.loop_obj)?; let core = self.core.clone(); diff --git a/src/transport/process/spawn.rs b/src/transport/process/spawn.rs index d0e3738..3ed013a 100644 --- a/src/transport/process/spawn.rs +++ b/src/transport/process/spawn.rs @@ -43,11 +43,16 @@ pub(super) struct ProcessPipes { stderr: Option, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn open_pipe_mask(has_stdin: bool, has_stdout: bool, has_stderr: bool) -> u8 { u8::from(has_stdin) | (u8::from(has_stdout) << 1) | (u8::from(has_stderr) << 2) } impl ProcessPipes { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessPipes") + )] pub(super) fn take_from( child: &mut Child, stdout_override: Option, @@ -70,6 +75,10 @@ impl ProcessPipes { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ProcessPipes") + )] pub(super) fn open_pipes(&self) -> HashSet { let mask = open_pipe_mask( self.stdin.is_some(), @@ -86,6 +95,7 @@ impl ProcessPipes { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn new_process_pipe_transport(py: Python<'_>, fd: i32) -> PyResult> { Ok(Py::new( py, @@ -99,6 +109,7 @@ pub(super) fn new_process_pipe_transport(py: Python<'_>, fd: i32) -> PyResult, text_config: Option<&ProcessTextConfig>, @@ -121,6 +132,7 @@ pub(super) fn process_text_extra_entries( }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn spawn_stdin_pipe_transport( py: Python<'_>, core: &Arc, @@ -156,6 +168,7 @@ pub(super) fn spawn_stdin_pipe_transport( Ok(transport.into_any()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn register_initial_pipe_transports( py: Python<'_>, core: &Arc, @@ -179,6 +192,7 @@ pub(super) fn register_initial_pipe_transports( Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn spawn_process_reader_thread( name: &str, core: Arc, @@ -192,6 +206,7 @@ pub(super) fn spawn_process_reader_thread( .map_err(|err| PyRuntimeError::new_err(format!("failed to spawn {name}: {err}"))) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn spawn_process_waiter_thread( core: Arc, child: Child, @@ -204,6 +219,7 @@ pub(super) fn spawn_process_waiter_thread( .map_err(|err| PyRuntimeError::new_err(format!("failed to spawn process waiter: {err}"))) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn spawn_process_workers( core: Arc, stdout: Option, @@ -220,6 +236,7 @@ pub(super) fn spawn_process_workers( spawn_process_waiter_thread(core, child, control_rx) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn spawn_process_transport( py: Python<'_>, params: ProcessTransportParams, @@ -273,6 +290,7 @@ pub fn spawn_process_transport( Ok(transport) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] pub(super) fn make_python_pipe_file_from_handle( py: Python<'_>, @@ -295,6 +313,7 @@ pub(super) fn make_python_pipe_file_from_handle( Ok(os.getattr("fdopen")?.call1((fd, mode, 0))?.unbind()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub(super) fn make_python_pipe_file( py: Python<'_>, diff --git a/src/transport/process/worker.rs b/src/transport/process/worker.rs index 338ff0e..1047238 100644 --- a/src/transport/process/worker.rs +++ b/src/transport/process/worker.rs @@ -26,6 +26,7 @@ use super::{ProcessCommand, ProcessTransportCore}; const PROCESS_READER_BUFFER_SIZE: usize = 65_536; const PROCESS_WAIT_POLL_INTERVAL: Duration = Duration::from_millis(20); +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn report_process_result( core: &Arc, result: PyResult<()>, @@ -36,6 +37,7 @@ pub(super) fn report_process_result( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub(super) fn report_process_io_error( core: &Arc, @@ -45,6 +47,7 @@ pub(super) fn report_process_io_error( core.report_error(PyRuntimeError::new_err(err.to_string()), message); } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub(super) fn send_process_signal(child: &Child, signal: i32) -> std::io::Result<()> { let pid = i32::try_from(child.id()).map_err(|_| { @@ -60,12 +63,14 @@ pub(super) fn send_process_signal(child: &Child, signal: i32) -> std::io::Result } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] fn process_exit_code_parts(code: Option, signal: Option) -> i32 { code.or_else(|| signal.and_then(i32::checked_neg)) .unwrap_or(-1) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn process_exit_code(status: std::process::ExitStatus) -> i32 { #[cfg(unix)] { @@ -77,6 +82,7 @@ pub(super) fn process_exit_code(status: std::process::ExitStatus) -> i32 { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn handle_process_exit(core: &Arc, code: i32) { if core .state @@ -98,6 +104,7 @@ pub(super) fn handle_process_exit(core: &Arc, code: i32) { ); } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn kill_process_child( core: &Arc, child: &mut Child, @@ -108,6 +115,7 @@ pub(super) fn kill_process_child( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn handle_process_command( core: &Arc, child: &mut Child, @@ -141,6 +149,7 @@ pub(super) fn handle_process_command( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn run_process_reader( core: Arc, fd: i32, @@ -180,6 +189,7 @@ pub(super) fn run_process_reader( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn run_process_waiter( core: Arc, mut child: Child, @@ -215,6 +225,7 @@ pub(super) fn run_process_waiter( mod verification { use super::process_exit_code_parts; + #[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[kani::proof] fn merge_process_exit_code_prefers_status_then_negated_signal() { let code: Option = kani::any(); diff --git a/src/transport/stream/accept.rs b/src/transport/stream/accept.rs index b2d024b..bf217e5 100644 --- a/src/transport/stream/accept.rs +++ b/src/transport/stream/accept.rs @@ -51,6 +51,7 @@ use crate::engine::{LoopCommand, LoopTransportCommand}; use crate::fd_ops; use crate::transport::tls::ServerTlsSettings; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn configure_accepted_tcp_stream( server: &Arc, stream: &StdTcpStream, @@ -67,6 +68,7 @@ pub(super) fn configure_accepted_tcp_stream( true } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub(super) fn configure_accepted_unix_stream( server: &Arc, @@ -80,6 +82,7 @@ pub(super) fn configure_accepted_unix_stream( true } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn report_server_io_error(server: &ServerCore, err: io::Error, message: &str) { if !server.is_closed() { server.report_error(PyRuntimeError::new_err(err.to_string()), message); @@ -93,6 +96,10 @@ pub(crate) struct BlockingAcceptLoop { } impl BlockingAcceptLoop { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "BlockingAcceptLoop") + )] pub(crate) fn new(server: Arc, listener: L, stop: Arc) -> Self { Self { server, @@ -102,6 +109,7 @@ impl BlockingAcceptLoop { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn server_spawn_context( py: Python<'_>, server: &Arc, @@ -117,6 +125,7 @@ pub(super) fn server_spawn_context( ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn server_tls_settings(py: Python<'_>, tls: &ServerTlsSettings) -> ServerTlsSettings { ServerTlsSettings { config: Arc::clone(&tls.config), @@ -126,6 +135,7 @@ pub(super) fn server_tls_settings(py: Python<'_>, tls: &ServerTlsSettings) -> Se } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn spawn_accepted_tcp_transport( py: Python<'_>, server: &Arc, @@ -148,6 +158,7 @@ pub(super) fn spawn_accepted_tcp_transport( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub(super) fn spawn_accepted_unix_transport( py: Python<'_>, @@ -171,6 +182,7 @@ pub(super) fn spawn_accepted_unix_transport( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(crate) fn spawn_accepted_transport_with_py( py: Python<'_>, server: &Arc, @@ -183,6 +195,7 @@ pub(crate) fn spawn_accepted_transport_with_py( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn schedule_accepted_transport( server: &Arc, stream: AcceptedStream, @@ -229,6 +242,7 @@ pub(super) fn schedule_accepted_transport( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn run_tcp_accept_loop(params: BlockingAcceptLoop) { let BlockingAcceptLoop { server, @@ -278,6 +292,7 @@ pub(super) fn run_tcp_accept_loop(params: BlockingAcceptLoop) { } } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] pub(crate) async fn run_server_accept_task( server: Arc, listener: ServerListener, @@ -291,6 +306,7 @@ pub(crate) async fn run_server_accept_task( drop(task_guard); } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] pub(super) async fn run_tcp_accept_task(server: Arc, listener: StdTcpListener) { #[cfg(windows)] { @@ -306,6 +322,10 @@ pub(super) struct WindowsAcceptPool(Vec>); #[cfg(windows)] impl Drop for WindowsAcceptPool { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { for lane in self.0.drain(..) { lane.cancel(); @@ -313,6 +333,7 @@ impl Drop for WindowsAcceptPool { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[cfg(windows)] pub(super) async fn run_windows_tcp_accept_pool(server: Arc, listener: StdTcpListener) { let lane_count = std::thread::available_parallelism() @@ -338,6 +359,7 @@ pub(super) async fn run_windows_tcp_accept_pool(server: Arc, listene std::future::pending::<()>().await; } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[cfg(windows)] pub(super) async fn run_windows_tcp_accept_lane( server: Arc, @@ -373,6 +395,7 @@ pub(super) async fn run_windows_tcp_accept_lane( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[cfg(not(windows))] pub(super) async fn run_tcp_accept_lane(server: Arc, listener: StdTcpListener) { let listener = match VibeTcpListener::from_std(listener) { @@ -413,6 +436,7 @@ pub(super) async fn run_tcp_accept_lane(server: Arc, listener: StdTc } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub(super) fn run_unix_accept_loop(params: BlockingAcceptLoop) { let BlockingAcceptLoop { @@ -464,6 +488,7 @@ pub(super) fn run_unix_accept_loop(params: BlockingAcceptLoop) } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[cfg(unix)] pub(super) async fn run_unix_accept_task(server: Arc, listener: StdUnixListener) { let listener = match VibeUnixListener::from_std(listener) { diff --git a/src/transport/stream/buffers.rs b/src/transport/stream/buffers.rs index b2166a9..03f4fe9 100644 --- a/src/transport/stream/buffers.rs +++ b/src/transport/stream/buffers.rs @@ -11,7 +11,7 @@ use futures::future::poll_fn; use futures::task::AtomicWaker; use super::tuning::{ - MAX_STREAM_READ_BUFFER_SIZE, READ_BUFFER_POOL_LIMIT, WRITE_BUFFER_BLOCK_SIZE, + MAX_STREAM_READ_BUFFER_SIZE, MIN_WRITE_BUFFER_CAPACITY, READ_BUFFER_POOL_LIMIT, WRITE_BUFFER_POOL_LIMIT, }; @@ -31,6 +31,10 @@ pub(super) struct OwnedReadBuffer { } impl OwnedReadBuffer { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OwnedReadBuffer") + )] pub(super) fn with_capacity(capacity: usize) -> Self { Self { bytes: Vec::with_capacity(capacity), @@ -38,6 +42,10 @@ impl OwnedReadBuffer { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OwnedReadBuffer") + )] pub(super) fn from_pooled(bytes: Vec, pool: &Arc) -> Self { Self { bytes, @@ -54,18 +62,30 @@ impl OwnedReadBuffer { impl Deref for OwnedReadBuffer { type Target = Vec; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn deref(&self) -> &Self::Target { &self.bytes } } impl DerefMut for OwnedReadBuffer { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn deref_mut(&mut self) -> &mut Self::Target { &mut self.bytes } } impl Drop for OwnedReadBuffer { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { if let Some(pool) = self.pool.take() { pool.release(std::mem::take(&mut self.bytes)); @@ -80,6 +100,10 @@ pub(super) struct PendingReadBuffer<'a> { } impl<'a> PendingReadBuffer<'a> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PendingReadBuffer") + )] pub(super) fn from_pooled( bytes: Vec, pool: &'a ReadBufferPool, @@ -92,16 +116,28 @@ impl<'a> PendingReadBuffer<'a> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PendingReadBuffer") + )] #[inline] pub(super) fn len(&self) -> usize { self.bytes.len() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PendingReadBuffer") + )] #[inline] pub(super) fn as_slice(&self) -> &[u8] { &self.bytes } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PendingReadBuffer") + )] pub(super) fn extend(&mut self, data: &[u8]) { // Most drains contain one read. Keep that allocation until delivery; // only acquire and copy into the coalescing buffer for a second chunk. @@ -117,6 +153,10 @@ impl<'a> PendingReadBuffer<'a> { } impl Drop for PendingReadBuffer<'_> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { if let Some(pool) = self.pool.take() { pool.release(std::mem::take(&mut self.bytes)); @@ -142,6 +182,10 @@ impl OwnedWriteBuffer { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OwnedWriteBuffer") + )] pub(super) fn from_pooled_slice(data: &[u8], pool: &Arc) -> Self { let (mut bytes, pooled) = pool.acquire(data.len()); bytes.extend_from_slice(data); @@ -152,6 +196,10 @@ impl OwnedWriteBuffer { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OwnedWriteBuffer") + )] pub(super) fn with_pooled_capacity(capacity: usize, pool: &Arc) -> Self { let (bytes, pooled) = pool.acquire(capacity); Self { @@ -161,10 +209,18 @@ impl OwnedWriteBuffer { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OwnedWriteBuffer") + )] pub(super) fn extend_from_slice(&mut self, data: &[u8]) { self.bytes.extend_from_slice(data); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OwnedWriteBuffer") + )] pub(super) fn try_append(&mut self, data: &[u8]) -> bool { if !can_append(self.offset, self.bytes.len(), data.len()) { return false; @@ -183,27 +239,44 @@ impl OwnedWriteBuffer { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OwnedWriteBuffer") + )] #[inline] pub(super) fn remaining(&self) -> &[u8] { &self.bytes[self.offset..] } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OwnedWriteBuffer") + )] #[inline] pub(super) fn len(&self) -> usize { self.remaining().len() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OwnedWriteBuffer") + )] #[inline] pub(super) fn advance(&mut self, written: usize) { self.offset += written; } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OwnedWriteBuffer") + )] #[inline] pub(super) fn is_empty(&self) -> bool { self.offset == self.bytes.len() } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn can_append(offset: usize, current_len: usize, incoming_len: usize) -> bool { offset == 0 @@ -212,11 +285,13 @@ fn can_append(offset: usize, current_len: usize, incoming_len: usize) -> bool { <= super::tuning::DEFAULT_WRITE_BUFFER_HIGH_WATER.saturating_sub(current_len) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn retain_write_buffer(capacity: usize) -> bool { capacity <= super::tuning::DEFAULT_WRITE_BUFFER_HIGH_WATER } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn retain_read_buffer(capacity: usize) -> bool { capacity <= MAX_STREAM_READ_BUFFER_SIZE @@ -236,6 +311,7 @@ enum PoolRelease { Discard, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn pool_acquire( available: usize, allocated: usize, @@ -253,6 +329,7 @@ fn pool_acquire( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn pool_release(retain: bool, available: usize, limit: usize) -> PoolRelease { if retain && available < limit { PoolRelease::Store @@ -262,6 +339,10 @@ fn pool_release(retain: bool, available: usize, limit: usize) -> PoolRelease { } impl Drop for OwnedWriteBuffer { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { if let Some(pool) = self.pool.take() { pool.release(std::mem::take(&mut self.bytes)); @@ -284,6 +365,10 @@ struct WriteBufferPoolState { } impl WriteBufferPoolState { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WriteBufferPoolState") + )] fn new() -> Self { Self { buffers: Vec::with_capacity(WRITE_BUFFER_POOL_LIMIT), @@ -291,6 +376,10 @@ impl WriteBufferPoolState { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WriteBufferPoolState") + )] fn acquire(&mut self, capacity: usize) -> (Vec, bool, usize, bool) { let (mut buffer, pooled, mut allocation_events, fallback) = match pool_acquire( self.buffers.len(), @@ -307,7 +396,7 @@ impl WriteBufferPoolState { PoolAcquire::Allocate => { self.allocated += 1; ( - Vec::with_capacity(capacity.max(WRITE_BUFFER_BLOCK_SIZE)), + Vec::with_capacity(capacity.max(MIN_WRITE_BUFFER_CAPACITY)), true, 1, false, @@ -324,6 +413,10 @@ impl WriteBufferPoolState { (buffer, pooled, allocation_events, fallback) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WriteBufferPoolState") + )] fn release(&mut self, mut buffer: Vec) { match pool_release( retain_write_buffer(buffer.capacity()), @@ -342,6 +435,10 @@ impl WriteBufferPoolState { } impl WriteBufferPool { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WriteBufferPool") + )] pub(super) fn new() -> Self { Self { state: Mutex::new(WriteBufferPoolState::new()), @@ -352,6 +449,10 @@ impl WriteBufferPool { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WriteBufferPool") + )] fn acquire(&self, capacity: usize) -> (Vec, bool) { let mut state = self.state.lock().expect("poisoned write buffer pool"); let (buffer, pooled, allocation_events, fallback) = state.acquire(capacity); @@ -367,6 +468,10 @@ impl WriteBufferPool { (buffer, pooled) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WriteBufferPool") + )] pub(super) fn release(&self, buffer: Vec) { let mut state = self.state.lock().expect("poisoned write buffer pool"); state.release(buffer); @@ -391,6 +496,10 @@ struct ReadBufferPoolState { } impl ReadBufferPoolState { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBufferPoolState") + )] fn new() -> Self { Self { buffers: Vec::with_capacity(READ_BUFFER_POOL_LIMIT), @@ -398,6 +507,10 @@ impl ReadBufferPoolState { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBufferPoolState") + )] fn try_acquire(&mut self, capacity: usize) -> (Option>, usize) { let (mut buffer, mut allocation_events) = match pool_acquire( self.buffers.len(), @@ -421,6 +534,10 @@ impl ReadBufferPoolState { (Some(buffer), allocation_events) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBufferPoolState") + )] fn release(&mut self, mut buffer: Vec) { match pool_release( retain_read_buffer(buffer.capacity()), @@ -439,6 +556,10 @@ impl ReadBufferPoolState { } impl ReadBufferPool { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBufferPool") + )] pub(super) fn new() -> Self { Self { state: Mutex::new(ReadBufferPoolState::new()), @@ -450,6 +571,10 @@ impl ReadBufferPool { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBufferPool") + )] pub(super) fn try_acquire(&self, capacity: usize) -> Option> { let mut state = self.state.lock().expect("poisoned stream read buffer pool"); let (buffer, allocation_events) = state.try_acquire(capacity); @@ -462,11 +587,19 @@ impl ReadBufferPool { buffer } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBufferPool") + )] fn has_available(&self) -> bool { let state = self.state.lock().expect("poisoned stream read buffer pool"); !state.buffers.is_empty() || state.allocated < READ_BUFFER_POOL_LIMIT } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBufferPool", future = true) + )] pub(super) async fn wait_async(&self) { poll_fn(|context| { if self.closed.load(Ordering::Acquire) || self.has_available() { @@ -482,6 +615,10 @@ impl ReadBufferPool { .await; } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBufferPool") + )] pub(super) fn wait_timeout(&self, timeout: Duration) { let state = self.state.lock().expect("poisoned stream read buffer pool"); if state.buffers.is_empty() && state.allocated >= READ_BUFFER_POOL_LIMIT { @@ -492,16 +629,28 @@ impl ReadBufferPool { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBufferPool") + )] pub(super) fn notify_all(&self) { self.available.notify_all(); self.async_available.wake(); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBufferPool") + )] pub(super) fn close(&self) { self.closed.store(true, Ordering::Release); self.notify_all(); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBufferPool") + )] pub(super) fn release(&self, buffer: Vec) { let mut state = self.state.lock().expect("poisoned stream read buffer pool"); // Waiters can sleep only when all slots are checked out. Capture that @@ -945,4 +1094,23 @@ mod tests { assert_eq!(pool.fallbacks.load(Ordering::Relaxed), 1); drop(held); } + + #[test] + fn small_pooled_writes_grow_without_losing_bytes_and_reuse_capacity() { + let pool = Arc::new(WriteBufferPool::new()); + let initial = vec![1; 1024]; + let mut buffer = OwnedWriteBuffer::from_pooled_slice(&initial, &pool); + assert_eq!(buffer.bytes.capacity(), 1024); + let tail = vec![2; 32 * 1024]; + buffer.extend_from_slice(&tail); + assert_eq!(&buffer.remaining()[..1024], &initial); + assert_eq!(&buffer.remaining()[1024..], &tail); + buffer.advance(1000); + assert_eq!(buffer.len(), 24 + tail.len()); + let pointer = buffer.bytes.as_ptr(); + drop(buffer); + let reused = OwnedWriteBuffer::from_pooled_slice(&initial, &pool); + assert_eq!(reused.bytes.as_ptr(), pointer); + assert_eq!(reused.remaining(), &initial); + } } diff --git a/src/transport/stream/builder.rs b/src/transport/stream/builder.rs index 13ed8db..af31de1 100644 --- a/src/transport/stream/builder.rs +++ b/src/transport/stream/builder.rs @@ -37,10 +37,12 @@ use crate::async_event::AsyncEvent; use crate::engine::LoopCore; use crate::fd_ops; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn task_locals_for_loop(py: Python<'_>, loop_obj: &Py) -> PyResult { TaskLocals::new(loop_obj.clone_ref(py).into_bound(py)).copy_context(py) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub(super) fn detached_socket_handle( py: Python<'_>, @@ -49,6 +51,7 @@ pub(super) fn detached_socket_handle( socket_obj.call_method0(py, "detach")?.extract(py) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn tcp_family(stream: &StdTcpStream) -> c_int { #[cfg(windows)] use windows_sys::Win32::Networking::WinSock::{AF_INET, AF_INET6}; @@ -82,6 +85,7 @@ pub(super) struct StreamTransportBuildParts { pub(super) state: StreamTransportState, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn stream_transport_state_parts( spawn_context: TransportSpawnContext, callbacks: ProtocolCallbacks, @@ -123,6 +127,7 @@ pub(super) fn stream_transport_state_parts( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn new_stream_transport_core( parts: StreamTransportBuildParts, writer_tx: WriterSender, @@ -185,6 +190,7 @@ pub(super) fn new_stream_transport_core( }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn new_py_stream_transport( py: Python<'_>, core: &Arc, @@ -196,6 +202,7 @@ pub(super) fn new_py_stream_transport( }, ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn merge_extra( mut base: HashMap>, extra: HashMap>, @@ -203,6 +210,7 @@ pub(super) fn merge_extra( base.extend(extra); base } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn fail_transport_worker_start( py: Python<'_>, core: &Arc, @@ -214,6 +222,7 @@ pub(super) fn fail_transport_worker_start( let _ = core.connection_lost_with_py(py, Some(PyRuntimeError::new_err(message.clone()))); PyRuntimeError::new_err(message) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn make_stream_extra( py: Python<'_>, fd: fd_ops::RawFd, diff --git a/src/transport/stream/connect.rs b/src/transport/stream/connect.rs index a920881..4c81518 100644 --- a/src/transport/stream/connect.rs +++ b/src/transport/stream/connect.rs @@ -12,6 +12,7 @@ use pyo3::prelude::*; use crate::engine::{LoopCommand, LoopCore}; use crate::fd_ops; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Waits for a connecting TCP socket to become writable on the vibeio reactor, /// then hands the outcome back to the loop thread via `ConnectCompleted`. The /// descriptor is duplicated so the vibeio stream owns (and closes) only the diff --git a/src/transport/stream/core_events.rs b/src/transport/stream/core_events.rs index e199f5b..ee90d44 100644 --- a/src/transport/stream/core_events.rs +++ b/src/transport/stream/core_events.rs @@ -52,6 +52,7 @@ enum ReadEventKind { Resume, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn read_event_kind(event: &PendingReadEvent) -> ReadEventKind { match event { PendingReadEvent::Data(_) => ReadEventKind::Data, @@ -62,6 +63,7 @@ fn read_event_kind(event: &PendingReadEvent) -> ReadEventKind { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn transition_read_event_state(state: u8, event: ReadEventKind) -> Option { match event { ReadEventKind::Data if state == READ_EVENT_OPEN => Some(READ_EVENT_OPEN), @@ -72,10 +74,12 @@ fn transition_read_event_state(state: u8, event: ReadEventKind) -> Option { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn can_coalesce_read_data(current: usize, incoming: usize, limit: usize) -> bool { incoming <= limit.saturating_sub(current) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn read_drain_budget_reached( drained_events: usize, drained_bytes: usize, @@ -86,12 +90,20 @@ fn read_drain_budget_reached( } impl StreamTransportCore { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] #[cfg(windows)] #[inline] pub(super) fn poll_reader_requested(&self) -> bool { self.poll_reader_requested.load(Ordering::Acquire) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] #[cfg(windows)] pub(super) fn request_poll_reader(&self) { if self.poll_reader_requested.swap(true, Ordering::AcqRel) { @@ -111,6 +123,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] #[cfg(windows)] pub(super) fn mark_poll_reader_ready(self: &Arc, rebound: bool) { if rebound && transport_stats_enabled() { @@ -125,6 +141,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] #[cfg(windows)] pub(super) fn wait_for_poll_reader(&self) -> io::Result<()> { if self.poll_reader_ready.load(Ordering::Acquire) { @@ -150,6 +170,10 @@ impl StreamTransportCore { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn close_extra_socket_with_py(&self, py: Python<'_>) { let socket = self .state @@ -163,6 +187,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] #[inline] pub(super) fn register_worker(&self, worker: WorkerThread) { self.workers @@ -171,6 +199,10 @@ impl StreamTransportCore { .push(worker); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn abort_workers(&self) { let workers = self .workers @@ -183,6 +215,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn ensure_writer_worker(self: &Arc) { let lazy = self .lazy_writer @@ -202,6 +238,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] /// Whether a socket can still close directly without bypassing a writer /// worker. Once the lazy target has been taken, all close/EOF commands /// must follow queued data through that worker; `writer_registered` is a @@ -213,6 +253,10 @@ impl StreamTransportCore { .is_some() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] #[inline] pub(super) fn server_ref(&self) -> Option> { self.state @@ -223,6 +267,10 @@ impl StreamTransportCore { .cloned() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn call_in_loop_context( &self, f: impl for<'py> FnOnce(Python<'py>) -> PyResult, @@ -235,6 +283,10 @@ impl StreamTransportCore { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn enqueue_pending_read_event(self: &Arc, event: PendingReadEvent) { // A start_tls handoff retires this core before reusing the socket. A // cancelled plaintext reader may still complete once; never deliver @@ -519,6 +571,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn reschedule_pending_read_events( self: &Arc, drained: &mut VecDeque, diff --git a/src/transport/stream/core_protocol.rs b/src/transport/stream/core_protocol.rs index 762ee93..0a44a4a 100644 --- a/src/transport/stream/core_protocol.rs +++ b/src/transport/stream/core_protocol.rs @@ -20,6 +20,7 @@ use super::tuning::{PENDING_READ_HIGH_WATER, PENDING_READ_LOW_WATER}; use super::{PendingReadEvent, PyStreamTransport, StreamTransportCore}; use crate::context::{run_in_context_noargs, run_in_context_onearg}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Copy raw bytes into a writable C-contiguous export, regardless of its /// element format. The export is released before calling buffer_updated, which /// is allowed to resize or replace the protocol's buffer. @@ -29,6 +30,7 @@ fn copy_to_protocol_buffer(buffer: &Bound<'_, PyAny>, source: &[u8]) -> PyResult }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn copy_to_protocol_buffer_locked(buffer: &Bound<'_, PyAny>, source: &[u8]) -> PyResult { let py = buffer.py(); // Exporters can make Py_buffer self-referential; keep its address stable. @@ -124,6 +126,10 @@ mod buffer_tests { } impl StreamTransportCore { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn apply_pending_read_backpressure(&self) { if self.pending_read_bytes.load(Ordering::Acquire) < PENDING_READ_HIGH_WATER { return; @@ -137,6 +143,10 @@ impl StreamTransportCore { self.reading.store(false, Ordering::Release); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn record_pending_read_drained(&self, len: usize) { let remaining = self .pending_read_bytes @@ -160,6 +170,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] #[inline] pub(super) fn call_protocol_method0( &self, @@ -171,6 +185,10 @@ impl StreamTransportCore { run_in_context_noargs(py, context, context_needs_run, callback) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] #[inline] pub(super) fn call_protocol_method1( &self, @@ -183,6 +201,10 @@ impl StreamTransportCore { run_in_context_onearg(py, context, context_needs_run, callback, arg.bind(py)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn flush_pending_data_with_py( &self, py: Python<'_>, @@ -202,6 +224,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn report_error_with_py( &self, py: Python<'_>, @@ -215,10 +241,18 @@ impl StreamTransportCore { .call_exception_handler(py, Some(&self.loop_obj), context.unbind().into_any()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn report_error(&self, err: PyErr, message: &str) { let _ = Python::try_attach(|py| self.report_error_with_py(py, err, message)); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub fn connection_made(&self, transport: Py) -> PyResult<()> { self.call_in_loop_context(|py| { let (callback, fast_path, context, context_needs_run) = { @@ -250,14 +284,26 @@ impl StreamTransportCore { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub fn data_received(&self, data: &[u8]) -> PyResult<()> { self.call_in_loop_context(|py| self.data_received_with_py(py, data)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub fn eof_received(&self) -> PyResult { self.call_in_loop_context(|py| self.eof_received_with_py(py)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub fn connection_lost(self: &Arc, exc: Option) -> PyResult<()> { // Always serialize loss with pending read data, even when the caller // is already on the loop thread. A direct callback here can overtake @@ -269,12 +315,20 @@ impl StreamTransportCore { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn report_connection_lost_result(&self, result: PyResult<()>) { if let Err(err) = result { self.report_error(err, "stream connection_lost callback failed"); } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn data_received_with_py(&self, py: Python<'_>, data: &[u8]) -> PyResult<()> { let fast_path = { let state = self.state.lock().expect("poisoned transport state"); @@ -292,6 +346,10 @@ impl StreamTransportCore { self.data_received_slow_path(py, data) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn data_received_slow_path(&self, py: Python<'_>, data: &[u8]) -> PyResult<()> { let (data_received, get_buffer, buffer_updated, context, context_needs_run) = { let state = self.state.lock().expect("poisoned transport state"); @@ -354,6 +412,10 @@ impl StreamTransportCore { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn eof_received_with_py(&self, py: Python<'_>) -> PyResult { let (callback, fast_path, context, context_needs_run) = { let state = self.state.lock().expect("poisoned transport state"); @@ -382,6 +444,10 @@ impl StreamTransportCore { result.bind(py).is_truthy() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn connection_lost_with_py( &self, py: Python<'_>, diff --git a/src/transport/stream/core_state.rs b/src/transport/stream/core_state.rs index 2ffce4f..82b16a4 100644 --- a/src/transport/stream/core_state.rs +++ b/src/transport/stream/core_state.rs @@ -20,6 +20,7 @@ use super::tuning::DEFAULT_WRITE_BUFFER_HIGH_WATER; use super::{StreamTransportCore, make_stream_extra}; use crate::fd_ops; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Resolve asyncio's optional write-buffer limits without overflowing when a /// caller supplies a very large low-water mark. fn normalize_write_buffer_limits( @@ -36,6 +37,10 @@ fn normalize_write_buffer_limits( } impl StreamTransportCore { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(crate) fn uses_native_stream_reader(&self) -> bool { matches!( self.state @@ -48,6 +53,10 @@ impl StreamTransportCore { ) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn set_protocol(&self, py: Python<'_>, protocol: Py) -> PyResult<()> { let callbacks = build_protocol_callbacks(py, &protocol)?; let mut state = self.state.lock().expect("poisoned transport state"); @@ -56,6 +65,10 @@ impl StreamTransportCore { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn get_protocol(&self, py: Python<'_>) -> Py { self.state .lock() @@ -64,6 +77,10 @@ impl StreamTransportCore { .clone_ref(py) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn get_extra(&self, py: Python<'_>, name: &str) -> Option> { let (cached, lazy_socket_family, transport_closed) = { let state = self.state.lock().expect("poisoned transport state"); @@ -101,6 +118,10 @@ impl StreamTransportCore { state.extra.get(name).map(|value| value.clone_ref(py)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] #[inline] pub(super) fn set_closing(&self) { self.state.lock().expect("poisoned transport state").closing = true; @@ -109,6 +130,10 @@ impl StreamTransportCore { self.read_buffer_pool.close(); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn runtime_socket_fd(&self) -> Option { let state = self.state.lock().expect("poisoned transport state"); if state.runtime_socket_io { @@ -118,6 +143,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn detach_underlying_stream(&self, py: Python<'_>) { self.detached.store(true, Ordering::Release); self.close_extra_socket_with_py(py); @@ -132,17 +161,29 @@ impl StreamTransportCore { self.read_buffer_pool.close(); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn release_direct_writer(&self) { if let Some(writer) = &self.direct_writer { writer.lock().expect("poisoned direct tasked writer").take(); } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn is_closing_or_lost(&self) -> bool { let state = self.state.lock().expect("poisoned transport state"); state.closing || state.lost_called } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn mark_write_eof(&self) { self.state .lock() @@ -150,10 +191,18 @@ impl StreamTransportCore { .write_eof_requested = true; } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn is_closing(&self) -> bool { self.state.lock().expect("poisoned transport state").closing } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn can_write_eof(&self) -> bool { self.state .lock() @@ -161,6 +210,10 @@ impl StreamTransportCore { .can_write_eof } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn pause_reading(&self) { let mut state = self.state.lock().expect("poisoned transport state"); state.read_paused = true; @@ -168,6 +221,10 @@ impl StreamTransportCore { self.reading.store(false, Ordering::Release); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn resume_reading(&self) { let mut state = self.state.lock().expect("poisoned transport state"); state.read_paused = false; @@ -181,10 +238,18 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn is_reading(&self) -> bool { self.reading.load(Ordering::Acquire) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn wait_until_readable(&self) { let mut state = self.state.lock().expect("poisoned transport state"); while (state.read_paused || state.read_backpressured) && !state.closing { @@ -198,6 +263,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore", future = true) + )] pub(super) async fn wait_until_async_readable(&self) { loop { if self.is_closing() || self.is_reading() { @@ -211,6 +280,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn acquire_read_buffer_blocking( &self, capacity: usize, @@ -228,6 +301,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore", future = true) + )] pub(super) async fn acquire_read_buffer_async(&self, capacity: usize) -> Option> { loop { if self.is_closing() { @@ -240,6 +317,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn is_writable(&self) -> bool { self.state .lock() @@ -247,6 +328,10 @@ impl StreamTransportCore { .writable } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn get_write_buffer_size(&self) -> usize { self.state .lock() @@ -255,11 +340,19 @@ impl StreamTransportCore { .size } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn get_write_buffer_limits(&self) -> (usize, usize) { let state = self.state.lock().expect("poisoned transport state"); (state.write_buffer.low_water, state.write_buffer.high_water) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn set_write_buffer_limits( self: &Arc, high: Option, diff --git a/src/transport/stream/core_write.rs b/src/transport/stream/core_write.rs index dc57d2b..7677fff 100644 --- a/src/transport/stream/core_write.rs +++ b/src/transport/stream/core_write.rs @@ -39,11 +39,13 @@ use super::{ use crate::engine::{LoopCommand, LoopTransportCommand}; use crate::fd_ops; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn is_write_batch_candidate(len: usize) -> bool { len > SMALL_WRITE_COALESCE_MIN_BYTES && len <= SMALL_WRITE_COALESCE_MAX_BYTES } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn stage_owned_buffer(pending: &mut Option, data: OwnedWriteBuffer) { if let Some(buffer) = pending { buffer.extend_from_slice(data.remaining()); @@ -59,6 +61,7 @@ pub(super) enum WriteBufferSignal { Resume, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn enqueue_write_buffer( state: &mut super::StreamWriteBufferState, len: usize, @@ -79,6 +82,7 @@ fn enqueue_write_buffer( Ok(WriteBufferSignal::None) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn drain_write_buffer(state: &mut super::StreamWriteBufferState, len: usize) -> WriteBufferSignal { state.size = state.size.saturating_sub(len); if state.protocol_paused && state.size <= state.low_water { @@ -89,6 +93,7 @@ fn drain_write_buffer(state: &mut super::StreamWriteBufferState, len: usize) -> } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn clear_write_buffer_state( state: &mut super::StreamWriteBufferState, resume_protocol: bool, @@ -103,6 +108,7 @@ fn clear_write_buffer_state( signal } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn reconcile_write_buffer_limits( state: &mut super::StreamWriteBufferState, low_water: usize, @@ -122,6 +128,10 @@ pub(super) fn reconcile_write_buffer_limits( } impl StreamTransportCore { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] #[inline] pub(super) fn write_backpressure_active(&self) -> bool { self.state @@ -130,6 +140,10 @@ impl StreamTransportCore { .writer_registered } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] #[inline] pub(super) fn set_write_backpressure_active(&self, active: bool) { self.state @@ -138,6 +152,10 @@ impl StreamTransportCore { .writer_registered = active; } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn close_on_write_eof(&self) -> bool { self.state .lock() @@ -182,6 +200,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn fail_write(self: &Arc, err: Option) { if self.is_closing() { return; @@ -203,6 +225,10 @@ impl StreamTransportCore { let _ = self.writer_tx.send(WriterCommand::Stop); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn queue_write(self: &Arc, data: OwnedWriteBuffer) -> io::Result<()> { let should_pause = self.record_write_buffer_enqueued(data.remaining().len())?; self.ensure_writer_worker(); @@ -217,6 +243,10 @@ impl StreamTransportCore { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn queue_recorded_write(self: &Arc, data: OwnedWriteBuffer) { self.ensure_writer_worker(); if self.writer_tx.send(WriterCommand::Data(data)).is_err() { @@ -225,6 +255,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn stage_direct_write(self: &Arc, data: &[u8]) -> io::Result<()> { if data.is_empty() { return Ok(()); @@ -245,6 +279,10 @@ impl StreamTransportCore { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] /// Retain an already-owned write allocation instead of copying it into a /// second pool slot. Only joining an existing batch requires a copy. fn stage_direct_write_buffer(self: &Arc, data: OwnedWriteBuffer) -> io::Result<()> { @@ -266,6 +304,10 @@ impl StreamTransportCore { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] fn finish_staged_write(self: &Arc, should_pause: bool) { if should_pause { self.notify_pause_writing(); @@ -331,6 +373,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] /// Hand staged bytes to the writer before a graceful shutdown. On Windows /// a normal flush can defer while the completion reader is being rebound; /// close/write_eof must not bypass those bytes via the lazy-writer shortcut. @@ -350,6 +396,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn discard_pending_direct_write(self: &Arc) { self.direct_write_scheduled.store(false, Ordering::Release); let discarded = { @@ -421,10 +471,18 @@ impl StreamTransportCore { )) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn new_pooled_write_buffer(&self, capacity: usize) -> OwnedWriteBuffer { OwnedWriteBuffer::with_pooled_capacity(capacity, &self.write_buffer_pool) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn try_write_buffer(self: &Arc, mut data: OwnedWriteBuffer) -> io::Result<()> { if data.is_empty() { return Ok(()); @@ -470,22 +528,42 @@ impl StreamTransportCore { self.queue_write(data) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore", future = true) + )] pub async fn wait_readable(self: &Arc) -> io::Result<()> { Err(io::Error::other( "transport readiness is not used in std transport mode", )) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore", future = true) + )] pub async fn wait_writable(self: &Arc) -> io::Result<()> { Err(io::Error::other( "transport readiness is not used in std transport mode", )) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub fn handle_read_ready_with_py(self: &Arc, _py: Python<'_>) {} + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub fn handle_write_ready_with_py(self: &Arc, _py: Python<'_>) {} + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn upgrade_stream( self: &Arc, py: Python<'_>, @@ -548,6 +626,10 @@ impl StreamTransportCore { )) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn pause_writing_with_py(&self, py: Python<'_>) -> PyResult<()> { let (callback, context, context_needs_run) = { let state = self.state.lock().expect("poisoned transport state"); @@ -564,6 +646,10 @@ impl StreamTransportCore { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn resume_writing_with_py(&self, py: Python<'_>) -> PyResult<()> { let (callback, context, context_needs_run) = { let state = self.state.lock().expect("poisoned transport state"); @@ -580,6 +666,10 @@ impl StreamTransportCore { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn notify_pause_writing(self: &Arc) { if self.loop_core.on_runtime_thread() { let _ = self.call_in_loop_context(|py| self.pause_writing_with_py(py)); @@ -589,6 +679,10 @@ impl StreamTransportCore { self.enqueue_pending_read_event(PendingReadEvent::PauseWriting); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn notify_resume_writing(self: &Arc) { if self.loop_core.on_runtime_thread() { let _ = self.call_in_loop_context(|py| self.resume_writing_with_py(py)); @@ -598,6 +692,10 @@ impl StreamTransportCore { self.enqueue_pending_read_event(PendingReadEvent::ResumeWriting); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn record_write_buffer_enqueued(&self, len: usize) -> io::Result { if len == 0 { return Ok(false); @@ -614,6 +712,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn record_write_buffer_drained(self: &Arc, len: usize) { if len == 0 { return; @@ -629,6 +731,10 @@ impl StreamTransportCore { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamTransportCore") + )] pub(super) fn clear_write_buffer(self: &Arc, resume_protocol: bool) { let should_resume = { let mut state = self.state.lock().expect("poisoned transport state"); diff --git a/src/transport/stream/fast.rs b/src/transport/stream/fast.rs index 909f6be..2bf1fa4 100644 --- a/src/transport/stream/fast.rs +++ b/src/transport/stream/fast.rs @@ -25,20 +25,33 @@ struct PyImmediateRead { #[pymethods] impl PyImmediateRead { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyImmediateRead") + )] fn __await__(slf: Py) -> Py { slf } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyImmediateRead") + )] fn __iter__(slf: Py) -> Py { slf } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyImmediateRead") + )] fn __next__(&mut self, py: Python<'_>) -> PyResult> { let value = self.value.take().unwrap_or_else(|| py.None()); Err(PyStopIteration::new_err(value)) } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn asyncio_iscoroutine(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { static ISCOROUTINE: PyOnceLock> = PyOnceLock::new(); Ok(ISCOROUTINE @@ -48,6 +61,7 @@ fn asyncio_iscoroutine(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { .bind(py)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Create a future on `loop_obj`, skipping the Python-level method dispatch /// when the loop is a native rsloop instance running on this thread. fn loop_create_future(py: Python<'_>, loop_obj: &Py) -> PyResult> { @@ -66,6 +80,10 @@ struct ReadBuffer { } impl ReadBuffer { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBuffer") + )] fn with_capacity(capacity: usize) -> Self { Self { bytes: OwnedReadBuffer::with_capacity(capacity), @@ -75,21 +93,37 @@ impl ReadBuffer { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBuffer") + )] #[inline] fn len(&self) -> usize { self.bytes.len().saturating_sub(self.start) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBuffer") + )] #[inline] fn is_empty(&self) -> bool { self.len() == 0 } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBuffer") + )] #[inline] fn unread(&self) -> &[u8] { &self.bytes[self.start..] } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBuffer") + )] fn extend(&mut self, data: &[u8]) { if data.is_empty() { return; @@ -99,6 +133,10 @@ impl ReadBuffer { self.bytes.extend_from_slice(data); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBuffer") + )] fn extend_owned(&mut self, data: OwnedReadBuffer) -> Option { if data.is_empty() { return Some(data); @@ -113,23 +151,39 @@ impl ReadBuffer { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBuffer") + )] fn consume(&mut self, n: usize) { self.start = self.start.saturating_add(n).min(self.bytes.len()); self.compact_if_needed(); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBuffer") + )] #[inline] fn consume_all(&mut self) { self.start = self.bytes.len(); self.compact_if_needed(); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBuffer") + )] fn replace(&mut self, data: &[u8]) { self.bytes.clear(); self.bytes.extend_from_slice(data); self.start = 0; } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadBuffer") + )] fn compact_if_needed(&mut self) { if self.start == 0 { return; @@ -302,6 +356,7 @@ mod read_buffer_tests { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// `bytes.find(needle, from)` over a slice. #[inline] fn find_from(haystack: &[u8], needle: &[u8], from: usize) -> Option { @@ -332,6 +387,7 @@ enum UntilScan { } impl UntilScan { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UntilScan"))] #[inline] fn is_pending(&self, eof: bool) -> bool { matches!(self, Self::NeedMore) && !eof @@ -357,6 +413,10 @@ enum Separators { } impl Separators { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "Separators") + )] fn single(bytes: &[u8]) -> Self { if bytes.len() > INLINE_SEPARATOR_LEN { return Self::Heap(vec![bytes.to_vec()]); @@ -369,6 +429,10 @@ impl Separators { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "Separators") + )] fn from_list(list: Vec>) -> Self { if let [only] = list.as_slice() { return Self::single(only); @@ -376,6 +440,10 @@ impl Separators { Self::Heap(list) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "Separators") + )] /// Order shortest-first, matching `sorted(separator, key=len)`. The sort has /// to stay stable so equal-length separators keep their given order. fn sort_by_length(&mut self) { @@ -384,6 +452,10 @@ impl Separators { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "Separators") + )] fn shortest(&self) -> Option<&[u8]> { match self { Self::Inline { bytes, len } => Some(&bytes[..*len]), @@ -391,6 +463,10 @@ impl Separators { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "Separators") + )] fn longest_len(&self) -> usize { match self { Self::Inline { len, .. } => *len, @@ -399,6 +475,7 @@ impl Separators { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn separator_match_replaces(best: Option<(usize, usize)>, candidate_end: usize) -> bool { best.is_none_or(|(_, best_end)| candidate_end < best_end) } @@ -418,6 +495,10 @@ struct UntilReadState { } impl UntilReadState { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UntilReadState") + )] fn new(mut separators: Separators, line_mode: bool) -> PyResult { // asyncio sorts shortest-first and keeps a strictly closer match end, so // the shortest separator wins when two of them end at the same byte. @@ -443,6 +524,10 @@ impl UntilReadState { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UntilReadState") + )] /// One iteration of asyncio's `readuntil` scan loop. fn scan(&mut self, buffer: &[u8], limit: usize) -> UntilScan { let buflen = buffer.len(); @@ -1055,6 +1140,7 @@ enum UntilOutcome { Exception(Py), } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Collect the separator argument, accepting the tuple form Python 3.13+ takes. fn extract_separators(separator: &Bound<'_, PyAny>) -> PyResult { if let Ok(tuple) = separator.cast::() { @@ -1072,6 +1158,7 @@ fn extract_separators(separator: &Bound<'_, PyAny>) -> PyResult { Ok(Separators::single(&separator.extract::>()?)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn extract_separator_bytes(value: &Bound<'_, PyAny>) -> PyResult> { if let Ok(bytes) = value.cast::() { return Ok(bytes.as_bytes().to_vec()); @@ -1103,12 +1190,17 @@ struct ExactReadAccumulator { expected: usize, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn exact_fill_amount(buffer_len: usize, filled: usize, expected: usize) -> usize { debug_assert!(filled <= expected); buffer_len.min(expected - filled) } impl ExactReadAccumulator { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ExactReadAccumulator") + )] fn new(py: Python<'_>, expected: usize) -> PyResult { let expected_size = ffi::Py_ssize_t::try_from(expected) .map_err(|_| pyo3::exceptions::PyOverflowError::new_err("read size is too large"))?; @@ -1131,6 +1223,10 @@ impl ExactReadAccumulator { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ExactReadAccumulator") + )] fn fill_from(&mut self, buffer: &mut ReadBuffer) { let amount = exact_fill_amount(buffer.len(), self.filled, self.expected); if amount == 0 { @@ -1148,6 +1244,10 @@ impl ExactReadAccumulator { buffer.consume(amount); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ExactReadAccumulator") + )] fn partial(&self) -> &[u8] { // SAFETY: The first `filled` bytes were initialized by `fill_from` and // the returned slice is consumed immediately while `self` is alive. @@ -1178,6 +1278,10 @@ pub struct PyFastStreamReader { } impl PyFastStreamReader { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn set_future_result_or_ignore_cancelled( py: Python<'_>, future: &Py, @@ -1199,6 +1303,10 @@ impl PyFastStreamReader { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn set_future_exception_or_ignore_cancelled( py: Python<'_>, future: &Py, @@ -1221,6 +1329,10 @@ impl PyFastStreamReader { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn new_with_loop(py: Python<'_>, loop_obj: Py, limit: usize) -> PyResult { if limit == 0 { return Err(PyValueError::new_err("Limit cannot be <= 0")); @@ -1240,15 +1352,27 @@ impl PyFastStreamReader { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] #[inline] fn create_future(&self, py: Python<'_>) -> PyResult> { loop_create_future(py, &self.loop_obj) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn ready_result_awaitable(&self, py: Python<'_>, value: Py) -> PyResult> { Ok(Py::new(py, PyImmediateRead { value: Some(value) })?.into_any()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn ready_exception_future(&self, py: Python<'_>, exc: Py) -> PyResult> { let future = self.create_future(py)?; python_names::call_method1( @@ -1260,11 +1384,19 @@ impl PyFastStreamReader { Ok(future) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] #[inline] fn bytes_object(py: Python<'_>, data: &[u8]) -> Py { PyBytes::new(py, data).unbind().into_any() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn unread_bytes_object(&mut self, py: Python<'_>, n: usize) -> Py { let len = self.buffer.len().min(n); let value = Self::bytes_object(py, &self.buffer.unread()[..len]); @@ -1272,12 +1404,20 @@ impl PyFastStreamReader { value } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn unread_all_bytes_object(&mut self, py: Python<'_>) -> Py { let value = Self::bytes_object(py, self.buffer.unread()); self.buffer.consume_all(); value } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn incomplete_read_error( py: Python<'_>, partial: &[u8], @@ -1290,6 +1430,10 @@ impl PyFastStreamReader { .unbind()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn maybe_resume_transport(&mut self, py: Python<'_>) -> PyResult<()> { if self.paused && self.buffer.len() <= self.limit && !self.transport.bind(py).is_none() { self.paused = false; @@ -1302,6 +1446,10 @@ impl PyFastStreamReader { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn maybe_pause_transport(&mut self, py: Python<'_>) -> PyResult<()> { // Never pause while a coroutine is waiting for data: a pending // readexactly()/read() waiter may need more than 2 * limit bytes to @@ -1330,6 +1478,10 @@ impl PyFastStreamReader { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn maybe_complete_waiter(&mut self, py: Python<'_>) -> PyResult<()> { let Some((future, kind)) = self .waiter @@ -1419,6 +1571,10 @@ impl PyFastStreamReader { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn start_waiter( &mut self, py: Python<'_>, @@ -1458,6 +1614,10 @@ impl PyFastStreamReader { Ok(future) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] pub(crate) fn set_transport_obj( &mut self, py: Python<'_>, @@ -1499,12 +1659,20 @@ impl PyFastStreamReader { self.maybe_pause_transport(py) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] #[inline] pub(crate) fn feed_eof_internal(&mut self, py: Python<'_>) -> PyResult<()> { self.eof = true; self.maybe_complete_waiter(py) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] pub(crate) fn set_exception_internal( &mut self, py: Python<'_>, @@ -1514,6 +1682,10 @@ impl PyFastStreamReader { self.maybe_complete_waiter(py) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn build_read_future(&mut self, py: Python<'_>, n: isize) -> PyResult> { if let Some(exc) = self.exception.as_ref() { return self.ready_exception_future(py, exc.clone_ref(py)); @@ -1561,6 +1733,10 @@ impl PyFastStreamReader { self.start_waiter(py, "readexactly", ReadWaitKind::Exact(n), None) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn limit_overrun_error(py: Python<'_>, message: &str, consumed: usize) -> PyResult> { let asyncio = py.import("asyncio")?; Ok(asyncio @@ -1569,6 +1745,10 @@ impl PyFastStreamReader { .unbind()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] /// `IncompleteReadError` with an undefined expected size, which is what /// `readuntil()` raises: the caller never said how many bytes it wanted. fn incomplete_until_error(py: Python<'_>, partial: Py) -> PyResult> { @@ -1579,6 +1759,10 @@ impl PyFastStreamReader { .unbind()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn until_limit_overrun( &mut self, py: Python<'_>, @@ -1609,6 +1793,10 @@ impl PyFastStreamReader { )) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] /// Turn a finished scan into the value or exception the awaitable carries. /// Only called once `UntilScan::is_pending` has ruled out waiting, so /// `NeedMore` here always means EOF arrived first. @@ -1656,6 +1844,10 @@ impl PyFastStreamReader { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn build_until_future( &mut self, py: Python<'_>, @@ -1681,6 +1873,10 @@ impl PyFastStreamReader { #[pymethods] impl PyFastStreamReader { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] #[new] #[pyo3(signature = (limit=DEFAULT_STREAM_LIMIT, loop_obj=None))] fn py_new(py: Python<'_>, limit: usize, loop_obj: Option>) -> PyResult { @@ -1694,21 +1890,37 @@ impl PyFastStreamReader { Self::new_with_loop(py, loop_obj, limit) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] #[getter(_rsloop_fast_reader)] fn get_rsloop_fast_reader(&self, py: Python<'_>) -> Py { py.None() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] #[getter(_loop)] fn get_loop_obj(&self, py: Python<'_>) -> Py { self.loop_obj.clone_ref(py) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] #[getter(_limit)] fn get_limit(&self) -> usize { self.limit } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] #[getter(_buffer)] fn get_buffer(&self, py: Python<'_>) -> Py { PyByteArray::new(py, self.buffer.unread()) @@ -1716,6 +1928,10 @@ impl PyFastStreamReader { .into_any() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] #[setter(_buffer)] fn set_buffer(&mut self, value: &Bound<'_, PyAny>) -> PyResult<()> { let data: Vec = value.extract()?; @@ -1723,6 +1939,10 @@ impl PyFastStreamReader { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] #[getter(_waiter)] fn get_waiter(&self, py: Python<'_>) -> Py { self.waiter @@ -1731,21 +1951,37 @@ impl PyFastStreamReader { .unwrap_or_else(|| py.None()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] #[getter(_transport)] fn get_transport(&self, py: Python<'_>) -> Py { self.transport.clone_ref(py) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] #[getter(_paused)] fn get_paused(&self) -> bool { self.paused } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] #[getter(_eof)] fn get_eof(&self) -> bool { self.eof } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] #[getter(_exception)] fn get_exception_obj(&self, py: Python<'_>) -> Py { self.exception @@ -1754,39 +1990,75 @@ impl PyFastStreamReader { .unwrap_or_else(|| py.None()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn exception(&self, py: Python<'_>) -> Py { self.get_exception_obj(py) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn set_exception(&mut self, py: Python<'_>, exc: Py) -> PyResult<()> { self.set_exception_internal(py, exc) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn set_transport_public(&mut self, py: Python<'_>, transport: Py) -> PyResult<()> { self.set_transport_obj(py, transport) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn feed_data(&mut self, py: Python<'_>, data: &[u8]) -> PyResult<()> { self.feed_data_internal(py, data) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn feed_eof(&mut self, py: Python<'_>) -> PyResult<()> { self.feed_eof_internal(py) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn at_eof(&self) -> bool { self.eof && self.buffer.is_empty() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] #[pyo3(signature = (n=-1))] fn read(&mut self, py: Python<'_>, n: isize) -> PyResult> { self.build_read_future(py, n) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn readexactly(&mut self, py: Python<'_>, n: usize) -> PyResult> { self.build_readexactly_future(py, n) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] #[pyo3(signature = (separator=None))] fn readuntil( &mut self, @@ -1806,6 +2078,10 @@ impl PyFastStreamReader { self.build_until_future(py, "readuntil", state) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamReader") + )] fn readline(&mut self, py: Python<'_>) -> PyResult> { let state = UntilReadState::new(Separators::single(b"\n"), true)?; self.build_until_future(py, "readline", state) @@ -1828,6 +2104,10 @@ pub struct PyFastStreamProtocol { } impl PyFastStreamProtocol { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] fn new_with_loop( py: Python<'_>, loop_obj: Py, @@ -1857,18 +2137,34 @@ impl PyFastStreamProtocol { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] fn has_client_connected_cb(&self, py: Python<'_>) -> bool { !self.client_connected_cb.bind(py).is_none() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] pub(crate) fn reader_ref(&self, py: Python<'_>) -> Py { self.reader.clone_ref(py) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] fn ready_none_future(&self, py: Python<'_>) -> PyResult> { Ok(self.ready_none.clone_ref(py)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] fn ready_exception_future(&self, py: Python<'_>, exc: Py) -> PyResult> { let future = loop_create_future(py, &self.loop_obj)?; python_names::call_method1( @@ -1880,12 +2176,20 @@ impl PyFastStreamProtocol { Ok(future) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] fn push_drain_waiter(&mut self, py: Python<'_>) -> PyResult> { let future = loop_create_future(py, &self.loop_obj)?; self.drain_waiters.push(future.clone_ref(py)); Ok(future) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] fn resolve_drain_waiters(&mut self, py: Python<'_>, exc: Option>) -> PyResult<()> { for future in self.drain_waiters.drain(..) { let future = future.bind(py); @@ -1917,6 +2221,10 @@ impl PyFastStreamProtocol { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] fn build_drain_future( &mut self, py: Python<'_>, @@ -1939,6 +2247,10 @@ impl PyFastStreamProtocol { self.push_drain_waiter(py) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] pub(crate) fn handle_connection_made( slf: Py, py: Python<'_>, @@ -2002,6 +2314,10 @@ impl PyFastStreamProtocol { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] pub(crate) fn handle_connection_lost( &mut self, py: Python<'_>, @@ -2055,6 +2371,10 @@ impl PyFastStreamProtocol { #[pymethods] impl PyFastStreamProtocol { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] #[new] #[pyo3(signature = (reader, client_connected_cb=None, loop_obj=None))] fn py_new( @@ -2078,38 +2398,70 @@ impl PyFastStreamProtocol { ) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] #[getter(_rsloop_fast_reader)] fn get_rsloop_fast_reader(&self, py: Python<'_>) -> Py { self.reader.clone_ref(py).into_any() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] fn connection_made(slf: Py, py: Python<'_>, transport: Py) -> PyResult<()> { Self::handle_connection_made(slf, py, transport) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] fn pause_writing(&mut self) { self.paused = true; } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] fn resume_writing(&mut self, py: Python<'_>) -> PyResult<()> { self.paused = false; self.resolve_drain_waiters(py, None) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] fn _drain_helper(&mut self, py: Python<'_>) -> PyResult> { self.build_drain_future(py, None) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] fn data_received(&mut self, py: Python<'_>, data: &[u8]) -> PyResult<()> { self.reader.borrow_mut(py).feed_data_internal(py, data) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] pub(super) fn eof_received(&mut self, py: Python<'_>) -> PyResult { self.reader.borrow_mut(py).feed_eof_internal(py)?; // TLS does not support keeping a write half open after peer EOF. Ok(!self.over_ssl) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamProtocol") + )] fn connection_lost(&mut self, py: Python<'_>, exc: Option>) -> PyResult<()> { self.handle_connection_lost(py, exc) } @@ -2128,11 +2480,19 @@ pub struct PyFastStreamWriter { #[pymethods] impl PyFastStreamWriter { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamWriter") + )] #[getter] fn transport(&self, py: Python<'_>) -> Py { self.transport.clone_ref(py) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamWriter") + )] fn write(&self, py: Python<'_>, data: &Bound<'_, PyAny>) -> PyResult<()> { // Native transports take the direct Rust path; only foreign // transports go through Python method dispatch. @@ -2143,6 +2503,10 @@ impl PyFastStreamWriter { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamWriter") + )] fn writelines(&self, py: Python<'_>, data: &Bound<'_, PyAny>) -> PyResult<()> { if let Ok(transport) = self.transport.bind(py).cast_exact::() { return transport.borrow().writelines(py, data); @@ -2151,24 +2515,44 @@ impl PyFastStreamWriter { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamWriter") + )] fn write_eof(&self, py: Python<'_>) -> PyResult> { self.transport.call_method0(py, "write_eof") } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamWriter") + )] fn can_write_eof(&self, py: Python<'_>) -> PyResult { self.transport .call_method0(py, "can_write_eof")? .extract(py) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamWriter") + )] fn close(&self, py: Python<'_>) -> PyResult> { self.transport.call_method0(py, "close") } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamWriter") + )] fn is_closing(&self, py: Python<'_>) -> PyResult { self.transport.call_method0(py, "is_closing")?.extract(py) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamWriter") + )] #[pyo3(signature = (name, default=None))] fn get_extra_info( &self, @@ -2183,10 +2567,18 @@ impl PyFastStreamWriter { ) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamWriter") + )] fn wait_closed(&self, py: Python<'_>) -> Py { self.protocol.borrow(py).closed.clone_ref(py) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastStreamWriter") + )] fn drain(&self, py: Python<'_>) -> PyResult> { let reader_exception = self .reader @@ -2208,6 +2600,10 @@ struct PyFastClientDoneCallback { #[pymethods] impl PyFastClientDoneCallback { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastClientDoneCallback") + )] fn __call__(&self, py: Python<'_>, task: Py) -> PyResult<()> { if task.call_method0(py, "cancelled")?.extract::(py)? { self.transport.call_method0(py, "close")?; @@ -2239,6 +2635,10 @@ struct PyFastProtocolFactory { #[pymethods] impl PyFastProtocolFactory { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyFastProtocolFactory") + )] fn __call__(&self, py: Python<'_>) -> PyResult> { let reader = Py::new( py, @@ -2256,6 +2656,7 @@ impl PyFastProtocolFactory { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn running_loop(py: Python<'_>) -> PyResult> { Ok(py .import("asyncio.events")? @@ -2263,6 +2664,7 @@ fn running_loop(py: Python<'_>) -> PyResult> { .unbind()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn copy_kwargs<'py>( py: Python<'py>, kwargs: Option<&Bound<'py, PyDict>>, @@ -2278,6 +2680,7 @@ fn copy_kwargs<'py>( Ok(Some(copied)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn native_stream_loop(py: Python<'_>) -> PyResult> { let loop_obj = running_loop(py)?; if !loop_obj.bind(py).is_instance_of::() { @@ -2288,6 +2691,7 @@ fn native_stream_loop(py: Python<'_>) -> PyResult> { Ok(loop_obj) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn host_port_objects( py: Python<'_>, host: Option>, @@ -2304,6 +2708,7 @@ fn host_port_objects( (host_obj, port_obj) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn fast_open_connection_awaitable( py: Python<'_>, loop_obj: &Py, @@ -2327,6 +2732,7 @@ fn fast_open_connection_awaitable( Ok((locals, awaitable)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn fast_open_connection_result(py: Python<'_>, created: Py) -> PyResult> { let result = created.bind(py).cast::()?; let transport = result.get_item(0)?.unbind(); @@ -2344,6 +2750,7 @@ fn fast_open_connection_result(py: Python<'_>, created: Py) -> PyResult

fd_ops::RawFd { match self { Self::Tcp(stream) => tcp_stream_raw_fd(stream), @@ -45,6 +49,10 @@ impl TaskedDirectWriter { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TaskedDirectWriter") + )] pub(super) fn shutdown_close(&self) -> io::Result<()> { match self { Self::Tcp(stream) => shutdown_tcp_stream(stream, Shutdown::Both), @@ -53,6 +61,10 @@ impl TaskedDirectWriter { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TaskedDirectWriter") + )] pub(super) fn shutdown_write(&self) -> io::Result<()> { match self { Self::Tcp(stream) => shutdown_tcp_stream(stream, Shutdown::Write), @@ -70,6 +82,10 @@ pub enum ReaderTarget { } impl ReaderTarget { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReaderTarget") + )] pub(super) fn fd(&self) -> fd_ops::RawFd { match self { Self::File(file) => file_raw_fd(file), @@ -79,11 +95,19 @@ impl ReaderTarget { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReaderTarget") + )] #[cfg(windows)] pub(super) fn pollable(&self) -> bool { !matches!(self, Self::File(_)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReaderTarget") + )] #[cfg(not(windows))] pub(super) fn pollable(&self) -> bool { true @@ -91,6 +115,10 @@ impl ReaderTarget { } impl Read for ReaderTarget { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn read(&mut self, buf: &mut [u8]) -> io::Result { match self { Self::File(file) => file.read(buf), @@ -121,6 +149,10 @@ pub(super) enum LazyWriterTarget { } impl LazyWriterTarget { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "LazyWriterTarget") + )] pub(super) fn materialize(self) -> PyResult { match self { Self::Tcp(fd) => duplicate_configured_tcp_stream(fd).map(WriterTarget::Tcp), @@ -131,6 +163,10 @@ impl LazyWriterTarget { } impl WriterTarget { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WriterTarget") + )] pub(super) fn fd(&self) -> Option { match self { Self::File(file) => Some(file_raw_fd(file)), @@ -141,16 +177,28 @@ impl WriterTarget { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WriterTarget") + )] #[cfg(windows)] pub(super) fn pollable(&self) -> bool { !matches!(self, Self::File(_)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WriterTarget") + )] #[cfg(not(windows))] pub(super) fn pollable(&self) -> bool { true } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WriterTarget") + )] pub(super) fn shutdown_write(&self) -> io::Result<()> { match self { Self::Tcp(stream) => shutdown_tcp_stream(stream, Shutdown::Write), @@ -160,6 +208,10 @@ impl WriterTarget { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WriterTarget") + )] pub(super) fn shutdown_close(&self) -> io::Result<()> { match self { Self::Tcp(stream) => shutdown_tcp_stream(stream, Shutdown::Both), @@ -177,6 +229,10 @@ pub(super) enum StreamKind { } impl StreamKind { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamKind") + )] pub(super) fn fd(&self) -> fd_ops::RawFd { match self { Self::Tcp(stream) => tcp_stream_raw_fd(stream), @@ -185,16 +241,28 @@ impl StreamKind { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamKind") + )] #[cfg(windows)] pub(super) fn pollable(&self) -> bool { true } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamKind") + )] #[cfg(not(windows))] pub(super) fn pollable(&self) -> bool { true } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamKind") + )] pub(super) fn shutdown_close(&self) -> io::Result<()> { match self { Self::Tcp(stream) => shutdown_tcp_stream(stream, Shutdown::Both), @@ -205,6 +273,10 @@ impl StreamKind { } impl Read for StreamKind { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn read(&mut self, buf: &mut [u8]) -> io::Result { match self { Self::Tcp(stream) => stream.read(buf), @@ -215,6 +287,10 @@ impl Read for StreamKind { } impl io::Write for StreamKind { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn write(&mut self, buf: &[u8]) -> io::Result { match self { Self::Tcp(stream) => stream.write(buf), @@ -223,6 +299,10 @@ impl io::Write for StreamKind { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn flush(&mut self) -> io::Result<()> { match self { Self::Tcp(stream) => stream.flush(), @@ -233,6 +313,10 @@ impl io::Write for StreamKind { } impl io::Write for WriterTarget { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn write(&mut self, buf: &[u8]) -> io::Result { match self { Self::File(file) => file.write(buf), @@ -243,6 +327,10 @@ impl io::Write for WriterTarget { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn flush(&mut self) -> io::Result<()> { match self { Self::File(file) => file.flush(), @@ -254,6 +342,7 @@ impl io::Write for WriterTarget { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn shutdown_tcp_stream(stream: &StdTcpStream, how: Shutdown) -> io::Result<()> { for attempt in 0..=100 { match stream.shutdown(how) { @@ -275,6 +364,7 @@ pub(super) fn shutdown_tcp_stream(stream: &StdTcpStream, how: Shutdown) -> io::R unreachable!("bounded shutdown retry loop always returns") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub(super) fn shutdown_unix_stream(stream: &StdUnixStream, how: Shutdown) -> io::Result<()> { for attempt in 0..=100 { @@ -297,6 +387,7 @@ pub(super) fn shutdown_unix_stream(stream: &StdUnixStream, how: Shutdown) -> io: unreachable!("bounded shutdown retry loop always returns") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub(super) fn is_no_buffer_space_code(raw_os_error: Option) -> bool { #[cfg(unix)] @@ -309,6 +400,7 @@ pub(super) fn is_no_buffer_space_code(raw_os_error: Option) -> bool { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn is_no_buffer_space(err: &io::Error) -> bool { is_no_buffer_space_code(err.raw_os_error()) diff --git a/src/transport/stream/mod.rs b/src/transport/stream/mod.rs index 0274f4c..1d7d09a 100644 --- a/src/transport/stream/mod.rs +++ b/src/transport/stream/mod.rs @@ -156,6 +156,10 @@ struct StreamWriteBufferState { } impl Default for StreamWriteBufferState { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn default() -> Self { Self { size: 0, @@ -243,6 +247,10 @@ pub struct ServerAcceptTaskGuard { } impl ServerAcceptTaskGuard { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ServerAcceptTaskGuard") + )] fn new(server: &Arc) -> Self { server.active_accept_tasks.fetch_add(1, Ordering::AcqRel); Self { @@ -252,6 +260,10 @@ impl ServerAcceptTaskGuard { } impl Drop for ServerAcceptTaskGuard { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { let previous = self .server @@ -267,6 +279,10 @@ struct PendingTlsHandshake { } impl Drop for PendingTlsHandshake { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { let previous = self .server diff --git a/src/transport/stream/pipe_transport.rs b/src/transport/stream/pipe_transport.rs index b8a2d16..0cb3915 100644 --- a/src/transport/stream/pipe_transport.rs +++ b/src/transport/stream/pipe_transport.rs @@ -31,6 +31,7 @@ use super::{ }; use crate::fd_ops; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn spawn_read_pipe_transport( py: Python<'_>, spawn_context: TransportSpawnContext, @@ -55,6 +56,7 @@ pub fn spawn_read_pipe_transport( Ok(transport) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn spawn_write_pipe_transport( py: Python<'_>, spawn_context: TransportSpawnContext, @@ -81,15 +83,24 @@ pub(super) enum PipeTransportMode { } impl PipeTransportMode { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PipeTransportMode") + )] pub(super) fn reading(&self) -> bool { matches!(self, Self::Read) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PipeTransportMode") + )] pub(super) fn writable(&self) -> bool { matches!(self, Self::Write) } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn pipe_transport_core( py: Python<'_>, spawn_context: TransportSpawnContext, @@ -125,6 +136,7 @@ pub(super) fn pipe_transport_core( Ok((core, transport, writer_rx)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn pipe_extra( py: Python<'_>, pipe_obj: &Py, @@ -139,6 +151,7 @@ pub(super) fn pipe_extra( extra } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(not(windows))] pub(super) fn pipe_file_from_obj(py: Python<'_>, pipe_obj: &Py) -> PyResult { let fd = fd_ops::fileobj_to_fd(py, pipe_obj.bind(py))?; @@ -146,6 +159,7 @@ pub(super) fn pipe_file_from_obj(py: Python<'_>, pipe_obj: &Py) -> PyResu from_owned_raw_fd(dup) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] pub(super) fn pipe_file_from_obj(py: Python<'_>, pipe_obj: &Py) -> PyResult { let fd = fd_ops::fileobj_to_fd(py, pipe_obj.bind(py))?; diff --git a/src/transport/stream/platform.rs b/src/transport/stream/platform.rs index 3ad6d61..d62a2d4 100644 --- a/src/transport/stream/platform.rs +++ b/src/transport/stream/platform.rs @@ -16,50 +16,59 @@ use std::os::windows::io::{ AsRawHandle, AsRawSocket, FromRawHandle, FromRawSocket, RawHandle, RawSocket, }; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub(super) fn file_raw_fd(file: &File) -> fd_ops::RawFd { fd_ops::RawFd::from(file.as_raw_fd()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] pub(super) fn file_raw_fd(file: &File) -> fd_ops::RawFd { file.as_raw_handle() as isize as fd_ops::RawFd } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] #[inline] pub(super) fn tcp_stream_raw_fd(stream: &TcpStream) -> fd_ops::RawFd { fd_ops::RawFd::from(stream.as_raw_fd()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] #[inline] pub(super) fn tcp_stream_raw_fd(stream: &TcpStream) -> fd_ops::RawFd { stream.as_raw_socket().cast_signed() } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub(super) fn tcp_listener_raw_fd(listener: &TcpListener) -> fd_ops::RawFd { fd_ops::RawFd::from(listener.as_raw_fd()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] pub(super) fn tcp_listener_raw_fd(listener: &TcpListener) -> fd_ops::RawFd { listener.as_raw_socket().cast_signed() } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] #[inline] pub(super) fn unix_raw_fd(fd: std::os::fd::RawFd) -> fd_ops::RawFd { fd_ops::RawFd::from(fd) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] fn raw_fd_for_std(fd: fd_ops::RawFd) -> PyResult { fd.try_into() .map_err(|_| PyRuntimeError::new_err("fd out of range")) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub(super) fn from_owned_raw_fd(fd: fd_ops::RawFd) -> PyResult { let fd = raw_fd_for_std(fd)?; @@ -67,23 +76,27 @@ pub(super) fn from_owned_raw_fd(fd: fd_ops::RawFd) -> PyResult Ok(unsafe { T::from_raw_fd(fd) }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] pub(super) fn from_owned_raw_socket(socket: RawSocket) -> T { // SAFETY: the caller transfers one owned socket to the returned IO object. unsafe { T::from_raw_socket(socket) } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] pub(super) fn from_owned_raw_handle(handle: RawHandle) -> T { // SAFETY: the caller transfers one owned handle to the returned IO object. unsafe { T::from_raw_handle(handle) } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub(super) fn socket_from_owned_raw(fd: fd_ops::RawFd) -> PyResult { from_owned_raw_fd(fd) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] pub(super) fn socket_from_owned_raw(fd: fd_ops::RawFd) -> PyResult { let fd: RawSocket = fd diff --git a/src/transport/stream/poll.rs b/src/transport/stream/poll.rs index 263d0f1..a5f7add 100644 --- a/src/transport/stream/poll.rs +++ b/src/transport/stream/poll.rs @@ -11,11 +11,13 @@ use std::time::Duration; use super::tuning::BLOCKING_POLL_INTERVAL_MS; use crate::fd_ops; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn clamp_poll_timeout_ms(remaining_ms: u128) -> i32 { let timeout = remaining_ms.clamp(1, u128::from(i32::MAX.unsigned_abs())); i32::try_from(timeout).expect("poll timeout is clamped to i32::MAX") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn wait_socket_ready( fd: fd_ops::RawFd, pollable: bool, @@ -40,6 +42,7 @@ pub(super) fn wait_socket_ready( Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn wait_socket_ready_once( fd: fd_ops::RawFd, pollable: bool, @@ -54,10 +57,12 @@ pub(super) fn wait_socket_ready_once( Ok(true) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn poll_read_ready(fd: fd_ops::RawFd) -> io::Result { fd_ops::poll_fd(fd, true, false, BLOCKING_POLL_INTERVAL_MS).map(|(ready, _)| ready) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn wait_socket_ready_until( fd: fd_ops::RawFd, pollable: bool, diff --git a/src/transport/stream/protocol.rs b/src/transport/stream/protocol.rs index 7d0dec0..6b304cf 100644 --- a/src/transport/stream/protocol.rs +++ b/src/transport/stream/protocol.rs @@ -46,6 +46,10 @@ pub(super) enum StreamReaderFastPath { } impl StreamReaderFastPath { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamReaderFastPath") + )] pub(super) fn clone_ref(&self, py: Python<'_>) -> Self { match self { Self::Native { protocol, reader } => Self::Native { @@ -66,6 +70,10 @@ impl StreamReaderFastPath { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamReaderFastPath") + )] pub(super) fn connection_made( &self, py: Python<'_>, @@ -107,6 +115,10 @@ impl StreamReaderFastPath { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamReaderFastPath") + )] pub(super) fn feed_data(&self, py: Python<'_>, data: &[u8]) -> PyResult<()> { match self { Self::Native { reader, .. } => reader.borrow_mut(py).feed_data_internal(py, data), @@ -177,6 +189,10 @@ impl StreamReaderFastPath { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamReaderFastPath") + )] pub(super) fn feed_owned_data( &self, py: Python<'_>, @@ -195,6 +211,10 @@ impl StreamReaderFastPath { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamReaderFastPath") + )] pub(super) fn feed_eof(&self, py: Python<'_>) -> PyResult<()> { match self { Self::Native { reader, .. } => reader.borrow_mut(py).feed_eof_internal(py), @@ -213,6 +233,10 @@ impl StreamReaderFastPath { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamReaderFastPath") + )] pub(super) fn connection_lost(&self, py: Python<'_>, exc: Option) -> PyResult<()> { match self { Self::Native { protocol, .. } => protocol.borrow_mut(py).handle_connection_lost( @@ -284,6 +308,10 @@ impl StreamReaderFastPath { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "StreamReaderFastPath") + )] pub(super) fn eof_received(&self, py: Python<'_>) -> PyResult { match self { Self::Native { protocol, .. } => protocol.borrow_mut(py).eof_received(py), @@ -295,6 +323,7 @@ impl StreamReaderFastPath { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn build_protocol_callbacks( py: Python<'_>, protocol: &Py, @@ -348,6 +377,7 @@ pub(super) fn build_protocol_callbacks( }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn stream_reader_fast_path( py: Python<'_>, protocol: &Bound<'_, PyAny>, @@ -361,6 +391,7 @@ pub(super) fn stream_reader_fast_path( asyncio_stream_reader_fast_path(py, protocol) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn native_stream_reader_fast_path( py: Python<'_>, protocol: &Bound<'_, PyAny>, @@ -375,6 +406,7 @@ pub(super) fn native_stream_reader_fast_path( })) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn generic_stream_reader_fast_path( protocol: &Bound<'_, PyAny>, ) -> PyResult> { @@ -387,6 +419,7 @@ pub(super) fn generic_stream_reader_fast_path( stream_reader_fast_path_from_reader(Some(protocol.clone().unbind()), reader) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn asyncio_stream_reader_fast_path( py: Python<'_>, protocol: &Bound<'_, PyAny>, @@ -404,6 +437,7 @@ pub(super) fn asyncio_stream_reader_fast_path( stream_reader_fast_path_from_reader(Some(protocol.clone().unbind()), reader) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn stream_reader_fast_path_from_reader( protocol: Option>, reader: Bound<'_, PyAny>, diff --git a/src/transport/stream/py_server.rs b/src/transport/stream/py_server.rs index 0532ac5..968606c 100644 --- a/src/transport/stream/py_server.rs +++ b/src/transport/stream/py_server.rs @@ -14,18 +14,22 @@ use super::PyServer; #[pymethods] impl PyServer { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyServer"))] fn close(&self) { self.core.close(); } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyServer"))] fn is_serving(&self) -> bool { self.core.is_serving() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyServer"))] fn get_loop(&self, py: Python<'_>) -> Py { self.core.loop_obj.clone_ref(py) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyServer"))] fn start_serving<'py>(&self, py: Python<'py>) -> PyResult> { let locals = self.core.locals(py)?; let core = Arc::clone(&self.core); @@ -35,6 +39,7 @@ impl PyServer { }) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyServer"))] fn wait_closed<'py>(&self, py: Python<'py>) -> PyResult> { let locals = self.core.locals(py)?; let core = Arc::clone(&self.core); @@ -60,6 +65,7 @@ impl PyServer { }) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyServer"))] fn serve_forever<'py>(&self, py: Python<'py>) -> PyResult> { let locals = self.core.locals(py)?; let core = Arc::clone(&self.core); @@ -78,6 +84,7 @@ impl PyServer { }) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyServer"))] #[getter] fn sockets(&self, py: Python<'_>) -> PyResult> { let tuple = PyTuple::new( @@ -91,6 +98,7 @@ impl PyServer { Ok(tuple.unbind().into_any()) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PyServer"))] fn __repr__(&self) -> String { format!( "", diff --git a/src/transport/stream/py_transport.rs b/src/transport/stream/py_transport.rs index 925fb0e..4b59e4d 100644 --- a/src/transport/stream/py_transport.rs +++ b/src/transport/stream/py_transport.rs @@ -61,10 +61,18 @@ impl PyStreamTransport { #[pymethods] impl PyStreamTransport { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] fn write(&self, py: Python<'_>, data: &Bound<'_, PyAny>) -> PyResult<()> { self.write_data(py, data) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] pub(super) fn writelines(&self, py: Python<'_>, seq: &Bound<'_, PyAny>) -> PyResult<()> { if self.core.has_text_encoding { for item in seq.try_iter()? { @@ -105,6 +113,10 @@ impl PyStreamTransport { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] fn close(&self) -> PyResult<()> { self.core.flush_pending_direct_write(); #[cfg(windows)] @@ -137,6 +149,10 @@ impl PyStreamTransport { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] fn abort(&self) -> PyResult<()> { self.core.discard_pending_direct_write(); self.core.set_closing(); @@ -158,14 +174,26 @@ impl PyStreamTransport { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] fn is_closing(&self) -> bool { self.core.is_closing() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] fn can_write_eof(&self) -> bool { self.core.can_write_eof() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] fn write_eof(&self) -> PyResult<()> { if !self.core.can_write_eof() { return Err(PyRuntimeError::new_err( @@ -199,6 +227,10 @@ impl PyStreamTransport { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] #[pyo3(signature=(name, default=None))] fn get_extra_info(&self, py: Python<'_>, name: &str, default: Option>) -> Py { self.core @@ -206,41 +238,77 @@ impl PyStreamTransport { .unwrap_or_else(|| default.unwrap_or_else(|| py.None())) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] fn get_protocol(&self, py: Python<'_>) -> Py { self.core.get_protocol(py) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] fn set_protocol(&self, py: Python<'_>, protocol: Py) { self.core .set_protocol(py, protocol) .expect("failed to update transport protocol"); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] fn pause_reading(&self) { self.core.pause_reading(); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] fn resume_reading(&self) { self.core.resume_reading(); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] fn is_reading(&self) -> bool { self.core.is_reading() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] fn get_write_buffer_size(&self) -> usize { self.core.get_write_buffer_size() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] fn get_write_buffer_limits(&self) -> (usize, usize) { self.core.get_write_buffer_limits() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] #[pyo3(signature=(high=None, low=None))] fn set_write_buffer_limits(&self, high: Option, low: Option) -> PyResult<()> { self.core.set_write_buffer_limits(high, low) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PyStreamTransport") + )] fn __repr__(&self) -> String { format!("", self.is_closing()) } diff --git a/src/transport/stream/reader.rs b/src/transport/stream/reader.rs index f31640d..d365a5e 100644 --- a/src/transport/stream/reader.rs +++ b/src/transport/stream/reader.rs @@ -31,6 +31,7 @@ use super::{PendingReadEvent, StreamTransportCore}; use crate::engine::{LoopCommand, LoopIoCommand}; use crate::fd_ops; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn spawn_reader_worker( core: Arc, reader: ReaderTarget, @@ -43,6 +44,7 @@ pub(super) fn spawn_reader_worker( Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn spawn_tls_reader_worker( core: Arc, tls_state: SharedTlsIoState, @@ -56,6 +58,7 @@ pub(super) fn spawn_tls_reader_worker( core.register_worker(worker); Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn run_stream_reader( core: Arc, mut reader: ReaderTarget, @@ -129,6 +132,7 @@ pub(super) fn run_stream_reader( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Retry non-blocking reads for a bounded window after a successful read. /// Returns `false` when the connection terminated (event already enqueued) /// and the reader loop must exit. @@ -186,6 +190,7 @@ pub(super) fn spin_read_stream( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Starts the socket reader for a stream transport. /// /// Active Unix loops route generic-protocol TCP readers to their loop-thread @@ -206,6 +211,7 @@ pub(super) fn spawn_socket_reader( })) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Stops a local reader immediately, or waits for coordination-thread cancellation. pub(super) fn stop_socket_reader(core: &StreamTransportCore, fd: fd_ops::RawFd) -> io::Result<()> { if core.loop_core.stop_io_task(fd) { @@ -226,6 +232,7 @@ pub(super) fn stop_socket_reader(core: &StreamTransportCore, fd: fd_ops::RawFd) .map_err(|err| io::Error::new(io::ErrorKind::TimedOut, err)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Requests reader cancellation without synchronously crossing to the runtime /// thread. Normal close/abort already mark the transport closing and shut down /// the socket, so waiting for the acknowledgement only adds teardown latency. @@ -246,6 +253,7 @@ pub(super) fn stop_socket_reader_nowait( })) .map_err(io::Error::other) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(crate) fn run_socket_reader_blocking( core: Arc, reader: ReaderTarget, @@ -254,6 +262,7 @@ pub(crate) fn run_socket_reader_blocking( run_stream_reader(core, reader, stop) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn run_tls_reader( core: Arc, tls_state: SharedTlsIoState, diff --git a/src/transport/stream/reader_task.rs b/src/transport/stream/reader_task.rs index f01430b..e4cb0eb 100644 --- a/src/transport/stream/reader_task.rs +++ b/src/transport/stream/reader_task.rs @@ -35,6 +35,7 @@ use super::tuning::SERVER_POLL_READER_TINY_TRIGGER_MAX_BYTES; use super::tuning::{MAX_STREAM_READ_BUFFER_SIZE, STREAM_READ_BUFFER_SIZE}; use super::{PendingReadEvent, StreamTransportCore}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[cfg(not(windows))] pub(crate) async fn run_tcp_socket_reader_task( core: Arc, @@ -96,6 +97,7 @@ pub(crate) async fn run_tcp_socket_reader_task( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[cfg(windows)] pub(crate) async fn run_tcp_socket_reader_task( core: Arc, @@ -264,6 +266,7 @@ pub(crate) async fn run_tcp_socket_reader_task( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[cfg(windows)] pub(super) async fn run_windows_poll_tcp_reader( core: Arc, @@ -307,6 +310,7 @@ pub(super) async fn run_windows_poll_tcp_reader( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[cfg(unix)] pub(crate) async fn run_unix_socket_reader_task( core: Arc, @@ -364,10 +368,12 @@ pub(crate) async fn run_unix_socket_reader_task( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn next_read_capacity(buf: &Vec) -> usize { next_read_capacity_for(buf.len(), buf.capacity()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn next_read_capacity_for(len: usize, allocated_capacity: usize) -> usize { let capacity = allocated_capacity.max(STREAM_READ_BUFFER_SIZE); if len == capacity && capacity < MAX_STREAM_READ_BUFFER_SIZE { diff --git a/src/transport/stream/server.rs b/src/transport/stream/server.rs index 7cc4e6c..5fc866a 100644 --- a/src/transport/stream/server.rs +++ b/src/transport/stream/server.rs @@ -21,6 +21,7 @@ use pyo3::prelude::*; use super::{PyServer, ServerCore, ServerCreateParams, ServerListener, ServerState}; use crate::async_event::AsyncEvent; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn create_server(py: Python<'_>, params: ServerCreateParams) -> PyResult> { let ServerCreateParams { loop_core, @@ -63,14 +64,17 @@ pub fn create_server(py: Python<'_>, params: ServerCreateParams) -> PyResult ServerListener { ServerListener::Tcp(listener) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub fn unix_server_listener(listener: StdUnixListener) -> ServerListener { ServerListener::Unix(listener) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub fn remove_unix_socket_if_present(path: impl AsRef) -> io::Result<()> { match fs::remove_file(path) { diff --git a/src/transport/stream/server_core.rs b/src/transport/stream/server_core.rs index fc8c388..6a2cfe1 100644 --- a/src/transport/stream/server_core.rs +++ b/src/transport/stream/server_core.rs @@ -37,6 +37,7 @@ use super::{ use crate::context::{ensure_running_loop, run_in_context}; use crate::engine::{LoopCommand, LoopIoCommand}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn reserve_tls_slot(current: usize, limit: usize, closed: bool) -> Option { (!closed && current < limit).then_some(current + 1) } @@ -46,6 +47,7 @@ fn release_tls_slot(current: usize) -> Option { current.checked_sub(1) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn close_server_flags(closed: &mut bool, serving: &mut bool) -> bool { if *closed { return false; @@ -56,6 +58,10 @@ fn close_server_flags(closed: &mut bool, serving: &mut bool) -> bool { } impl ServerCore { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ServerCore") + )] pub(super) fn close_python_sockets(&self) { let _ = Python::try_attach(|py| -> PyResult<()> { for socket in &self.sockets { @@ -65,6 +71,10 @@ impl ServerCore { }); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ServerCore") + )] pub(crate) fn report_error(&self, err: PyErr, message: &str) { let _ = Python::try_attach(|py| -> PyResult<()> { let context = PyDict::new(py); @@ -78,6 +88,10 @@ impl ServerCore { }); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ServerCore") + )] pub(super) fn create_protocol_with_py(&self, py: Python<'_>) -> PyResult> { ensure_running_loop(py, &self.loop_obj)?; let callback = self.protocol_factory.bind(py).clone().unbind(); @@ -85,32 +99,56 @@ impl ServerCore { run_in_context(py, &self.context, self.context_needs_run, &callback, &args) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ServerCore") + )] #[inline] pub(super) fn locals(&self, py: Python<'_>) -> PyResult { task_locals_for_loop(py, &self.loop_obj) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ServerCore") + )] #[inline] pub(super) fn is_closed(&self) -> bool { self.state.lock().expect("poisoned server state").closed } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ServerCore") + )] pub(super) fn is_serving(&self) -> bool { let state = self.state.lock().expect("poisoned server state"); state.serving && !state.closed } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ServerCore") + )] #[inline] pub(super) fn connection_opened(&self) { self.active_connections.fetch_add(1, Ordering::SeqCst); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ServerCore") + )] #[inline] pub(super) fn connection_lost(&self) { self.active_connections.fetch_sub(1, Ordering::SeqCst); self.closed_notify.notify_all(); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ServerCore") + )] pub(super) fn reserve_tls_handshake(self: &Arc) -> Option { if self.is_closed() { return None; @@ -133,6 +171,10 @@ impl ServerCore { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ServerCore") + )] pub(super) fn close(&self) { { let mut state = self.state.lock().expect("poisoned server state"); @@ -197,6 +239,10 @@ impl ServerCore { self.closed_notify.notify_all(); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ServerCore") + )] pub fn spawn_accept_tasks(self: &Arc) { let listeners = { let mut state = self.state.lock().expect("poisoned server state"); diff --git a/src/transport/stream/server_types.rs b/src/transport/stream/server_types.rs index 2fe8392..a6427b1 100644 --- a/src/transport/stream/server_types.rs +++ b/src/transport/stream/server_types.rs @@ -33,6 +33,10 @@ pub struct TransportSpawnContext { } impl TransportSpawnContext { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TransportSpawnContext") + )] pub fn new( py: Python<'_>, loop_core: Arc, @@ -65,6 +69,10 @@ pub struct ServerCreateParams { } impl ServerCreateParams { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ServerCreateParams") + )] pub fn new( spawn_context: TransportSpawnContext, sockets: Vec>, @@ -91,12 +99,20 @@ impl ServerCreateParams { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ServerCreateParams") + )] #[cfg(unix)] pub fn with_cleanup_path(mut self, cleanup_path: Option) -> Self { self.cleanup_path = cleanup_path; self } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ServerCreateParams") + )] pub fn with_tls(mut self, tls: Option>) -> Self { self.tls = tls; self diff --git a/src/transport/stream/socket_transport.rs b/src/transport/stream/socket_transport.rs index 4008a0d..2eb42dd 100644 --- a/src/transport/stream/socket_transport.rs +++ b/src/transport/stream/socket_transport.rs @@ -40,6 +40,7 @@ use super::{PyStreamTransport, ServerCore, TransportSpawnContext, spawn_socket_r use crate::fd_ops; use crate::transport::tls::{ClientTlsSettings, ServerTlsSettings}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn transport_from_socket( py: Python<'_>, spawn_context: TransportSpawnContext, @@ -67,6 +68,7 @@ pub fn transport_from_socket( ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn transport_from_socket_tls( py: Python<'_>, spawn_context: TransportSpawnContext, @@ -99,6 +101,7 @@ pub fn transport_from_socket_tls( ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn transport_from_socket_server_tls( py: Python<'_>, spawn_context: TransportSpawnContext, @@ -130,11 +133,13 @@ pub fn transport_from_socket_server_tls( true, ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub fn tcp_stream_from_owned_socket_fd(fd: fd_ops::RawFd) -> PyResult { configured_tcp_stream_from_owned_fd(fd) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub fn unix_stream_from_owned_socket_fd(fd: fd_ops::RawFd) -> PyResult { let stream = from_owned_raw_fd::(fd)?; @@ -144,10 +149,12 @@ pub fn unix_stream_from_owned_socket_fd(fd: fd_ops::RawFd) -> PyResult PyResult { configured_tcp_listener_from_owned_fd(fd) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub fn unix_listener_from_owned_socket_fd(fd: fd_ops::RawFd) -> PyResult { let listener = from_owned_raw_fd::(fd)?; @@ -157,11 +164,13 @@ pub fn unix_listener_from_owned_socket_fd(fd: fd_ops::RawFd) -> PyResult PyResult { let dup = fd_ops::dup_raw_fd(fd).map_err(|err| PyRuntimeError::new_err(err.to_string()))?; configured_tcp_stream_from_owned_fd(dup) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn configured_tcp_stream_from_owned_fd(fd: fd_ops::RawFd) -> PyResult { let socket = socket_from_owned_raw(fd)?; socket @@ -173,6 +182,7 @@ pub(super) fn configured_tcp_stream_from_owned_fd(fd: fd_ops::RawFd) -> PyResult Ok(socket.into()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn configured_tcp_listener_from_owned_fd(fd: fd_ops::RawFd) -> PyResult { let socket = socket_from_owned_raw(fd)?; socket @@ -181,6 +191,7 @@ pub(super) fn configured_tcp_listener_from_owned_fd(fd: fd_ops::RawFd) -> PyResu Ok(socket.into()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub(super) fn duplicate_unix_direct_writer(raw_fd: fd_ops::RawFd) -> PyResult { let writer_fd = @@ -191,6 +202,7 @@ pub(super) fn duplicate_unix_direct_writer(raw_fd: fd_ops::RawFd) -> PyResult, mut spawn_context: TransportSpawnContext, @@ -245,6 +257,7 @@ pub fn spawn_tcp_transport( Ok(transport) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub fn spawn_unix_transport( py: Python<'_>, diff --git a/src/transport/stream/stats.rs b/src/transport/stream/stats.rs index 78d0d17..d35e1d5 100644 --- a/src/transport/stream/stats.rs +++ b/src/transport/stream/stats.rs @@ -17,6 +17,7 @@ pub(super) static TRANSPORT_STAGED_WRITES: AtomicU64 = AtomicU64::new(0); pub(super) static TRANSPORT_DIRECT_WRITE_ATTEMPTS: AtomicU64 = AtomicU64::new(0); pub(super) static TRANSPORT_POLL_REBINDS: AtomicU64 = AtomicU64::new(0); +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn transport_stats_enabled() -> bool { static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); *ENABLED.get_or_init(|| { @@ -26,6 +27,7 @@ pub(super) fn transport_stats_enabled() -> bool { }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[pyfunction] pub fn transport_stats(py: Python<'_>) -> PyResult> { let stats = PyDict::new(py); @@ -55,6 +57,7 @@ pub fn transport_stats(py: Python<'_>) -> PyResult> { Ok(stats.unbind()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[pyfunction] pub fn reset_transport_stats() { TRANSPORT_READ_EVENTS.store(0, Ordering::Relaxed); diff --git a/src/transport/stream/tls_session.rs b/src/transport/stream/tls_session.rs index fbcf4db..5914640 100644 --- a/src/transport/stream/tls_session.rs +++ b/src/transport/stream/tls_session.rs @@ -25,6 +25,10 @@ pub(super) enum TlsConnectionKind { } impl TlsConnectionKind { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TlsConnectionKind") + )] pub(super) fn is_handshaking(&self) -> bool { match self { Self::Client(conn) => conn.is_handshaking(), @@ -32,6 +36,10 @@ impl TlsConnectionKind { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TlsConnectionKind") + )] pub(super) fn wants_read(&self) -> bool { match self { Self::Client(conn) => conn.wants_read(), @@ -39,6 +47,10 @@ impl TlsConnectionKind { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TlsConnectionKind") + )] pub(super) fn wants_write(&self) -> bool { match self { Self::Client(conn) => conn.wants_write(), @@ -46,6 +58,10 @@ impl TlsConnectionKind { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TlsConnectionKind") + )] pub(super) fn read_tls(&mut self, stream: &mut StreamKind) -> io::Result { match self { Self::Client(conn) => conn.read_tls(stream), @@ -53,6 +69,10 @@ impl TlsConnectionKind { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TlsConnectionKind") + )] pub(super) fn write_tls(&mut self, stream: &mut StreamKind) -> io::Result { match self { Self::Client(conn) => conn.write_tls(stream), @@ -60,6 +80,10 @@ impl TlsConnectionKind { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TlsConnectionKind") + )] pub(super) fn process_new_packets(&mut self) -> Result<(), rustls::Error> { match self { Self::Client(conn) => conn.process_new_packets().map(|_| ()), @@ -67,6 +91,10 @@ impl TlsConnectionKind { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TlsConnectionKind") + )] pub(super) fn reader_read(&mut self, buf: &mut [u8]) -> io::Result { match self { Self::Client(conn) => conn.reader().read(buf), @@ -74,6 +102,10 @@ impl TlsConnectionKind { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TlsConnectionKind") + )] fn writer_write(&mut self, data: &[u8]) -> io::Result { match self { Self::Client(conn) => conn.writer().write(data), @@ -81,6 +113,10 @@ impl TlsConnectionKind { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TlsConnectionKind") + )] pub(super) fn send_close_notify(&mut self) { match self { Self::Client(conn) => conn.send_close_notify(), @@ -98,6 +134,10 @@ pub(super) struct TlsIoState { pub(super) type SharedTlsIoState = Arc>; impl TlsIoState { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TlsIoState") + )] pub(super) fn write_plaintext_all(&mut self, mut data: &[u8]) -> io::Result<()> { while !data.is_empty() { let written = self.connection.writer_write(data)?; @@ -117,25 +157,45 @@ impl TlsIoState { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TlsIoState") + )] #[inline] pub(super) fn fd(&self) -> fd_ops::RawFd { self.stream.fd() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TlsIoState") + )] pub(super) fn pollable(&self) -> bool { self.stream.pollable() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TlsIoState") + )] #[inline] pub(super) fn shutdown_close(&self) -> io::Result<()> { self.stream.shutdown_close() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TlsIoState") + )] #[inline] pub(super) fn read_tls(&mut self) -> io::Result { self.connection.read_tls(&mut self.stream) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TlsIoState") + )] #[inline] pub(super) fn write_tls(&mut self) -> io::Result { self.connection.write_tls(&mut self.stream) @@ -148,10 +208,12 @@ pub(super) enum TlsReadOutcome { ConnectionLost(String), } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn tls_server_closed(server: Option<&Weak>) -> bool { server.is_some_and(|server| server.upgrade().is_none_or(|server| server.is_closed())) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn complete_tls_handshake( tls_state: &SharedTlsIoState, timeout: Duration, @@ -178,6 +240,7 @@ pub(super) fn complete_tls_handshake( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn tls_handshake_step(tls_state: &SharedTlsIoState) -> io::Result { let mut state = tls_state.lock().expect("poisoned tls state"); if !state.connection.is_handshaking() { @@ -204,6 +267,7 @@ pub(super) fn tls_handshake_step(tls_state: &SharedTlsIoState) -> io::Result, state: &mut TlsIoState, @@ -252,6 +317,7 @@ pub(super) fn drain_tls_plaintext_locked( Ok(saw_data) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn drain_buffered_tls_plaintext( core: &Arc, tls_state: &SharedTlsIoState, @@ -265,6 +331,7 @@ pub(super) fn drain_buffered_tls_plaintext( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn read_tls_records( core: &Arc, tls_state: &SharedTlsIoState, @@ -304,11 +371,13 @@ pub(super) fn read_tls_records( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn tls_socket_wait_target(tls_state: &SharedTlsIoState) -> (fd_ops::RawFd, bool) { let state = tls_state.lock().expect("poisoned tls state"); (state.fd(), state.pollable()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn close_tls_writer(tls_state: &SharedTlsIoState) -> io::Result<()> { let mut state = tls_state.lock().expect("poisoned tls state"); let shutdown_timeout = state.shutdown_timeout; @@ -318,11 +387,13 @@ pub(super) fn close_tls_writer(tls_state: &SharedTlsIoState) -> io::Result<()> { result.and(close_result) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn abort_tls_writer(tls_state: &SharedTlsIoState) -> io::Result<()> { let state = tls_state.lock().expect("poisoned tls state"); state.shutdown_close() } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn flush_tls_io_locked(state: &mut TlsIoState) -> io::Result<()> { while state.connection.wants_write() { match state.write_tls() { @@ -343,6 +414,7 @@ pub(super) fn flush_tls_io_locked(state: &mut TlsIoState) -> io::Result<()> { Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn flush_tls_close_io_locked( state: &mut TlsIoState, timeout: Duration, @@ -369,6 +441,7 @@ pub(super) fn flush_tls_close_io_locked( Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn tls_io_error(err: rustls::Error) -> io::Error { io::Error::new(io::ErrorKind::InvalidData, err.to_string()) } diff --git a/src/transport/stream/tls_transport.rs b/src/transport/stream/tls_transport.rs index 07b32e5..7e3b2af 100644 --- a/src/transport/stream/tls_transport.rs +++ b/src/transport/stream/tls_transport.rs @@ -45,6 +45,7 @@ pub struct PreparedTlsTransport { stream: StreamKind, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Retires plaintext I/O before a TLS handshake is allowed to touch the socket. pub fn prepare_start_tls_transport( py: Python<'_>, @@ -59,6 +60,7 @@ pub fn prepare_start_tls_transport( }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn start_tls_transport( py: Python<'_>, prepared: PreparedTlsTransport, @@ -76,6 +78,7 @@ pub fn start_tls_transport( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn spawn_tls_client_transport( py: Python<'_>, spawn_context: TransportSpawnContext, @@ -101,6 +104,7 @@ pub(super) fn spawn_tls_client_transport( ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn spawn_tls_server_transport( py: Python<'_>, spawn_context: TransportSpawnContext, @@ -135,6 +139,7 @@ pub(super) struct TlsTransportConfig { pub(super) call_connection_made: bool, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn tls_stream_extra( py: Python<'_>, stream: &StreamKind, @@ -153,6 +158,7 @@ pub(super) fn tls_stream_extra( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn tls_io_state( stream: StreamKind, connection: TlsConnectionKind, @@ -165,6 +171,7 @@ pub(super) fn tls_io_state( })) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn spawn_tls_transport( py: Python<'_>, mut spawn_context: TransportSpawnContext, diff --git a/src/transport/stream/tuning.rs b/src/transport/stream/tuning.rs index 930273b..94863e0 100644 --- a/src/transport/stream/tuning.rs +++ b/src/transport/stream/tuning.rs @@ -38,20 +38,26 @@ pub(super) const SERVER_POLL_READER_TINY_TRIGGER_MAX_BYTES: usize = 16; // loop-thread backlog. Buffers are allocated lazily; once all slots exist, // readers wait for recycling rather than allocating beyond this bound. pub(super) const READ_BUFFER_POOL_LIMIT: usize = 4; +// Amortize tiny headers without reserving a full 16 KiB block per connection. +pub(super) const MIN_WRITE_BUFFER_CAPACITY: usize = 256; +// This unit defines the pool's slot budget, not its initial allocation size. pub(super) const WRITE_BUFFER_BLOCK_SIZE: usize = 16 * 1024; pub(super) const WRITE_BUFFER_POOL_LIMIT: usize = DEFAULT_WRITE_BUFFER_HIGH_WATER / WRITE_BUFFER_BLOCK_SIZE + 1; pub(super) const TLS_WORKER_STACK_SIZE: usize = 256 * 1024; const DEFAULT_MAX_PENDING_TLS_HANDSHAKES: usize = 256; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn positive_usize_or_default(value: Option, default: usize) -> usize { value.filter(|value| *value > 0).unwrap_or(default) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn reader_spin_micros(value: Option) -> u64 { value.unwrap_or(30).min(1_000) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn parse_positive_usize(value: Option<&str>, default: usize) -> usize { positive_usize_or_default( value.and_then(|value| value.trim().parse::().ok()), @@ -59,12 +65,14 @@ fn parse_positive_usize(value: Option<&str>, default: usize) -> usize { ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn parse_reader_spin_window(value: Option<&str>) -> Duration { Duration::from_micros(reader_spin_micros( value.and_then(|value| value.trim().parse::().ok()), )) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn max_pending_tls_handshakes() -> usize { static LIMIT: std::sync::OnceLock = std::sync::OnceLock::new(); *LIMIT.get_or_init(|| { @@ -73,6 +81,7 @@ pub(super) fn max_pending_tls_handshakes() -> usize { }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn max_write_buffer_size() -> usize { static LIMIT: std::sync::OnceLock = std::sync::OnceLock::new(); *LIMIT.get_or_init(|| { @@ -85,6 +94,7 @@ pub(super) fn max_write_buffer_size() -> usize { // reads for this long before falling back to poll(). Request/response peers // usually answer within microseconds, and skipping the poll() sleep/wake // halves per-roundtrip latency on actively chatting connections. +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn reader_spin_window() -> Duration { static WINDOW: std::sync::OnceLock = std::sync::OnceLock::new(); *WINDOW.get_or_init(|| { diff --git a/src/transport/stream/worker.rs b/src/transport/stream/worker.rs index 8a0fca2..c208082 100644 --- a/src/transport/stream/worker.rs +++ b/src/transport/stream/worker.rs @@ -22,6 +22,10 @@ pub(super) struct WorkerThread { } impl WorkerThread { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WorkerThread") + )] pub(super) fn spawn( name: &'static str, task: impl FnOnce(Arc) + Send + 'static, @@ -29,6 +33,10 @@ impl WorkerThread { Self::spawn_with_stack(name, None, task) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WorkerThread") + )] pub(super) fn spawn_with_stack( name: &'static str, stack_size: Option, @@ -58,6 +66,10 @@ impl WorkerThread { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WorkerThread") + )] pub(super) fn spawn_interruptible( name: &'static str, wake: impl FnOnce() + Send + 'static, @@ -68,10 +80,18 @@ impl WorkerThread { Ok(worker) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WorkerThread") + )] pub(super) fn abort(self) { self.abort_with_timeout(WORKER_JOIN_TIMEOUT); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WorkerThread") + )] fn abort_with_timeout(mut self, join_timeout: Duration) { self.stop.store(true, Ordering::Release); if let Some(wake) = self.wake.take() { diff --git a/src/transport/stream/write_queue.rs b/src/transport/stream/write_queue.rs index 0256316..c49658d 100644 --- a/src/transport/stream/write_queue.rs +++ b/src/transport/stream/write_queue.rs @@ -14,10 +14,18 @@ struct QueueState { } impl QueueState { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "QueueState") + )] fn new() -> Self { Self::with_capacity(INITIAL_WRITER_QUEUE_CAPACITY) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "QueueState") + )] fn with_capacity(capacity: usize) -> Self { Self { commands: VecDeque::with_capacity(capacity), @@ -26,6 +34,10 @@ impl QueueState { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "QueueState") + )] fn enqueue(&mut self, command: WriterCommand) -> Result<(), WriterCommand> { if !self.receiver_alive { return Err(command); @@ -40,6 +52,10 @@ impl QueueState { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "QueueState") + )] fn try_dequeue(&mut self) -> Result { if let Some(command) = self.commands.pop_front() { Ok(command) @@ -69,6 +85,7 @@ pub(super) enum TryRecvError { Disconnected, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn channel() -> (WriterSender, WriterReceiver) { let shared = Arc::new(SharedQueue { state: Mutex::new(QueueState::new()), @@ -83,6 +100,10 @@ pub(super) fn channel() -> (WriterSender, WriterReceiver) { } impl WriterSender { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WriterSender") + )] pub(super) fn send(&self, command: WriterCommand) -> Result<(), WriterCommand> { let mut state = self.shared.state.lock().expect("poisoned writer queue"); state.enqueue(command)?; @@ -93,6 +114,10 @@ impl WriterSender { } impl Drop for WriterSender { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { // Serialize disconnection with recv's predicate check and Condvar wait // to prevent a lost shutdown notification. No I/O or wait is performed @@ -108,6 +133,10 @@ impl Drop for WriterSender { } impl WriterReceiver { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WriterReceiver") + )] pub(super) fn recv(&self) -> Result { let mut state = self.shared.state.lock().expect("poisoned writer queue"); loop { @@ -125,6 +154,10 @@ impl WriterReceiver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WriterReceiver") + )] pub(super) fn try_recv(&self) -> Result { self.shared .state @@ -135,6 +168,10 @@ impl WriterReceiver { } impl Drop for WriterReceiver { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { // Serialize disconnection with enqueue so later sends are rejected. // This guard only updates queue metadata; it does not wait for the diff --git a/src/transport/stream/writer.rs b/src/transport/stream/writer.rs index 16e6af8..33ebd34 100644 --- a/src/transport/stream/writer.rs +++ b/src/transport/stream/writer.rs @@ -29,6 +29,7 @@ use super::write_queue::{TryRecvError, WriterReceiver}; use super::{StreamTransportCore, WriterCommand}; use crate::fd_ops; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn spawn_writer_worker( core: Arc, writer: WriterTarget, @@ -42,6 +43,7 @@ pub(super) fn spawn_writer_worker( Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn spawn_tls_writer_worker( core: Arc, tls_state: SharedTlsIoState, @@ -57,6 +59,7 @@ pub(super) fn spawn_tls_writer_worker( Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn run_stream_writer( core: Arc, mut writer: WriterTarget, @@ -92,6 +95,7 @@ pub(super) fn run_stream_writer( core.report_connection_lost_result(core.connection_lost(None)); } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn handle_stream_writer_command( core: &Arc, writer: &mut WriterTarget, @@ -116,6 +120,7 @@ pub(super) fn handle_stream_writer_command( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn write_stream_data_batch( core: &Arc, writer: &mut WriterTarget, @@ -155,6 +160,7 @@ pub(super) fn write_stream_data_batch( true } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn write_one_stream_buffer( core: &Arc, writer: &mut WriterTarget, @@ -169,6 +175,7 @@ pub(super) fn write_one_stream_buffer( true } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn handle_stream_write_eof( core: &Arc, writer: &mut WriterTarget, @@ -184,12 +191,14 @@ pub(super) fn handle_stream_write_eof( true } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn report_writer_io_error(core: &Arc, err: io::Error) { core.report_connection_lost_result( core.connection_lost(Some(PyRuntimeError::new_err(err.to_string()))), ); } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn report_writer_close_result(core: &Arc, result: io::Result<()>) { core.report_connection_lost_result( core.connection_lost( @@ -200,6 +209,7 @@ pub(super) fn report_writer_close_result(core: &Arc, result ); } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn run_tls_writer( core: Arc, tls_state: SharedTlsIoState, @@ -229,6 +239,7 @@ pub(super) fn run_tls_writer( core.report_connection_lost_result(core.connection_lost(None)); } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn handle_tls_writer_command( core: &Arc, tls_state: &SharedTlsIoState, @@ -253,6 +264,7 @@ pub(super) fn handle_tls_writer_command( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn write_tls_data_batch( core: &Arc, tls_state: &SharedTlsIoState, @@ -311,6 +323,7 @@ pub(super) fn write_tls_data_batch( true } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn report_tls_close_result(core: &Arc, result: io::Result<()>) { match result { Ok(()) => core.report_connection_lost_result(core.connection_lost(None)), @@ -321,6 +334,7 @@ pub(super) fn report_tls_close_result(core: &Arc, result: i } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn write_all_owned( writer: &mut WriterTarget, data: &mut OwnedWriteBuffer, @@ -362,6 +376,7 @@ pub(super) fn write_all_owned( Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub(super) fn is_transient_write_backpressure(err: &io::Error) -> bool { if err.kind() == io::ErrorKind::WouldBlock { diff --git a/src/transport/tls/material.rs b/src/transport/tls/material.rs index aae2883..14bbffa 100644 --- a/src/transport/tls/material.rs +++ b/src/transport/tls/material.rs @@ -12,6 +12,7 @@ use rustls::pki_types::{CertificateDer, PrivateKeyDer}; static NATIVE_ROOTS: OnceLock>, String>> = OnceLock::new(); +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn native_root_certificates() -> PyResult<&'static [CertificateDer<'static>]> { let result = NATIVE_ROOTS.get_or_init(|| { let native = rustls_native_certs::load_native_certs(); @@ -25,6 +26,7 @@ fn native_root_certificates() -> PyResult<&'static [CertificateDer<'static>]> { }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn root_store_from_context( py: Python<'_>, ssl_context: &Py, @@ -57,6 +59,7 @@ pub(super) fn root_store_from_context( Ok(roots) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn load_cert_chain_metadata( py: Python<'_>, ssl_context: &Py, @@ -79,6 +82,7 @@ pub(super) fn load_cert_chain_metadata( load_pem_identity(&certfile, &keyfile, password.as_deref()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn load_pem_identity( certfile: &str, keyfile: &str, @@ -105,6 +109,7 @@ fn load_pem_identity( Ok(Some((certs, key))) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn load_private_key(key_data: Vec) -> PyResult> { let mut pkcs8_reader = BufReader::new(Cursor::new(key_data.clone())); if let Some(key) = rustls_pemfile::pkcs8_private_keys(&mut pkcs8_reader) @@ -136,18 +141,22 @@ fn load_private_key(key_data: Vec) -> PyResult> { Err(PyRuntimeError::new_err("no supported private key found")) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn verify_mode_value(py: Python<'_>, ssl_context: &Py) -> PyResult { ssl_context.getattr(py, "verify_mode")?.extract::(py) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn ssl_verify_constant(py: Python<'_>, name: &str) -> PyResult { py.import("ssl")?.getattr(name)?.extract::() } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn io_err_to_py(err: io::Error) -> PyErr { PyRuntimeError::new_err(err.to_string()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn to_py_tls_err(err: impl std::fmt::Display) -> PyErr { PyRuntimeError::new_err(err.to_string()) } diff --git a/src/transport/tls/mod.rs b/src/transport/tls/mod.rs index c77694d..59177c1 100644 --- a/src/transport/tls/mod.rs +++ b/src/transport/tls/mod.rs @@ -46,6 +46,7 @@ pub struct ServerTlsSettings { pub ssl_context: Py, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn client_tls_settings( py: Python<'_>, ssl: &Bound<'_, PyAny>, @@ -69,6 +70,7 @@ pub fn client_tls_settings( }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn cached_client_config(py: Python<'_>, ssl_context: &Py) -> PyResult> { // The Python `SSLContext` owns the capsule, so repeated connections can // reuse an `Arc` until compatibility metadata changes. @@ -109,6 +111,7 @@ fn cached_client_config(py: Python<'_>, ssl_context: &Py) -> PyResult, ssl: &Bound<'_, PyAny>, @@ -128,6 +131,7 @@ pub fn server_tls_settings( }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn handshake_timeout(value: Option) -> PyResult { let secs = value.unwrap_or(DEFAULT_HANDSHAKE_TIMEOUT_SECS); if !secs.is_finite() || secs <= 0.0 { @@ -138,6 +142,7 @@ fn handshake_timeout(value: Option) -> PyResult { Ok(Duration::from_secs_f64(secs)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn shutdown_timeout(value: Option) -> PyResult { let secs = value.unwrap_or(DEFAULT_SHUTDOWN_TIMEOUT_SECS); if !secs.is_finite() || secs <= 0.0 { @@ -148,6 +153,7 @@ fn shutdown_timeout(value: Option) -> PyResult { Ok(Duration::from_secs_f64(secs)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub fn tls_extra( py: Python<'_>, ssl_context: &Py, @@ -161,6 +167,7 @@ pub fn tls_extra( extra } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn normalize_client_ssl_context(py: Python<'_>, ssl: &Bound<'_, PyAny>) -> PyResult> { if ssl.is_none() { return Err(PyTypeError::new_err("ssl must not be None")); @@ -177,6 +184,7 @@ fn normalize_client_ssl_context(py: Python<'_>, ssl: &Bound<'_, PyAny>) -> PyRes Ok(ssl.clone().unbind()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn normalize_server_ssl_context(py: Python<'_>, ssl: &Bound<'_, PyAny>) -> PyResult> { if ssl.is_none() { return Err(PyTypeError::new_err("ssl must not be None")); @@ -190,6 +198,7 @@ fn normalize_server_ssl_context(py: Python<'_>, ssl: &Bound<'_, PyAny>) -> PyRes Ok(ssl.clone().unbind()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn ensure_ssl_context(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult<()> { let ssl_mod = py.import("ssl")?; let cls = ssl_mod.getattr("SSLContext")?; @@ -201,6 +210,7 @@ fn ensure_ssl_context(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult<()> )) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn resolve_server_hostname( py: Python<'_>, ssl_context: &Py, @@ -227,6 +237,7 @@ fn resolve_server_hostname( Ok("localhost".to_owned()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn build_client_config(py: Python<'_>, ssl_context: &Py) -> PyResult { let roots = root_store_from_context(py, ssl_context)?; let maybe_identity = load_cert_chain_metadata(py, ssl_context)?; @@ -258,6 +269,10 @@ fn build_client_config(py: Python<'_>, ssl_context: &Py) -> PyResult, @@ -269,6 +284,10 @@ impl ServerCertVerifier for NoServerCertVerifier { Ok(ServerCertVerified::assertion()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn verify_tls12_signature( &self, _message: &[u8], @@ -278,6 +297,10 @@ impl ServerCertVerifier for NoServerCertVerifier { Ok(HandshakeSignatureValid::assertion()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn verify_tls13_signature( &self, _message: &[u8], @@ -287,6 +310,10 @@ impl ServerCertVerifier for NoServerCertVerifier { Ok(HandshakeSignatureValid::assertion()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn supported_verify_schemes(&self) -> Vec { vec![ SignatureScheme::RSA_PKCS1_SHA1, @@ -305,6 +332,7 @@ impl ServerCertVerifier for NoServerCertVerifier { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn build_server_config(py: Python<'_>, ssl_context: &Py) -> PyResult { let (certs, key) = load_cert_chain_metadata(py, ssl_context)?.ok_or_else(|| { PyRuntimeError::new_err( @@ -334,6 +362,7 @@ fn build_server_config(py: Python<'_>, ssl_context: &Py) -> PyResult &'static [&'static SupportedProtocolVersion] { rustls::DEFAULT_VERSIONS } diff --git a/src/vibeio/batch_allocator.rs b/src/vibeio/batch_allocator.rs index c2bb471..180f3e1 100644 --- a/src/vibeio/batch_allocator.rs +++ b/src/vibeio/batch_allocator.rs @@ -33,6 +33,10 @@ mod native { // deallocated blocks enter the cache; live blocks remain disjoint and are // never inspected. Cell operations cannot unwind or invoke user code. unsafe impl Allocator for BatchAllocator { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn allocate(&self, layout: Layout) -> Result, AllocError> { if let Some(block) = self.cached.get() @@ -47,6 +51,10 @@ mod native { System.allocate(layout) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { if layout.size() != 0 && layout.size() <= MAX_RETAINED && self.cached.get().is_none() { @@ -60,6 +68,10 @@ mod native { } impl Drop for BatchAllocator { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { if let Some(block) = self.cached.take() { // SAFETY: the cached block was returned by its previous owner; @@ -71,6 +83,7 @@ mod native { pub(crate) type Batch<'a, T> = Vec; + #[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub(crate) fn batch(allocator: &BatchAllocator, capacity: usize) -> Batch<'_, T> { Vec::with_capacity_in(capacity, allocator) @@ -293,6 +306,7 @@ mod draining { remaining: &'a mut [T], } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(crate) fn drain_batch<'a, T>(batch: &'a mut Batch<'_, T>) -> BatchDrain<'a, T> { let len = batch.len(); // SAFETY: the old len elements are initialized. Setting len to zero transfers @@ -309,6 +323,10 @@ mod draining { impl Iterator for BatchDrain<'_, T> { type Item = T; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn next(&mut self) -> Option { let (first, remaining) = std::mem::take(&mut self.remaining).split_first_mut()?; @@ -318,6 +336,10 @@ mod draining { Some(unsafe { std::ptr::read(first) }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn size_hint(&self) -> (usize, Option) { (self.remaining.len(), Some(self.remaining.len())) } @@ -326,6 +348,10 @@ mod draining { impl ExactSizeIterator for BatchDrain<'_, T> {} impl Drop for BatchDrain<'_, T> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { // SAFETY: only initialized, unyielded elements remain. Slice drop glue // also drops later elements if one destructor panics. @@ -341,10 +367,12 @@ mod ordinary { #[derive(Default)] pub(crate) struct BatchAllocator {} pub(crate) type Batch<'a, T> = Vec; + #[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub(crate) fn batch(_: &BatchAllocator, capacity: usize) -> Batch<'_, T> { Vec::with_capacity(capacity) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub(crate) fn drain_batch<'a, T>(batch: &'a mut Batch<'_, T>) -> std::vec::Drain<'a, T> { batch.drain(..) diff --git a/src/vibeio/blocking/default.rs b/src/vibeio/blocking/default.rs index 6869382..f85a00f 100644 --- a/src/vibeio/blocking/default.rs +++ b/src/vibeio/blocking/default.rs @@ -6,12 +6,20 @@ pub struct DefaultBlockingThreadPool { } impl DefaultBlockingThreadPool { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DefaultBlockingThreadPool") + )] /// Creates a new `DefaultBlockingThreadPool` with the default maximum number of threads. #[inline] pub fn new() -> Self { Self::with_max_threads(512) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DefaultBlockingThreadPool") + )] /// Creates a new `DefaultBlockingThreadPool` with the specified maximum number of threads. /// /// # Panics @@ -36,6 +44,10 @@ impl DefaultBlockingThreadPool { } impl BlockingThreadPool for DefaultBlockingThreadPool { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn spawn(&self, task: Box) { self.inner.execute(move || { @@ -45,6 +57,10 @@ impl BlockingThreadPool for DefaultBlockingThreadPool { } impl Default for DefaultBlockingThreadPool { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn default() -> Self { Self::new() } diff --git a/src/vibeio/blocking/mod.rs b/src/vibeio/blocking/mod.rs index 0e39651..69ff5e4 100644 --- a/src/vibeio/blocking/mod.rs +++ b/src/vibeio/blocking/mod.rs @@ -16,6 +16,10 @@ pub use default::*; pub struct SpawnBlockingError; impl fmt::Display for SpawnBlockingError { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "failed to spawn blocking task") } @@ -23,6 +27,7 @@ impl fmt::Display for SpawnBlockingError { impl std::error::Error for SpawnBlockingError {} +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Offload a borrowed operation while retaining ownership through worker unwind. /// Cancellation drops the caller's share; a queued/running worker retains its /// share until it stops using the buffer. Partial mutations are not rolled back. @@ -124,6 +129,7 @@ pub trait BlockingThreadPool: 'static { fn spawn(&self, task: Box); } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Spawns a blocking task onto a blocking thread pool. /// /// This function is a convenience wrapper around a blocking thread pool. diff --git a/src/vibeio/builder.rs b/src/vibeio/builder.rs index 3b30af8..347bd70 100644 --- a/src/vibeio/builder.rs +++ b/src/vibeio/builder.rs @@ -5,6 +5,7 @@ use crate::vibeio::blocking::DefaultBlockingThreadPool; use crate::vibeio::{blocking::BlockingThreadPool, driver::AnyDriver}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(target_os = "linux")] fn ensure_rsloop_platform() -> Result<(), std::io::Error> { // Linux vendors frequently backport io_uring features, while containers @@ -13,6 +14,7 @@ fn ensure_rsloop_platform() -> Result<(), std::io::Error> { Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(target_os = "macos")] fn ensure_rsloop_platform() -> Result<(), std::io::Error> { let name = c"kern.osproductversion"; @@ -35,6 +37,7 @@ fn ensure_rsloop_platform() -> Result<(), std::io::Error> { ensure_supported_macos_release(&buffer, length) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(any(target_os = "macos", test))] fn ensure_supported_macos_release(buffer: &[u8], length: usize) -> std::io::Result<()> { let invalid = || { @@ -62,6 +65,7 @@ fn ensure_supported_macos_release(buffer: &[u8], length: usize) -> std::io::Resu Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn ensure_rsloop_platform() -> Result<(), std::io::Error> { use windows_sys::Wdk::System::SystemServices::RtlGetVersion; @@ -82,6 +86,7 @@ fn ensure_rsloop_platform() -> Result<(), std::io::Error> { ensure_supported_windows_version(info.dwMajorVersion, info.dwMinorVersion, info.dwBuildNumber) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(any(windows, test))] fn ensure_supported_windows_version( major: u32, @@ -97,6 +102,7 @@ fn ensure_supported_windows_version( Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] fn ensure_rsloop_platform() -> Result<(), std::io::Error> { Err(std::io::Error::new( @@ -127,6 +133,10 @@ pub enum DriverKind { } impl DriverKind { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DriverKind") + )] /// Creates a new runtime I/O driver from this kind. #[inline] pub(crate) fn into_driver(self) -> Result { @@ -162,6 +172,10 @@ pub struct RuntimeBuilder { } impl RuntimeBuilder { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeBuilder") + )] /// Creates a new runtime builder with default configuration. /// /// By default, the builder will select the best available driver for the platform. @@ -175,6 +189,10 @@ impl RuntimeBuilder { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeBuilder") + )] /// Selects the scheduler profile used by rsloop. /// /// The profile keeps bounded task batches and polls timers during long @@ -187,12 +205,20 @@ impl RuntimeBuilder { self } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeBuilder") + )] /// Sets the I/O driver for the runtime. pub fn driver(mut self, driver_kind: DriverKind) -> Self { self.driver_kind = Some(driver_kind); self } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeBuilder") + )] /// Enables or disables the timer for the runtime. /// /// By default, the timer is disabled. @@ -201,6 +227,10 @@ impl RuntimeBuilder { self } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeBuilder") + )] /// Selects the runtime's configured blocking pool for filesystem operations. /// /// When disabled (the default), filesystem path operations use the shared @@ -211,12 +241,20 @@ impl RuntimeBuilder { self } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeBuilder") + )] /// Sets the blocking thread pool for the runtime. pub fn blocking_pool(mut self, blocking_pool: Box) -> Self { self.blocking_pool = Some(blocking_pool); self } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeBuilder") + )] /// Sets the default blocking thread pool for the runtime with specified maximum number of threads. #[cfg(feature = "blocking-default")] pub fn default_blocking_pool(mut self, max_threads: usize) -> Self { @@ -226,6 +264,10 @@ impl RuntimeBuilder { self } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeBuilder") + )] /// Builds the async runtime with the configured settings. /// /// If no driver was explicitly set, selects the best available driver for the platform. @@ -249,6 +291,10 @@ impl RuntimeBuilder { } impl Default for RuntimeBuilder { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn default() -> Self { Self::new() } diff --git a/src/vibeio/driver/iocp.rs b/src/vibeio/driver/iocp.rs index 266afa4..c94753c 100644 --- a/src/vibeio/driver/iocp.rs +++ b/src/vibeio/driver/iocp.rs @@ -419,6 +419,10 @@ struct AfdPollInfo { } impl AfdPollInfo { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "AfdPollInfo") + )] #[inline] fn new(socket: SOCKET, events: u32) -> Self { Self { @@ -439,6 +443,10 @@ pub struct IocpInterruptor { } impl Interruptor for IocpInterruptor { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn interrupt(&self) { if let Some(port) = self.port.upgrade() { @@ -505,6 +513,10 @@ struct DriverState { } impl DriverState { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DriverState") + )] fn retain_cancelled( &mut self, token: usize, @@ -542,6 +554,10 @@ pub struct IocpDriver { } impl Drop for IocpDriver { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { if self.quiesce(Duration::from_secs(1)).is_err() { // CancelIoEx/NtCancelIoFileEx are requests, not acknowledgements. @@ -558,6 +574,10 @@ impl Drop for IocpDriver { } impl IocpDriver { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] fn quiesce(&mut self, timeout: Duration) -> io::Result<()> { let mut wakers = Vec::new(); let state = self.state.get_mut(); @@ -608,6 +628,10 @@ impl IocpDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] pub(crate) fn new() -> Result { // SAFETY: INVALID_HANDLE_VALUE plus a null existing port requests a new @@ -633,6 +657,10 @@ impl IocpDriver { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn update_waiter(waiter_slot: &mut Option, waker: Waker) -> Option { if !waiter_slot @@ -645,16 +673,28 @@ impl IocpDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn iocp_handle(&self) -> HANDLE { self.port.as_raw_handle() as HANDLE } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn status_is_success(status: NTSTATUS) -> bool { status >= 0 } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn ntstatus_to_io_error(status: NTSTATUS) -> io::Error { // SAFETY: this conversion takes only an integer status, with no pointer @@ -667,6 +707,10 @@ impl IocpDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn raw_os_handle_to_windows_handle(handle: RawOsHandle) -> HANDLE { match handle { @@ -675,6 +719,10 @@ impl IocpDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn raw_os_handle_to_socket(handle: RawOsHandle) -> Result { match handle { @@ -686,11 +734,19 @@ impl IocpDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn duration_to_timeout_ms(timeout: Option) -> u32 { super::iocp_timeout_ms(timeout) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn completion_result_from_entry(entry: &OVERLAPPED_ENTRY) -> i32 { if entry.Internal == 0 { @@ -709,6 +765,10 @@ impl IocpDriver { super::encode_completion_error(win32_error).unwrap_or(-(ERROR_ARITHMETIC_OVERFLOW as i32)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn interest_to_afd_events(interest: Interest) -> u32 { let mut events = AFD_POLL_DISCONNECT | AFD_POLL_ABORT | AFD_POLL_LOCAL_CLOSE; @@ -727,6 +787,10 @@ impl IocpDriver { events } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn get_base_socket(socket: SOCKET, ioctl: u32) -> Result { let mut base_socket: SOCKET = INVALID_SOCKET; @@ -763,6 +827,10 @@ impl IocpDriver { Ok(base_socket) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn resolve_base_socket(socket: SOCKET) -> Result { super::resolve_base_socket_with( @@ -772,6 +840,10 @@ impl IocpDriver { ) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn open_afd_handle() -> Result { let device_name = format!("\\Device\\Afd\\vibeio-{}", std::process::id()); @@ -829,6 +901,10 @@ impl IocpDriver { Ok(unsafe { OwnedHandle::from_raw_handle(afd_handle as RawHandle) }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn ensure_afd_handle(&self) -> Result { { @@ -871,6 +947,10 @@ impl IocpDriver { .as_raw_handle() as HANDLE) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn arm_poll_operation( &self, @@ -978,6 +1058,10 @@ impl IocpDriver { Err(Self::ntstatus_to_io_error(status)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn cancel_poll_operation(&self, poll_token: usize) { let afd_handle = { @@ -1004,6 +1088,10 @@ impl IocpDriver { let _ = unsafe { NtCancelIoFileEx(afd_handle, io_status_ptr, &mut cancel_status) }; } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn disassociate_iocp_handle(&self, handle: &InnerRawHandle) -> io::Result<()> { let windows_handle = Self::raw_os_handle_to_windows_handle(handle.handle); @@ -1033,6 +1121,10 @@ impl IocpDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn process_entries(&self, entries: &[OVERLAPPED_ENTRY]) { let mut wakers = Vec::new(); @@ -1101,6 +1193,10 @@ impl IocpDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn process_batch(&self, timeout_ms: u32) -> Result { // GetQueuedCompletionStatusEx initializes exactly the entries it @@ -1158,6 +1254,10 @@ impl IocpDriver { Ok(entries_removed) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IocpDriver") + )] #[inline] fn process_ready_completions(&self) -> Result<(), io::Error> { for _ in 0..IOCP_DRAIN_BATCHES { @@ -1172,6 +1272,10 @@ impl IocpDriver { impl Driver for IocpDriver { type Interruptor = IocpInterruptor; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn flush(&self) { match self.process_ready_completions() { @@ -1181,6 +1285,10 @@ impl Driver for IocpDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn wait(&self, timeout: Option) { let timeout_ms = Self::duration_to_timeout_ms(timeout); @@ -1196,6 +1304,10 @@ impl Driver for IocpDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn register_handle( &self, @@ -1205,6 +1317,10 @@ impl Driver for IocpDriver { self.register_handle_with_mode(handle, interest, RegistrationMode::Completion) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn register_handle_with_mode( &self, @@ -1264,6 +1380,10 @@ impl Driver for IocpDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn reregister_handle( &self, @@ -1287,6 +1407,10 @@ impl Driver for IocpDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn deregister_handle(&self, handle: &InnerRawHandle) -> Result<(), io::Error> { let is_completion = matches!( @@ -1324,11 +1448,19 @@ impl Driver for IocpDriver { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn supports_completion(&self) -> bool { true } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn submit_completion(&self, op: &mut O, waker: Waker) -> CompletionIoResult where @@ -1369,6 +1501,10 @@ impl Driver for IocpDriver { CompletionIoResult::Retry(completion_token) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn submit_poll( &self, @@ -1420,6 +1556,10 @@ impl Driver for IocpDriver { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn get_completion_result(&self, token: usize) -> Option { let mut state = self.state.borrow_mut(); @@ -1433,6 +1573,10 @@ impl Driver for IocpDriver { completed } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn set_completion_waker(&self, token: usize, waker: Waker) { let mut state = self.state.borrow_mut(); @@ -1445,6 +1589,10 @@ impl Driver for IocpDriver { drop(old_waker); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn get_interruptor(&self) -> Self::Interruptor { IocpInterruptor { @@ -1452,12 +1600,20 @@ impl Driver for IocpDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn ignore_completion(&self, token: usize, data: Box) { let (retired, _) = self.state.borrow_mut().retain_cancelled(token, data); drop(retired); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn cancel_completion(&self, token: usize, handle: RawOsHandle, data: Box) { let (retired, overlapped) = self.state.borrow_mut().retain_cancelled(token, data); diff --git a/src/vibeio/driver/kqueue.rs b/src/vibeio/driver/kqueue.rs index 3bdd502..2b8438a 100644 --- a/src/vibeio/driver/kqueue.rs +++ b/src/vibeio/driver/kqueue.rs @@ -24,6 +24,10 @@ pub struct KqueueInterruptor { } impl Interruptor for KqueueInterruptor { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn interrupt(&self) { if let Some(waker) = self.waker.upgrade() { @@ -38,6 +42,10 @@ struct DriverWaker { } impl DriverWaker { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DriverWaker") + )] fn new() -> io::Result { let (sender, receiver) = UnixDatagram::pair()?; sender.set_nonblocking(true)?; @@ -45,11 +53,19 @@ impl DriverWaker { Ok(Self { sender, receiver }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DriverWaker") + )] #[inline] fn wake(&self) -> io::Result<()> { super::send_wake_notification(|| self.sender.send(&[1])) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DriverWaker") + )] fn acknowledge(&self) { let mut buffer = [0_u8; 256]; loop { @@ -79,6 +95,10 @@ struct DriverState { } impl Registration { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "Registration") + )] fn record_readiness(&mut self, filter: i16) -> Option { let (waiter, ready) = if filter == libc::EVFILT_READ && self.registered_read { (&mut self.read_waiter, &mut self.read_ready) @@ -106,6 +126,10 @@ pub struct KqueueDriver { } impl KqueueDriver { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "KqueueDriver") + )] pub(crate) fn new() -> io::Result { // SAFETY: kqueue takes no pointers and returns a newly owned descriptor. let kqueue = unsafe { libc::kqueue() }; @@ -134,6 +158,10 @@ impl KqueueDriver { Ok(driver) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "KqueueDriver") + )] #[inline] fn change(fd: RawFd, filter: i16, flags: u16, key: usize) -> libc::kevent { libc::kevent { @@ -146,20 +174,36 @@ impl KqueueDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "KqueueDriver") + )] #[inline] fn encode_key(token: Token, generation: u32) -> usize { ((generation as usize) << 32) | (token.0 & u32::MAX as usize) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "KqueueDriver") + )] #[inline] fn decode_key(key: usize) -> (Token, u32) { (Token(key & u32::MAX as usize), (key >> 32) as u32) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "KqueueDriver") + )] fn apply_change(&self, change: libc::kevent) -> io::Result<()> { self.apply_changes(std::slice::from_ref(&change)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "KqueueDriver") + )] fn apply_changes(&self, changes: &[libc::kevent]) -> io::Result<()> { // SAFETY: changes is live input storage for this synchronous call. // Internal callers supply at most two entries; there is no output array. @@ -180,10 +224,18 @@ impl KqueueDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "KqueueDriver") + )] fn delete_filter(&self, fd: RawFd, filter: i16) -> io::Result<()> { Self::delete_filter_with(|| self.apply_change(Self::change(fd, filter, libc::EV_DELETE, 0))) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "KqueueDriver") + )] fn delete_filter_with(mut delete: impl FnMut() -> io::Result<()>) -> io::Result<()> { loop { match delete() { @@ -203,6 +255,10 @@ impl KqueueDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "KqueueDriver") + )] fn install_registration_with( &self, changes: &[libc::kevent], @@ -231,6 +287,10 @@ impl KqueueDriver { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "KqueueDriver") + )] fn wait_events(&self, timeout: Option) -> io::Result<()> { let timeout = timeout.map(|duration| duration.min(MAX_WAIT)); let timespec = timeout.map(|duration| libc::timespec { @@ -297,6 +357,10 @@ impl KqueueDriver { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "KqueueDriver") + )] #[inline] fn filter(interest: Interest) -> i16 { if interest.is_readable() { @@ -306,6 +370,10 @@ impl KqueueDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "KqueueDriver") + )] fn deregister_with( &self, handle: &InnerRawHandle, @@ -364,6 +432,10 @@ impl KqueueDriver { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "KqueueDriver") + )] fn reregister_with( &self, handle: &InnerRawHandle, @@ -435,11 +507,19 @@ impl KqueueDriver { impl Driver for KqueueDriver { type Interruptor = KqueueInterruptor; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn should_flush(&self) -> bool { false } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn wait(&self, timeout: Option) { if let Err(err) = self.wait_events(timeout) { @@ -447,6 +527,10 @@ impl Driver for KqueueDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn register_handle(&self, handle: &InnerRawHandle, interest: Interest) -> io::Result { let (token, generation) = { let mut state = self.state.borrow_mut(); @@ -496,6 +580,10 @@ impl Driver for KqueueDriver { Ok(token) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn reregister_handle(&self, handle: &InnerRawHandle, interest: Interest) -> io::Result<()> { self.reregister_with(handle, interest, |fd, filter, add, key| { if add { @@ -511,10 +599,18 @@ impl Driver for KqueueDriver { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn deregister_handle(&self, handle: &InnerRawHandle) -> io::Result<()> { self.deregister_with(handle, |fd, filter| self.delete_filter(fd, filter)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn submit_poll( &self, handle: &InnerRawHandle, @@ -560,6 +656,10 @@ impl Driver for KqueueDriver { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn get_interruptor(&self) -> Self::Interruptor { KqueueInterruptor { diff --git a/src/vibeio/driver/mio.rs b/src/vibeio/driver/mio.rs index 327ba55..f63f051 100644 --- a/src/vibeio/driver/mio.rs +++ b/src/vibeio/driver/mio.rs @@ -26,6 +26,7 @@ use crate::vibeio::{driver::Driver, fd_inner::InnerRawHandle}; const MAX_POLL_TIMEOUT: Duration = Duration::from_secs(24 * 60 * 60); const WAKE_TOKEN: Token = Token(usize::MAX); +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn bounded_poll_timeout(timeout: Option) -> Option { timeout.map(|timeout| timeout.min(MAX_POLL_TIMEOUT)) @@ -36,6 +37,10 @@ pub struct MioInterruptor { } impl Interruptor for MioInterruptor { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn interrupt(&self) { if let Some(waker) = self.waker.upgrade() { @@ -49,15 +54,27 @@ struct DriverWaker(MioWaker); #[cfg(not(target_vendor = "apple"))] impl DriverWaker { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DriverWaker") + )] fn new(registry: &Registry) -> io::Result { MioWaker::new(registry, WAKE_TOKEN).map(Self) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DriverWaker") + )] #[inline] fn wake(&self) -> io::Result<()> { self.0.wake() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DriverWaker") + )] #[inline] fn acknowledge(&self) {} } @@ -73,6 +90,10 @@ struct DriverWaker { #[cfg(target_vendor = "apple")] impl DriverWaker { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DriverWaker") + )] fn new(registry: &Registry) -> io::Result { let (sender, receiver) = UnixDatagram::pair()?; sender.set_nonblocking(true)?; @@ -86,11 +107,19 @@ impl DriverWaker { Ok(Self { sender, receiver }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DriverWaker") + )] #[inline] fn wake(&self) -> io::Result<()> { super::send_wake_notification(|| self.sender.send(&[1])) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DriverWaker") + )] fn acknowledge(&self) { let mut buffer = [0_u8; 256]; loop { @@ -124,6 +153,7 @@ pub struct MioDriver { } impl MioDriver { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "MioDriver"))] #[inline] pub(crate) fn new() -> Result { let poll = Poll::new()?; @@ -142,6 +172,7 @@ impl MioDriver { }) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "MioDriver"))] #[inline] fn update_waiter(waiter_slot: &mut Option, waker: Waker) -> Option { if !waiter_slot @@ -154,6 +185,7 @@ impl MioDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "MioDriver"))] #[inline] pub(crate) fn wait_timeout(&self, timeout: Option) { let mut poll = self.poll.borrow_mut(); @@ -196,11 +228,19 @@ impl MioDriver { impl Driver for MioDriver { type Interruptor = MioInterruptor; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn flush(&self) { self.wait_timeout(Some(Duration::ZERO)); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn should_flush(&self) -> bool { // Registration and re-registration are applied synchronously. Polling @@ -209,11 +249,19 @@ impl Driver for MioDriver { false } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn wait(&self, timeout: Option) { self.wait_timeout(timeout); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn get_interruptor(&self) -> Self::Interruptor { MioInterruptor { @@ -221,6 +269,10 @@ impl Driver for MioDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn register_handle( &self, @@ -249,6 +301,10 @@ impl Driver for MioDriver { Ok(token) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn reregister_handle( &self, @@ -273,6 +329,10 @@ impl Driver for MioDriver { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn deregister_handle(&self, handle: &InnerRawHandle) -> Result<(), io::Error> { let fd = { @@ -301,6 +361,10 @@ impl Driver for MioDriver { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn submit_poll( &self, diff --git a/src/vibeio/driver/mod.rs b/src/vibeio/driver/mod.rs index 8295627..2528806 100644 --- a/src/vibeio/driver/mod.rs +++ b/src/vibeio/driver/mod.rs @@ -47,6 +47,7 @@ pub enum CompletionIoResult { #[cfg(any(target_os = "linux", windows, test))] struct RetainedCompletionData(Vec>); +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Preserve stable payload allocations without building a recursive drop chain. #[cfg(any(target_os = "linux", windows, test))] fn retain_completion_data( @@ -105,6 +106,7 @@ mod retained_completion_tests { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Follow provider handles without looping forever on a cyclic fallback chain. #[cfg(any(windows, test))] fn resolve_base_socket_with( @@ -208,6 +210,7 @@ mod base_socket_tests { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Encode only errors representable by the driver's negative i32 result. #[cfg(any(windows, test))] fn encode_completion_error(code: u32) -> Option { @@ -234,6 +237,7 @@ mod completion_encoding_tests { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Reserve Windows INFINITE for an explicitly unbounded wait. #[cfg(any(windows, test))] fn iocp_timeout_ms(timeout: Option) -> u32 { @@ -262,6 +266,7 @@ mod iocp_timeout_tests { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Decode the driver's negative error representation without signed overflow. pub(crate) fn completion_error(result: i32) -> io::Error { match result.checked_neg().filter(|code| *code > 0) { @@ -273,6 +278,7 @@ pub(crate) fn completion_error(result: i32) -> io::Error { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(any(target_vendor = "apple", target_os = "linux", test))] #[inline] fn send_wake_notification(mut send: impl FnMut() -> io::Result) -> io::Result<()> { @@ -288,6 +294,7 @@ fn send_wake_notification(mut send: impl FnMut() -> io::Result) -> io::Re } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn unsupported_completion_error() -> io::Error { io::Error::new( @@ -296,6 +303,7 @@ fn unsupported_completion_error() -> io::Error { ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn unsupported_poll_error() -> io::Error { io::Error::new( @@ -322,6 +330,10 @@ pub enum AnyInterruptor { } impl AnyInterruptor { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "AnyInterruptor") + )] pub(crate) fn interrupt(&self) { match self { AnyInterruptor::Mock(interruptor) => interruptor.interrupt(), @@ -340,10 +352,12 @@ impl AnyInterruptor { pub trait Driver { type Interruptor: Interruptor; + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Driver"))] /// Flushes the driver's I/O. #[inline] fn flush(&self) {} + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Driver"))] /// Returns whether the executor should call `flush` after polling a task batch. #[inline] fn should_flush(&self) -> bool { @@ -360,6 +374,7 @@ pub trait Driver { interest: Interest, ) -> Result; + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Driver"))] /// Registers an I/O source with the requested mode. fn register_handle_with_mode( &self, @@ -380,12 +395,14 @@ pub trait Driver { /// Removes an I/O source from the poller. fn deregister_handle(&self, handle: &InnerRawHandle) -> Result<(), std::io::Error>; + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Driver"))] /// Returns whether the driver supports completion-based I/O operations. #[inline] fn supports_completion(&self) -> bool { false } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Driver"))] /// Submits a completion-based I/O operation. #[inline] fn submit_completion(&self, _op: &mut O, _waker: Waker) -> CompletionIoResult @@ -395,6 +412,7 @@ pub trait Driver { CompletionIoResult::SubmitErr(unsupported_completion_error()) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Driver"))] /// Re-registers interest and submits a waker for poll-based I/O. #[inline] fn submit_poll( @@ -406,12 +424,14 @@ pub trait Driver { Err(unsupported_poll_error()) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Driver"))] /// Obtains the result for a completion-based I/O operation. #[inline] fn get_completion_result(&self, _token: usize) -> Option { None } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Driver"))] /// Polls a Linux multishot accept stream. #[cfg(target_os = "linux")] #[inline] @@ -426,14 +446,17 @@ pub trait Driver { ))) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Driver"))] /// Sets the waker for a completion-based I/O operation. #[inline] fn set_completion_waker(&self, _token: usize, _waker: Waker) {} + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Driver"))] /// Cancels a completion-based I/O operation. #[inline] fn ignore_completion(&self, _token: usize, _data: Box) {} + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Driver"))] /// Cancels a Windows completion operation while retaining its owned data /// until the completion packet is observed. #[cfg(windows)] @@ -468,35 +491,41 @@ pub enum AnyDriver { } impl AnyDriver { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[cfg(unix)] #[inline] pub(crate) fn new_mio() -> Result { Ok(AnyDriver::Mio(MioDriver::new()?)) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[cfg(target_vendor = "apple")] #[inline] pub(crate) fn new_kqueue() -> Result { Ok(AnyDriver::Kqueue(KqueueDriver::new()?)) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[inline] pub(crate) fn new_mock() -> Self { AnyDriver::Mock(MockDriver::new()) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[cfg(windows)] #[inline] pub(crate) fn new_iocp() -> Result { Ok(AnyDriver::Iocp(IocpDriver::new()?)) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[cfg(target_os = "linux")] #[inline] pub(crate) fn new_uring_custom(builder: io_uring::Builder) -> Result { Ok(AnyDriver::IoUring(UringDriver::new(1024, builder)?)) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[cfg(target_os = "linux")] #[inline] pub(crate) fn new_uring() -> Result { @@ -514,6 +543,7 @@ impl AnyDriver { .or_else(|_| Self::new_uring_custom(io_uring::IoUring::builder())) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[inline] pub(crate) fn new_best() -> Result { #[cfg(target_os = "linux")] @@ -538,6 +568,7 @@ impl AnyDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[inline] pub(crate) fn flush(&self) { match self { @@ -553,6 +584,7 @@ impl AnyDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[inline] pub(crate) fn should_flush(&self) -> bool { match self { @@ -568,6 +600,7 @@ impl AnyDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[inline] pub(crate) fn wait(&self, timeout: Option) { match self { @@ -583,6 +616,7 @@ impl AnyDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[allow(dead_code)] #[inline] pub(crate) fn register_handle( @@ -603,6 +637,7 @@ impl AnyDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[inline] pub(crate) fn register_handle_with_mode( &self, @@ -623,6 +658,7 @@ impl AnyDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[inline] pub(crate) fn reregister_handle( &self, @@ -642,6 +678,7 @@ impl AnyDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[inline] pub(crate) fn deregister_handle(&self, handle: &InnerRawHandle) -> Result<(), std::io::Error> { match self { @@ -657,6 +694,7 @@ impl AnyDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[inline] pub(crate) fn supports_completion(&self) -> bool { match self { @@ -672,6 +710,7 @@ impl AnyDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[inline] pub(crate) fn submit_completion(&self, op: &mut O, waker: Waker) -> CompletionIoResult where @@ -690,6 +729,7 @@ impl AnyDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[inline] pub(crate) fn submit_poll( &self, @@ -710,6 +750,7 @@ impl AnyDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[inline] pub(crate) fn get_completion_result(&self, token: usize) -> Option { match self { @@ -725,6 +766,7 @@ impl AnyDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[cfg(target_os = "linux")] #[inline] pub(crate) fn poll_multishot_accept( @@ -739,6 +781,7 @@ impl AnyDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[inline] pub(crate) fn set_completion_waker(&self, token: usize, waker: Waker) { match self { @@ -754,6 +797,7 @@ impl AnyDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[cfg(not(windows))] #[inline] pub(crate) fn ignore_completion(&self, token: usize, data: Box) { @@ -768,6 +812,7 @@ impl AnyDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[cfg(windows)] #[inline] pub(crate) fn cancel_completion( @@ -782,6 +827,7 @@ impl AnyDriver { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AnyDriver"))] #[inline] pub(crate) fn get_interruptor(&self) -> AnyInterruptor { match self { diff --git a/src/vibeio/driver/uring.rs b/src/vibeio/driver/uring.rs index 2347255..81d1cb1 100644 --- a/src/vibeio/driver/uring.rs +++ b/src/vibeio/driver/uring.rs @@ -27,6 +27,7 @@ const COMPLETION_KEY_KIND: u8 = 1; const ACCEPT_KEY_KIND: u8 = 2; const MEMORY_FALLBACK_ENTRIES: [u32; 2] = [256, 64]; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn build_with_memory_fallback( entries: u32, mut build: impl FnMut(u32) -> io::Result, @@ -61,6 +62,10 @@ pub struct UringInterruptor { } impl Interruptor for UringInterruptor { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn interrupt(&self) { if let Some(eventfd) = self.eventfd.upgrade() { @@ -119,6 +124,10 @@ struct Completion { } impl Drop for Completion { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { if self.returns_fd && let Some(fd) = self.completed.take().filter(|fd| *fd >= 0) @@ -145,6 +154,10 @@ struct CompletionBatch { } impl CompletionBatch { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "CompletionBatch") + )] fn dispatch(self) { // Arbitrary payload destructors and wakers must run outside ring/state // borrows; either can reenter the driver or submit another operation. @@ -159,6 +172,10 @@ impl CompletionBatch { } impl DriverState { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DriverState") + )] fn ignore_completion( &mut self, token: usize, @@ -194,6 +211,10 @@ pub struct UringDriver { } impl Drop for UringDriver { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { if self.quiesce().is_err() { // Ring close can defer cancellation. If the kernel cannot confirm @@ -209,6 +230,10 @@ impl Drop for UringDriver { } impl UringDriver { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] /// Stop all submitted work before retained operation storage is released. fn quiesce(&mut self) -> io::Result<()> { let ring = self.ring.get_mut(); @@ -251,6 +276,10 @@ impl UringDriver { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] fn drain_shutdown_cq(ring: &mut IoUring, state: &mut DriverState) { for cqe in ring.completion() { let key = cqe.user_data(); @@ -272,6 +301,10 @@ impl UringDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] #[inline] pub(crate) fn new(entries: u32, builder: io_uring::Builder) -> Result { // Ring teardown is deferred by the kernel. Rapid runtime churn can @@ -312,6 +345,10 @@ impl UringDriver { Ok(driver) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] #[inline] fn update_waiter(waiter_slot: &mut Option, waker: Waker) -> Option { if !waiter_slot @@ -324,11 +361,19 @@ impl UringDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] #[inline] fn encode_completion_key(token: usize) -> u64 { ((token as u64) << KEY_KIND_BITS) | COMPLETION_KEY_KIND as u64 } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] #[inline] fn encode_poll_key(token: Token, generation: u32) -> u64 { ((u64::from(generation) & 0x3fff_ffff) << 34) @@ -336,16 +381,28 @@ impl UringDriver { | POLL_KEY_KIND as u64 } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] #[inline] fn decode_token(key: u64) -> Token { Token(((key >> KEY_KIND_BITS) & u64::from(u32::MAX)) as usize) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] #[inline] fn decode_poll_generation(key: u64) -> u32 { (key >> 34) as u32 } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] #[inline] fn encode_accept_key(token: Token, generation: u32) -> u64 { ((u64::from(generation) & 0x3fff_ffff) << 34) @@ -353,11 +410,19 @@ impl UringDriver { | ACCEPT_KEY_KIND as u64 } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] #[inline] fn decode_key_kind(key: u64) -> u8 { (key & KEY_KIND_MASK) as u8 } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] #[inline] fn interest_to_poll_mask(interest: Interest) -> u32 { let mut mask = 0; @@ -370,6 +435,10 @@ impl UringDriver { mask } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] #[inline] fn submitter_call_result(result: Result) -> Result<(), io::Error> { match result { @@ -380,6 +449,10 @@ impl UringDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] #[inline] fn push_entry(&self, entry: squeue::Entry) -> Result<(), io::Error> { let mut ring = self.ring.borrow_mut(); @@ -404,6 +477,10 @@ impl UringDriver { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] #[inline] fn push_poll_add( &self, @@ -419,6 +496,10 @@ impl UringDriver { self.push_entry(entry) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] #[inline] fn collect_completions( &self, @@ -467,6 +548,10 @@ impl UringDriver { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] /// Drain the completion queue, deferring user callbacks until borrows end. #[inline] fn drain_cq(ring: &mut IoUring, state: &mut DriverState) -> CompletionBatch { @@ -582,6 +667,10 @@ impl UringDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UringDriver") + )] #[inline] fn submit_interrupt(&self) { use io_uring::{opcode, types}; @@ -798,6 +887,10 @@ mod memory_fallback_tests { impl Driver for UringDriver { type Interruptor = UringInterruptor; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn flush(&self) { match self.collect_completions(false, None) { @@ -807,11 +900,19 @@ impl Driver for UringDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn should_flush(&self) -> bool { self.pending_submissions.load(Ordering::Acquire) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn wait(&self, timeout: Option) { match self.collect_completions(true, timeout) { @@ -821,6 +922,10 @@ impl Driver for UringDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn get_interruptor(&self) -> Self::Interruptor { UringInterruptor { @@ -828,6 +933,10 @@ impl Driver for UringDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn register_handle( &self, @@ -837,6 +946,10 @@ impl Driver for UringDriver { self.register_handle_with_mode(handle, interest, RegistrationMode::Completion) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn register_handle_with_mode( &self, @@ -876,6 +989,10 @@ impl Driver for UringDriver { Ok(token) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn reregister_handle( &self, @@ -899,6 +1016,10 @@ impl Driver for UringDriver { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn deregister_handle(&self, handle: &InnerRawHandle) -> Result<(), io::Error> { { @@ -928,11 +1049,19 @@ impl Driver for UringDriver { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn supports_completion(&self) -> bool { true } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn submit_poll( &self, @@ -996,6 +1125,10 @@ impl Driver for UringDriver { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn submit_completion(&self, op: &mut O, waker: Waker) -> super::CompletionIoResult where @@ -1028,6 +1161,10 @@ impl Driver for UringDriver { CompletionIoResult::Retry(token) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn get_completion_result(&self, token: usize) -> Option { let mut state = self.state.borrow_mut(); @@ -1041,6 +1178,10 @@ impl Driver for UringDriver { completed } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn poll_multishot_accept( &self, handle: &InnerRawHandle, @@ -1116,6 +1257,10 @@ impl Driver for UringDriver { Poll::Pending } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn set_completion_waker(&self, token: usize, waker: Waker) { let mut state = self.state.borrow_mut(); @@ -1128,6 +1273,10 @@ impl Driver for UringDriver { drop(old_waker); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn ignore_completion(&self, token: usize, data: Box) { let retired = self.state.borrow_mut().ignore_completion(token, data); diff --git a/src/vibeio/executor.rs b/src/vibeio/executor.rs index f342320..f502519 100644 --- a/src/vibeio/executor.rs +++ b/src/vibeio/executor.rs @@ -40,6 +40,7 @@ use crate::vibeio::driver::{AnyDriver, AnyInterruptor}; use crate::vibeio::process::{ZombieReaperMessage, start_zombie_reaper}; use crate::vibeio::task::{RemoteWakeContext, Task, TaskWake}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(crate) fn enqueue_local_wake(wake: &Arc, remote: &Arc) -> bool { CURRENT_RUNTIME.with(|current| { let Ok(current) = current.try_borrow() else { @@ -107,6 +108,10 @@ where { type Output = (); + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.project(); @@ -155,12 +160,20 @@ pub struct JoinHandle { } impl JoinHandle { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "JoinHandle") + )] /// Creates a new `JoinHandle` with the given state. #[inline] fn new(state: Rc>>) -> Self { Self { state } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "JoinHandle") + )] /// Cancels the task associated with this handle. /// /// The task will be interrupted and not resumed. @@ -189,6 +202,10 @@ impl JoinHandle { impl Future for JoinHandle { type Output = T; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { { @@ -229,6 +246,10 @@ struct BlockOnNotify { } impl BlockOnNotify { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "BlockOnNotify") + )] #[inline] fn new( interruptor: AnyInterruptor, @@ -244,22 +265,38 @@ impl BlockOnNotify { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "BlockOnNotify") + )] #[inline] fn is_ready(&self) -> bool { self.ready.load(Ordering::Acquire) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "BlockOnNotify") + )] #[inline] fn take_ready(&self) -> bool { self.ready.swap(false, Ordering::AcqRel) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "BlockOnNotify") + )] #[cfg(any(target_vendor = "apple", windows))] #[inline] fn force_ready(&self) { self.ready.store(true, Ordering::Release); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "BlockOnNotify") + )] #[inline] fn notify(&self) { self.ready.store(true, Ordering::Release); @@ -272,6 +309,10 @@ impl BlockOnNotify { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "BlockOnNotify") + )] #[inline] fn waker(self: &Arc) -> Waker { Waker::from(Arc::clone(self)) @@ -279,11 +320,19 @@ impl BlockOnNotify { } impl Wake for BlockOnNotify { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn wake(self: Arc) { self.notify(); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn wake_by_ref(self: &Arc) { self.notify(); @@ -293,6 +342,10 @@ impl Wake for BlockOnNotify { struct CurrentRuntimeGuard; impl CurrentRuntimeGuard { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "CurrentRuntimeGuard") + )] /// Enter a runtime, making it available for spawning tasks. /// /// Panics if called while already inside a runtime. @@ -312,6 +365,10 @@ impl CurrentRuntimeGuard { } impl Drop for CurrentRuntimeGuard { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] /// Exit the runtime, clearing the current runtime reference. #[inline] fn drop(&mut self) { @@ -322,6 +379,7 @@ impl Drop for CurrentRuntimeGuard { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Get the I/O driver for the current runtime. /// /// Returns `None` if called outside a runtime context. @@ -334,6 +392,7 @@ pub(crate) fn current_driver() -> Option> { }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Get the timer for the current runtime. /// /// Returns `None` if called outside a runtime context or if timers are not enabled. @@ -346,6 +405,7 @@ pub(crate) fn current_timer() -> Option> { }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Get the zombie reaper channel for the current runtime. /// /// Returns `None` if called outside a runtime context or if process support is not enabled. @@ -362,6 +422,7 @@ pub(crate) async fn current_zombie_reaper() -> Option(&self, future: impl Future + 'static) -> JoinHandle @@ -496,6 +563,10 @@ impl RuntimeInner { JoinHandle::new(state) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeInner", future = true) + )] /// Spawn a blocking task on this runtime's thread pool. #[inline] pub(crate) async fn spawn_blocking(&self, f: F) -> Result @@ -507,12 +578,20 @@ impl RuntimeInner { crate::vibeio::blocking::spawn_blocking(pool.as_ref(), f).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeInner") + )] /// Enqueue a task for polling. #[inline] fn enqueue(&self, task: Rc) { self.queue.borrow_mut().push_back(task); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeInner") + )] /// Drain ready tasks into the given batch. #[inline] fn drain_ready(&self, batch: &mut Batch<'_, Rc>, mut budget: usize) { @@ -544,6 +623,10 @@ impl RuntimeInner { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeInner") + )] #[inline] fn stop_waiting(&self) { self.remote_wake.waiting.store(false, Ordering::Release); @@ -552,6 +635,10 @@ impl RuntimeInner { .store(false, Ordering::Release); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeInner") + )] #[inline] fn should_skip_wait(&self) -> bool { if self.next_task.borrow().is_some() || !self.remote_wake.queue.is_empty() { @@ -561,6 +648,10 @@ impl RuntimeInner { !self.queue.borrow().is_empty() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RuntimeInner") + )] /// Take the next task to run, if any. #[inline] fn take_next_task(&self) -> Option> { @@ -573,6 +664,7 @@ impl RuntimeInner { } impl Runtime { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Runtime"))] /// Create a new runtime with the given driver. /// /// By default, this enables the timer and file I/O offload. @@ -587,6 +679,7 @@ impl Runtime { Self::with_options(driver, true, blocking_pool, true, false) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Runtime"))] /// Create a new runtime with the given driver and options. #[inline] pub(crate) fn with_options( @@ -636,6 +729,7 @@ impl Runtime { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Runtime"))] /// Spawn a task on this runtime. /// /// Returns a `JoinHandle` that can be awaited to get the task's output. @@ -650,6 +744,10 @@ impl Runtime { .spawn(future) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "Runtime", future = true) + )] /// Spawn a blocking task on this runtime's thread pool. #[inline] pub async fn spawn_blocking(&self, f: F) -> Result @@ -686,6 +784,7 @@ impl Runtime { })); } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Runtime"))] /// Run the runtime and execute the given future to completion. /// /// This method blocks the current thread and drives the runtime until @@ -822,6 +921,10 @@ impl Runtime { } impl Drop for Runtime { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { // Drop all tasks with current runtime entered let inner = self.inner.take().expect("runtime has been dropped"); diff --git a/src/vibeio/fd_inner.rs b/src/vibeio/fd_inner.rs index de15d5e..63726c5 100644 --- a/src/vibeio/fd_inner.rs +++ b/src/vibeio/fd_inner.rs @@ -32,6 +32,7 @@ pub struct InnerRawHandle { // only after registration succeeds, and relinquishes it before re-registering. const UNREGISTERED: Token = Token(usize::MAX); +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Set the descriptor's blocking mode without changing unrelated status flags. #[cfg(all( unix, @@ -44,6 +45,7 @@ const UNREGISTERED: Token = Token(usize::MAX); ) ))] pub(crate) fn set_nonblocking(fd: RawOsHandle, nonblocking: bool) -> io::Result<()> { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn fcntl(fd: RawOsHandle, command: libc::c_int, value: libc::c_int) -> io::Result { loop { // SAFETY: F_GETFL and F_SETFL take integer arguments, not pointers. @@ -85,12 +87,20 @@ impl InnerRawHandle { driver, } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "InnerRawHandle") + )] /// Share ownership with operations using this registration's driver. #[cfg(all(target_os = "linux", any(feature = "fs", feature = "splice")))] pub(crate) fn driver_owner(&self) -> Rc { self.driver.clone() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "InnerRawHandle") + )] /// Retain operation storage on the registration's owner until completion. #[inline] pub(crate) fn cancel_completion(&self, token: usize, data: Box) { @@ -100,6 +110,10 @@ impl InnerRawHandle { self.driver.ignore_completion(token, data); } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "InnerRawHandle") + )] #[inline] pub(crate) fn new(handle: RawOsHandle, interest: Interest) -> Result { let default_mode = if current_driver() @@ -114,6 +128,10 @@ impl InnerRawHandle { Self::new_with_mode(handle, interest, default_mode) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "InnerRawHandle") + )] #[inline] pub(crate) fn new_with_mode( handle: RawOsHandle, @@ -129,6 +147,10 @@ impl InnerRawHandle { Self::new_with_driver_and_mode(&driver, handle, interest, mode) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "InnerRawHandle") + )] #[inline] pub(crate) fn new_with_driver_and_mode( driver: &Rc, @@ -154,34 +176,58 @@ impl InnerRawHandle { Ok(inner) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "InnerRawHandle") + )] #[cfg(unix)] #[inline] pub(crate) fn token(&self) -> Token { self.token } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "InnerRawHandle") + )] #[allow(dead_code)] #[inline] pub(crate) fn reregister(&self, interest: Interest) -> Result<(), io::Error> { self.driver.reregister_handle(self, interest) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "InnerRawHandle") + )] #[inline] pub(crate) fn supports_completion(&self) -> bool { self.driver.supports_completion() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "InnerRawHandle") + )] #[inline] pub(crate) fn uses_completion(&self) -> bool { self.supports_completion() && matches!(self.mode, RegistrationMode::Completion) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "InnerRawHandle") + )] #[allow(dead_code)] #[inline] pub(crate) fn mode(&self) -> RegistrationMode { self.mode } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "InnerRawHandle") + )] /// Replace the registration. If deregistration fails, return its error /// without attempting a replacement. If acquiring the new registration fails, the /// handle is unregistered: callers must drop it or retry before doing I/O. @@ -213,6 +259,10 @@ impl InnerRawHandle { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "InnerRawHandle") + )] #[inline] pub(crate) fn poll_op( &self, @@ -229,6 +279,10 @@ impl InnerRawHandle { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "InnerRawHandle") + )] #[inline] pub(crate) fn poll_op_poll( &self, @@ -249,6 +303,10 @@ impl InnerRawHandle { } impl Drop for InnerRawHandle { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { if self.token != UNREGISTERED { diff --git a/src/vibeio/fs/file.rs b/src/vibeio/fs/file.rs index ca77eeb..e42d9a3 100644 --- a/src/vibeio/fs/file.rs +++ b/src/vibeio/fs/file.rs @@ -48,6 +48,10 @@ pub struct File { } impl File { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "File", future = true) + )] /// Opens a file for reading. /// /// This is the async version of [`std::fs::File::open`]. @@ -73,6 +77,10 @@ impl File { OpenOptions::new().read(true).open(path).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "File", future = true) + )] /// Opens a file for writing, creating it if it does not exist. /// /// This is the async version of [`std::fs::File::create`]. @@ -103,6 +111,7 @@ impl File { .await } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "File"))] /// Returns a new `OpenOptions` builder. /// /// This is a convenience method equivalent to `OpenOptions::new()`. @@ -111,6 +120,7 @@ impl File { OpenOptions::new() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "File"))] /// Creates a new `File` from a standard library file. /// /// This is a convenience method equivalent to `File::from_std_with_cursor(inner, 0)`. @@ -119,6 +129,7 @@ impl File { Self::from_std_with_cursor(inner, 0) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "File"))] /// Creates a new `File` from a standard library file with a specified cursor position. /// /// This is an internal method used to create a `File` with a custom cursor position. @@ -150,6 +161,7 @@ impl File { Ok(Self { inner, io, cursor }) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "File"))] /// Converts the `File` back into a standard library `std::fs::File`. #[inline] pub fn into_std(self) -> std::fs::File { @@ -158,6 +170,7 @@ impl File { inner } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "File"))] /// Returns the completion handle if this file is using io_uring completion. #[inline] fn completion_handle(&self) -> Option<&InnerRawHandle> { @@ -167,6 +180,10 @@ impl File { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "File", future = true) + )] /// Reads bytes from the file at a specific offset. /// /// This method reads into the provided buffer starting at the given offset. @@ -208,6 +225,10 @@ impl File { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "File", future = true) + )] /// Reads bytes from the file at a specific offset, filling the entire buffer. /// /// This method fills the provided buffer's writable capacity starting at the @@ -241,6 +262,10 @@ impl File { .await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "File", future = true) + )] /// Writes bytes to the file at a specific offset. /// /// This method writes from the provided buffer starting at the given offset. @@ -285,6 +310,10 @@ impl File { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "File", future = true) + )] /// Writes bytes to the file at a specific offset, writing the entire buffer. /// /// This method writes from the provided buffer starting at the given offset, @@ -317,6 +346,10 @@ impl File { .await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "File", future = true) + )] /// Synchronizes all data and metadata to disk. /// /// This is the async version of [`std::fs::File::sync_all`]. @@ -356,6 +389,10 @@ impl File { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "File", future = true) + )] /// Synchronizes file data to disk without necessarily syncing metadata. /// /// This is the async version of [`std::fs::File::sync_data`]. @@ -395,6 +432,10 @@ impl File { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "File", future = true) + )] /// Returns the metadata for this file. /// /// This is the async version of [`std::fs::File::metadata`]. @@ -547,6 +588,7 @@ mod exact_at_tests { } impl ExactAt { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "ExactAt"))] fn remaining(&self, buf: &impl IoBuf) -> usize { match self { Self::Read => buf.buf_capacity(), @@ -555,6 +597,7 @@ impl ExactAt { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] async fn exact_at( buf: B, mut offset: u64, @@ -612,6 +655,7 @@ where (Ok(()), buf.into_inner()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] #[inline] fn read_at_blocking(file: &std::fs::File, buf: &mut [u8], offset: u64) -> io::Result { @@ -619,6 +663,7 @@ fn read_at_blocking(file: &std::fs::File, buf: &mut [u8], offset: u64) -> io::Re file.read_at(buf, offset) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] #[inline] fn read_at_blocking(file: &std::fs::File, buf: &mut [u8], offset: u64) -> io::Result { @@ -626,6 +671,7 @@ fn read_at_blocking(file: &std::fs::File, buf: &mut [u8], offset: u64) -> io::Re file.seek_read(buf, offset) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] #[inline] fn write_at_blocking(file: &std::fs::File, buf: &[u8], offset: u64) -> io::Result { @@ -633,6 +679,7 @@ fn write_at_blocking(file: &std::fs::File, buf: &[u8], offset: u64) -> io::Resul file.write_at(buf, offset) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] #[inline] fn write_at_blocking(file: &std::fs::File, buf: &[u8], offset: u64) -> io::Result { @@ -640,26 +687,31 @@ fn write_at_blocking(file: &std::fs::File, buf: &[u8], offset: u64) -> io::Resul file.seek_write(buf, offset) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn sync_all_blocking(file: &std::fs::File) -> io::Result<()> { file.sync_all() } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn sync_data_blocking(file: &std::fs::File) -> io::Result<()> { file.sync_data() } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn metadata_blocking(file: &std::fs::File) -> io::Result { Ok(Metadata::from_std(file.metadata()?)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub(crate) fn blocking_pool_io_error() -> io::Error { io::Error::other("can't spawn blocking task for file I/O") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[inline] async fn read_at_in_blocking_pool( file: &std::fs::File, @@ -680,6 +732,7 @@ async fn read_at_in_blocking_pool( ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[inline] async fn write_at_in_blocking_pool( file: &std::fs::File, @@ -700,6 +753,7 @@ async fn write_at_in_blocking_pool( ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[inline] async fn sync_all_in_blocking_pool(file: &std::fs::File) -> io::Result<()> { let file = file.try_clone()?; @@ -708,6 +762,7 @@ async fn sync_all_in_blocking_pool(file: &std::fs::File) -> io::Result<()> { .map_err(|_| blocking_pool_io_error())? } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[inline] async fn sync_data_in_blocking_pool(file: &std::fs::File) -> io::Result<()> { let file = file.try_clone()?; @@ -716,6 +771,7 @@ async fn sync_data_in_blocking_pool(file: &std::fs::File) -> io::Result<()> { .map_err(|_| blocking_pool_io_error())? } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[inline] async fn metadata_in_blocking_pool(file: &std::fs::File) -> io::Result { let file = file.try_clone()?; @@ -725,6 +781,10 @@ async fn metadata_in_blocking_pool(file: &std::fs::File) -> io::Result } impl AsyncRead for File { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn read(&mut self, buf: B) -> (Result, B) { let (read, buf) = self.read_at(buf, self.cursor).await; @@ -736,6 +796,10 @@ impl AsyncRead for File { } impl AsyncWrite for File { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn write(&mut self, buf: B) -> (Result, B) { let (written, buf) = self.write_at(buf, self.cursor).await; @@ -745,6 +809,10 @@ impl AsyncWrite for File { (written, buf) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn flush(&mut self) -> Result<(), io::Error> { Ok(()) @@ -753,6 +821,10 @@ impl AsyncWrite for File { #[cfg(unix)] impl AsRawFd for File { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_fd(&self) -> RawFd { self.inner.as_raw_fd() @@ -761,6 +833,10 @@ impl AsRawFd for File { #[cfg(unix)] impl IntoRawFd for File { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_fd(self) -> RawFd { self.into_std().into_raw_fd() @@ -769,6 +845,10 @@ impl IntoRawFd for File { #[cfg(windows)] impl AsRawHandle for File { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_handle(&self) -> RawHandle { self.inner.as_raw_handle() @@ -777,6 +857,10 @@ impl AsRawHandle for File { #[cfg(windows)] impl IntoRawHandle for File { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_handle(self) -> RawHandle { self.into_std().into_raw_handle() diff --git a/src/vibeio/fs/metadata.rs b/src/vibeio/fs/metadata.rs index 7089c01..2ff8ed8 100644 --- a/src/vibeio/fs/metadata.rs +++ b/src/vibeio/fs/metadata.rs @@ -33,6 +33,7 @@ enum MetadataInner { } impl Metadata { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Metadata"))] /// Creates a new `Metadata` from a standard library `std::fs::Metadata`. #[inline] pub(crate) fn from_std(md: std::fs::Metadata) -> Self { @@ -41,6 +42,7 @@ impl Metadata { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Metadata"))] /// Creates a new `Metadata` from a `libc::statx` structure. #[cfg(all(target_os = "linux", any(target_env = "gnu", musl_v1_2_3)))] #[inline] @@ -50,6 +52,7 @@ impl Metadata { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Metadata"))] /// Returns the size of the file in bytes. #[allow(clippy::len_without_is_empty)] #[inline] @@ -61,6 +64,7 @@ impl Metadata { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Metadata"))] /// Returns the file permissions. #[inline] pub fn permissions(&self) -> std::fs::Permissions { @@ -74,6 +78,7 @@ impl Metadata { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Metadata"))] /// Returns the file type. #[inline] pub fn file_type(&self) -> FileType { @@ -84,6 +89,7 @@ impl Metadata { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Metadata"))] /// Returns `true` if this metadata is for a directory. #[inline] pub fn is_dir(&self) -> bool { @@ -94,6 +100,7 @@ impl Metadata { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Metadata"))] /// Returns `true` if this metadata is for a regular file. #[inline] pub fn is_file(&self) -> bool { @@ -104,6 +111,7 @@ impl Metadata { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Metadata"))] /// Returns `true` if this metadata is for a symbolic link. #[inline] pub fn is_symlink(&self) -> bool { @@ -114,6 +122,7 @@ impl Metadata { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Metadata"))] /// Returns the time the file was last accessed. /// /// # Errors @@ -129,6 +138,7 @@ impl Metadata { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Metadata"))] /// Returns the time the file was created. /// /// # Errors @@ -144,6 +154,7 @@ impl Metadata { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Metadata"))] /// Returns the time the file was last modified. /// /// # Errors @@ -160,6 +171,7 @@ impl Metadata { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Converts a `libc::statx_timestamp` to a `SystemTime`. #[cfg(all(target_os = "linux", any(target_env = "gnu", musl_v1_2_3)))] #[inline] @@ -224,6 +236,7 @@ pub struct FileType { } impl FileType { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "FileType"))] /// Test whether this file type represents a directory. /// /// # Examples @@ -234,6 +247,7 @@ impl FileType { self.is_dir } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "FileType"))] /// Test whether this file type represents a regular file. /// /// # Examples @@ -244,6 +258,7 @@ impl FileType { self.is_file } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "FileType"))] /// Test whether this file type represents a symbolic link. /// /// # Examples diff --git a/src/vibeio/fs/mod.rs b/src/vibeio/fs/mod.rs index d0c8280..cb6273a 100644 --- a/src/vibeio/fs/mod.rs +++ b/src/vibeio/fs/mod.rs @@ -47,6 +47,7 @@ use crate::vibeio::op::SymlinkOp; #[cfg(target_os = "linux")] use crate::vibeio::op::UnlinkOp; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Run a filesystem operation away from the async executor thread. /// /// A caller-selected runtime pool takes precedence. The shared async-std pool @@ -66,6 +67,7 @@ where } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Creates a symbolic link to a directory on Windows. /// /// Creates the link at `path`, pointing to `target`, using the standard library. @@ -87,6 +89,7 @@ pub fn windows_symlink_dir(path: String, target: String) -> std::io::Result<()> std::os::windows::fs::symlink_dir(target, path) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Creates a symbolic link to a file on Windows. /// /// Creates the link at `path`, pointing to `target`, using the standard library. @@ -108,6 +111,7 @@ pub fn windows_symlink_file(path: String, target: String) -> std::io::Result<()> std::os::windows::fs::symlink_file(target, path) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Returns the canonical form of a path with all components normalized. /// /// This is the async version of [`std::fs::canonicalize`]. @@ -128,6 +132,7 @@ pub async fn canonicalize>(path: P) -> std::io::Result run_blocking_fs(move || path.canonicalize()).await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Reads the entire contents of a file into a vector of bytes. /// /// This is the async version of [`std::fs::read`]. @@ -162,6 +167,7 @@ pub async fn read(path: impl AsRef) -> std::io::Result> Ok(bytes) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Reads the entire contents of a file into a string. /// /// This is the async version of [`std::fs::read_to_string`]. @@ -182,6 +188,7 @@ pub async fn read_to_string(path: impl AsRef) -> std::io::Resul .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err.utf8_error())) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Writes a byte slice to a file, creating it if necessary. /// /// This is the async version of [`std::fs::write`]. @@ -214,6 +221,7 @@ pub async fn write( file.flush().await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Creates a hard link at the destination path pointing to the source. /// /// This is the async version of [`std::fs::hard_link`]. @@ -264,6 +272,7 @@ pub async fn hard_link( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Creates a hard link at the destination path pointing to the source. /// /// This is the async version of [`std::fs::hard_link`]. @@ -290,6 +299,7 @@ pub async fn hard_link( run_blocking_fs(move || std::fs::hard_link(src, dst)).await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Creates a symbolic link to a directory. /// /// This is the async version of [std::os::unix::fs::symlink](https://doc.rust-lang.org/std/os/unix/fs/fn.symlink.html) (on Unix) or @@ -319,6 +329,7 @@ pub async fn symlink_dir( run_blocking_fs(move || std::os::windows::fs::symlink_dir(src, dst)).await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Creates a symbolic link to a directory. /// /// This is the async version of [`std::os::unix::fs::symlink`]. @@ -369,6 +380,7 @@ pub async fn symlink_dir( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Creates a symbolic link to a directory. /// /// This is the async version of [`std::os::unix::fs::symlink`]. @@ -395,6 +407,7 @@ pub async fn symlink_dir( run_blocking_fs(move || std::os::unix::fs::symlink(src, dst)).await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Creates a symbolic link to a file. /// /// This is the async version of [std::os::unix::fs::symlink](https://doc.rust-lang.org/std/os/unix/fs/fn.symlink.html) (on Unix) or @@ -424,6 +437,7 @@ pub async fn symlink_file( run_blocking_fs(move || std::os::windows::fs::symlink_file(src, dst)).await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Creates a symbolic link to a file. /// /// This is the async version of [`std::os::unix::fs::symlink`]. @@ -474,6 +488,7 @@ pub async fn symlink_file( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Creates a symbolic link to a file. /// /// This is the async version of [`std::os::unix::fs::symlink`]. @@ -500,6 +515,7 @@ pub async fn symlink_file( run_blocking_fs(move || std::os::unix::fs::symlink(src, dst)).await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Creates a symbolic link. /// /// This is a convenience function that calls [`symlink_file`]. Use this when you @@ -521,6 +537,7 @@ pub async fn symlink( symlink_file(src, dst).await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Renames a file or directory to a new location. /// /// This is the async version of [`std::fs::rename`]. @@ -570,6 +587,7 @@ pub async fn rename( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Renames a file or directory to a new location. /// /// This is the async version of [`std::fs::rename`]. @@ -596,6 +614,7 @@ pub async fn rename( run_blocking_fs(move || std::fs::rename(from, to)).await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Removes an empty directory. /// /// This is the async version of [`std::fs::remove_dir`]. @@ -634,6 +653,7 @@ pub async fn remove_dir(path: impl AsRef) -> std::io::Result<() } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Removes an empty directory. /// /// This is the async version of [`std::fs::remove_dir`]. @@ -656,6 +676,7 @@ pub async fn remove_dir(path: impl AsRef) -> std::io::Result<() run_blocking_fs(move || std::fs::remove_dir(path)).await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Removes a file. /// /// This is the async version of [`std::fs::remove_file`]. @@ -692,6 +713,7 @@ pub async fn remove_file(path: impl AsRef) -> std::io::Result<( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Removes a file. /// /// This is the async version of [`std::fs::remove_file`]. @@ -712,6 +734,7 @@ pub async fn remove_file(path: impl AsRef) -> std::io::Result<( run_blocking_fs(move || std::fs::remove_file(path)).await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Creates a directory. /// /// This is the async version of [`std::fs::create_dir`]. @@ -751,6 +774,7 @@ pub async fn create_dir(path: impl AsRef) -> std::io::Result<() } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Creates a directory. /// /// This is the async version of [`std::fs::create_dir`]. @@ -773,6 +797,7 @@ pub async fn create_dir(path: impl AsRef) -> std::io::Result<() run_blocking_fs(move || std::fs::create_dir(path)).await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Creates a new, empty directory and all its parent components if they don't exist. /// /// This is the async version of [`std::fs::create_dir_all`]. @@ -838,6 +863,7 @@ pub async fn create_dir_all(path: impl AsRef) -> std::io::Resul Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Returns metadata about a file or directory. /// /// This is the async version of [`std::fs::metadata`]. @@ -884,6 +910,7 @@ pub async fn metadata(path: impl AsRef) -> std::io::Result) -> std::io::Result) -> std::io::Res } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Returns metadata about a file or directory without following symlinks. /// /// This is the async version of [`std::fs::symlink_metadata`]. diff --git a/src/vibeio/fs/open_options.rs b/src/vibeio/fs/open_options.rs index 7434471..bf38439 100644 --- a/src/vibeio/fs/open_options.rs +++ b/src/vibeio/fs/open_options.rs @@ -39,6 +39,10 @@ pub struct OpenOptions { } impl OpenOptions { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OpenOptions") + )] /// Creates a new `OpenOptions` with default values. /// /// By default, all options are set to `false`. @@ -54,6 +58,10 @@ impl OpenOptions { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OpenOptions") + )] /// Sets whether the file should be opened for reading. #[inline] pub fn read(&mut self, read: bool) -> &mut Self { @@ -61,6 +69,10 @@ impl OpenOptions { self } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OpenOptions") + )] /// Sets whether the file should be opened for writing. #[inline] pub fn write(&mut self, write: bool) -> &mut Self { @@ -68,6 +80,10 @@ impl OpenOptions { self } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OpenOptions") + )] /// Sets whether the file should be opened in append mode. #[inline] pub fn append(&mut self, append: bool) -> &mut Self { @@ -75,6 +91,10 @@ impl OpenOptions { self } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OpenOptions") + )] /// Sets whether the file should be truncated if it already exists. #[inline] pub fn truncate(&mut self, truncate: bool) -> &mut Self { @@ -82,6 +102,10 @@ impl OpenOptions { self } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OpenOptions") + )] /// Sets whether the file should be created if it does not exist. #[inline] pub fn create(&mut self, create: bool) -> &mut Self { @@ -89,6 +113,10 @@ impl OpenOptions { self } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OpenOptions") + )] /// Sets whether the file should be created exclusively (fails if it already exists). #[inline] pub fn create_new(&mut self, create_new: bool) -> &mut Self { @@ -96,6 +124,10 @@ impl OpenOptions { self } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OpenOptions") + )] /// Validates the open options. /// /// This is an internal method used to ensure the options are valid. @@ -125,6 +157,10 @@ impl OpenOptions { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OpenOptions") + )] /// Returns the initial cursor position for the file. /// /// If append mode is enabled, the cursor is set to the end of the file. @@ -137,6 +173,10 @@ impl OpenOptions { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OpenOptions", future = true) + )] /// Opens a file with the configured options. /// /// This is the async version of [`std::fs::OpenOptions::open`]. @@ -194,6 +234,10 @@ impl OpenOptions { File::from_std_with_cursor(std_file, cursor) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OpenOptions", future = true) + )] /// Opens a file in the blocking thread pool. /// /// This is an internal method used when io_uring is not available. @@ -229,6 +273,10 @@ impl OpenOptions { .map_err(|_| crate::vibeio::fs::file::blocking_pool_io_error())? } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OpenOptions") + )] /// Opens a file synchronously. /// /// This is an internal method used when io_uring is not available. @@ -252,6 +300,10 @@ impl OpenOptions { options.open(path) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "OpenOptions") + )] /// Builds an `OpenOp` for use with io_uring. /// /// This is an internal method used on Linux with io_uring support. @@ -305,6 +357,10 @@ impl OpenOptions { } impl Default for OpenOptions { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] /// Returns the default `OpenOptions`. /// /// This is equivalent to `OpenOptions::new()`. diff --git a/src/vibeio/io/buf.rs b/src/vibeio/io/buf.rs index 2236c5c..bd02651 100644 --- a/src/vibeio/io/buf.rs +++ b/src/vibeio/io/buf.rs @@ -78,16 +78,28 @@ pub unsafe trait IoBufMut: IoBuf { // SAFETY: Vec owns its allocation; len describes initialized bytes and never // exceeds capacity. Operations do not resize it while its pointer is in use. unsafe impl IoBuf for Vec { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_buf_ptr(&self) -> *const u8 { self.as_ptr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn buf_len(&self) -> usize { self.len() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn buf_capacity(&self) -> usize { self.capacity() @@ -97,11 +109,19 @@ unsafe impl IoBuf for Vec { // SAFETY: &mut Vec grants exclusive access to its allocation, including spare // capacity. set_buf_init exposes only the prefix initialized by its caller. unsafe impl IoBufMut for Vec { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_buf_mut_ptr(&mut self) -> *mut u8 { self.as_mut_ptr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] unsafe fn set_buf_init(&mut self, len: usize) { // SAFETY: the IoBufMut caller guarantees initialization and capacity. @@ -112,16 +132,28 @@ unsafe impl IoBufMut for Vec { // SAFETY: String owns initialized UTF-8 bytes. Only read access to its len-byte // prefix is exposed, so neither UTF-8 validity nor spare capacity is modified. unsafe impl IoBuf for String { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_buf_ptr(&self) -> *const u8 { self.as_ptr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn buf_len(&self) -> usize { self.len() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn buf_capacity(&self) -> usize { self.capacity() @@ -130,16 +162,28 @@ unsafe impl IoBuf for String { // SAFETY: the initialized slice is immutable and outlives every operation. unsafe impl IoBuf for &'static [u8] { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "<& 'static [u8] as IoBuf>") + )] #[inline] fn as_buf_ptr(&self) -> *const u8 { self.as_ptr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "<& 'static [u8] as IoBuf>") + )] #[inline] fn buf_len(&self) -> usize { self.len() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "<& 'static [u8] as IoBuf>") + )] #[inline] fn buf_capacity(&self) -> usize { self.len() @@ -148,16 +192,28 @@ unsafe impl IoBuf for &'static [u8] { // SAFETY: the initialized UTF-8 bytes are immutable and have static storage. unsafe impl IoBuf for &'static str { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "<& 'static str as IoBuf>") + )] #[inline] fn as_buf_ptr(&self) -> *const u8 { self.as_bytes().as_ptr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "<& 'static str as IoBuf>") + )] #[inline] fn buf_len(&self) -> usize { self.len() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "<& 'static str as IoBuf>") + )] #[inline] fn buf_capacity(&self) -> usize { self.len() @@ -167,16 +223,28 @@ unsafe impl IoBuf for &'static str { // SAFETY: all N bytes are initialized. The operation keeps the array stationary // while using its pointer (completion operations box their buffer storage). unsafe impl IoBuf for [u8; N] { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "<[u8 ; N] as IoBuf>") + )] #[inline] fn as_buf_ptr(&self) -> *const u8 { self.as_ptr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "<[u8 ; N] as IoBuf>") + )] #[inline] fn buf_len(&self) -> usize { N } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "<[u8 ; N] as IoBuf>") + )] #[inline] fn buf_capacity(&self) -> usize { N @@ -186,26 +254,46 @@ unsafe impl IoBuf for [u8; N] { // SAFETY: the exclusive array borrow covers all N writable initialized bytes. // Partial writes do not make any of the remaining array elements uninitialized. unsafe impl IoBufMut for [u8; N] { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "<[u8 ; N] as IoBufMut>") + )] #[inline] fn as_buf_mut_ptr(&mut self) -> *mut u8 { self.as_mut_ptr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "<[u8 ; N] as IoBufMut>") + )] unsafe fn set_buf_init(&mut self, _len: usize) {} } // SAFETY: the box owns len initialized bytes at a stable allocation address. unsafe impl IoBuf for Box<[u8]> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_buf_ptr(&self) -> *const u8 { self.as_ptr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn buf_len(&self) -> usize { self.len() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn buf_capacity(&self) -> usize { self.len() @@ -215,11 +303,19 @@ unsafe impl IoBuf for Box<[u8]> { // SAFETY: the box exclusively owns its initialized slice; partial writes leave // the other bytes valid and do not change the allocation size. unsafe impl IoBufMut for Box<[u8]> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_buf_mut_ptr(&mut self) -> *mut u8 { self.as_mut_ptr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] unsafe fn set_buf_init(&mut self, _len: usize) {} } @@ -230,12 +326,20 @@ pub(crate) struct IoBufWithCursor { } impl IoBufWithCursor { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IoBufWithCursor") + )] /// Create a new `IoBufWithCursor` with the given buffer. #[inline] pub(crate) fn new(buf: I) -> Self { IoBufWithCursor { buf, cursor: 0 } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IoBufWithCursor") + )] /// Advance the cursor by `n` bytes. #[inline] pub(crate) fn advance(&mut self, n: usize) { @@ -246,6 +350,10 @@ impl IoBufWithCursor { self.cursor += n; } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IoBufWithCursor") + )] /// Consume the wrapper and return the inner buffer. #[inline] pub(crate) fn into_inner(self) -> I { @@ -256,6 +364,10 @@ impl IoBufWithCursor { // SAFETY: private fields and checked advance keep cursor within initialized // data. The underlying IoBuf retains ownership and supplies stable storage. unsafe impl IoBuf for IoBufWithCursor { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_buf_ptr(&self) -> *const u8 { // SAFETY: cursor is at most the initialized length, hence in bounds or @@ -263,11 +375,19 @@ unsafe impl IoBuf for IoBufWithCursor { unsafe { self.buf.as_buf_ptr().add(self.cursor) } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn buf_len(&self) -> usize { self.buf.buf_len() - self.cursor } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn buf_capacity(&self) -> usize { self.buf.buf_capacity() - self.cursor @@ -277,12 +397,20 @@ unsafe impl IoBuf for IoBufWithCursor { // SAFETY: the suffix is exclusively borrowed from the underlying IoBufMut; // its writable capacity is reduced by the checked cursor offset. unsafe impl IoBufMut for IoBufWithCursor { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_buf_mut_ptr(&mut self) -> *mut u8 { // SAFETY: cursor never exceeds the underlying initialized length. unsafe { self.buf.as_buf_mut_ptr().add(self.cursor) } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] unsafe fn set_buf_init(&mut self, len: usize) { // SAFETY: caller initialized len suffix bytes within suffix capacity; // the prefix before cursor was already initialized. @@ -298,6 +426,10 @@ pub(crate) struct IoBufTemporaryPoll { } impl IoBufTemporaryPoll { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IoBufTemporaryPoll") + )] /// Create a new `IoBufTemporaryPoll` with the given pointer and length. /// /// # Safety @@ -315,6 +447,10 @@ impl IoBufTemporaryPoll { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IoBufTemporaryPoll") + )] /// Wrap writable storage without claiming that any bytes are initialized. /// /// # Safety @@ -334,16 +470,28 @@ impl IoBufTemporaryPoll { // SAFETY: constructors distinguish initialized length from writable capacity; // their caller keeps the allocation stable for the entire synchronous poll. unsafe impl IoBuf for IoBufTemporaryPoll { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_buf_ptr(&self) -> *const u8 { self.ptr as *const u8 } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn buf_len(&self) -> usize { self.len } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn buf_capacity(&self) -> usize { self.capacity @@ -353,11 +501,19 @@ unsafe impl IoBuf for IoBufTemporaryPoll { // SAFETY: mutable use requires exclusive storage under the constructor contract. // Only the initialized prefix supplied by a successful read is exposed. unsafe impl IoBufMut for IoBufTemporaryPoll { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_buf_mut_ptr(&mut self) -> *mut u8 { self.ptr } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] unsafe fn set_buf_init(&mut self, len: usize) { self.len = len; @@ -391,6 +547,10 @@ pub unsafe trait IoVectoredBuf: 'static { /// Returns owned descriptors for this value's readable memory regions. fn as_iovecs(&self) -> Box<[IoVec]>; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IoVectoredBuf") + )] /// Returns `true` if there are no vector descriptors. /// /// This does not test the total initialized byte count. Input operations @@ -423,6 +583,10 @@ pub unsafe trait IoVectoredBufMut: IoVectoredBuf { // SAFETY: each box owns initialized bytes at an address stable across moves of // the vector. Unlike a collection of libc::iovec, it owns the pointed-to storage. unsafe impl IoVectoredBuf for Vec> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_iovecs(&self) -> Box<[IoVec]> { self.iter() @@ -433,6 +597,10 @@ unsafe impl IoVectoredBuf for Vec> { .collect() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn is_empty(&self) -> bool { self.is_empty() @@ -442,6 +610,10 @@ unsafe impl IoVectoredBuf for Vec> { // SAFETY: distinct owned boxes cannot overlap; &mut self grants exclusive access // to every initialized buffer for the duration of the operation. unsafe impl IoVectoredBufMut for Vec> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_iovecs_mut(&mut self) -> Box<[IoVec]> { self.iter_mut() @@ -459,6 +631,10 @@ pub(crate) struct IoVectoredBufTemporaryPoll { } impl IoVectoredBufTemporaryPoll { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IoVectoredBufTemporaryPoll") + )] /// Create a new `IoVectoredBufTemporaryPoll` from immutable slices. /// /// # Safety @@ -475,6 +651,10 @@ impl IoVectoredBufTemporaryPoll { Self { iovecs } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "IoVectoredBufTemporaryPoll") + )] /// Create a new `IoVectoredBufTemporaryPoll` from mutable slices. /// /// # Safety @@ -495,6 +675,10 @@ impl IoVectoredBufTemporaryPoll { // SAFETY: the unsafe constructors require stable, initialized borrowed storage // and forbid retaining pointers beyond the synchronous polling operation. unsafe impl IoVectoredBuf for IoVectoredBufTemporaryPoll { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_iovecs(&self) -> Box<[IoVec]> { self.iovecs @@ -503,6 +687,10 @@ unsafe impl IoVectoredBuf for IoVectoredBufTemporaryPoll { .collect() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn is_empty(&self) -> bool { self.iovecs.is_empty() @@ -512,12 +700,17 @@ unsafe impl IoVectoredBuf for IoVectoredBufTemporaryPoll { // SAFETY: mutable I/O is permitted only for new_mut, whose contract preserves // the non-overlapping exclusive IoSliceMut borrows throughout the poll. unsafe impl IoVectoredBufMut for IoVectoredBufTemporaryPoll { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_iovecs_mut(&mut self) -> Box<[IoVec]> { self.as_iovecs() } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(any(test, feature = "fs", feature = "process", feature = "stdio"))] #[inline] pub(crate) fn iobuf_to_slice(buf: &impl IoBuf) -> &[u8] { @@ -526,6 +719,7 @@ pub(crate) fn iobuf_to_slice(buf: &impl IoBuf) -> &[u8] { unsafe { std::slice::from_raw_parts(buf.as_buf_ptr(), buf.buf_len()) } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(any(test, feature = "fs", feature = "process", feature = "stdio"))] #[inline] pub(crate) fn read_into_buf( diff --git a/src/vibeio/io/mod.rs b/src/vibeio/io/mod.rs index 0115e30..7d87c28 100644 --- a/src/vibeio/io/mod.rs +++ b/src/vibeio/io/mod.rs @@ -69,6 +69,10 @@ pub trait AsyncRead { /// of this read. A zero count indicates EOF unless capacity was zero. async fn read(&mut self, buf: B) -> (Result, B); + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "AsyncRead", future = true) + )] /// Read data into vectored buffers. #[inline] async fn read_vectored( @@ -94,6 +98,10 @@ pub trait AsyncWrite { /// The returned count must not exceed the supplied initialized length. async fn write(&mut self, buf: B) -> (Result, B); + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "AsyncWrite", future = true) + )] /// Write data from vectored buffers. #[inline] async fn write_vectored(&mut self, bufs: V) -> (Result, V) { @@ -106,6 +114,10 @@ pub trait AsyncWrite { ) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "AsyncWrite", future = true) + )] /// Flush the writer. #[inline] async fn flush(&mut self) -> Result<(), io::Error> { @@ -120,6 +132,10 @@ pub trait AsInnerRawHandle<'a> { } impl<'a> AsInnerRawHandle<'a> for InnerRawHandle { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = ">") + )] #[inline] fn as_inner_raw_handle(&'a self) -> &'a InnerRawHandle { self @@ -131,6 +147,10 @@ pub trait AsyncReadPoll { /// Polls to check if the reader is readable. fn poll_readable(&self, cx: &mut std::task::Context) -> std::task::Poll>; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "AsyncReadPoll", future = true) + )] /// Returns a future that resolves when the reader becomes readable. #[inline] async fn readable(&self) -> Result<(), io::Error> { @@ -143,6 +163,10 @@ pub trait AsyncWritePoll { /// Polls to check if the writer is writable. fn poll_writable(&self, cx: &mut std::task::Context) -> std::task::Poll>; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "AsyncWritePoll", future = true) + )] /// Returns a future that resolves when the writer becomes writable. #[inline] async fn writable(&self) -> Result<(), io::Error> { diff --git a/src/vibeio/io/pipe.rs b/src/vibeio/io/pipe.rs index 6dd942c..0a2f0ea 100644 --- a/src/vibeio/io/pipe.rs +++ b/src/vibeio/io/pipe.rs @@ -33,6 +33,7 @@ use crate::vibeio::{ io::{AsyncRead, AsyncWrite}, }; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn pipe_inner() -> std::io::Result<(OwnedFd, OwnedFd)> { let (reader, writer) = std::io::pipe()?; Ok((reader.into(), writer.into())) @@ -220,6 +221,7 @@ mod setup_tests { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Create a new async-aware pipe. /// /// Returns a tuple of `(reader, writer)` pipe endpoints. @@ -245,6 +247,7 @@ pub struct PollPipe { } impl Pipe { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Pipe"))] /// Create a `Pipe` from a standard library `OwnedFd` with the given registration mode. #[inline] pub(crate) fn from_std_with_mode( @@ -261,6 +264,7 @@ impl Pipe { Ok(Self { inner, handle }) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Pipe"))] /// Convert this `Pipe` to a `PollPipe` for readiness-based operations. #[inline] pub fn into_poll(self) -> Result { @@ -272,12 +276,14 @@ impl Pipe { } impl PollPipe { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PollPipe"))] /// Convert this `PollPipe` back to an adaptive `Pipe`. #[inline] pub fn into_adaptive(self) -> Pipe { self.stream } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "PollPipe"))] /// Convert this `PollPipe` to a completion-based `Pipe`. #[inline] pub fn into_completion(self) -> Result { @@ -289,6 +295,10 @@ impl PollPipe { } impl AsRawFd for Pipe { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_fd(&self) -> RawFd { self.inner.as_raw_fd() @@ -296,6 +306,10 @@ impl AsRawFd for Pipe { } impl AsRawFd for PollPipe { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_fd(&self) -> RawFd { self.stream.inner.as_raw_fd() @@ -303,6 +317,10 @@ impl AsRawFd for PollPipe { } impl IntoRawFd for Pipe { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_fd(self) -> RawFd { let Self { handle, inner } = self; @@ -312,6 +330,10 @@ impl IntoRawFd for Pipe { } impl IntoRawFd for PollPipe { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_fd(self) -> RawFd { self.stream.into_raw_fd() @@ -319,6 +341,10 @@ impl IntoRawFd for PollPipe { } impl<'a> AsInnerRawHandle<'a> for Pipe { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = ">") + )] #[inline] fn as_inner_raw_handle(&'a self) -> &'a InnerRawHandle { &self.handle @@ -326,6 +352,10 @@ impl<'a> AsInnerRawHandle<'a> for Pipe { } impl<'a> AsInnerRawHandle<'a> for PollPipe { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = ">") + )] #[inline] fn as_inner_raw_handle(&'a self) -> &'a InnerRawHandle { self.stream.as_inner_raw_handle() @@ -333,6 +363,10 @@ impl<'a> AsInnerRawHandle<'a> for PollPipe { } impl AsyncRead for Pipe { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn read(&mut self, buf: B) -> (Result, B) { let handle = &self.handle; @@ -341,6 +375,10 @@ impl AsyncRead for Pipe { (result, op.take_bufs()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn read_vectored( &mut self, @@ -357,6 +395,10 @@ impl AsyncRead for Pipe { } impl TokioAsyncRead for PollPipe { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_read( self: Pin<&mut Self>, @@ -391,6 +433,10 @@ impl TokioAsyncRead for PollPipe { } impl AsyncWrite for Pipe { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn write(&mut self, buf: B) -> (Result, B) { let handle = &self.handle; @@ -399,11 +445,19 @@ impl AsyncWrite for Pipe { (result, op.take_bufs()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn flush(&mut self) -> Result<(), io::Error> { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn write_vectored(&mut self, bufs: B) -> (Result, B) { if bufs.is_empty() { @@ -417,6 +471,10 @@ impl AsyncWrite for Pipe { } impl TokioAsyncWrite for PollPipe { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_write( self: Pin<&mut Self>, @@ -432,6 +490,10 @@ impl TokioAsyncWrite for PollPipe { this.stream.handle.poll_op_poll(cx, &mut op) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_write_vectored( self: Pin<&mut Self>, @@ -450,16 +512,28 @@ impl TokioAsyncWrite for PollPipe { this.stream.handle.poll_op_poll(cx, &mut op) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn is_write_vectored(&self) -> bool { true } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { Poll::Ready(Ok(())) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { Poll::Ready(Ok(())) diff --git a/src/vibeio/io/splice.rs b/src/vibeio/io/splice.rs index e1900cd..fff4dd8 100644 --- a/src/vibeio/io/splice.rs +++ b/src/vibeio/io/splice.rs @@ -18,6 +18,7 @@ use mio::Interest; use crate::vibeio::{fd_inner::InnerRawHandle, io::AsInnerRawHandle, op::SpliceOp}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Transfer data from one file descriptor to another using `splice`. /// /// This function uses the kernel's `splice` system call to transfer data @@ -49,6 +50,7 @@ pub async fn splice<'a, 'b>( result } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Transfer exactly `len` bytes from one file descriptor to another using `splice`. /// /// This function calls `splice()` repeatedly until `len` bytes have been transferred @@ -71,6 +73,7 @@ pub async fn splice_exact<'a, 'b>( Ok(total) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Transfer data from a file to a socket using `sendfile` semantics. /// /// This function implements `sendfile`-like behavior using `splice` with an @@ -99,6 +102,7 @@ pub async fn sendfile_exact<'a, 'b>( .await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] async fn retry_splice(mut transfer: impl FnMut() -> F) -> std::io::Result where F: std::future::Future>, @@ -111,6 +115,7 @@ where } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] async fn transfer_batches( len: u64, mut fill: Fill, @@ -153,6 +158,10 @@ struct WriteOwnedFd { } impl WriteOwnedFd { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "WriteOwnedFd") + )] fn new(writer: OwnedFd) -> std::io::Result { let handle = InnerRawHandle::new(writer.as_raw_fd(), Interest::WRITABLE)?; crate::vibeio::fd_inner::set_nonblocking(writer.as_raw_fd(), !handle.uses_completion())?; @@ -164,6 +173,10 @@ impl WriteOwnedFd { } impl<'a> AsInnerRawHandle<'a> for WriteOwnedFd { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = ">") + )] #[inline] fn as_inner_raw_handle(&'a self) -> &'a InnerRawHandle { &self.handle diff --git a/src/vibeio/io/stdio.rs b/src/vibeio/io/stdio.rs index e6fe128..ce43191 100644 --- a/src/vibeio/io/stdio.rs +++ b/src/vibeio/io/stdio.rs @@ -38,64 +38,75 @@ pub struct Stderr { _private: (), } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Get an async-aware stdin reader. #[inline] pub fn stdin() -> Stdin { Stdin { _private: () } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Get an async-aware stdout writer. #[inline] pub fn stdout() -> Stdout { Stdout { _private: () } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Get an async-aware stderr writer. #[inline] pub fn stderr() -> Stderr { Stderr { _private: () } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn read_stdin_blocking(buf: &mut [u8]) -> io::Result { let mut stdin = std::io::stdin(); stdin.read(buf) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn write_stdout_blocking(buf: &[u8]) -> io::Result { let mut stdout = std::io::stdout(); stdout.write(buf) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn write_stderr_blocking(buf: &[u8]) -> io::Result { let mut stderr = std::io::stderr(); stderr.write(buf) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn flush_stdout_blocking() -> io::Result<()> { let mut stdout = std::io::stdout(); stdout.flush() } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn flush_stderr_blocking() -> io::Result<()> { let mut stderr = std::io::stderr(); stderr.flush() } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn blocking_pool_io_error() -> io::Error { io::Error::other("can't spawn blocking task for stdio I/O") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[inline] async fn read_in_blocking_pool(buf: B) -> (io::Result, B) { stdio_in_blocking_pool(buf, |buf| read_into_buf(buf, read_stdin_blocking)).await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] async fn stdio_in_blocking_pool( buf: B, operation: impl FnOnce(&mut B) -> io::Result + Send + 'static, @@ -107,11 +118,13 @@ async fn stdio_in_blocking_pool( ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[inline] async fn write_stdout_in_blocking_pool(buf: B) -> (io::Result, B) { stdio_in_blocking_pool(buf, |buf| write_stdout_blocking(iobuf_to_slice(buf))).await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[inline] async fn write_stderr_in_blocking_pool(buf: B) -> (io::Result, B) { stdio_in_blocking_pool(buf, |buf| write_stderr_blocking(iobuf_to_slice(buf))).await @@ -216,6 +229,7 @@ mod tests { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[inline] async fn flush_stdout_in_blocking_pool() -> io::Result<()> { crate::vibeio::spawn_blocking(flush_stdout_blocking) @@ -223,6 +237,7 @@ async fn flush_stdout_in_blocking_pool() -> io::Result<()> { .map_err(|_| blocking_pool_io_error())? } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[inline] async fn flush_stderr_in_blocking_pool() -> io::Result<()> { crate::vibeio::spawn_blocking(flush_stderr_blocking) @@ -231,6 +246,10 @@ async fn flush_stderr_in_blocking_pool() -> io::Result<()> { } impl AsyncRead for Stdin { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn read(&mut self, mut buf: B) -> (Result, B) { if buf.buf_capacity() == 0 { @@ -247,6 +266,10 @@ impl AsyncRead for Stdin { } impl AsyncWrite for Stdout { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn write(&mut self, buf: B) -> (Result, B) { if buf.buf_len() == 0 { @@ -261,6 +284,10 @@ impl AsyncWrite for Stdout { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn flush(&mut self) -> Result<(), io::Error> { if current_driver().is_some() { @@ -272,6 +299,10 @@ impl AsyncWrite for Stdout { } impl AsyncWrite for Stderr { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn write(&mut self, buf: B) -> (Result, B) { if buf.buf_len() == 0 { @@ -286,6 +317,10 @@ impl AsyncWrite for Stderr { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn flush(&mut self) -> Result<(), io::Error> { if current_driver().is_some() { diff --git a/src/vibeio/io/util.rs b/src/vibeio/io/util.rs index d72af7e..dc451d8 100644 --- a/src/vibeio/io/util.rs +++ b/src/vibeio/io/util.rs @@ -21,6 +21,7 @@ use super::{AsyncRead, AsyncWrite}; use crate::vibeio::io::IoBufMut; use crate::vibeio::io::{IoBuf, IoBufWithCursor}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Copy data from a reader to a writer. /// /// This function reads from `reader` and writes to `writer` until EOF is reached. @@ -114,6 +115,7 @@ pub struct WriteHalf { inner: Arc>, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Split an object implementing both `AsyncRead` and `AsyncWrite` into two /// independently usable halves. /// @@ -144,6 +146,7 @@ impl ReadHalf where T: AsyncRead + AsyncWrite + 'static, { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "ReadHalf"))] /// Consume the half and return the shared inner `Arc>`. pub fn into_inner(self) -> Arc> { self.inner @@ -154,6 +157,7 @@ impl WriteHalf where T: AsyncRead + AsyncWrite + 'static, { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "WriteHalf"))] /// Consume the half and return the shared inner `Arc>`. pub fn into_inner(self) -> Arc> { self.inner @@ -164,6 +168,10 @@ impl AsyncRead for ReadHalf where T: AsyncRead + AsyncWrite + 'static, { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] async fn read( &mut self, buf: B, @@ -173,6 +181,10 @@ where (*guard).read(buf).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] async fn read_vectored( &mut self, bufs: B, @@ -185,6 +197,10 @@ impl AsyncWrite for WriteHalf where T: AsyncRead + AsyncWrite + 'static, { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] async fn write( &mut self, buf: B, @@ -193,6 +209,10 @@ where (*guard).write(buf).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] async fn write_vectored( &mut self, bufs: B, @@ -200,6 +220,10 @@ where self.inner.lock().await.write_vectored(bufs).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] async fn flush(&mut self) -> Result<(), io::Error> { let mut guard = self.inner.lock().await; (*guard).flush().await @@ -207,6 +231,10 @@ where } impl AsyncRead for Box { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn read_vectored( &mut self, @@ -214,6 +242,10 @@ impl AsyncRead for Box { ) -> (io::Result, B) { (**self).read_vectored(bufs).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn read( &mut self, @@ -224,6 +256,10 @@ impl AsyncRead for Box { } impl AsyncRead for &mut R { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "<& mut R as AsyncRead>", future = true) + )] #[inline] async fn read_vectored( &mut self, @@ -231,6 +267,10 @@ impl AsyncRead for &mut R { ) -> (io::Result, B) { (**self).read_vectored(bufs).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "<& mut R as AsyncRead>", future = true) + )] #[inline] async fn read( &mut self, @@ -241,6 +281,10 @@ impl AsyncRead for &mut R { } impl AsyncWrite for Box { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn write_vectored( &mut self, @@ -248,6 +292,10 @@ impl AsyncWrite for Box { ) -> (io::Result, B) { (**self).write_vectored(bufs).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn write( &mut self, @@ -256,6 +304,10 @@ impl AsyncWrite for Box { (**self).write(buf).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn flush(&mut self) -> Result<(), std::io::Error> { (**self).flush().await @@ -263,6 +315,10 @@ impl AsyncWrite for Box { } impl AsyncWrite for &mut W { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "<& mut W as AsyncWrite>", future = true) + )] #[inline] async fn write_vectored( &mut self, @@ -270,6 +326,10 @@ impl AsyncWrite for &mut W { ) -> (io::Result, B) { (**self).write_vectored(bufs).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "<& mut W as AsyncWrite>", future = true) + )] #[inline] async fn write( &mut self, @@ -278,12 +338,17 @@ impl AsyncWrite for &mut W { (**self).write(buf).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "<& mut W as AsyncWrite>", future = true) + )] #[inline] async fn flush(&mut self) -> Result<(), std::io::Error> { (**self).flush().await } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Copy bytes concurrently in both directions, shutting down each destination's /// write half when its source reaches EOF. /// diff --git a/src/vibeio/net/mod.rs b/src/vibeio/net/mod.rs index 32a5453..ce355a1 100644 --- a/src/vibeio/net/mod.rs +++ b/src/vibeio/net/mod.rs @@ -38,6 +38,7 @@ //! See "Unix socket exchange and path cleanup" in //! `tools/vibeio-check/EXAMPLES.md` for an executable, Unix-gated example. +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn try_io_ready( ready: &std::cell::RefCell, diff --git a/src/vibeio/net/tcp/listener.rs b/src/vibeio/net/tcp/listener.rs index 5335e6a..d6fde3e 100644 --- a/src/vibeio/net/tcp/listener.rs +++ b/src/vibeio/net/tcp/listener.rs @@ -24,6 +24,7 @@ use mio::Interest; use crate::vibeio::op::AcceptOp; use crate::vibeio::{fd_inner::InnerRawHandle, net::TcpStream}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn bind_one(address: SocketAddr) -> Result { let socket = socket2::Socket::new( socket2::Domain::for_address(address), @@ -70,6 +71,10 @@ pub struct TcpListener { } impl TcpListener { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TcpListener") + )] /// Creates a new `TcpListener` which will be bound to the specified address. /// /// This is the async version of [`std::net::TcpListener::bind`]. @@ -96,6 +101,10 @@ impl TcpListener { .unwrap_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "no addresses"))) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TcpListener") + )] /// Creates a new `TcpListener` from a standard library `TcpListener`. /// /// # Errors @@ -114,6 +123,10 @@ impl TcpListener { Ok(Self { inner, handle }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TcpListener") + )] /// Creates a new `TcpListener` from a standard library `TcpListener` in poll mode. /// /// This could be useful when using cloned `TcpListener` on Windows. @@ -133,6 +146,10 @@ impl TcpListener { Ok(Self { inner, handle }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TcpListener") + )] /// Returns the local address of this listener. /// /// # Errors @@ -143,6 +160,10 @@ impl TcpListener { self.inner.local_addr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TcpListener", future = true) + )] /// Accepts a new incoming connection from this listener. /// /// This is the async version of [`std::net::TcpListener::accept`]. @@ -163,6 +184,10 @@ impl TcpListener { #[cfg(unix)] impl AsRawFd for TcpListener { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_fd(&self) -> RawFd { self.inner.as_raw_fd() @@ -171,6 +196,10 @@ impl AsRawFd for TcpListener { #[cfg(unix)] impl IntoRawFd for TcpListener { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_fd(self) -> RawFd { let Self { handle, inner } = self; @@ -181,6 +210,10 @@ impl IntoRawFd for TcpListener { #[cfg(windows)] impl AsRawSocket for TcpListener { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_socket(&self) -> RawSocket { self.inner.as_raw_socket() @@ -189,6 +222,10 @@ impl AsRawSocket for TcpListener { #[cfg(windows)] impl IntoRawSocket for TcpListener { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_socket(self) -> RawSocket { let Self { handle, inner } = self; diff --git a/src/vibeio/net/tcp/stream.rs b/src/vibeio/net/tcp/stream.rs index 7dcdc9f..cedf90a 100644 --- a/src/vibeio/net/tcp/stream.rs +++ b/src/vibeio/net/tcp/stream.rs @@ -44,6 +44,7 @@ use crate::vibeio::{ io::{AsyncRead, AsyncWrite}, }; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] fn new_socket( address: SocketAddr, @@ -57,6 +58,7 @@ fn new_socket( Ok((socket.into(), raw_addr, raw_addr_len)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn new_socket( address: SocketAddr, @@ -109,6 +111,10 @@ pub struct PollTcpStream { } impl TcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TcpStream", future = true) + )] /// Connects to the specified address. /// /// This is the async version of [`std::net::TcpStream::connect`]. @@ -134,6 +140,10 @@ impl TcpStream { .unwrap_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "no addresses"))) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TcpStream", future = true) + )] #[inline] async fn connect_one(address: SocketAddr) -> Result { let (inner, raw_addr, raw_addr_len) = new_socket(address)?; @@ -146,6 +156,7 @@ impl TcpStream { Ok(stream) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TcpStream"))] /// Returns the local address of this connection. /// /// # Errors @@ -156,6 +167,7 @@ impl TcpStream { self.inner.local_addr() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TcpStream"))] /// Returns the remote address of this connection. /// /// # Errors @@ -166,6 +178,7 @@ impl TcpStream { self.inner.peer_addr() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TcpStream"))] /// Returns the current state of the TCP_NODELAY option for this socket. /// /// # Errors @@ -176,6 +189,7 @@ impl TcpStream { self.inner.nodelay() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TcpStream"))] /// Sets the value of the TCP_NODELAY option for this socket. /// /// When set, this disables the Nagle algorithm, which means that small @@ -189,6 +203,7 @@ impl TcpStream { self.inner.set_nodelay(nodelay) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TcpStream"))] /// Shuts down the connection. #[inline] pub fn shutdown(&self, how: Shutdown) -> Result<(), io::Error> { @@ -199,6 +214,10 @@ impl TcpStream { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "TcpStream", future = true) + )] /// Peeks at data from the socket without removing it from the buffer. /// /// # Errors @@ -212,6 +231,7 @@ impl TcpStream { (result, op.take_bufs()) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TcpStream"))] /// Creates a new `TcpStream` from a standard library `TcpStream`. /// /// # Errors @@ -222,6 +242,7 @@ impl TcpStream { Self::from_std_with_mode(inner, RegistrationMode::Completion) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TcpStream"))] /// Creates a new `TcpStream` from a standard library `TcpStream` with a specific registration mode. #[inline] pub(crate) fn from_std_with_mode( @@ -231,6 +252,7 @@ impl TcpStream { Self::from_shared_with_mode(Arc::new(inner), mode) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TcpStream"))] /// Creates a stream registration that shares ownership of a standard TCP /// stream with another I/O path. #[inline] @@ -241,6 +263,7 @@ impl TcpStream { Self::from_shared_with_mode(inner, mode) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TcpStream"))] #[inline] fn from_shared_with_mode( inner: Arc, @@ -262,6 +285,7 @@ impl TcpStream { Ok(Self { inner, handle }) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TcpStream"))] /// Converts this stream into a poll-only variant. /// /// The returned `PollTcpStream` will always use readiness-based I/O. @@ -281,6 +305,10 @@ impl TcpStream { } impl PollTcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollTcpStream", future = true) + )] /// Connects to the specified address using poll-based I/O. #[inline] pub async fn connect(address: impl ToSocketAddrs) -> Result { @@ -296,6 +324,10 @@ impl PollTcpStream { .unwrap_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "no addresses"))) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollTcpStream", future = true) + )] #[inline] async fn connect_one(address: SocketAddr) -> Result { let (inner, raw_addr, raw_addr_len) = new_socket(address)?; @@ -308,6 +340,10 @@ impl PollTcpStream { Ok(stream) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollTcpStream") + )] /// Creates a new `PollTcpStream` from a standard library `TcpStream`. #[inline] pub fn from_std(inner: std::net::TcpStream) -> Result { @@ -318,6 +354,10 @@ impl PollTcpStream { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollTcpStream") + )] /// Creates a poll stream that shares ownership of a standard TCP stream. #[inline] pub fn from_shared(inner: Arc) -> Result { @@ -328,12 +368,20 @@ impl PollTcpStream { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollTcpStream") + )] /// Converts this poll stream into an adaptive `TcpStream`. #[inline] pub fn into_adaptive(self) -> TcpStream { self.stream } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollTcpStream") + )] /// Converts this poll stream into a completion-based `TcpStream`. #[inline] pub fn into_completion(self) -> Result { @@ -345,36 +393,60 @@ impl PollTcpStream { Ok(stream) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollTcpStream") + )] /// Returns the local address of this connection. #[inline] pub fn local_addr(&self) -> Result { self.stream.local_addr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollTcpStream") + )] /// Returns the remote address of this connection. #[inline] pub fn peer_addr(&self) -> Result { self.stream.peer_addr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollTcpStream") + )] /// Returns the current state of the TCP_NODELAY option for this socket. #[inline] pub fn nodelay(&self) -> Result { self.stream.nodelay() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollTcpStream") + )] /// Sets the value of the TCP_NODELAY option for this socket. #[inline] pub fn set_nodelay(&self, nodelay: bool) -> Result<(), io::Error> { self.stream.set_nodelay(nodelay) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollTcpStream") + )] /// Shuts down the connection. #[inline] pub fn shutdown(&self, how: Shutdown) -> Result<(), io::Error> { self.stream.shutdown(how) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollTcpStream", future = true) + )] /// Peeks at data from the socket without removing it from the buffer. /// /// This method uses readiness-based I/O and is compatible with `tokio::io`. @@ -392,6 +464,10 @@ impl PollTcpStream { .await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollTcpStream") + )] /// Tries to perform an I/O operation on the socket, returning an error if it is not ready. #[inline] pub fn try_io_readable(&self, io: Io) -> io::Result @@ -401,6 +477,10 @@ impl PollTcpStream { crate::vibeio::net::try_io_ready(&self.read_ready, "read not ready", io) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollTcpStream") + )] /// Tries to perform an I/O operation on the socket, returning an error if it is not ready. #[inline] pub fn try_io_writable(&self, io: Io) -> io::Result @@ -413,6 +493,10 @@ impl PollTcpStream { #[cfg(unix)] impl AsRawFd for TcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_fd(&self) -> RawFd { self.inner.as_raw_fd() @@ -421,6 +505,10 @@ impl AsRawFd for TcpStream { #[cfg(unix)] impl AsRawFd for PollTcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_fd(&self) -> RawFd { self.stream.inner.as_raw_fd() @@ -429,6 +517,10 @@ impl AsRawFd for PollTcpStream { #[cfg(unix)] impl IntoRawFd for TcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_fd(self) -> RawFd { let Self { handle, inner } = self; @@ -445,6 +537,10 @@ impl IntoRawFd for TcpStream { #[cfg(unix)] impl IntoRawFd for PollTcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_fd(self) -> RawFd { self.stream.into_raw_fd() @@ -453,6 +549,10 @@ impl IntoRawFd for PollTcpStream { #[cfg(windows)] impl AsRawSocket for TcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_socket(&self) -> RawSocket { self.inner.as_raw_socket() @@ -461,6 +561,10 @@ impl AsRawSocket for TcpStream { #[cfg(windows)] impl IntoRawSocket for TcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_socket(self) -> RawSocket { let Self { handle, inner } = self; @@ -477,6 +581,10 @@ impl IntoRawSocket for TcpStream { #[cfg(windows)] impl AsRawSocket for PollTcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_socket(&self) -> RawSocket { self.stream.as_raw_socket() @@ -485,6 +593,10 @@ impl AsRawSocket for PollTcpStream { #[cfg(windows)] impl IntoRawSocket for PollTcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_socket(self) -> RawSocket { self.stream.into_raw_socket() @@ -492,6 +604,10 @@ impl IntoRawSocket for PollTcpStream { } impl<'a> AsInnerRawHandle<'a> for TcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = ">") + )] #[inline] fn as_inner_raw_handle(&'a self) -> &'a InnerRawHandle { &self.handle @@ -499,6 +615,10 @@ impl<'a> AsInnerRawHandle<'a> for TcpStream { } impl<'a> AsInnerRawHandle<'a> for PollTcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = ">") + )] #[inline] fn as_inner_raw_handle(&'a self) -> &'a InnerRawHandle { self.stream.as_inner_raw_handle() @@ -506,6 +626,10 @@ impl<'a> AsInnerRawHandle<'a> for PollTcpStream { } impl AsyncRead for TcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn read(&mut self, buf: B) -> (Result, B) { let handle = &self.handle; @@ -514,6 +638,10 @@ impl AsyncRead for TcpStream { (result, op.take_bufs()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn read_vectored( &mut self, @@ -530,6 +658,10 @@ impl AsyncRead for TcpStream { } impl TokioAsyncRead for PollTcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_read( self: Pin<&mut Self>, @@ -564,6 +696,10 @@ impl TokioAsyncRead for PollTcpStream { } impl AsyncWrite for TcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn write(&mut self, buf: B) -> (Result, B) { let handle = &self.handle; @@ -572,11 +708,19 @@ impl AsyncWrite for TcpStream { (result, op.take_bufs()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn flush(&mut self) -> Result<(), io::Error> { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn write_vectored(&mut self, bufs: B) -> (Result, B) { if bufs.is_empty() { @@ -590,6 +734,10 @@ impl AsyncWrite for TcpStream { } impl TokioAsyncWrite for PollTcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_write( self: Pin<&mut Self>, @@ -605,6 +753,10 @@ impl TokioAsyncWrite for PollTcpStream { this.stream.handle.poll_op_poll(cx, &mut op) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_write_vectored( self: Pin<&mut Self>, @@ -623,16 +775,28 @@ impl TokioAsyncWrite for PollTcpStream { this.stream.handle.poll_op_poll(cx, &mut op) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn is_write_vectored(&self) -> bool { true } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { Poll::Ready(Ok(())) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { Poll::Ready(self.get_mut().shutdown(Shutdown::Write)) @@ -640,6 +804,10 @@ impl TokioAsyncWrite for PollTcpStream { } impl AsyncReadPoll for PollTcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_readable(&self, cx: &mut std::task::Context) -> std::task::Poll> { if *self.read_ready.borrow() { @@ -655,6 +823,10 @@ impl AsyncReadPoll for PollTcpStream { } impl AsyncWritePoll for PollTcpStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_writable(&self, cx: &mut std::task::Context) -> std::task::Poll> { if *self.write_ready.borrow() { diff --git a/src/vibeio/net/udp.rs b/src/vibeio/net/udp.rs index 4cd25ae..c28b0e6 100644 --- a/src/vibeio/net/udp.rs +++ b/src/vibeio/net/udp.rs @@ -35,6 +35,7 @@ use crate::vibeio::io::{ use crate::vibeio::op::{ConnectOp, socket_addr_to_raw}; use crate::vibeio::op::{ReadinessOp, RecvOp, RecvfromOp, SendOp, SendtoOp}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[cfg(unix)] #[inline] async fn connect_one(handle: &InnerRawHandle, address: SocketAddr) -> Result<(), io::Error> { @@ -68,6 +69,7 @@ pub struct UdpSocket { } impl UdpSocket { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Creates a new `UdpSocket` which will be bound to the specified address. /// /// This is the async version of [`std::net::UdpSocket::bind`]. @@ -85,6 +87,7 @@ impl UdpSocket { Self::from_std(inner) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Creates a new `UdpSocket` from a standard library `UdpSocket`. /// /// # Errors @@ -95,6 +98,7 @@ impl UdpSocket { Self::from_std_with_mode(inner, RegistrationMode::Completion) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Creates a new `UdpSocket` from a standard library `UdpSocket` with a specific registration mode. #[inline] pub(crate) fn from_std_with_mode( @@ -118,6 +122,7 @@ impl UdpSocket { Ok(Self { inner, handle }) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Converts this socket into a poll-only variant. /// /// The returned `PollUdpSocket` will always use readiness-based I/O. @@ -135,6 +140,7 @@ impl UdpSocket { }) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Converts this `UdpSocket` into the standard library `UdpSocket`. #[inline] pub fn into_std(self) -> StdUdpSocket { @@ -143,6 +149,7 @@ impl UdpSocket { inner } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Returns the local address of this socket. /// /// # Errors @@ -153,6 +160,7 @@ impl UdpSocket { self.inner.local_addr() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Returns the remote address of this socket. /// /// # Errors @@ -163,6 +171,10 @@ impl UdpSocket { self.inner.peer_addr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UdpSocket", future = true) + )] /// Connects this UDP socket to a remote address. /// /// This is the async version of [`std::net::UdpSocket::connect`]. @@ -195,6 +207,10 @@ impl UdpSocket { .unwrap_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "no addresses"))) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UdpSocket", future = true) + )] /// Receives a single datagram message. /// /// This is the async version of [`std::net::UdpSocket::recv`]. @@ -212,6 +228,10 @@ impl UdpSocket { (result, op.take_bufs()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UdpSocket", future = true) + )] /// Receives a single datagram message, returning the sender's address. /// /// This is the async version of [`std::net::UdpSocket::recv_from`]. @@ -232,6 +252,10 @@ impl UdpSocket { (result, op.take_bufs()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UdpSocket", future = true) + )] /// Sends data on a connected socket. /// /// This is the async version of [`std::net::UdpSocket::send`]. @@ -249,6 +273,10 @@ impl UdpSocket { (result, op.take_bufs()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UdpSocket", future = true) + )] /// Sends data to the specified address. /// /// This is the async version of [`std::net::UdpSocket::send_to`]. @@ -290,6 +318,10 @@ impl UdpSocket { ) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UdpSocket", future = true) + )] /// Receives data without removing it from the socket's receive queue. /// /// This is the async version of [`std::net::UdpSocket::peek`]. @@ -307,6 +339,10 @@ impl UdpSocket { (result, op.take_bufs()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UdpSocket", future = true) + )] /// Receives data without removing it from the socket's receive queue, /// returning the sender's address. /// @@ -328,6 +364,7 @@ impl UdpSocket { (result, op.take_bufs()) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Returns a new `UdpSocket` that shares the same underlying file descriptor. /// /// # Errors @@ -338,6 +375,7 @@ impl UdpSocket { Self::from_std(self.inner.try_clone()?) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Sets the broadcast flag. /// /// When set, the socket can send broadcast packets. @@ -350,6 +388,7 @@ impl UdpSocket { self.inner.set_broadcast(broadcast) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Returns the current value of the broadcast flag. /// /// # Errors @@ -360,6 +399,7 @@ impl UdpSocket { self.inner.broadcast() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Sets the time-to-live (TTL) value. /// /// This controls how many hops a packet can traverse before being discarded. @@ -372,6 +412,7 @@ impl UdpSocket { self.inner.set_ttl(ttl) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Returns the current TTL value. /// /// # Errors @@ -382,6 +423,7 @@ impl UdpSocket { self.inner.ttl() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Sets the multicast loop flag for IPv4. /// /// When set, multicast packets are looped back to the local socket. @@ -394,6 +436,7 @@ impl UdpSocket { self.inner.set_multicast_loop_v4(multicast_loop_v4) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Returns the current IPv4 multicast loop flag. /// /// # Errors @@ -404,6 +447,7 @@ impl UdpSocket { self.inner.multicast_loop_v4() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Sets the multicast TTL for IPv4. /// /// This controls how many hops multicast packets can traverse. @@ -416,6 +460,7 @@ impl UdpSocket { self.inner.set_multicast_ttl_v4(multicast_ttl_v4) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Returns the current IPv4 multicast TTL. /// /// # Errors @@ -426,6 +471,7 @@ impl UdpSocket { self.inner.multicast_ttl_v4() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Sets the multicast loop flag for IPv6. /// /// When set, multicast packets are looped back to the local socket. @@ -438,6 +484,7 @@ impl UdpSocket { self.inner.set_multicast_loop_v6(multicast_loop_v6) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Returns the current IPv6 multicast loop flag. /// /// # Errors @@ -448,6 +495,7 @@ impl UdpSocket { self.inner.multicast_loop_v6() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Joins a multicast group for IPv4. /// /// # Errors @@ -462,6 +510,7 @@ impl UdpSocket { self.inner.join_multicast_v4(multiaddr, interface) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Joins a multicast group for IPv6. /// /// # Errors @@ -472,6 +521,7 @@ impl UdpSocket { self.inner.join_multicast_v6(multiaddr, interface) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Leaves a multicast group for IPv4. /// /// # Errors @@ -486,6 +536,7 @@ impl UdpSocket { self.inner.leave_multicast_v4(multiaddr, interface) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Leaves a multicast group for IPv6. /// /// # Errors @@ -500,6 +551,7 @@ impl UdpSocket { self.inner.leave_multicast_v6(multiaddr, interface) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Takes the pending error from the socket. /// /// # Errors @@ -510,6 +562,7 @@ impl UdpSocket { self.inner.take_error() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Sets the read timeout for the socket. /// /// # Errors @@ -520,6 +573,7 @@ impl UdpSocket { self.inner.set_read_timeout(dur) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Sets the write timeout for the socket. /// /// # Errors @@ -530,6 +584,7 @@ impl UdpSocket { self.inner.set_write_timeout(dur) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Returns the read timeout for the socket. /// /// # Errors @@ -540,6 +595,7 @@ impl UdpSocket { self.inner.read_timeout() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UdpSocket"))] /// Returns the write timeout for the socket. /// /// # Errors @@ -553,6 +609,10 @@ impl UdpSocket { #[cfg(unix)] impl AsRawFd for UdpSocket { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_fd(&self) -> RawFd { self.inner.as_raw_fd() @@ -561,6 +621,10 @@ impl AsRawFd for UdpSocket { #[cfg(unix)] impl IntoRawFd for UdpSocket { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_fd(self) -> RawFd { self.into_std().into_raw_fd() @@ -569,6 +633,10 @@ impl IntoRawFd for UdpSocket { #[cfg(windows)] impl AsRawSocket for UdpSocket { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_socket(&self) -> RawSocket { self.inner.as_raw_socket() @@ -577,6 +645,10 @@ impl AsRawSocket for UdpSocket { #[cfg(windows)] impl IntoRawSocket for UdpSocket { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_socket(self) -> RawSocket { self.into_std().into_raw_socket() @@ -584,6 +656,10 @@ impl IntoRawSocket for UdpSocket { } impl<'a> AsInnerRawHandle<'a> for UdpSocket { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = ">") + )] #[inline] fn as_inner_raw_handle(&'a self) -> &'a InnerRawHandle { &self.handle @@ -610,6 +686,10 @@ pub struct PollUdpSocket { } impl PollUdpSocket { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Creates a new `PollUdpSocket` bound to the specified address. /// /// # Errors @@ -625,6 +705,10 @@ impl PollUdpSocket { Self::from_std(inner) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Creates a new `PollUdpSocket` from a standard library `UdpSocket`. /// /// # Errors @@ -639,12 +723,20 @@ impl PollUdpSocket { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Converts this poll socket into an adaptive `UdpSocket`. #[inline] pub fn into_adaptive(self) -> UdpSocket { self.socket } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Converts this poll socket into a completion-based `UdpSocket`. /// /// # Errors @@ -660,6 +752,10 @@ impl PollUdpSocket { Ok(socket) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket", future = true) + )] /// Connects this UDP socket to a remote address. /// /// # Errors @@ -673,6 +769,10 @@ impl PollUdpSocket { self.socket.connect(address).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Returns the local address of this socket. /// /// # Errors @@ -683,6 +783,10 @@ impl PollUdpSocket { self.socket.local_addr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Returns the remote address of this socket. /// /// # Errors @@ -693,6 +797,10 @@ impl PollUdpSocket { self.socket.peer_addr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket", future = true) + )] /// Receives a single datagram message. /// /// This is the poll-based version of [`UdpSocket::recv`]. @@ -701,6 +809,10 @@ impl PollUdpSocket { self.socket.recv(buf).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket", future = true) + )] /// Receives a single datagram message, returning the sender's address. /// /// This is the poll-based version of [`UdpSocket::recv_from`]. @@ -712,6 +824,10 @@ impl PollUdpSocket { self.socket.recv_from(buf).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket", future = true) + )] /// Sends data on a connected socket. /// /// This is the poll-based version of [`UdpSocket::send`]. @@ -720,6 +836,10 @@ impl PollUdpSocket { self.socket.send(buf).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket", future = true) + )] /// Sends data to the specified address. /// /// This is the poll-based version of [`UdpSocket::send_to`]. @@ -732,6 +852,10 @@ impl PollUdpSocket { self.socket.send_to(buf, address).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket", future = true) + )] /// Receives data without removing it from the socket's receive queue. /// /// This is the poll-based version of [`UdpSocket::peek`]. @@ -740,6 +864,10 @@ impl PollUdpSocket { self.socket.peek(buf).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket", future = true) + )] /// Receives data without removing it from the socket's receive queue, /// returning the sender's address. /// @@ -752,6 +880,10 @@ impl PollUdpSocket { self.socket.peek_from(buf).await } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Returns a new `PollUdpSocket` that shares the same underlying file descriptor. /// /// # Errors @@ -766,6 +898,10 @@ impl PollUdpSocket { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Sets the broadcast flag. /// /// When set, the socket can send broadcast packets. @@ -778,6 +914,10 @@ impl PollUdpSocket { self.socket.set_broadcast(broadcast) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Returns the current value of the broadcast flag. /// /// # Errors @@ -788,6 +928,10 @@ impl PollUdpSocket { self.socket.broadcast() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Sets the time-to-live (TTL) value. /// /// This controls how many hops a packet can traverse before being discarded. @@ -800,6 +944,10 @@ impl PollUdpSocket { self.socket.set_ttl(ttl) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Returns the current TTL value. /// /// # Errors @@ -810,6 +958,10 @@ impl PollUdpSocket { self.socket.ttl() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Sets the multicast loop flag for IPv4. /// /// When set, multicast packets are looped back to the local socket. @@ -822,6 +974,10 @@ impl PollUdpSocket { self.socket.set_multicast_loop_v4(multicast_loop_v4) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Returns the current IPv4 multicast loop flag. /// /// # Errors @@ -832,6 +988,10 @@ impl PollUdpSocket { self.socket.multicast_loop_v4() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Sets the multicast TTL for IPv4. /// /// This controls how many hops multicast packets can traverse. @@ -844,6 +1004,10 @@ impl PollUdpSocket { self.socket.set_multicast_ttl_v4(multicast_ttl_v4) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Returns the current IPv4 multicast TTL. /// /// # Errors @@ -854,6 +1018,10 @@ impl PollUdpSocket { self.socket.multicast_ttl_v4() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Sets the multicast loop flag for IPv6. /// /// When set, multicast packets are looped back to the local socket. @@ -866,6 +1034,10 @@ impl PollUdpSocket { self.socket.set_multicast_loop_v6(multicast_loop_v6) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Returns the current IPv6 multicast loop flag. /// /// # Errors @@ -876,6 +1048,10 @@ impl PollUdpSocket { self.socket.multicast_loop_v6() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Joins a multicast group for IPv4. /// /// # Errors @@ -890,6 +1066,10 @@ impl PollUdpSocket { self.socket.join_multicast_v4(multiaddr, interface) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Joins a multicast group for IPv6. /// /// # Errors @@ -900,6 +1080,10 @@ impl PollUdpSocket { self.socket.join_multicast_v6(multiaddr, interface) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Leaves a multicast group for IPv4. /// /// # Errors @@ -914,6 +1098,10 @@ impl PollUdpSocket { self.socket.leave_multicast_v4(multiaddr, interface) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Leaves a multicast group for IPv6. /// /// # Errors @@ -928,6 +1116,10 @@ impl PollUdpSocket { self.socket.leave_multicast_v6(multiaddr, interface) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Takes the pending error from the socket. /// /// # Errors @@ -938,6 +1130,10 @@ impl PollUdpSocket { self.socket.take_error() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Sets the read timeout for the socket. /// /// # Errors @@ -948,6 +1144,10 @@ impl PollUdpSocket { self.socket.set_read_timeout(dur) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Sets the write timeout for the socket. /// /// # Errors @@ -958,6 +1158,10 @@ impl PollUdpSocket { self.socket.set_write_timeout(dur) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Returns the read timeout for the socket. /// /// # Errors @@ -968,6 +1172,10 @@ impl PollUdpSocket { self.socket.read_timeout() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Returns the write timeout for the socket. /// /// # Errors @@ -978,6 +1186,10 @@ impl PollUdpSocket { self.socket.write_timeout() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Polls to receive a single datagram message from the socket. /// /// This is the poll-based counterpart to [`UdpSocket::recv`]. @@ -996,6 +1208,10 @@ impl PollUdpSocket { handle.poll_op_poll(cx, &mut op) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Polls to receive a single datagram message, returning the sender's address. /// /// This is the poll-based counterpart to [`UdpSocket::recv_from`]. @@ -1014,6 +1230,10 @@ impl PollUdpSocket { handle.poll_op_poll(cx, &mut op) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Polls to send data on a connected socket. /// /// This is the poll-based counterpart to [`UdpSocket::send`]. @@ -1032,6 +1252,10 @@ impl PollUdpSocket { handle.poll_op_poll(cx, &mut op) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Polls to send data to the specified address. /// /// This is the poll-based counterpart to [`UdpSocket::send_to`]. @@ -1052,6 +1276,10 @@ impl PollUdpSocket { handle.poll_op_poll(cx, &mut op) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Polls to peek at data from the socket without removing it. /// /// This is the poll-based counterpart to [`UdpSocket::peek`]. @@ -1070,6 +1298,10 @@ impl PollUdpSocket { handle.poll_op_poll(cx, &mut op) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Polls to peek at data from the socket without removing it, /// returning the sender's address. /// @@ -1090,6 +1322,10 @@ impl PollUdpSocket { handle.poll_op_poll(cx, &mut op) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Tries to perform an I/O operation on the socket, returning an error if it is not ready. #[inline] pub fn try_io_readable(&self, io: Io) -> io::Result @@ -1099,6 +1335,10 @@ impl PollUdpSocket { crate::vibeio::net::try_io_ready(&self.read_ready, "read not ready", io) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUdpSocket") + )] /// Tries to perform an I/O operation on the socket, returning an error if it is not ready. #[inline] pub fn try_io_writable(&self, io: Io) -> io::Result @@ -1110,6 +1350,10 @@ impl PollUdpSocket { } impl AsyncReadPoll for PollUdpSocket { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_readable(&self, cx: &mut Context<'_>) -> Poll> { if *self.read_ready.borrow() { @@ -1125,6 +1369,10 @@ impl AsyncReadPoll for PollUdpSocket { } impl AsyncWritePoll for PollUdpSocket { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_writable(&self, cx: &mut Context<'_>) -> Poll> { if *self.write_ready.borrow() { @@ -1140,6 +1388,10 @@ impl AsyncWritePoll for PollUdpSocket { } impl<'a> AsInnerRawHandle<'a> for PollUdpSocket { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = ">") + )] #[inline] fn as_inner_raw_handle(&'a self) -> &'a InnerRawHandle { self.socket.as_inner_raw_handle() @@ -1148,6 +1400,10 @@ impl<'a> AsInnerRawHandle<'a> for PollUdpSocket { #[cfg(unix)] impl AsRawFd for PollUdpSocket { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_fd(&self) -> RawFd { self.socket.inner.as_raw_fd() @@ -1156,6 +1412,10 @@ impl AsRawFd for PollUdpSocket { #[cfg(unix)] impl IntoRawFd for PollUdpSocket { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_fd(self) -> RawFd { self.socket.into_std().into_raw_fd() @@ -1164,6 +1424,10 @@ impl IntoRawFd for PollUdpSocket { #[cfg(windows)] impl AsRawSocket for PollUdpSocket { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_socket(&self) -> RawSocket { self.socket.inner.as_raw_socket() @@ -1172,6 +1436,10 @@ impl AsRawSocket for PollUdpSocket { #[cfg(windows)] impl IntoRawSocket for PollUdpSocket { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_socket(self) -> RawSocket { self.socket.into_std().into_raw_socket() diff --git a/src/vibeio/net/unix/listener.rs b/src/vibeio/net/unix/listener.rs index 31b02c3..55b020d 100644 --- a/src/vibeio/net/unix/listener.rs +++ b/src/vibeio/net/unix/listener.rs @@ -49,6 +49,10 @@ pub struct UnixListener { } impl UnixListener { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UnixListener") + )] /// Creates a new `UnixListener` which will be bound to the specified path. /// /// Binding is synchronous; the returned listener supports async accepts. @@ -80,6 +84,10 @@ impl UnixListener { Self::from_std(inner) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UnixListener") + )] /// Creates a new `UnixListener` from a standard library `UnixListener`. /// /// # Errors @@ -92,6 +100,10 @@ impl UnixListener { Ok(Self { inner, handle }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UnixListener") + )] /// Returns the local address of this listener. /// /// # Errors @@ -102,6 +114,10 @@ impl UnixListener { self.inner.local_addr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UnixListener", future = true) + )] /// Accepts a new incoming connection from this listener. /// /// This is the async version of [`std::os::unix::net::UnixListener::accept`]. @@ -123,6 +139,10 @@ impl UnixListener { } impl AsRawFd for UnixListener { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_fd(&self) -> RawFd { self.inner.as_raw_fd() @@ -130,6 +150,10 @@ impl AsRawFd for UnixListener { } impl IntoRawFd for UnixListener { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_fd(self) -> RawFd { let Self { handle, inner } = self; diff --git a/src/vibeio/net/unix/stream.rs b/src/vibeio/net/unix/stream.rs index e99601e..17dbb6b 100644 --- a/src/vibeio/net/unix/stream.rs +++ b/src/vibeio/net/unix/stream.rs @@ -39,6 +39,7 @@ use crate::vibeio::{ io::{AsyncRead, AsyncWrite}, }; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn socket_addr_to_raw(path: &Path) -> Result<(libc::sockaddr_un, libc::socklen_t), io::Error> { let bytes = path.as_os_str().as_bytes(); @@ -91,6 +92,7 @@ fn socket_addr_to_raw(path: &Path) -> Result<(libc::sockaddr_un, libc::socklen_t Ok((sockaddr, addr_len)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn new_socket( path: &Path, @@ -140,6 +142,10 @@ pub struct PollUnixStream { } impl UnixStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UnixStream", future = true) + )] /// Connects to the specified Unix domain socket path. /// /// This is the async version of [`std::os::unix::net::UnixStream::connect`]. @@ -163,6 +169,10 @@ impl UnixStream { Ok(stream) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UnixStream") + )] /// Returns the local address of this connection. /// /// # Errors @@ -173,6 +183,10 @@ impl UnixStream { self.inner.local_addr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UnixStream") + )] /// Returns the remote address of this connection. /// /// # Errors @@ -183,6 +197,10 @@ impl UnixStream { self.inner.peer_addr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UnixStream") + )] /// Shuts down the connection. #[inline] pub fn shutdown(&self, how: Shutdown) -> Result<(), io::Error> { @@ -193,6 +211,10 @@ impl UnixStream { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UnixStream") + )] /// Creates a new `UnixStream` from a standard library `UnixStream`. /// /// # Errors @@ -203,6 +225,10 @@ impl UnixStream { Self::from_std_with_mode(inner, RegistrationMode::Completion) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UnixStream") + )] /// Creates a new `UnixStream` from a standard library `UnixStream` with a specific registration mode. #[inline] pub(crate) fn from_std_with_mode( @@ -218,6 +244,10 @@ impl UnixStream { Ok(Self { inner, handle }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UnixStream") + )] /// Converts this stream into a poll-only variant. /// /// The returned `PollUnixStream` will always use readiness-based I/O. @@ -237,6 +267,10 @@ impl UnixStream { } impl PollUnixStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUnixStream", future = true) + )] /// Connects to the specified Unix domain socket path using poll-based I/O. #[inline] pub async fn connect(path: impl AsRef) -> Result { @@ -250,6 +284,10 @@ impl PollUnixStream { Ok(stream) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUnixStream") + )] /// Creates a new `PollUnixStream` from a standard library `UnixStream`. #[inline] pub fn from_std(inner: StdUnixStream) -> Result { @@ -260,12 +298,20 @@ impl PollUnixStream { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUnixStream") + )] /// Converts this poll stream into an adaptive `UnixStream`. #[inline] pub fn into_adaptive(self) -> UnixStream { self.stream } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUnixStream") + )] /// Converts this poll stream into a completion-based `UnixStream`. #[inline] pub fn into_completion(self) -> Result { @@ -277,24 +323,40 @@ impl PollUnixStream { Ok(stream) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUnixStream") + )] /// Returns the local address of this connection. #[inline] pub fn local_addr(&self) -> Result { self.stream.local_addr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUnixStream") + )] /// Returns the remote address of this connection. #[inline] pub fn peer_addr(&self) -> Result { self.stream.peer_addr() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUnixStream") + )] /// Shuts down the connection. #[inline] pub fn shutdown(&self, how: Shutdown) -> Result<(), io::Error> { self.stream.shutdown(how) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUnixStream") + )] /// Tries to perform an I/O operation on the socket, returning an error if it is not ready. #[inline] pub fn try_io_readable(&self, io: Io) -> io::Result @@ -304,6 +366,10 @@ impl PollUnixStream { crate::vibeio::net::try_io_ready(&self.read_ready, "read not ready", io) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PollUnixStream") + )] /// Tries to perform an I/O operation on the socket, returning an error if it is not ready. #[inline] pub fn try_io_writable(&self, io: Io) -> io::Result @@ -315,6 +381,10 @@ impl PollUnixStream { } impl AsRawFd for PollUnixStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_fd(&self) -> RawFd { self.stream.inner.as_raw_fd() @@ -322,6 +392,10 @@ impl AsRawFd for PollUnixStream { } impl IntoRawFd for PollUnixStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_fd(self) -> RawFd { self.stream.into_raw_fd() @@ -329,6 +403,10 @@ impl IntoRawFd for PollUnixStream { } impl TokioAsyncRead for PollUnixStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_read( self: Pin<&mut Self>, @@ -363,6 +441,10 @@ impl TokioAsyncRead for PollUnixStream { } impl TokioAsyncWrite for PollUnixStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_write( self: Pin<&mut Self>, @@ -378,6 +460,10 @@ impl TokioAsyncWrite for PollUnixStream { this.stream.handle.poll_op_poll(cx, &mut op) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_write_vectored( self: Pin<&mut Self>, @@ -396,16 +482,28 @@ impl TokioAsyncWrite for PollUnixStream { this.stream.handle.poll_op_poll(cx, &mut op) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn is_write_vectored(&self) -> bool { true } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { Poll::Ready(Ok(())) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { Poll::Ready(self.get_mut().shutdown(Shutdown::Write)) @@ -413,6 +511,10 @@ impl TokioAsyncWrite for PollUnixStream { } impl UnixStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "UnixStream") + )] /// Creates a new `UnixStream` that is ready to use with readiness-based I/O. /// /// This is useful when you want to create a Unix stream that uses poll-based I/O. @@ -427,6 +529,10 @@ impl UnixStream { } impl AsRawFd for UnixStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_fd(&self) -> RawFd { self.inner.as_raw_fd() @@ -434,6 +540,10 @@ impl AsRawFd for UnixStream { } impl<'a> AsInnerRawHandle<'a> for UnixStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = ">") + )] #[inline] fn as_inner_raw_handle(&'a self) -> &'a InnerRawHandle { &self.handle @@ -441,6 +551,10 @@ impl<'a> AsInnerRawHandle<'a> for UnixStream { } impl<'a> AsInnerRawHandle<'a> for PollUnixStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = ">") + )] #[inline] fn as_inner_raw_handle(&'a self) -> &'a InnerRawHandle { self.stream.as_inner_raw_handle() @@ -448,6 +562,10 @@ impl<'a> AsInnerRawHandle<'a> for PollUnixStream { } impl IntoRawFd for UnixStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_fd(self) -> RawFd { let Self { handle, inner } = self; @@ -457,6 +575,10 @@ impl IntoRawFd for UnixStream { } impl AsyncRead for UnixStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn read(&mut self, buf: B) -> (Result, B) { let handle = &self.handle; @@ -465,6 +587,10 @@ impl AsyncRead for UnixStream { (result, op.take_bufs()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn read_vectored( &mut self, @@ -481,6 +607,10 @@ impl AsyncRead for UnixStream { } impl AsyncWrite for UnixStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn write(&mut self, buf: B) -> (Result, B) { let handle = &self.handle; @@ -489,11 +619,19 @@ impl AsyncWrite for UnixStream { (result, op.take_bufs()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn flush(&mut self) -> Result<(), io::Error> { Ok(()) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn write_vectored(&mut self, bufs: B) -> (Result, B) { if bufs.is_empty() { @@ -507,6 +645,10 @@ impl AsyncWrite for UnixStream { } impl AsyncReadPoll for PollUnixStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_readable(&self, cx: &mut std::task::Context) -> std::task::Poll> { if *self.read_ready.borrow() { @@ -522,6 +664,10 @@ impl AsyncReadPoll for PollUnixStream { } impl AsyncWritePoll for PollUnixStream { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_writable(&self, cx: &mut std::task::Context) -> std::task::Poll> { if *self.write_ready.borrow() { diff --git a/src/vibeio/op/accept.rs b/src/vibeio/op/accept.rs index 310f2a0..7e159bc 100644 --- a/src/vibeio/op/accept.rs +++ b/src/vibeio/op/accept.rs @@ -43,6 +43,7 @@ type OwnedAcceptSocket = OwnedFd; #[cfg(windows)] type OwnedAcceptSocket = OwnedSocket; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn load_accept_ex(socket: SOCKET) -> Result { let mut bytes_returned: u32 = 0; @@ -79,6 +80,7 @@ fn load_accept_ex(socket: SOCKET) -> Result { Ok(accept_ex) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn load_get_accept_ex_sockaddrs( socket: SOCKET, @@ -117,6 +119,7 @@ fn load_get_accept_ex_sockaddrs( Ok(get_accept_ex_sockaddrs) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn listener_socket_family(listener_socket: SOCKET) -> Result { let mut addr = SOCKADDR_STORAGE::default(); @@ -139,6 +142,7 @@ fn listener_socket_family(listener_socket: SOCKET) -> Result { Ok(if address.is_ipv4() { AF_INET } else { AF_INET6 } as i32) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn create_accept_socket(listener_socket: SOCKET) -> Result { let family = listener_socket_family(listener_socket)?; @@ -152,6 +156,7 @@ fn create_accept_socket(listener_socket: SOCKET) -> Result Result<(), io::Error> { // SAFETY: the option value points to a live SOCKET of the exact supplied @@ -176,6 +181,7 @@ const ACCEPTEX_ADDR_LEN: usize = std::mem::size_of::() + 16; #[cfg(windows)] const ACCEPTEX_OUTPUT_BUFFER_LEN: usize = ACCEPTEX_ADDR_LEN * 2; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] fn finish_unix_accept( owned: OwnedFd, @@ -204,6 +210,7 @@ fn finish_unix_accept( Ok((owned, address)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn finish_windows_accept(owned: OwnedSocket) -> io::Result<(OwnedAcceptSocket, SocketAddr)> { let mut peer = SOCKADDR_STORAGE::default(); @@ -224,6 +231,7 @@ fn finish_windows_accept(owned: OwnedSocket) -> io::Result<(OwnedAcceptSocket, S Ok((owned, address)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn last_socket_error() -> io::Error { // SAFETY: reads this thread's Winsock error state without pointer arguments. @@ -246,6 +254,7 @@ pub struct AcceptOp<'a> { } impl<'a> AcceptOp<'a> { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AcceptOp"))] #[inline] pub fn new(handle: &'a InnerRawHandle) -> Self { Self { @@ -266,12 +275,20 @@ impl<'a> AcceptOp<'a> { } impl Op for AcceptOp<'_> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(target_os = "linux")] fn completion_returns_fd(&self) -> bool { true } type Output = (OwnedAcceptSocket, SocketAddr); + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_poll( @@ -353,6 +370,10 @@ impl Op for AcceptOp<'_> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_completion( @@ -477,6 +498,10 @@ impl Op for AcceptOp<'_> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(windows)] #[inline] fn submit_windows(&mut self, overlapped: *mut OVERLAPPED) -> Result<(), io::Error> { @@ -548,6 +573,10 @@ impl Op for AcceptOp<'_> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(target_os = "linux")] #[inline] fn build_completion_entry( @@ -570,6 +599,10 @@ impl Op for AcceptOp<'_> { } impl Drop for AcceptOp<'_> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { diff --git a/src/vibeio/op/accept_unix.rs b/src/vibeio/op/accept_unix.rs index 50974ab..f8498d0 100644 --- a/src/vibeio/op/accept_unix.rs +++ b/src/vibeio/op/accept_unix.rs @@ -19,6 +19,10 @@ pub struct AcceptUnixOp<'a> { } impl<'a> AcceptUnixOp<'a> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "AcceptUnixOp") + )] #[inline] pub fn new(handle: &'a InnerRawHandle) -> Self { Self { @@ -29,12 +33,20 @@ impl<'a> AcceptUnixOp<'a> { } impl Op for AcceptUnixOp<'_> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(target_os = "linux")] fn completion_returns_fd(&self) -> bool { true } type Output = OwnedFd; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_poll( &mut self, @@ -88,6 +100,10 @@ impl Op for AcceptUnixOp<'_> { Poll::Ready(Ok(owned)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_completion( &mut self, @@ -132,6 +148,10 @@ impl Op for AcceptUnixOp<'_> { Poll::Ready(Ok(owned)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(target_os = "linux")] #[inline] fn build_completion_entry( @@ -153,6 +173,10 @@ impl Op for AcceptUnixOp<'_> { } impl Drop for AcceptUnixOp<'_> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { if let Some(completion_token) = self.completion_token { diff --git a/src/vibeio/op/connect.rs b/src/vibeio/op/connect.rs index b69c812..0020fe0 100644 --- a/src/vibeio/op/connect.rs +++ b/src/vibeio/op/connect.rs @@ -27,6 +27,7 @@ use crate::vibeio::fd_inner::InnerRawHandle; use crate::vibeio::fd_inner::RawOsHandle; use crate::vibeio::op::Op; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] fn start_nonblocking_connect( fd: std::os::fd::RawFd, @@ -50,6 +51,7 @@ fn start_nonblocking_connect( Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn start_nonblocking_connect( socket: std::os::windows::io::RawSocket, @@ -71,6 +73,7 @@ fn start_nonblocking_connect( Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn connectex_bind_error(err_code: i32) -> io::Result<()> { // bind documents WSAEINVAL as "already bound". WSAEADDRINUSE instead @@ -83,6 +86,7 @@ fn connectex_bind_error(err_code: i32) -> io::Result<()> { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn ensure_connectex_bound(socket: SOCKET, address: &ConnectAddress) -> Result<(), io::Error> { let AddressStorage::Inet(addr) = &address.storage; @@ -135,6 +139,7 @@ fn ensure_connectex_bound(socket: SOCKET, address: &ConnectAddress) -> Result<() Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn load_connect_ex(socket: SOCKET) -> Result { let mut bytes_returned: u32 = 0; @@ -173,6 +178,7 @@ fn load_connect_ex(socket: SOCKET) -> Result Ok(connect_ex) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn set_connect_context(socket: SOCKET) -> Result<(), io::Error> { // SAFETY: this option requires no payload; null and zero provide none. @@ -218,6 +224,10 @@ struct ConnectAddress { } impl ConnectAddress { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ConnectAddress") + )] fn raw(&self) -> (AddressPointer, AddressLength) { let ptr = match &self.storage { AddressStorage::Inet(addr) => std::ptr::from_ref(addr.as_ref()).cast(), @@ -227,6 +237,10 @@ impl ConnectAddress { (ptr, self.len) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ConnectAddress") + )] fn validate_len(len: AddressLength, capacity: usize) -> io::Result<()> { let len = usize::try_from(len).map_err(|_| { io::Error::new( @@ -257,6 +271,7 @@ pub struct ConnectOp<'a> { } impl<'a> ConnectOp<'a> { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "ConnectOp"))] /// Own aligned address storage; callers cannot submit a dangling raw pointer. pub fn new( handle: &'a InnerRawHandle, @@ -275,6 +290,7 @@ impl<'a> ConnectOp<'a> { )) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "ConnectOp"))] #[cfg(unix)] pub fn new_unix( handle: &'a InnerRawHandle, @@ -297,6 +313,7 @@ impl<'a> ConnectOp<'a> { )) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "ConnectOp"))] fn with_address(handle: &'a InnerRawHandle, addr: ConnectAddress) -> Self { Self { handle, @@ -310,6 +327,7 @@ impl<'a> ConnectOp<'a> { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "ConnectOp"))] #[cfg(any(target_os = "linux", windows, test))] fn address(&self) -> (AddressPointer, AddressLength) { self.addr.as_ref().expect("connect address missing").raw() @@ -319,6 +337,10 @@ impl<'a> ConnectOp<'a> { impl Op for ConnectOp<'_> { type Output = (); + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_poll( @@ -508,6 +530,10 @@ impl Op for ConnectOp<'_> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_completion( @@ -560,6 +586,10 @@ impl Op for ConnectOp<'_> { Poll::Ready(Ok(())) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(windows)] #[inline] fn submit_windows(&mut self, overlapped: *mut OVERLAPPED) -> Result<(), io::Error> { @@ -615,6 +645,10 @@ impl Op for ConnectOp<'_> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(target_os = "linux")] #[inline] fn build_completion_entry( @@ -634,6 +668,10 @@ impl Op for ConnectOp<'_> { } impl Drop for ConnectOp<'_> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { // Retain the original allocation, not a copy: the kernel may still diff --git a/src/vibeio/op/fsync.rs b/src/vibeio/op/fsync.rs index ff00ce1..5720af5 100644 --- a/src/vibeio/op/fsync.rs +++ b/src/vibeio/op/fsync.rs @@ -13,6 +13,7 @@ pub struct FsyncOp<'a> { } impl<'a> FsyncOp<'a> { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "FsyncOp"))] #[inline] pub fn new(handle: &'a InnerRawHandle, data_only: bool) -> Self { Self { @@ -26,6 +27,10 @@ impl<'a> FsyncOp<'a> { impl Op for FsyncOp<'_> { type Output = (); + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_completion( &mut self, @@ -61,6 +66,10 @@ impl Op for FsyncOp<'_> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn build_completion_entry( &mut self, @@ -79,6 +88,10 @@ impl Op for FsyncOp<'_> { } impl Drop for FsyncOp<'_> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { if let Some(completion_token) = self.completion_token { diff --git a/src/vibeio/op/hard_link.rs b/src/vibeio/op/hard_link.rs index d76679e..79acfbd 100644 --- a/src/vibeio/op/hard_link.rs +++ b/src/vibeio/op/hard_link.rs @@ -15,6 +15,10 @@ pub struct HardLinkOp { } impl HardLinkOp { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "HardLinkOp") + )] #[inline] pub fn new(driver: Rc, old_path: CString, new_path: CString) -> Self { Self { @@ -29,6 +33,10 @@ impl HardLinkOp { impl Op for HardLinkOp { type Output = (); + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_completion( &mut self, @@ -70,6 +78,10 @@ impl Op for HardLinkOp { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn build_completion_entry( &mut self, @@ -97,6 +109,10 @@ impl Op for HardLinkOp { } impl Drop for HardLinkOp { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { // Paths and result storage remain owned until the kernel acknowledges diff --git a/src/vibeio/op/io_util.rs b/src/vibeio/op/io_util.rs index e45a6c0..e18621e 100644 --- a/src/vibeio/op/io_util.rs +++ b/src/vibeio/op/io_util.rs @@ -25,6 +25,7 @@ mod completion_error_tests { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Normalize EOF reported either during submission or by a completed read. pub(super) fn read_error_result(error: io::Error) -> io::Result { // Overlapped ReadFile can report EOF immediately or through its completion. @@ -36,6 +37,7 @@ pub(super) fn read_error_result(error: io::Error) -> io::Result { Err(error) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(all(target_os = "linux", feature = "fs"))] pub(super) fn positional_offset(offset: u64) -> io::Result { // Linux file offsets are signed. In particular, io_uring treats all-one @@ -49,6 +51,7 @@ pub(super) fn positional_offset(offset: u64) -> io::Result { Ok(offset) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] pub(super) fn iovec_count>(count: usize) -> io::Result { T::try_from(count).map_err(|_| { io::Error::new( @@ -58,6 +61,7 @@ pub(super) fn iovec_count>(count: usize) -> io::Result { }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub(super) fn iovec_to_system(bufs: &[crate::vibeio::io::IoVec]) -> Box<[libc::iovec]> { bufs.iter() @@ -68,6 +72,7 @@ pub(super) fn iovec_to_system(bufs: &[crate::vibeio::io::IoVec]) -> Box<[libc::i .collect() } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(any(target_os = "linux", windows, test))] #[inline] pub(super) fn completion_len(capacity: usize) -> io::Result { @@ -81,6 +86,7 @@ pub(super) fn completion_len(capacity: usize) -> io::Result { }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(any(windows, test))] pub(super) fn completion_vectored_len(mut lengths: impl Iterator) -> io::Result { let total = lengths @@ -89,6 +95,7 @@ pub(super) fn completion_vectored_len(mut lengths: impl Iterator) completion_len(total) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] pub(super) fn set_cloexec(fd: std::os::fd::RawFd) -> Result<(), io::Error> { // SAFETY: F_GETFD has no pointer arguments and only queries descriptor flags. @@ -543,6 +550,10 @@ mod storage_tests { } impl CompletionBuffer { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "CompletionBuffer") + )] #[inline] pub(crate) fn new(buf: B, stable: bool) -> Self { if stable { @@ -552,6 +563,10 @@ impl CompletionBuffer { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "CompletionBuffer") + )] #[inline] pub(crate) fn as_ref(&self) -> &B { match self { @@ -560,6 +575,10 @@ impl CompletionBuffer { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "CompletionBuffer") + )] #[inline] pub(crate) fn as_mut(&mut self) -> &mut B { match self { @@ -568,6 +587,10 @@ impl CompletionBuffer { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "CompletionBuffer") + )] #[inline] pub(crate) fn into_inner(self) -> B { match self { @@ -576,6 +599,10 @@ impl CompletionBuffer { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "CompletionBuffer") + )] #[inline] pub(crate) fn into_stable_box(self) -> Box { match self { @@ -585,6 +612,7 @@ impl CompletionBuffer { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub(crate) fn poll_result_or_wait( result: io::Result, diff --git a/src/vibeio/op/mkdir.rs b/src/vibeio/op/mkdir.rs index a70a85d..b273698 100644 --- a/src/vibeio/op/mkdir.rs +++ b/src/vibeio/op/mkdir.rs @@ -15,6 +15,7 @@ pub struct MkDirOp { } impl MkDirOp { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "MkDirOp"))] #[inline] pub fn new(driver: Rc, path: CString, mode: libc::mode_t) -> Self { Self { @@ -29,6 +30,10 @@ impl MkDirOp { impl Op for MkDirOp { type Output = (); + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_completion( &mut self, @@ -70,6 +75,10 @@ impl Op for MkDirOp { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn build_completion_entry( &mut self, @@ -90,6 +99,10 @@ impl Op for MkDirOp { } impl Drop for MkDirOp { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { // Paths and result storage remain owned until the kernel acknowledges diff --git a/src/vibeio/op/mod.rs b/src/vibeio/op/mod.rs index 3823b5a..07496b4 100644 --- a/src/vibeio/op/mod.rs +++ b/src/vibeio/op/mod.rs @@ -97,6 +97,7 @@ pub trait Op { /// I/O operation return type type Output; + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Op"))] /// Whether a successful Linux completion transfers ownership of a new fd. /// The driver must close an unclaimed result after cancellation. #[cfg(target_os = "linux")] @@ -104,6 +105,7 @@ pub trait Op { false } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Op"))] /// Polls the operation for readiness (poll-based I/O). #[inline] fn poll_poll( @@ -117,6 +119,7 @@ pub trait Op { ))) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Op"))] /// Polls the operation for readiness (completion-based I/O). #[inline] fn poll_completion( @@ -130,6 +133,7 @@ pub trait Op { ))) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Op"))] /// Polls the operation for readiness (automatically determined I/O). #[cfg(any(feature = "fs", feature = "process"))] #[allow(dead_code)] @@ -142,6 +146,7 @@ pub trait Op { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Op"))] /// Builds an io_uring submission entry for this operation. Returns the /// constructed SQE. #[cfg(target_os = "linux")] @@ -156,6 +161,7 @@ pub trait Op { )) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Op"))] /// Submits a Windows overlapped I/O operation for this operation. #[cfg(windows)] #[inline] diff --git a/src/vibeio/op/open.rs b/src/vibeio/op/open.rs index 5cbeb03..ba2cbb9 100644 --- a/src/vibeio/op/open.rs +++ b/src/vibeio/op/open.rs @@ -19,6 +19,7 @@ pub struct OpenOp { } impl OpenOp { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "OpenOp"))] #[inline] pub fn new(driver: Rc, path: CString, flags: i32, mode: libc::mode_t) -> Self { Self { @@ -34,10 +35,18 @@ impl OpenOp { impl Op for OpenOp { type Output = OwnedFd; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn completion_returns_fd(&self) -> bool { true } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_completion( &mut self, @@ -83,6 +92,10 @@ impl Op for OpenOp { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn build_completion_entry( &mut self, @@ -104,6 +117,10 @@ impl Op for OpenOp { } impl Drop for OpenOp { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { // Paths and result storage remain owned until the kernel acknowledges diff --git a/src/vibeio/op/read.rs b/src/vibeio/op/read.rs index 561774f..7c67026 100644 --- a/src/vibeio/op/read.rs +++ b/src/vibeio/op/read.rs @@ -23,6 +23,7 @@ use crate::vibeio::op::Op; use crate::vibeio::op::io_util::completion_len; use crate::vibeio::op::io_util::{CompletionBuffer, poll_result_or_wait}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] #[inline] fn socket_read(socket: SOCKET, buf: &mut impl IoBufMut) -> io::Result { @@ -74,6 +75,7 @@ pub struct ReadOp<'a, B: IoBufMut> { } impl<'a, B: IoBufMut> ReadOp<'a, B> { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "ReadOp"))] #[inline] pub fn new(handle: &'a InnerRawHandle, buf: B) -> Self { Self { @@ -83,6 +85,7 @@ impl<'a, B: IoBufMut> ReadOp<'a, B> { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "ReadOp"))] #[inline] pub fn take_bufs(mut self) -> B { assert!( @@ -96,6 +99,10 @@ impl<'a, B: IoBufMut> ReadOp<'a, B> { impl Op for ReadOp<'_, B> { type Output = usize; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_poll( @@ -144,6 +151,10 @@ impl Op for ReadOp<'_, B> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_completion( @@ -192,6 +203,10 @@ impl Op for ReadOp<'_, B> { Poll::Ready(Ok(read)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(windows)] #[inline] fn submit_windows(&mut self, overlapped: *mut OVERLAPPED) -> Result<(), io::Error> { @@ -262,6 +277,10 @@ impl Op for ReadOp<'_, B> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(target_os = "linux")] #[inline] fn build_completion_entry( @@ -285,6 +304,10 @@ impl Op for ReadOp<'_, B> { } impl Drop for ReadOp<'_, B> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { diff --git a/src/vibeio/op/readat.rs b/src/vibeio/op/readat.rs index 1050896..c007594 100644 --- a/src/vibeio/op/readat.rs +++ b/src/vibeio/op/readat.rs @@ -30,6 +30,7 @@ pub struct ReadAtOp<'a, B: IoBufMut> { } impl<'a, B: IoBufMut> ReadAtOp<'a, B> { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "ReadAtOp"))] #[inline] pub fn new(handle: &'a InnerRawHandle, buf: B, offset: u64) -> Self { Self { @@ -40,6 +41,7 @@ impl<'a, B: IoBufMut> ReadAtOp<'a, B> { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "ReadAtOp"))] #[inline] pub fn take_bufs(mut self) -> B { assert!( @@ -53,6 +55,10 @@ impl<'a, B: IoBufMut> ReadAtOp<'a, B> { impl Op for ReadAtOp<'_, B> { type Output = usize; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_completion( @@ -100,6 +106,10 @@ impl Op for ReadAtOp<'_, B> { Poll::Ready(Ok(read)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(windows)] #[inline] fn submit_windows(&mut self, overlapped: *mut OVERLAPPED) -> Result<(), io::Error> { @@ -154,6 +164,10 @@ impl Op for ReadAtOp<'_, B> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(target_os = "linux")] #[inline] fn build_completion_entry( @@ -185,6 +199,10 @@ impl Op for ReadAtOp<'_, B> { } impl Drop for ReadAtOp<'_, B> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { diff --git a/src/vibeio/op/readiness.rs b/src/vibeio/op/readiness.rs index b272564..e74fb71 100644 --- a/src/vibeio/op/readiness.rs +++ b/src/vibeio/op/readiness.rs @@ -11,6 +11,10 @@ pub struct ReadinessOp<'a> { } impl<'a> ReadinessOp<'a> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadinessOp") + )] #[inline] pub fn new_readable(handle: &'a InnerRawHandle) -> Self { Self { @@ -19,6 +23,10 @@ impl<'a> ReadinessOp<'a> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ReadinessOp") + )] #[inline] pub fn new_writable(handle: &'a InnerRawHandle) -> Self { Self { @@ -31,6 +39,10 @@ impl<'a> ReadinessOp<'a> { impl Op for ReadinessOp<'_> { type Output = (); + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_poll( &mut self, diff --git a/src/vibeio/op/readv.rs b/src/vibeio/op/readv.rs index 4d5a069..cd429e4 100644 --- a/src/vibeio/op/readv.rs +++ b/src/vibeio/op/readv.rs @@ -22,6 +22,7 @@ use crate::vibeio::op::io_util::iovec_to_system; use crate::vibeio::op::io_util::{iovec_count, poll_result_or_wait}; use crate::vibeio::{driver::CompletionIoResult, io::IoVectoredBufMut}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] #[inline] fn socket_read_vectored(socket: SOCKET, bufs: &mut B) -> io::Result { @@ -79,6 +80,7 @@ pub struct ReadvOp<'a, B: IoVectoredBufMut> { } impl<'a, B: IoVectoredBufMut> ReadvOp<'a, B> { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "ReadvOp"))] #[inline] pub fn new(handle: &'a InnerRawHandle, bufs: B) -> Self { Self { @@ -92,6 +94,7 @@ impl<'a, B: IoVectoredBufMut> ReadvOp<'a, B> { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "ReadvOp"))] #[inline] pub fn take_bufs(mut self) -> B { assert!( @@ -105,6 +108,10 @@ impl<'a, B: IoVectoredBufMut> ReadvOp<'a, B> { impl Op for ReadvOp<'_, B> { type Output = usize; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_poll( @@ -145,6 +152,10 @@ impl Op for ReadvOp<'_, B> { poll_result_or_wait(result, self.handle, cx, driver, Interest::READABLE) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_completion( @@ -222,6 +233,10 @@ impl Op for ReadvOp<'_, B> { Poll::Ready(Ok(result as usize)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(windows)] #[inline] fn submit_windows(&mut self, overlapped: *mut OVERLAPPED) -> Result<(), io::Error> { @@ -325,6 +340,10 @@ impl Op for ReadvOp<'_, B> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(target_os = "linux")] #[inline] fn build_completion_entry( @@ -359,6 +378,10 @@ impl Op for ReadvOp<'_, B> { } impl Drop for ReadvOp<'_, B> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { diff --git a/src/vibeio/op/recv.rs b/src/vibeio/op/recv.rs index 54ade8d..34bb535 100644 --- a/src/vibeio/op/recv.rs +++ b/src/vibeio/op/recv.rs @@ -21,6 +21,7 @@ use crate::vibeio::op::Op; use crate::vibeio::op::io_util::completion_len; use crate::vibeio::op::io_util::{CompletionBuffer, poll_result_or_wait}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] #[inline] fn socket_recv(socket: SOCKET, buf: &mut impl IoBufMut, peek: bool) -> io::Result { @@ -74,6 +75,7 @@ pub struct RecvOp<'a, B: IoBufMut> { } impl<'a, B: IoBufMut> RecvOp<'a, B> { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "RecvOp"))] #[inline] pub fn new(handle: &'a InnerRawHandle, buf: B) -> Self { Self { @@ -84,6 +86,7 @@ impl<'a, B: IoBufMut> RecvOp<'a, B> { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "RecvOp"))] pub fn new_peek(handle: &'a InnerRawHandle, buf: B) -> Self { Self { handle, @@ -93,6 +96,7 @@ impl<'a, B: IoBufMut> RecvOp<'a, B> { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "RecvOp"))] #[inline] pub fn take_bufs(mut self) -> B { assert!( @@ -106,6 +110,10 @@ impl<'a, B: IoBufMut> RecvOp<'a, B> { impl Op for RecvOp<'_, B> { type Output = usize; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_poll( @@ -154,6 +162,10 @@ impl Op for RecvOp<'_, B> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_completion( @@ -196,6 +208,10 @@ impl Op for RecvOp<'_, B> { Poll::Ready(Ok(read)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(windows)] #[inline] fn submit_windows(&mut self, overlapped: *mut OVERLAPPED) -> Result<(), io::Error> { @@ -249,6 +265,10 @@ impl Op for RecvOp<'_, B> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(target_os = "linux")] #[inline] fn build_completion_entry( @@ -273,6 +293,10 @@ impl Op for RecvOp<'_, B> { } impl Drop for RecvOp<'_, B> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { diff --git a/src/vibeio/op/recvfrom.rs b/src/vibeio/op/recvfrom.rs index b107ebe..e88ed4e 100644 --- a/src/vibeio/op/recvfrom.rs +++ b/src/vibeio/op/recvfrom.rs @@ -26,6 +26,7 @@ use crate::vibeio::op::Op; use crate::vibeio::op::io_util::CompletionBuffer; use crate::vibeio::op::socket_addr::sockaddr_storage_to_socketaddr; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] #[inline] fn socket_recvfrom( @@ -107,6 +108,10 @@ pub struct RecvfromOp<'a, B: IoBufMut> { } impl<'a, B: IoBufMut> RecvfromOp<'a, B> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RecvfromOp") + )] #[inline] pub fn new(handle: &'a InnerRawHandle, buf: B) -> Self { Self { @@ -121,6 +126,10 @@ impl<'a, B: IoBufMut> RecvfromOp<'a, B> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RecvfromOp") + )] #[inline] pub fn new_peek(handle: &'a InnerRawHandle, buf: B) -> Self { Self { @@ -135,6 +144,10 @@ impl<'a, B: IoBufMut> RecvfromOp<'a, B> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "RecvfromOp") + )] #[inline] pub fn take_bufs(mut self) -> B { assert!( @@ -148,6 +161,10 @@ impl<'a, B: IoBufMut> RecvfromOp<'a, B> { impl Op for RecvfromOp<'_, B> { type Output = (usize, SocketAddr); + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_poll( @@ -212,6 +229,10 @@ impl Op for RecvfromOp<'_, B> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_completion( @@ -292,6 +313,10 @@ impl Op for RecvfromOp<'_, B> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(windows)] #[inline] fn submit_windows(&mut self, overlapped: *mut OVERLAPPED) -> Result<(), io::Error> { @@ -359,6 +384,10 @@ impl Op for RecvfromOp<'_, B> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(target_os = "linux")] #[inline] fn build_completion_entry( @@ -411,6 +440,10 @@ impl Op for RecvfromOp<'_, B> { } impl Drop for RecvfromOp<'_, B> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { diff --git a/src/vibeio/op/rename.rs b/src/vibeio/op/rename.rs index 275a230..9e96edd 100644 --- a/src/vibeio/op/rename.rs +++ b/src/vibeio/op/rename.rs @@ -15,6 +15,7 @@ pub struct RenameOp { } impl RenameOp { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "RenameOp"))] #[inline] pub fn new(driver: Rc, old_path: CString, new_path: CString) -> Self { Self { @@ -29,6 +30,10 @@ impl RenameOp { impl Op for RenameOp { type Output = (); + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_completion( &mut self, @@ -70,6 +75,10 @@ impl Op for RenameOp { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn build_completion_entry( &mut self, @@ -97,6 +106,10 @@ impl Op for RenameOp { } impl Drop for RenameOp { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { // Paths and result storage remain owned until the kernel acknowledges diff --git a/src/vibeio/op/send.rs b/src/vibeio/op/send.rs index 58131e7..49b4bed 100644 --- a/src/vibeio/op/send.rs +++ b/src/vibeio/op/send.rs @@ -21,6 +21,7 @@ use crate::vibeio::op::Op; use crate::vibeio::op::io_util::completion_len; use crate::vibeio::op::io_util::{CompletionBuffer, poll_result_or_wait}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] #[inline] fn socket_send(socket: SOCKET, buf: &B) -> io::Result { @@ -70,6 +71,7 @@ pub struct SendOp<'a, B: IoBuf> { } impl<'a, B: IoBuf> SendOp<'a, B> { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "SendOp"))] #[inline] pub fn new(handle: &'a InnerRawHandle, buf: B) -> Self { Self { @@ -79,6 +81,7 @@ impl<'a, B: IoBuf> SendOp<'a, B> { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "SendOp"))] #[inline] pub fn take_bufs(mut self) -> B { assert!( @@ -92,6 +95,10 @@ impl<'a, B: IoBuf> SendOp<'a, B> { impl Op for SendOp<'_, B> { type Output = usize; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_poll( @@ -132,6 +139,10 @@ impl Op for SendOp<'_, B> { poll_result_or_wait(result, self.handle, cx, driver, Interest::WRITABLE) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_completion( @@ -167,6 +178,10 @@ impl Op for SendOp<'_, B> { Poll::Ready(Ok(written)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(windows)] #[inline] fn submit_windows(&mut self, overlapped: *mut OVERLAPPED) -> Result<(), io::Error> { @@ -221,6 +236,10 @@ impl Op for SendOp<'_, B> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(target_os = "linux")] #[inline] fn build_completion_entry( @@ -244,6 +263,10 @@ impl Op for SendOp<'_, B> { } impl Drop for SendOp<'_, B> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { diff --git a/src/vibeio/op/sendto.rs b/src/vibeio/op/sendto.rs index c1a743e..25aa171 100644 --- a/src/vibeio/op/sendto.rs +++ b/src/vibeio/op/sendto.rs @@ -23,6 +23,7 @@ use crate::vibeio::io::IoBuf; use crate::vibeio::op::io_util::{CompletionBuffer, poll_result_or_wait}; use crate::vibeio::op::{Op, socket_addr_to_raw}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] #[inline] fn socket_sendto(socket: SOCKET, buf: &B, addr: SocketAddr) -> io::Result { @@ -94,6 +95,7 @@ pub struct SendtoOp<'a, B: IoBuf> { } impl<'a, B: IoBuf> SendtoOp<'a, B> { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "SendtoOp"))] #[inline] pub fn new(handle: &'a InnerRawHandle, buf: B, addr: SocketAddr) -> Self { Self { @@ -108,6 +110,7 @@ impl<'a, B: IoBuf> SendtoOp<'a, B> { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "SendtoOp"))] #[inline] pub fn take_bufs(mut self) -> B { assert!( @@ -121,6 +124,10 @@ impl<'a, B: IoBuf> SendtoOp<'a, B> { impl Op for SendtoOp<'_, B> { type Output = usize; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_poll( @@ -164,6 +171,10 @@ impl Op for SendtoOp<'_, B> { poll_result_or_wait(result, self.handle, cx, driver, Interest::WRITABLE) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_completion( @@ -199,6 +210,10 @@ impl Op for SendtoOp<'_, B> { Poll::Ready(Ok(written)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(windows)] #[inline] fn submit_windows(&mut self, overlapped: *mut OVERLAPPED) -> Result<(), io::Error> { @@ -265,6 +280,10 @@ impl Op for SendtoOp<'_, B> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(target_os = "linux")] #[inline] fn build_completion_entry( @@ -317,6 +336,10 @@ impl Op for SendtoOp<'_, B> { } impl Drop for SendtoOp<'_, B> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { diff --git a/src/vibeio/op/socket_addr.rs b/src/vibeio/op/socket_addr.rs index e0cbd81..99cb6c2 100644 --- a/src/vibeio/op/socket_addr.rs +++ b/src/vibeio/op/socket_addr.rs @@ -8,6 +8,7 @@ type NativeAddress = libc::sockaddr_storage; #[cfg(windows)] type NativeAddress = windows_sys::Win32::Networking::WinSock::SOCKADDR_STORAGE; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub(crate) fn socket_addr_to_raw(address: SocketAddr) -> (NativeAddress, socket2::socklen_t) { let address = socket2::SockAddr::from(address); @@ -24,6 +25,7 @@ use windows_sys::Win32::Networking::WinSock::{ AF_INET, AF_INET6, SOCKADDR_IN, SOCKADDR_IN6, SOCKADDR_STORAGE, }; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn validate_address_length(length: usize, expected: usize, capacity: usize) -> io::Result<()> { if length < expected || length > capacity { return Err(io::Error::new( @@ -34,6 +36,7 @@ fn validate_address_length(length: usize, expected: usize, capacity: usize) -> i Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] #[inline] pub(super) fn sockaddr_storage_to_socketaddr( @@ -82,6 +85,7 @@ pub(super) fn sockaddr_storage_to_socketaddr( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] #[inline] pub(super) fn sockaddr_storage_to_socketaddr( @@ -130,6 +134,7 @@ pub(super) fn sockaddr_storage_to_socketaddr( } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(any(windows, test))] pub(super) fn socketaddr_from_buffer( buffer: &[u8], diff --git a/src/vibeio/op/splice.rs b/src/vibeio/op/splice.rs index 7da6c3e..9922716 100644 --- a/src/vibeio/op/splice.rs +++ b/src/vibeio/op/splice.rs @@ -23,6 +23,7 @@ pub struct SpliceOp<'a> { // AsRawFd does not promise the validity required by BorrowedFd::borrow_raw. // Ask the kernel to validate and duplicate the descriptor instead. +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn duplicate_fd(fd: RawFd) -> io::Result { loop { // SAFETY: F_DUPFD_CLOEXEC takes an integer minimum descriptor number. @@ -46,6 +47,7 @@ struct SourceRegistration { } impl<'a> SpliceOp<'a> { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "SpliceOp"))] #[inline] pub fn new(fd_in: RawFd, fd_out: &'a InnerRawHandle, len: usize) -> Self { Self { @@ -61,6 +63,7 @@ impl<'a> SpliceOp<'a> { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "SpliceOp"))] fn source_ready(&self) -> io::Result { let mut descriptor = libc::pollfd { fd: self.fd_in, @@ -83,6 +86,7 @@ impl<'a> SpliceOp<'a> { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "SpliceOp"))] fn source_handle(&mut self) -> io::Result<&InnerRawHandle> { if self.source_registration.is_none() { let fd = duplicate_fd(self.fd_in)?; @@ -101,6 +105,10 @@ impl<'a> SpliceOp<'a> { impl Op for SpliceOp<'_> { type Output = usize; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_poll( &mut self, @@ -147,6 +155,10 @@ impl Op for SpliceOp<'_> { poll_result_or_wait(result, self.fd_out, cx, driver, Interest::WRITABLE) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_completion( &mut self, @@ -182,6 +194,10 @@ impl Op for SpliceOp<'_> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn build_completion_entry( &mut self, @@ -212,6 +228,10 @@ impl Op for SpliceOp<'_> { } impl Drop for SpliceOp<'_> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { if let Some(completion_token) = self.completion_token { diff --git a/src/vibeio/op/statx.rs b/src/vibeio/op/statx.rs index 3c3a3c1..764edf0 100644 --- a/src/vibeio/op/statx.rs +++ b/src/vibeio/op/statx.rs @@ -21,6 +21,7 @@ pub struct StatxOp { } impl StatxOp { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "StatxOp"))] #[inline] pub fn new( driver: Rc, @@ -44,6 +45,10 @@ impl StatxOp { impl Op for StatxOp { type Output = libc::statx; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_completion( &mut self, @@ -90,6 +95,10 @@ impl Op for StatxOp { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn build_completion_entry( &mut self, @@ -123,6 +132,10 @@ impl Op for StatxOp { } impl Drop for StatxOp { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { // Paths and result storage remain owned until the kernel acknowledges diff --git a/src/vibeio/op/symlink.rs b/src/vibeio/op/symlink.rs index 6387781..6c0e16b 100644 --- a/src/vibeio/op/symlink.rs +++ b/src/vibeio/op/symlink.rs @@ -15,6 +15,7 @@ pub struct SymlinkOp { } impl SymlinkOp { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "SymlinkOp"))] #[inline] pub fn new(driver: Rc, old_path: CString, new_path: CString) -> Self { Self { @@ -29,6 +30,10 @@ impl SymlinkOp { impl Op for SymlinkOp { type Output = (); + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_completion( &mut self, @@ -70,6 +75,10 @@ impl Op for SymlinkOp { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn build_completion_entry( &mut self, @@ -96,6 +105,10 @@ impl Op for SymlinkOp { } impl Drop for SymlinkOp { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { // Paths and result storage remain owned until the kernel acknowledges diff --git a/src/vibeio/op/unlink.rs b/src/vibeio/op/unlink.rs index fe3e6e7..d73e685 100644 --- a/src/vibeio/op/unlink.rs +++ b/src/vibeio/op/unlink.rs @@ -15,6 +15,7 @@ pub struct UnlinkOp { } impl UnlinkOp { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "UnlinkOp"))] #[inline] pub fn new(driver: Rc, path: CString, is_dir: bool) -> Self { Self { @@ -29,6 +30,10 @@ impl UnlinkOp { impl Op for UnlinkOp { type Output = (); + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_completion( &mut self, @@ -70,6 +75,10 @@ impl Op for UnlinkOp { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn build_completion_entry( &mut self, @@ -91,6 +100,10 @@ impl Op for UnlinkOp { } impl Drop for UnlinkOp { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { // Paths and result storage remain owned until the kernel acknowledges diff --git a/src/vibeio/op/waitpid.rs b/src/vibeio/op/waitpid.rs index f8dd482..63ce9d1 100644 --- a/src/vibeio/op/waitpid.rs +++ b/src/vibeio/op/waitpid.rs @@ -102,6 +102,7 @@ pub struct WaitPidOp { } impl WaitPidOp { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "WaitPidOp"))] /// Create a new `WaitPidOp` for the given child PID. #[inline] pub fn new(pid: u32) -> Self { @@ -112,6 +113,7 @@ impl WaitPidOp { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "WaitPidOp"))] /// Open a pidfd for `pid` via the `pidfd_open` syscall (Linux 5.3+). #[inline] fn open_pidfd(pid: libc::pid_t) -> io::Result { @@ -126,6 +128,7 @@ impl WaitPidOp { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "WaitPidOp"))] /// Check for exit and reap without blocking, before arming pidfd readiness. #[inline] fn reap(pid: libc::pid_t) -> io::Result { @@ -154,6 +157,10 @@ impl WaitPidOp { impl Op for WaitPidOp { type Output = i32; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn poll_poll( &mut self, cx: &mut Context<'_>, @@ -198,6 +205,10 @@ impl Op for WaitPidOp { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn poll_completion( &mut self, cx: &mut Context<'_>, diff --git a/src/vibeio/op/write.rs b/src/vibeio/op/write.rs index 73a611e..3adc210 100644 --- a/src/vibeio/op/write.rs +++ b/src/vibeio/op/write.rs @@ -22,6 +22,7 @@ use crate::vibeio::op::Op; use crate::vibeio::op::io_util::completion_len; use crate::vibeio::op::io_util::{CompletionBuffer, poll_result_or_wait}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] #[inline] fn socket_write(socket: SOCKET, buf: &impl IoBuf) -> io::Result { @@ -72,6 +73,7 @@ pub struct WriteOp<'a, B: IoBuf> { } impl<'a, B: IoBuf> WriteOp<'a, B> { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "WriteOp"))] #[inline] pub fn new(handle: &'a InnerRawHandle, buf: B) -> Self { Self { @@ -81,6 +83,7 @@ impl<'a, B: IoBuf> WriteOp<'a, B> { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "WriteOp"))] #[inline] pub fn take_bufs(mut self) -> B { assert!( @@ -94,6 +97,10 @@ impl<'a, B: IoBuf> WriteOp<'a, B> { impl Op for WriteOp<'_, B> { type Output = usize; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_poll( @@ -133,6 +140,10 @@ impl Op for WriteOp<'_, B> { poll_result_or_wait(result, self.handle, cx, driver, Interest::WRITABLE) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_completion( @@ -170,6 +181,10 @@ impl Op for WriteOp<'_, B> { Poll::Ready(Ok(result as usize)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(windows)] #[inline] fn submit_windows(&mut self, overlapped: *mut OVERLAPPED) -> Result<(), io::Error> { @@ -253,6 +268,10 @@ impl Op for WriteOp<'_, B> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(target_os = "linux")] #[inline] fn build_completion_entry( @@ -276,6 +295,10 @@ impl Op for WriteOp<'_, B> { } impl Drop for WriteOp<'_, B> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { diff --git a/src/vibeio/op/writeat.rs b/src/vibeio/op/writeat.rs index 7deeccf..4bebcee 100644 --- a/src/vibeio/op/writeat.rs +++ b/src/vibeio/op/writeat.rs @@ -19,6 +19,7 @@ use crate::vibeio::io::IoBuf; use crate::vibeio::op::Op; use crate::vibeio::op::io_util::CompletionBuffer; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(any(windows, test))] pub(crate) fn validate_windows_write_offset(offset: u64) -> io::Result<()> { // WriteFile interprets both offset words set to all-one bits as append. @@ -43,6 +44,7 @@ pub struct WriteAtOp<'a, B: IoBuf> { } impl<'a, B: IoBuf> WriteAtOp<'a, B> { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "WriteAtOp"))] #[inline] pub fn new(handle: &'a InnerRawHandle, buf: B, offset: u64) -> Self { Self { @@ -53,6 +55,7 @@ impl<'a, B: IoBuf> WriteAtOp<'a, B> { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "WriteAtOp"))] #[inline] pub fn take_bufs(mut self) -> B { assert!( @@ -66,6 +69,10 @@ impl<'a, B: IoBuf> WriteAtOp<'a, B> { impl Op for WriteAtOp<'_, B> { type Output = usize; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_completion( @@ -101,6 +108,10 @@ impl Op for WriteAtOp<'_, B> { Poll::Ready(Ok(written)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(windows)] #[inline] fn submit_windows(&mut self, overlapped: *mut OVERLAPPED) -> Result<(), io::Error> { @@ -152,6 +163,10 @@ impl Op for WriteAtOp<'_, B> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(target_os = "linux")] #[inline] fn build_completion_entry( @@ -179,6 +194,10 @@ impl Op for WriteAtOp<'_, B> { } impl Drop for WriteAtOp<'_, B> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { diff --git a/src/vibeio/op/writev.rs b/src/vibeio/op/writev.rs index 54b93f7..2a1bb6e 100644 --- a/src/vibeio/op/writev.rs +++ b/src/vibeio/op/writev.rs @@ -23,6 +23,7 @@ use crate::vibeio::op::Op; use crate::vibeio::op::io_util::iovec_to_system; use crate::vibeio::op::io_util::{iovec_count, poll_result_or_wait}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] #[inline] fn socket_write_vectored(socket: SOCKET, bufs: &B) -> io::Result { @@ -79,6 +80,7 @@ pub struct WritevOp<'a, B: IoVectoredBuf> { } impl<'a, B: IoVectoredBuf> WritevOp<'a, B> { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "WritevOp"))] #[inline] pub fn new(handle: &'a InnerRawHandle, bufs: B) -> Self { Self { @@ -92,6 +94,7 @@ impl<'a, B: IoVectoredBuf> WritevOp<'a, B> { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "WritevOp"))] #[inline] pub fn take_bufs(mut self) -> B { assert!( @@ -105,6 +108,10 @@ impl<'a, B: IoVectoredBuf> WritevOp<'a, B> { impl Op for WritevOp<'_, B> { type Output = usize; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_poll( @@ -146,6 +153,10 @@ impl Op for WritevOp<'_, B> { poll_result_or_wait(result, self.handle, cx, driver, Interest::WRITABLE) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(any(unix, windows))] #[inline] fn poll_completion( @@ -193,6 +204,10 @@ impl Op for WritevOp<'_, B> { Poll::Ready(Ok(result as usize)) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(windows)] #[inline] fn submit_windows(&mut self, overlapped: *mut OVERLAPPED) -> Result<(), io::Error> { @@ -307,6 +322,10 @@ impl Op for WritevOp<'_, B> { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[cfg(target_os = "linux")] #[inline] fn build_completion_entry( @@ -341,6 +360,10 @@ impl Op for WritevOp<'_, B> { } impl Drop for WritevOp<'_, B> { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { if let Some(token) = self.completion_token.take() { diff --git a/src/vibeio/process/mod.rs b/src/vibeio/process/mod.rs index 6423a02..15ffc87 100644 --- a/src/vibeio/process/mod.rs +++ b/src/vibeio/process/mod.rs @@ -70,16 +70,19 @@ enum ChildIo { Blocking, } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn stdio_closed_error() -> io::Error { io::Error::new(io::ErrorKind::BrokenPipe, "child stdio is closed") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn command_consumed_error() -> io::Error { io::Error::new(io::ErrorKind::InvalidInput, "command has been consumed") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn child_consumed_error() -> io::Error { io::Error::new( @@ -88,11 +91,13 @@ fn child_consumed_error() -> io::Error { ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn blocking_pool_io_error() -> io::Error { io::Error::other("can't spawn blocking task for process I/O") } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(unix)] #[inline] fn make_child_io(fd: RawFd, interest: Interest) -> io::Result { @@ -114,6 +119,7 @@ fn make_child_io(fd: RawFd, interest: Interest) -> io::Result { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[inline] async fn read_in_blocking_pool(inner: R, buf: B) -> (io::Result, R, B) where @@ -132,6 +138,7 @@ where ) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[inline] async fn write_in_blocking_pool(inner: W, buf: B) -> (io::Result, W, B) where @@ -199,6 +206,10 @@ pub struct ChildStderr { } impl ChildStdin { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ChildStdin") + )] /// Create a new `ChildStdin` from a standard library `ChildStdin`. #[inline] pub(crate) fn from_std(inner: std::process::ChildStdin) -> io::Result { @@ -213,12 +224,20 @@ impl ChildStdin { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ChildStdin") + )] /// Consume this `ChildStdin` and return the underlying `std::process::ChildStdin`. #[inline] pub fn into_std(mut self) -> std::process::ChildStdin { self.inner.take().expect("child stdin is already taken") } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ChildStdin") + )] #[inline] fn drop_handle(&mut self) { // Deregister before the standard stream closes its descriptor. @@ -228,6 +247,10 @@ impl ChildStdin { } impl ChildStdout { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ChildStdout") + )] /// Create a new `ChildStdout` from a standard library `ChildStdout`. #[inline] pub(crate) fn from_std(inner: std::process::ChildStdout) -> io::Result { @@ -242,12 +265,20 @@ impl ChildStdout { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ChildStdout") + )] /// Consume this `ChildStdout` and return the underlying `std::process::ChildStdout`. #[inline] pub fn into_std(mut self) -> std::process::ChildStdout { self.inner.take().expect("child stdout is already taken") } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ChildStdout") + )] #[inline] fn drop_handle(&mut self) { // Deregister before the standard stream closes its descriptor. @@ -257,6 +288,10 @@ impl ChildStdout { } impl ChildStderr { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ChildStderr") + )] /// Create a new `ChildStderr` from a standard library `ChildStderr`. #[inline] pub(crate) fn from_std(inner: std::process::ChildStderr) -> io::Result { @@ -271,12 +306,20 @@ impl ChildStderr { }) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ChildStderr") + )] /// Consume this `ChildStderr` and return the underlying `std::process::ChildStderr`. #[inline] pub fn into_std(mut self) -> std::process::ChildStderr { self.inner.take().expect("child stderr is already taken") } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ChildStderr") + )] #[inline] fn drop_handle(&mut self) { // Deregister before the standard stream closes its descriptor. @@ -286,6 +329,10 @@ impl ChildStderr { } impl Drop for ChildStdin { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { self.drop_handle(); @@ -293,6 +340,10 @@ impl Drop for ChildStdin { } impl Drop for ChildStdout { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { self.drop_handle(); @@ -300,6 +351,10 @@ impl Drop for ChildStdout { } impl Drop for ChildStderr { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { self.drop_handle(); @@ -307,6 +362,10 @@ impl Drop for ChildStderr { } impl AsyncWrite for ChildStdin { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn write(&mut self, buf: B) -> (Result, B) { if buf.buf_len() == 0 { @@ -338,6 +397,10 @@ impl AsyncWrite for ChildStdin { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn flush(&mut self) -> Result<(), io::Error> { #[cfg(unix)] @@ -361,6 +424,10 @@ impl AsyncWrite for ChildStdin { } impl AsyncRead for ChildStdout { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn read(&mut self, buf: B) -> (Result, B) { if buf.buf_capacity() == 0 { @@ -395,6 +462,10 @@ impl AsyncRead for ChildStdout { } impl AsyncRead for ChildStderr { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "", future = true) + )] #[inline] async fn read(&mut self, buf: B) -> (Result, B) { if buf.buf_capacity() == 0 { @@ -430,6 +501,10 @@ impl AsyncRead for ChildStderr { #[cfg(unix)] impl AsRawFd for ChildStdin { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_fd(&self) -> RawFd { self.inner @@ -441,6 +516,10 @@ impl AsRawFd for ChildStdin { #[cfg(unix)] impl IntoRawFd for ChildStdin { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_fd(self) -> RawFd { self.into_std().into_raw_fd() @@ -449,6 +528,10 @@ impl IntoRawFd for ChildStdin { #[cfg(unix)] impl AsRawFd for ChildStdout { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_fd(&self) -> RawFd { self.inner @@ -460,6 +543,10 @@ impl AsRawFd for ChildStdout { #[cfg(unix)] impl IntoRawFd for ChildStdout { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_fd(self) -> RawFd { self.into_std().into_raw_fd() @@ -468,6 +555,10 @@ impl IntoRawFd for ChildStdout { #[cfg(unix)] impl AsRawFd for ChildStderr { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_fd(&self) -> RawFd { self.inner @@ -479,6 +570,10 @@ impl AsRawFd for ChildStderr { #[cfg(unix)] impl IntoRawFd for ChildStderr { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_fd(self) -> RawFd { self.into_std().into_raw_fd() @@ -487,6 +582,10 @@ impl IntoRawFd for ChildStderr { #[cfg(windows)] impl AsRawHandle for ChildStdin { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_handle(&self) -> RawHandle { self.inner @@ -498,6 +597,10 @@ impl AsRawHandle for ChildStdin { #[cfg(windows)] impl IntoRawHandle for ChildStdin { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_handle(self) -> RawHandle { self.into_std().into_raw_handle() @@ -506,6 +609,10 @@ impl IntoRawHandle for ChildStdin { #[cfg(windows)] impl AsRawHandle for ChildStdout { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_handle(&self) -> RawHandle { self.inner @@ -517,6 +624,10 @@ impl AsRawHandle for ChildStdout { #[cfg(windows)] impl IntoRawHandle for ChildStdout { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_handle(self) -> RawHandle { self.into_std().into_raw_handle() @@ -525,6 +636,10 @@ impl IntoRawHandle for ChildStdout { #[cfg(windows)] impl AsRawHandle for ChildStderr { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn as_raw_handle(&self) -> RawHandle { self.inner @@ -536,6 +651,10 @@ impl AsRawHandle for ChildStderr { #[cfg(windows)] impl IntoRawHandle for ChildStderr { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn into_raw_handle(self) -> RawHandle { self.into_std().into_raw_handle() @@ -564,6 +683,7 @@ pub struct Child { } impl Child { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Child"))] /// Create a new `Child` from a standard library `Child`. #[inline] pub(crate) fn from_std(child: std::process::Child) -> io::Result { @@ -599,28 +719,36 @@ impl Child { Ok(wrapped) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Child"))] /// Returns the OS-assigned process identifier. #[inline] pub fn id(&self) -> u32 { self.id } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Child"))] #[inline] fn inner_mut(&mut self) -> io::Result<&mut std::process::Child> { self.inner.as_mut().ok_or_else(child_consumed_error) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Child"))] #[inline] fn take_inner(&mut self) -> io::Result { self.inner.take().ok_or_else(child_consumed_error) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Child"))] /// Force kill the process. #[inline] pub fn kill(&mut self) -> io::Result<()> { self.inner_mut()?.kill() } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "Child", future = true) + )] /// Asynchronously wait for the process to exit. /// /// This method returns a future that resolves to the process's exit status. @@ -632,6 +760,7 @@ impl Child { self.reaper.wait(child).await } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Child"))] /// Check if the process has exited without blocking. /// /// Returns `Ok(Some(status))` if the process has exited, `Ok(None)` if it @@ -643,6 +772,10 @@ impl Child { } impl Drop for Child { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn drop(&mut self) { let _ = self.stdin.take(); // Similarly to std::process::Child::wait @@ -669,6 +802,7 @@ pub struct Command { } impl Command { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Command"))] /// Create a new `Command` for the given program. #[inline] pub fn new(program: impl AsRef) -> Self { @@ -677,11 +811,13 @@ impl Command { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Command"))] #[inline] fn inner_mut(&mut self) -> &mut std::process::Command { self.inner.as_mut().expect("command has been consumed") } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Command"))] /// Add an argument to pass to the program. #[inline] pub fn arg(&mut self, arg: impl AsRef) -> &mut Self { @@ -689,6 +825,7 @@ impl Command { self } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Command"))] /// Add multiple arguments to pass to the program. #[inline] pub fn args(&mut self, args: I) -> &mut Self @@ -700,6 +837,7 @@ impl Command { self } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Command"))] /// Set an environment variable for the process. #[inline] pub fn env(&mut self, key: K, val: V) -> &mut Self @@ -711,6 +849,7 @@ impl Command { self } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Command"))] /// Set multiple environment variables for the process. #[inline] pub fn envs(&mut self, vars: I) -> &mut Self @@ -723,6 +862,7 @@ impl Command { self } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Command"))] /// Remove an environment variable for the process. #[inline] pub fn env_remove>(&mut self, key: K) -> &mut Self { @@ -730,6 +870,7 @@ impl Command { self } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Command"))] /// Clear all environment variables for the process. #[inline] pub fn env_clear(&mut self) -> &mut Self { @@ -737,6 +878,7 @@ impl Command { self } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Command"))] /// Set the working directory for the process. #[inline] pub fn current_dir(&mut self, dir: impl AsRef) -> &mut Self { @@ -744,6 +886,7 @@ impl Command { self } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Command"))] /// Configure the standard input for the process. #[inline] pub fn stdin(&mut self, cfg: Stdio) -> &mut Self { @@ -751,6 +894,7 @@ impl Command { self } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Command"))] /// Configure the standard output for the process. #[inline] pub fn stdout(&mut self, cfg: Stdio) -> &mut Self { @@ -758,6 +902,7 @@ impl Command { self } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Command"))] /// Configure the standard error for the process. #[inline] pub fn stderr(&mut self, cfg: Stdio) -> &mut Self { @@ -765,6 +910,7 @@ impl Command { self } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Command"))] /// Spawn the process and return a `Child` handle. #[inline] pub fn spawn(&mut self) -> io::Result { @@ -776,6 +922,10 @@ impl Command { Child::from_std(child) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "Command", future = true) + )] /// Run the process to completion and return its exit status. /// /// This is an async version of `std::process::Command::status`. @@ -795,6 +945,10 @@ impl Command { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "Command", future = true) + )] /// Run the process to completion and return its output. /// /// This is an async version of `std::process::Command::output`. @@ -814,12 +968,14 @@ impl Command { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Command"))] /// Get a mutable reference to the underlying `std::process::Command`. #[inline] pub fn as_std(&mut self) -> &mut std::process::Command { self.inner_mut() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Command"))] /// Consume this `Command` and return the underlying `std::process::Command`. #[inline] pub fn into_std(mut self) -> std::process::Command { diff --git a/src/vibeio/process/reaper.rs b/src/vibeio/process/reaper.rs index 732e263..676f9e2 100644 --- a/src/vibeio/process/reaper.rs +++ b/src/vibeio/process/reaper.rs @@ -23,18 +23,30 @@ pub(crate) struct ReapChild(Option); impl std::ops::Deref for ReapChild { type Target = std::process::Child; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn deref(&self) -> &Self::Target { self.0.as_ref().expect("child already consumed") } } impl std::ops::DerefMut for ReapChild { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn deref_mut(&mut self) -> &mut Self::Target { self.0.as_mut().expect("child already consumed") } } impl Drop for ReapChild { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { let Some(mut child) = self.0.take() else { return; @@ -53,6 +65,7 @@ impl Drop for ReapChild { type PendingChild = std::sync::Arc>>; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn wait_pending_child(pending: &PendingChild) { // This mutex only transfers ownership; never retain its guard while waiting. let child = pending.lock().unwrap().take(); @@ -61,6 +74,7 @@ fn wait_pending_child(pending: &PendingChild) { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn reap_with_worker( child: std::process::Child, start: impl FnOnce(PendingChild) -> io::Result<()>, @@ -76,12 +90,20 @@ fn reap_with_worker( /// Zombie reaper process that waits on child processes asynchronously. impl ZombieReaper { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ZombieReaper") + )] /// Creates a new zombie reaper instance. #[inline] pub(crate) fn new() -> Self { Self } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ZombieReaper", future = true) + )] /// Waits on a child process asynchronously. #[inline] pub(crate) async fn wait(&self, child: std::process::Child) -> io::Result { @@ -100,6 +122,10 @@ impl ZombieReaper { .map_err(|_| io::Error::other("zombie reaper error"))? } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "ZombieReaper") + )] /// Reaps a child process on drop, waiting asynchronously if possible. #[inline] pub(crate) fn reap_on_drop(&self, mut child: std::process::Child) { @@ -117,6 +143,7 @@ impl ZombieReaper { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn wait_in_background((mut child, sender): ZombieReaperMessage) { // On spawn failure, dropping the closure still drops the reaping guard. let _ = std::thread::Builder::new() @@ -315,6 +342,7 @@ mod ownership_tests { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] pub(crate) fn start_zombie_reaper() -> async_channel::Sender { let (tx, rx) = async_channel::unbounded(); @@ -334,6 +362,10 @@ struct WaitContext { #[cfg(windows)] impl Drop for WaitContext { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { let handle = self.wait_handle.load(std::sync::atomic::Ordering::Acquire); if !handle.is_null() { @@ -365,6 +397,7 @@ unsafe extern "system" fn wait_callback(ctx: *mut std::ffi::c_void, _timed_out: // returned, even when this callback finishes before RegisterWait returns. } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn register_process_wait(message: ZombieReaperMessage) { use std::sync::{ @@ -411,6 +444,7 @@ fn register_process_wait(message: ZombieReaperMessage) { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[inline] #[cfg(windows)] async fn zombie_reaper_fn(rx: async_channel::Receiver) { @@ -476,6 +510,7 @@ mod windows_wait_tests { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[inline] #[cfg(unix)] async fn zombie_reaper_fn(rx: async_channel::Receiver) { @@ -496,6 +531,7 @@ async fn zombie_reaper_fn(rx: async_channel::Receiver) { // Linux pidfd-based reaper (kernel ≥ 5.3) // --------------------------------------------------------------------------- +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Probe whether `pidfd_open` is supported on this kernel. #[cfg(target_os = "linux")] #[inline] @@ -521,6 +557,7 @@ fn pidfd_available() -> bool { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Convert a raw `waitpid` status into a `std::process::ExitStatus`. #[cfg(target_os = "linux")] #[inline] @@ -529,6 +566,7 @@ fn exit_status_from_raw(raw: i32) -> ExitStatus { ExitStatus::from_raw(raw) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[cfg(target_os = "linux")] #[inline] async fn zombie_reaper_fn_linux_pidfd(rx: async_channel::Receiver) { @@ -598,6 +636,7 @@ async fn zombie_reaper_fn_linux_pidfd(rx: async_channel::Receiver) { @@ -662,12 +701,14 @@ async fn zombie_reaper_fn_unix(rx: async_channel::Receiver) } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[inline] #[cfg(all(unix, not(feature = "signal")))] async fn zombie_reaper_fn_unix(rx: async_channel::Receiver) { zombie_reaper_fn_threads(rx).await; } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] #[cfg(unix)] async fn zombie_reaper_fn_threads(rx: async_channel::Receiver) { // Process support does not require an optional runtime blocking pool. diff --git a/src/vibeio/signal/unix.rs b/src/vibeio/signal/unix.rs index debb3fa..38599cc 100644 --- a/src/vibeio/signal/unix.rs +++ b/src/vibeio/signal/unix.rs @@ -109,6 +109,10 @@ impl SignalKind { } impl From for libc::c_int { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = ">") + )] #[inline] fn from(kind: SignalKind) -> libc::c_int { kind.0 @@ -159,6 +163,10 @@ impl PendingSignals { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "PendingSignals") + )] fn take(&self, signum: usize) -> bool { self.0[signum].swap(false, Ordering::SeqCst) } @@ -183,6 +191,7 @@ pub struct Signal { } impl Signal { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Signal"))] /// Register for a Unix signal. /// /// Creates a new signal listener for the given signal kind. If this is the @@ -199,12 +208,17 @@ impl Signal { }) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Signal"))] /// Returns the signal kind being listened to. #[inline] pub fn kind(&self) -> SignalKind { self.kind } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "Signal", future = true) + )] /// Wait for the next occurrence of the signal. /// /// This method returns a future that resolves when the signal is received. @@ -215,6 +229,7 @@ impl Signal { poll_fn(|cx| self.poll_recv(cx)).await } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Signal"))] fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll> { // Serialize checking the counter and registering with dispatcher wakeup. // Each listener owns one slot so its drop can release its waker. @@ -252,6 +267,10 @@ impl Signal { } impl Drop for Signal { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { // Synchronously retire this slot under the dispatcher lock. Deferring // removal could retain a cancelled task's waker indefinitely. User @@ -263,6 +282,7 @@ impl Drop for Signal { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Convenience builder for Unix signals. /// /// This is a wrapper around `Signal::new()` that provides a more ergonomic API. @@ -280,6 +300,7 @@ pub struct CtrlC { } impl CtrlC { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "CtrlC"))] /// Create a new Ctrl-C listener. pub fn new() -> io::Result { Ok(Self { @@ -291,12 +312,17 @@ impl CtrlC { impl Future for CtrlC { type Output = io::Result<()>; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.get_mut(); this.signal.poll_recv(cx) } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Cross-platform Ctrl-C support. /// /// Returns a future that resolves when Ctrl-C is received. @@ -305,6 +331,7 @@ pub fn ctrl_c() -> io::Result { CtrlC::new() } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn register_signal(kind: SignalKind) -> io::Result> { if !(1..SIGNAL_SLOTS as libc::c_int).contains(&kind.0) { return Err(io::Error::new( @@ -337,6 +364,7 @@ fn register_signal(kind: SignalKind) -> io::Result> { Ok(state) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn unregister_signal(kind: SignalKind) { let Some(registry) = REGISTRY.get() else { return; @@ -362,14 +390,17 @@ fn unregister_signal(kind: SignalKind) { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn registry() -> io::Result<&'static Arc> { REGISTRY.get_or_try_init(init_registry) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn init_registry() -> io::Result> { init_registry_with_start(start_dispatch_thread) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn init_registry_with_start( start: impl FnOnce(Arc) -> io::Result<()>, ) -> io::Result> { @@ -387,6 +418,7 @@ fn init_registry_with_start( Ok(registry) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn start_dispatch_thread(registry: Arc) -> io::Result<()> { std::thread::Builder::new() .name("vibeio-signal-dispatch".to_string()) @@ -395,6 +427,7 @@ fn start_dispatch_thread(registry: Arc) -> io::Result<()> { Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn dispatch_loop(registry: Arc) { let read_fd = registry.read_fd.as_raw_fd(); let mut buf = [0u8; 128]; @@ -423,6 +456,7 @@ fn dispatch_loop(registry: Arc) { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn dispatch_signal(registry: &Registry, signum: libc::c_int) { let state = { let signals = registry.signals.lock().unwrap(); @@ -469,6 +503,7 @@ fn write_signal_notification(fd: RawFd, signum: libc::c_int) { errno::set_errno(saved_errno); } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn install_handler(signum: libc::c_int) -> io::Result { // SAFETY: sigaction's C fields admit zero initialization. The handler, // flags and mask are set below before the structure is passed to the OS. @@ -492,6 +527,7 @@ fn install_handler(signum: libc::c_int) -> io::Result { Ok(prev) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// # Safety /// `prev` must be the action saved for this signal, with any handler it refers /// to still valid for subsequent signal delivery. @@ -505,6 +541,7 @@ unsafe fn restore_handler(signum: libc::c_int, prev: &libc::sigaction) -> io::Re Ok(()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn create_pipe() -> io::Result<(OwnedFd, OwnedFd)> { let (reader, writer) = std::io::pipe()?; crate::vibeio::fd_inner::set_nonblocking(writer.as_raw_fd(), true)?; diff --git a/src/vibeio/signal/windows.rs b/src/vibeio/signal/windows.rs index 0ba60d9..ce84c57 100644 --- a/src/vibeio/signal/windows.rs +++ b/src/vibeio/signal/windows.rs @@ -43,6 +43,7 @@ pub struct CtrlC { } impl CtrlC { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "CtrlC"))] /// Create a new Ctrl-C listener. #[cfg(windows)] pub fn new() -> io::Result { @@ -56,6 +57,7 @@ impl CtrlC { }) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "CtrlC"))] fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll> { let mut replacement = None; loop { @@ -91,6 +93,10 @@ impl CtrlC { } impl Drop for CtrlC { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { // Synchronously retire this slot under the dispatcher lock. Deferring // removal could retain a cancelled task's waker indefinitely. User @@ -104,11 +110,16 @@ impl Drop for CtrlC { impl Future for CtrlC { type Output = io::Result<()>; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { self.get_mut().poll_recv(cx) } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Cross-platform Ctrl-C support. /// /// Returns a future that resolves when Ctrl-C is received. @@ -118,6 +129,7 @@ pub fn ctrl_c() -> io::Result { CtrlC::new() } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn dispatch_ctrl_c(state: &CtrlCState) { let wakers = { let mut wakers = state.wakers.lock().unwrap(); @@ -132,6 +144,7 @@ fn dispatch_ctrl_c(state: &CtrlCState) { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(windows)] fn ctrl_c_state() -> io::Result<&'static Arc> { initialize_ctrl_c_state(&CTRL_C_STATE, &CTRL_C_HANDLER_INSTALLED, || { @@ -146,6 +159,7 @@ fn ctrl_c_state() -> io::Result<&'static Arc> { }) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] fn initialize_ctrl_c_state<'a>( state_cell: &'a OnceCell>, installed: &OnceCell<()>, diff --git a/src/vibeio/task.rs b/src/vibeio/task.rs index 560ed95..1543260 100644 --- a/src/vibeio/task.rs +++ b/src/vibeio/task.rs @@ -34,6 +34,7 @@ pub struct Task { } impl Task { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Task"))] #[inline] pub fn waker_ref(self: &Rc) -> WakerRef<'_> { // Borrowing the proxy does not increment its reference count. Only @@ -41,11 +42,13 @@ impl Task { waker_ref(&self.wake) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Task"))] #[inline] pub fn waker(self: &Rc) -> Waker { waker(self.wake.clone()) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Task"))] #[inline] pub fn mark_dequeued(&self) { self.wake.queued.store(false, Ordering::Relaxed); @@ -53,6 +56,10 @@ impl Task { } impl ArcWake for TaskWake { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn wake_by_ref(wake: &Arc) { Self::enqueue_if_needed(wake); @@ -60,6 +67,7 @@ impl ArcWake for TaskWake { } impl TaskWake { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "TaskWake"))] fn enqueue_if_needed(wake: &Arc) { let Some(remote) = wake.remote_wake.upgrade() else { return; diff --git a/src/vibeio/time/interval.rs b/src/vibeio/time/interval.rs index 7dfc5b7..f74b426 100644 --- a/src/vibeio/time/interval.rs +++ b/src/vibeio/time/interval.rs @@ -4,6 +4,7 @@ use std::time::{Duration, Instant}; use super::sleep::Sleep; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[inline] fn duration_remainder(duration: Duration, divisor: Duration) -> Duration { let remainder = duration.as_nanos() % divisor.as_nanos(); @@ -39,6 +40,7 @@ pub struct Interval { } impl Interval { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Interval"))] #[inline] pub fn new(period: Duration) -> Self { Self { @@ -48,12 +50,14 @@ impl Interval { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Interval"))] /// Configure how missed ticks are handled. #[inline] pub fn set_missed_tick_behavior(&mut self, behavior: MissedTickBehavior) { self.missed_tick_behavior = behavior; } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Interval"))] /// Reset the interval schedule so the next tick is computed relative to /// the time when `tick()` is next called (useful when you want to restart /// the cadence). @@ -62,6 +66,10 @@ impl Interval { self.next_deadline = None; } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "Interval", future = true) + )] /// Await the next tick. Returns the number of ticks that should be processed: /// - For `MissedTickBehavior::Skip` this will be `1`. /// - For `MissedTickBehavior::CatchUp` this may be `> 1` if several periods were missed. @@ -72,6 +80,10 @@ impl Interval { // Keep the scheduling clock explicit so catch-up boundaries can be tested // without scheduler latency changing the expected number of missed ticks. + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "Interval", future = true) + )] async fn tick_at(&mut self, now: Instant) -> u64 { // Determine base next (the previous next_deadline or now+period) let base_next = self diff --git a/src/vibeio/time/mod.rs b/src/vibeio/time/mod.rs index 0edc58a..acab89d 100644 --- a/src/vibeio/time/mod.rs +++ b/src/vibeio/time/mod.rs @@ -17,6 +17,7 @@ mod interval; mod sleep; mod timeout; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Add a duration, saturating at the platform's last representable Instant. #[inline] pub(crate) fn deadline_after( @@ -29,6 +30,7 @@ pub(crate) fn deadline_after( saturating_deadline(base, duration) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cold] fn saturating_deadline( base: std::time::Instant, @@ -64,30 +66,35 @@ pub use sleep::{Sleep, ZeroBehavior}; #[allow(unused_imports)] pub use timeout::{Timeout, TimeoutError, timeout}; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Convenience builder: returns a `Sleep` future. #[inline] pub fn sleep(duration: std::time::Duration) -> Sleep { Sleep::new(duration) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Convenience builder allowing zero-behavior control for tiny durations. #[inline] pub fn sleep_with_zero_behavior(duration: std::time::Duration, behavior: ZeroBehavior) -> Sleep { Sleep::new_with_zero_behavior(duration, behavior) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Convenience builder: returns an `Interval`. #[inline] pub fn interval(period: std::time::Duration) -> Interval { Interval::new(period) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Convenience builder: returns a `Sleep` that completes at the provided absolute `Instant`. #[inline] pub fn sleep_until(deadline: std::time::Instant) -> Sleep { Sleep::sleep_until(deadline) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Convenience async function that awaits `future` but returns an error if it /// does not complete before the absolute `deadline` Instant. /// The inner future is polled first; an immediately ready result wins even if @@ -100,6 +107,7 @@ pub async fn timeout_at( Timeout::new_at(future, deadline).await } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Convenience builder: returns an `Interval` with the first tick scheduled to /// complete at `first_tick_instant` and subsequent ticks every `period`. #[inline] diff --git a/src/vibeio/time/sleep.rs b/src/vibeio/time/sleep.rs index 58d37bb..44485ed 100644 --- a/src/vibeio/time/sleep.rs +++ b/src/vibeio/time/sleep.rs @@ -43,6 +43,7 @@ pub struct Sleep { } impl Sleep { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Sleep"))] /// Create a new Sleep instance for the provided `duration`. /// Deadlines beyond the platform's Instant range saturate at its upper limit. #[inline] @@ -50,6 +51,7 @@ impl Sleep { Self::sleep_until(super::deadline_after(Instant::now(), duration)) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Sleep"))] /// Create a Sleep with custom behavior for zero-length waits. #[inline] pub fn new_with_zero_behavior(duration: Duration, zero_behavior: ZeroBehavior) -> Self { @@ -59,6 +61,7 @@ impl Sleep { ) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Sleep"))] /// Create a Sleep that completes at the specified absolute `deadline`. /// /// Preserves the absolute deadline without converting through a duration. @@ -67,6 +70,7 @@ impl Sleep { Self::sleep_until_with_zero_behavior(deadline, ZeroBehavior::Immediate) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Sleep"))] #[inline] pub(crate) fn sleep_until_with_zero_behavior( deadline: Instant, @@ -82,6 +86,7 @@ impl Sleep { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Sleep"))] /// Reset the sleep to a new absolute `deadline` (`Instant`). /// /// If a timer was previously scheduled, cancel it. The timer will be @@ -108,6 +113,10 @@ impl Sleep { impl Future for Sleep { type Output = (); + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.get_mut(); @@ -202,6 +211,10 @@ impl Future for Sleep { } impl Drop for Sleep { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn drop(&mut self) { // If we still have an outstanding timer handle, cancel it so the timer // won't hold onto our waker. diff --git a/src/vibeio/time/timeout.rs b/src/vibeio/time/timeout.rs index e63f870..6066cef 100644 --- a/src/vibeio/time/timeout.rs +++ b/src/vibeio/time/timeout.rs @@ -15,6 +15,10 @@ use super::sleep::Sleep; pub struct TimeoutError; impl fmt::Display for TimeoutError { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "operation timed out") } @@ -40,12 +44,14 @@ pin_project_lite::pin_project! { } impl Timeout { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Timeout"))] /// Create a new `Timeout` future. #[inline] pub fn new(future: F, duration: Duration) -> Self { Self::new_at(future, super::deadline_after(Instant::now(), duration)) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Timeout"))] /// Create a timeout with an absolute deadline, without rebasing it on now. #[inline] pub fn new_at(future: F, deadline: Instant) -> Self { @@ -63,6 +69,10 @@ where { type Output = Result; + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.project(); @@ -100,6 +110,7 @@ where } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure(future = true))] /// Convenience async function that awaits `future` but returns an error if it /// does not complete within `duration`. /// diff --git a/src/vibeio/timer.rs b/src/vibeio/timer.rs index 272c330..20ed053 100644 --- a/src/vibeio/timer.rs +++ b/src/vibeio/timer.rs @@ -28,6 +28,10 @@ struct DeadlineHeap { } impl DeadlineHeap { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DeadlineHeap") + )] #[inline] fn new() -> Self { Self { @@ -37,6 +41,10 @@ impl DeadlineHeap { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DeadlineHeap") + )] #[inline] fn less(&self, left: usize, right: usize) -> bool { let left = &self.entries[self.heap[left]]; @@ -44,6 +52,10 @@ impl DeadlineHeap { (left.deadline, left.generation) < (right.deadline, right.generation) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DeadlineHeap") + )] #[inline] fn swap_nodes(&mut self, left: usize, right: usize) { self.heap.swap(left, right); @@ -51,6 +63,10 @@ impl DeadlineHeap { self.entries[self.heap[right]].heap_index = right; } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DeadlineHeap") + )] fn sift_up(&mut self, mut index: usize) { while index != 0 { let parent = (index - 1) / 4; @@ -62,6 +78,10 @@ impl DeadlineHeap { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DeadlineHeap") + )] fn sift_down(&mut self, mut index: usize) { loop { let first_child = index * 4 + 1; @@ -83,6 +103,10 @@ impl DeadlineHeap { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DeadlineHeap") + )] #[inline] fn insert(&mut self, deadline: Instant, waker: Waker) -> TimerHandle { self.next_generation = self.next_generation.wrapping_add(1); @@ -105,11 +129,19 @@ impl DeadlineHeap { } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DeadlineHeap") + )] #[inline] fn deadline(&self) -> Option { self.heap.first().map(|index| self.entries[*index].deadline) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DeadlineHeap") + )] fn remove(&mut self, handle: TimerHandle) -> Option { let entry = self.entries.get(handle.slab_index)?; if entry.generation != handle.generation { @@ -135,6 +167,10 @@ impl DeadlineHeap { Some(entry.waker) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "DeadlineHeap") + )] #[inline] fn pop_expired(&mut self, now: Instant, output: &mut Vec) { while self.deadline().is_some_and(|deadline| deadline <= now) { @@ -158,6 +194,7 @@ pub struct Timer { } impl Timer { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Timer"))] #[inline] pub fn new() -> Self { Self { @@ -168,6 +205,7 @@ impl Timer { } } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Timer"))] #[inline] fn now(&self) -> Instant { #[cfg(test)] @@ -177,6 +215,7 @@ impl Timer { Instant::now() } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Timer"))] #[inline] pub fn submit(&self, deadline: Instant, waker: Waker) -> Option { if deadline <= self.now() { @@ -186,6 +225,7 @@ impl Timer { Some(self.deadlines.borrow_mut().insert(deadline, waker)) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Timer"))] #[inline] pub fn cancel(&self, handle: TimerHandle) { let waker = self.deadlines.borrow_mut().remove(handle); @@ -193,6 +233,7 @@ impl Timer { drop(waker); } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Timer"))] /// Replace a live registration's waiter without changing its heap position. /// Returns false if the handle has expired, been cancelled, or been reused. pub(crate) fn update_waker(&self, handle: TimerHandle, waker: &Waker) -> bool { @@ -228,6 +269,7 @@ impl Timer { updated } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "Timer"))] /// Wakes every expired timer and returns the exact duration until the next /// deadline. Unlike the old millisecond wheel this never discards partial /// elapsed time, so frequent scheduler spins cannot freeze timer progress. @@ -260,6 +302,10 @@ impl Timer { } impl Default for Timer { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] fn default() -> Self { Self::new() } diff --git a/src/vibeio/util/async_wrap.rs b/src/vibeio/util/async_wrap.rs index 65d60aa..0da3a94 100644 --- a/src/vibeio/util/async_wrap.rs +++ b/src/vibeio/util/async_wrap.rs @@ -75,6 +75,7 @@ pub struct AsyncWrap { } impl AsyncWrap { + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AsyncWrap"))] fn poll_pending_write(&mut self, cx: &mut Context<'_>) -> Poll> { if let Some(kind) = self.write_error { return Poll::Ready(Err(std::io::Error::new( @@ -94,6 +95,7 @@ impl AsyncWrap { Poll::Ready(result) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AsyncWrap"))] fn poll_pending_flush(&mut self, cx: &mut Context<'_>) -> Poll> { let Some(future) = self.flush_fut.as_mut() else { return Poll::Ready(Ok(())); @@ -104,6 +106,7 @@ impl AsyncWrap { Poll::Ready(result) } + #[cfg_attr(feature = "hotpath-profile", hotpath::measure(impl_type = "AsyncWrap"))] /// Create a new `AsyncWrap` wrapping the given inner value. #[inline] pub fn new(inner: T) -> Self { @@ -122,6 +125,10 @@ impl AsyncRead for AsyncWrap where T: crate::vibeio::io::AsyncRead + 'static, { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_read( self: Pin<&mut Self>, @@ -190,6 +197,10 @@ impl AsyncWrite for AsyncWrap where T: crate::vibeio::io::AsyncWrite + 'static, { + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_write( self: Pin<&mut Self>, @@ -244,6 +255,10 @@ where } } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.get_mut(); @@ -264,6 +279,10 @@ where this.poll_pending_flush(cx) } + #[cfg_attr( + feature = "hotpath-profile", + hotpath::measure(impl_type = "") + )] #[inline] fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { self.poll_flush(cx) diff --git a/src/vibeio/util/mod.rs b/src/vibeio/util/mod.rs index 582f1f3..4a68d39 100644 --- a/src/vibeio/util/mod.rs +++ b/src/vibeio/util/mod.rs @@ -16,6 +16,7 @@ use crate::vibeio::current_driver; #[cfg(target_os = "linux")] use std::sync::OnceLock; +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Check if the current driver supports completion-based I/O operations. /// /// This function returns `true` when the runtime is using a driver that can @@ -25,6 +26,7 @@ pub fn supports_completion() -> bool { current_driver().is_some_and(|driver| driver.supports_completion()) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] /// Check if the system supports io_uring with required operations. /// /// This function performs a runtime probe to verify that io_uring is available @@ -46,6 +48,7 @@ pub fn supports_io_uring() -> bool { } } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(target_os = "linux")] fn detect_io_uring_support() -> bool { use io_uring::opcode; @@ -96,6 +99,7 @@ fn detect_io_uring_support() -> bool { required_ops.iter().all(|op| probe.is_supported(*op)) } +#[cfg_attr(feature = "hotpath-profile", hotpath::measure)] #[cfg(target_os = "linux")] fn build_probe_ring() -> std::io::Result { let mut builder = io_uring::IoUring::builder(); diff --git a/tests/test_hotpath_profile.py b/tests/test_hotpath_profile.py new file mode 100644 index 0000000..d3892be --- /dev/null +++ b/tests/test_hotpath_profile.py @@ -0,0 +1,82 @@ +"""Exercise the feature-only native profiler in a fresh interpreter.""" + +import json +import os +import subprocess +import sys + +import pytest +from rsloop import _loop + +pytestmark = pytest.mark.skipif( + not hasattr(_loop, "hotpath_start"), reason="requires a hotpath-profile build" +) + + +def test_profile_lifecycle_and_complete_native_report(tmp_path): + output = tmp_path / "profile.json" + program = r""" +import asyncio +import sys +import rsloop +from rsloop import _loop + +def raises(kind, function, *args, **kwargs): + try: + function(*args, **kwargs) + except kind: + return + raise AssertionError(f"expected {kind.__name__}") + +raises(RuntimeError, _loop.hotpath_stop) +raises(ValueError, _loop.hotpath_start, sys.argv[1], format="invalid") +raises(ValueError, _loop.hotpath_start, sys.argv[1], time_sampling_rate=float("nan")) +_loop.hotpath_start(sys.argv[1]) +raises(RuntimeError, _loop.hotpath_start, sys.argv[1]) + +async def main(): + loop = asyncio.get_running_loop() + seen = [] + for i in range(100): + loop.call_soon(seen.append, i) + await asyncio.sleep(0.001) + assert seen == list(range(100)) + async def echo(reader, writer): + writer.write(await reader.readexactly(4)) + await writer.drain() + writer.close() + await writer.wait_closed() + server = await asyncio.start_server(echo, "127.0.0.1", 0) + try: + reader, writer = await asyncio.open_connection("127.0.0.1", server.sockets[0].getsockname()[1]) + try: + writer.write(b"test") + assert await reader.readexactly(4) == b"test" + finally: + writer.close() + await writer.wait_closed() + finally: + server.close() + await server.wait_closed() + +try: + rsloop.run(asyncio.wait_for(main(), 10)) +finally: + _loop.hotpath_stop() +raises(RuntimeError, _loop.hotpath_stop) +raises(RuntimeError, _loop.hotpath_start, sys.argv[1]) +""" + env = {k: v for k, v in os.environ.items() if not k.startswith("HOTPATH_")} + subprocess.run( + [sys.executable, "-c", program, str(output)], env=env, check=True, timeout=30 + ) + report = json.loads(output.read_text()) + section = report.get("functions_timing", report.get("functions_alloc")) + assert section["total_count"] == section["included_count"] > 50 + names = [row["name"] for row in section["data"]] + for namespace in ("context", "engine", "bindings", "transport::stream", "vibeio"): + assert any(name.startswith(f"rsloop::{namespace}::") for name in names) + assert report["futures"]["data"] + assert all( + row["sampled_polls"] == row["total_polls"] for row in report["futures"]["data"] + ) diff --git a/tests/test_run.py b/tests/test_run.py index 4da7732..ac32b7c 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -99,8 +99,12 @@ def test_build_info_describes_native_extension(self) -> None: "runtime_profile", "minimum_os", "tls_backend", + "hotpath_profile", + "hotpath_alloc_profile", } assert info["version"] == rsloop.__version__ + assert isinstance(info["hotpath_profile"], bool) + assert isinstance(info["hotpath_alloc_profile"], bool) assert info["profile"] in {"debug", "release"} assert info["target_os"] assert info["target_arch"] diff --git a/tests/test_stream_reader.py b/tests/test_stream_reader.py index 9e41496..0602ab2 100644 --- a/tests/test_stream_reader.py +++ b/tests/test_stream_reader.py @@ -276,6 +276,57 @@ async def drive(reader, script, call): class TestFastStreamReaderCompat: + @pytest.mark.parametrize("finish", ["exact", "overflow", "eof", "error"]) + def test_fragmented_exact_read_preserves_result_and_error(self, finish): + async def main(): + loop = asyncio.get_running_loop() + outcomes = [] + for reader in ( + PyFastStreamReader(64, loop), + asyncio.StreamReader(limit=64), + ): + # Exercise existing buffered data, then multiple direct feeds. + reader.feed_data(b"ab") + pending = asyncio.ensure_future(reader.readexactly(8)) + await asyncio.sleep(0) + reader.feed_data(b"cd") + await asyncio.sleep(0) + reader.feed_data(b"ef") + await asyncio.sleep(0) + assert not pending.done() + if finish == "eof": + reader.feed_eof() + elif finish == "error": + reader.set_exception(ConnectionResetError("reset")) + else: + reader.feed_data(b"ghTAIL" if finish == "overflow" else b"gh") + try: + outcomes.append(("ok", await asyncio.wait_for(pending, 5))) + except asyncio.IncompleteReadError as exc: + outcomes.append(("eof", exc.partial, exc.expected)) + except ConnectionResetError as exc: + outcomes.append(("error", str(exc))) + if finish == "overflow": + assert await reader.readexactly(4) == b"TAIL" + assert outcomes[0] == outcomes[1] + + rsloop.run(main()) + + def test_cancelled_exact_result_is_not_exposed_by_later_feeds(self): + async def main(): + reader = PyFastStreamReader(64, asyncio.get_running_loop()) + pending = asyncio.ensure_future(reader.readexactly(8)) + await asyncio.sleep(0) + reader.feed_data(b"abcd") + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + reader.feed_data(b"efgh") + reader.feed_data(b"next") + assert await reader.readexactly(4) == b"next" + + rsloop.run(main()) + def _run_case(self, limit, script, call): results = {} @@ -383,6 +434,43 @@ async def main() -> None: class TestFastStreamReaderNetwork: """The reported break: `readline()` over a real `asyncio.open_connection()`.""" + def test_fragmented_large_exact_read_keeps_trailing_bytes(self): + payload = bytes(range(251)) * 8192 + + async def main(): + async def send(reader, writer): + try: + await reader.readexactly(1) + for start in range(0, len(payload), 16381): + writer.write(payload[start : start + 16381]) + await writer.drain() + writer.write(b"TAIL") + finally: + writer.close() + await writer.wait_closed() + + server = await asyncio.start_server(send, "127.0.0.1", 0) + try: + reader, writer = await asyncio.open_connection( + "127.0.0.1", server.sockets[0].getsockname()[1] + ) + try: + writer.write(b"!") + assert ( + await asyncio.wait_for(reader.readexactly(len(payload)), 5) + == payload + ) + assert await reader.readexactly(4) == b"TAIL" + assert await reader.read() == b"" + finally: + writer.close() + await writer.wait_closed() + finally: + server.close() + await server.wait_closed() + + rsloop.run(main()) + @staticmethod async def _echo_lines(reader, writer) -> None: try: diff --git a/tests/tooling/test_benchmark_loops.py b/tests/tooling/test_benchmark_loops.py index 201984a..8bcca0a 100644 --- a/tests/tooling/test_benchmark_loops.py +++ b/tests/tooling/test_benchmark_loops.py @@ -18,7 +18,8 @@ class TestBenchmarkLoop: @pytest.mark.parametrize( - "mode", ["timers_retained", "timers_discarded", "timers_cancelled"] + "mode", + ["timers_retained", "timers_discarded", "timers_cancelled", "timers_mixed"], ) @pytest.mark.parametrize("loop_name", ["asyncio", "rsloop"]) def test_timer_workloads_drain_partial_final_batches(self, mode, loop_name): diff --git a/tests/tooling/test_profile_with_hotpath.py b/tests/tooling/test_profile_with_hotpath.py new file mode 100644 index 0000000..9baee28 --- /dev/null +++ b/tests/tooling/test_profile_with_hotpath.py @@ -0,0 +1,161 @@ +"""Contracts for reproducible, complete hotpath collection.""" + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +pytestmark = pytest.mark.tooling +SPEC = importlib.util.spec_from_file_location( + "profile_with_hotpath", + Path(__file__).resolve().parents[2] / "scripts/profile_with_hotpath.py", +) +assert SPEC and SPEC.loader +profile = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(profile) + + +@pytest.mark.parametrize( + "option,value", + [ + ("--callbacks", "0"), + ("--probe-iterations", "-1"), + ("--time-sampling-rate", "nan"), + ("--time-sampling-rate", "1.1"), + ("--timeout", "inf"), + ], +) +def test_invalid_profile_parameters(option, value): + with pytest.raises(SystemExit): + profile.parse_args(["callbacks", "--output", "report.json", option, value]) + + +def test_environment_cannot_silently_filter_or_redirect_reports(monkeypatch): + monkeypatch.setenv("HOTPATH_OUTPUT_PATH", "wrong.json") + monkeypatch.setenv("HOTPATH_FUNCTIONS_LIMIT", "2") + monkeypatch.setenv("HOTPATH_FOCUS", "irrelevant") + monkeypatch.setenv("HOTPATH_OUTPUT_FORMAT", "none") + env = profile.profile_environment(SimpleNamespace(focus="tls")) + assert env["HOTPATH_LIMIT"] == "0" + assert env["HOTPATH_FOCUS"] == "tls" + assert "HOTPATH_OUTPUT_PATH" not in env + assert "HOTPATH_FUNCTIONS_LIMIT" not in env + assert "HOTPATH_OUTPUT_FORMAT" not in env + + +def test_child_preserves_module_arguments_and_profiler_settings(tmp_path): + args = profile.parse_args( + [ + "module", + "--output", + str(tmp_path / "profile.json"), + "--module", + "pytest", + "--focus", + "stream", + "--time-sampling-rate", + "0.2", + "--module-args", + "-q", + "tests/test_tls.py", + ] + ) + command = profile.child_command(args, "module", args.output) + assert profile.__file__ is not None + assert command[:2] == [sys.executable, str(Path(profile.__file__).resolve())] + parsed = profile.parse_args(command[2:]) + assert parsed.child + assert parsed.time_sampling_rate == 0.2 + assert parsed.module_args == ["-q", "tests/test_tls.py"] + + +def test_coverage_keeps_unobserved_and_excluded_paths(): + entries = [ + {"name": "rsloop::seen", "status": "instrumented"}, + {"name": "rsloop::other_platform", "status": "instrumented"}, + {"name": "rsloop::signal_handler", "status": "ffi_callback"}, + ] + report = { + "functions_timing": { + "total_count": 2, + "included_count": 2, + "data": [{"name": "rsloop"}, {"name": "rsloop::seen"}], + } + } + result = profile.coverage(entries, [report]) + assert result["observed_names"] == 1 + assert result["instrumented_names"] == 2 + assert [r["observed"] for r in result["functions"]] == [True, False, False] + assert not result["unmapped_observed_names"] + assert json.loads(json.dumps(result)) == result + + +@pytest.mark.parametrize( + "section", ["functions_timing", "functions_alloc", "futures", "threads"] +) +def test_truncated_reports_are_rejected(section): + with pytest.raises(ValueError, match="truncated"): + profile.report_names({section: {"total_count": 100, "included_count": 50}}) + + +def test_existing_output_is_preserved(tmp_path): + output = tmp_path / "report.json" + output.write_text("existing") + with pytest.raises(SystemExit, match="refusing to overwrite"): + profile.main(["callbacks", "--output", str(output)]) + assert output.read_text() == "existing" + + +def test_suite_retains_failures_and_successful_reports(tmp_path, monkeypatch): + monkeypatch.setattr(profile, "WORKLOADS", ("callbacks", "tasks")) + monkeypatch.setattr(profile, "inventory", list) + monkeypatch.setattr(profile.subprocess, "check_output", lambda *a, **k: "revision") + + def run(command, **kwargs): + if command[2] == "callbacks": + return SimpleNamespace(returncode=1, stdout="", stderr="workload failed") + output = Path(command[command.index("--output") + 1]) + output.write_text( + json.dumps( + { + "functions_timing": { + "total_count": 1, + "included_count": 1, + "data": [{"name": "rsloop::tasks"}], + } + } + ) + ) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(profile.subprocess, "run", run) + output = tmp_path / "suite" + with pytest.raises(SystemExit, match="profiling failed: callbacks"): + profile.main(["all", "--output", str(output)]) + manifest = json.loads((output / "manifest.json").read_text()) + assert [r["status"] for r in manifest["runs"]] == ["failed", "ok"] + assert "workload failed" in (output / "callbacks.log").read_text() + assert (output / "tasks.json").exists() + assert (output / "coverage.json").exists() + assert "callbacks | failed" in (output / "summary.md").read_text() + + +def test_timeout_retains_child_output(tmp_path, monkeypatch): + monkeypatch.setattr(profile, "inventory", list) + monkeypatch.setattr(profile.subprocess, "check_output", lambda *a, **k: "revision") + + def run(command, **kwargs): + raise subprocess.TimeoutExpired( + command, 1, output=b"partial output", stderr=b"error" + ) + + monkeypatch.setattr(profile.subprocess, "run", run) + output = tmp_path / "timeout.json" + with pytest.raises(SystemExit, match="profiling failed"): + profile.main(["callbacks", "--output", str(output)]) + log = output.with_suffix(".log").read_text() + assert "partial output" in log and "timed out" in log diff --git a/tools/hotpath-coverage/Cargo.lock b/tools/hotpath-coverage/Cargo.lock new file mode 100644 index 0000000..1f0d24e --- /dev/null +++ b/tools/hotpath-coverage/Cargo.lock @@ -0,0 +1,121 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rsloop-hotpath-coverage" +version = "0.0.0" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.119", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tools/hotpath-coverage/Cargo.toml b/tools/hotpath-coverage/Cargo.toml new file mode 100644 index 0000000..0df15db --- /dev/null +++ b/tools/hotpath-coverage/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "rsloop-hotpath-coverage" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +proc-macro2 = { version = "1", features = ["span-locations"] } +quote = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +syn = { version = "2", features = ["full", "visit"] } diff --git a/tools/hotpath-coverage/src/main.rs b/tools/hotpath-coverage/src/main.rs new file mode 100644 index 0000000..51a3f57 --- /dev/null +++ b/tools/hotpath-coverage/src/main.rs @@ -0,0 +1,346 @@ +//! Audit explicit hotpath hooks using Rust syntax, including inactive platforms. +//! Run from any directory. --instrument adds missing attributes; default checks. +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use quote::ToTokens; +use serde::Serialize; +use syn::{ + Attribute, Signature, + spanned::Spanned, + visit::{self, Visit}, +}; + +#[derive(Serialize)] +struct Entry { + file: String, + line: usize, + name: String, + status: &'static str, + asynchronous: bool, +} + +struct Audit { + file: String, + module: String, + owner: Option, + entries: Vec, + insertions: Vec<(usize, String)>, +} + +fn test_only(attrs: &[Attribute]) -> bool { + fn condition(meta: &syn::Meta) -> bool { + match meta { + syn::Meta::Path(p) => p.is_ident("test") || p.is_ident("kani"), + syn::Meta::List(list) => { + let Ok(items) = list.parse_args_with( + syn::punctuated::Punctuated::::parse_terminated, + ) else { + return false; + }; + if list.path.is_ident("all") { + items.iter().any(condition) + } else if list.path.is_ident("any") { + !items.is_empty() && items.iter().all(condition) + } else { + false + } + } + _ => false, + } + } + attrs.iter().any(|a| { + a.path().is_ident("test") + || (a.path().is_ident("cfg") + && a.parse_args::().is_ok_and(|m| condition(&m))) + }) +} + +impl Audit { + fn function(&mut self, sig: &Signature, attrs: &[Attribute], line: usize) { + if test_only(attrs) { + return; + } + let method = self + .owner + .as_ref() + .map(|s| format!("{s}::")) + .unwrap_or_default(); + let name = format!("{}::{method}{}", self.module, sig.ident); + let measured = attrs.iter().any(|a| { + let text = a.meta.to_token_stream().to_string(); + text.contains("hotpath :: measure") + }); + let mut status = if sig.constness.is_some() { + "const_fn" + } else if sig.abi.is_some() { + "ffi_callback" + } else if self.file == "src/profile.rs" { + "profiler_control" + } else if self.file == "src/bindings/loop_api/pre_exec.rs" { + "post_fork" + } else if self.file == "src/vibeio/signal/unix.rs" + && (sig.ident == "write_signal_notification" + || (self.owner.as_deref() == Some("PendingSignals") && sig.ident == "notify")) + { + "signal_safe" + } else if sig.ident == "__traverse__" { + // A CPython GC visitor must not allocate or run arbitrary callbacks. + "gc_visitor" + } else if measured { + "instrumented" + } else { + let mut args = Vec::new(); + if let Some(owner) = &self.owner { + args.push(format!("impl_type = {owner:?}")); + } + if sig.asyncness.is_some() { + args.push("future = true".into()); + } + let args = if args.is_empty() { + String::new() + } else { + format!("({})", args.join(", ")) + }; + self.insertions.push(( + line, + format!("#[cfg_attr(feature = \"hotpath-profile\", hotpath::measure{args})]"), + )); + "missing" + }; + if measured && status != "instrumented" { + status = "forbidden_hook"; + } + if measured + && sig.asyncness.is_some() + && !attrs.iter().any(|a| { + a.meta + .to_token_stream() + .to_string() + .contains("future = true") + }) + { + status = "missing_future_tracking"; + } + self.entries.push(Entry { + file: self.file.clone(), + line: sig.ident.span().start().line, + name, + status, + asynchronous: sig.asyncness.is_some(), + }); + } +} + +impl<'ast> Visit<'ast> for Audit { + fn visit_item_mod(&mut self, node: &'ast syn::ItemMod) { + if test_only(&node.attrs) { + return; + } + let old = self.module.clone(); + self.module.push_str(&format!("::{}", node.ident)); + visit::visit_item_mod(self, node); + self.module = old; + } + fn visit_item_impl(&mut self, node: &'ast syn::ItemImpl) { + if test_only(&node.attrs) { + return; + } + let ty = match node.self_ty.as_ref() { + syn::Type::Path(p) => p.path.segments.last().unwrap().ident.to_string(), + other => other.to_token_stream().to_string(), + }; + let owner = if let Some((_, path, _)) = &node.trait_ { + format!("<{ty} as {}>", path.to_token_stream()) + } else { + ty + }; + let old = self.owner.replace(owner); + visit::visit_item_impl(self, node); + self.owner = old; + } + fn visit_item_fn(&mut self, node: &'ast syn::ItemFn) { + self.function(&node.sig, &node.attrs, node.span().start().line); + // Nested helpers are separately inventoried, but have module-scoped labels. + if !test_only(&node.attrs) { + visit::visit_block(self, &node.block); + } + } + fn visit_impl_item_fn(&mut self, node: &'ast syn::ImplItemFn) { + self.function(&node.sig, &node.attrs, node.span().start().line); + if !test_only(&node.attrs) { + visit::visit_block(self, &node.block); + } + } + fn visit_item_trait(&mut self, node: &'ast syn::ItemTrait) { + let old = self.owner.replace(node.ident.to_string()); + visit::visit_item_trait(self, node); + self.owner = old; + } + fn visit_trait_item_fn(&mut self, node: &'ast syn::TraitItemFn) { + if node.default.is_some() { + self.function(&node.sig, &node.attrs, node.span().start().line); + } + } +} + +fn sources(dir: &Path, paths: &mut Vec) -> std::io::Result<()> { + for entry in fs::read_dir(dir)? { + let path = entry?.path(); + if path.is_dir() { + sources(&path, paths)?; + } else if path.extension().is_some_and(|e| e == "rs") { + paths.push(path); + } + } + Ok(()) +} + +fn main() -> Result<(), Box> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let args: Vec<_> = std::env::args().skip(1).collect(); + if args.iter().any(|s| s != "--instrument" && s != "--json") { + return Err("usage: rsloop-hotpath-coverage [--instrument] [--json]".into()); + } + let instrument = args.iter().any(|s| s == "--instrument"); + let mut paths = Vec::new(); + sources(&root.join("src"), &mut paths)?; + paths.sort(); + let mut entries = Vec::new(); + let mut missing = 0; + for path in paths { + let file = path + .strip_prefix(&root)? + .to_string_lossy() + .replace('\\', "/"); + if file == "src/verification.rs" + || file.ends_with("/test_support.rs") + || file.ends_with("/mock.rs") + { + continue; + } + let source = fs::read_to_string(&path)?; + let syntax = syn::parse_file(&source)?; + let mut parts: Vec<_> = file.trim_end_matches(".rs").split('/').skip(1).collect(); + if matches!(parts.last(), Some(&"mod" | &"lib")) { + parts.pop(); + } + let module = std::iter::once("rsloop") + .chain(parts) + .collect::>() + .join("::"); + let mut audit = Audit { + file, + module, + owner: None, + entries: Vec::new(), + insertions: Vec::new(), + }; + audit.visit_file(&syntax); + missing += audit.insertions.len(); + if instrument && !audit.insertions.is_empty() { + let mut lines: Vec = source.lines().map(str::to_string).collect(); + audit.insertions.sort_by_key(|(line, _)| *line); + for (line, attr) in audit.insertions.into_iter().rev() { + let indent: String = lines[line - 1] + .chars() + .take_while(|c| c.is_whitespace()) + .collect(); + lines.insert(line - 1, format!("{indent}{attr}")); + } + fs::write(&path, lines.join("\n") + "\n")?; + } + entries.extend(audit.entries); + } + if args.iter().any(|s| s == "--json") { + println!("{}", serde_json::to_string_pretty(&entries)?); + } + eprintln!( + "{} function definitions audited; {missing} {} hooks", + entries.len(), + if instrument { "added" } else { "missing" } + ); + let invalid: Vec<_> = entries + .iter() + .filter(|e| matches!(e.status, "forbidden_hook" | "missing_future_tracking")) + .collect(); + for entry in &invalid { + eprintln!( + "{}:{}: {} ({})", + entry.file, entry.line, entry.status, entry.name + ); + } + if (missing > 0 && !instrument) || !invalid.is_empty() { + std::process::exit(1); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn audit(source: &str, file: &str) -> Audit { + let mut audit = Audit { + file: file.into(), + module: "rsloop::fixture".into(), + owner: None, + entries: Vec::new(), + insertions: Vec::new(), + }; + audit.visit_file(&syn::parse_file(source).unwrap()); + audit + } + + #[test] + fn skips_tests_but_keeps_optional_runtime_code() { + let a = audit( + r#" + #[cfg(all(unix, test))] mod tests { fn helper() {} } + #[cfg(any(test, feature = "fs"))] fn optional() {} + #[cfg(any(test, kani))] fn proof() {} + const fn constant() -> u8 { 1 } + extern "C" fn callback() {} + "#, + "src/example.rs", + ); + assert_eq!(a.insertions.len(), 1); + assert_eq!( + a.entries.iter().map(|e| e.status).collect::>(), + ["missing", "const_fn", "ffi_callback"] + ); + } + + #[test] + fn methods_keep_type_and_trait_identity_and_async_tracking() { + let a = audit( + "impl A { async fn run() {} } impl Future for A { fn poll() {} }", + "src/example.rs", + ); + assert!( + a.insertions[0] + .1 + .contains("impl_type = \"A\", future = true") + ); + assert_ne!(a.entries[0].name, a.entries[1].name); + assert!(a.entries[1].name.contains("")); + } + + #[test] + fn rejects_hooks_on_signal_safe_helpers_and_missing_future_tracking() { + let a = audit( + r#" + #[cfg_attr(feature = "hotpath-profile", hotpath::measure)] + fn write_signal_notification() {} + #[cfg_attr(feature = "hotpath-profile", hotpath::measure)] + async fn receive() {} + "#, + "src/vibeio/signal/unix.rs", + ); + assert_eq!(a.entries[0].status, "forbidden_hook"); + assert_eq!(a.entries[1].status, "missing_future_tracking"); + assert!(a.insertions.is_empty()); + } +} diff --git a/tools/vibeio-check/Cargo.toml b/tools/vibeio-check/Cargo.toml index d12ac54..ecede96 100644 --- a/tools/vibeio-check/Cargo.toml +++ b/tools/vibeio-check/Cargo.toml @@ -58,4 +58,4 @@ style = { level = "allow", priority = -1 } suspicious = { level = "warn", priority = -1 } [lints.rust] -unexpected_cfgs = { level = "warn", check-cfg = ["cfg(kani)", "cfg(musl_v1_2_3)", "cfg(syscall_accept4)"] } +unexpected_cfgs = { level = "warn", check-cfg = ["cfg(kani)", "cfg(musl_v1_2_3)", "cfg(syscall_accept4)", "cfg(feature, values(\"hotpath-profile\"))"] }