Skip to content

per-host http client pool #7301

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Mzack9999 wants to merge 11 commits into
base: dev
Choose a base branch
from
Draft

per-host http client pool #7301

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
95739a2
attempting per-host http client pool
Mzack9999 Mar 25, 2026
fa6ee67
close idle conns
Mzack9999 Mar 25, 2026
5516e1c
fixing idle conn leak
Mzack9999 Mar 25, 2026
ae6b21f
lint
Mzack9999 Mar 25, 2026
96360b8
ignore http internal routines
Mzack9999 Mar 25, 2026
ab8cf23
merge dev into 5453-per-host-http-client-pool
Mzack9999 Mar 25, 2026
7024544
fixing tests
Mzack9999 Mar 25, 2026
6c3d9b8
lowering flaky http requirement
Mzack9999 Mar 25, 2026
60d0627
fixing open gate issue
Mzack9999 Mar 25, 2026
32134d5
Merge branch 'dev' into 5453-per-host-http-client-pool
Mzack9999 Apr 22, 2026
cef06fe
adding perf tests
Mzack9999 Apr 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion internal/runner/runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ func New(options *types.Options) (*Runner, error) {
var httpclient *retryablehttp.Client
if options.ProxyInternal && options.AliveHttpProxy != "" || options.AliveSocksProxy != "" {
var err error
httpclient, err = httpclientpool.Get(options, &httpclientpool.Configuration{})
httpclient, err = httpclientpool.Get(options, &httpclientpool.Configuration{}, "")
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -427,6 +427,11 @@ func (r *Runner) Close() {
if r.httpStats != nil {
r.httpStats.DisplayTopStats(r.options.NoColor)
}
if newConns, reusedConns := httpclientpool.GetConnectionStats(); newConns+reusedConns > 0 {
total := newConns + reusedConns
ratio := float64(reusedConns) / float64(total) * 100
gologger.Info().Msgf("HTTP connections: %d total, %d new, %d reused (%.1f%%)", total, newConns, reusedConns, ratio)
}
Comment on lines +430 to +434
Copy link
Copy Markdown
Contributor

@coderabbitai coderabbitai Bot Mar 25, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Connection stats will be cumulative across multiple in-process scans.

GetConnectionStats() reads package-global counters, so a second SDK/embedded run in the same process will log totals from earlier executions too. If this is meant to diagnose one scan, reset or scope these stats by ExecutionId when a run starts or ends.

🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@internal/runner/runner.go` around lines 430 - 434, GetConnectionStats()
returns package-global cumulative counters, so per-scan logs in runner.go show
aggregated totals across in-process runs; update the httpclientpool usage to
produce per-execution stats by either calling a reset or using a scoped API:
add/ call httpclientpool.ResetConnectionStats() at the start (or end) of a run
inside the runner (around where GetConnectionStats() is invoked) or change to an
API like httpclientpool.GetConnectionStatsForExecution(executionId) /
httpclientpool.ScopedStats(executionId) and pass the run's ExecutionId so the
logged totals (from GetConnectionStats / new scoped method) reflect only the
current scan. Ensure you reference and modify the call site in runner.go where
GetConnectionStats() is used and wire in ExecutionId from the current Runner
context.

// dump hosterrors cache
if r.hostErrors != nil {
r.hostErrors.Close()
Expand Down Expand Up @@ -507,6 +512,11 @@ func (r *Runner) setupPDCPUpload(writer output.Writer) output.Writer {
// RunEnumeration sets up the input layer for giving input nuclei.
// binary and runs the actual enumeration
func (r *Runner) RunEnumeration() error {
// Reset connection-reuse counters so the summary logged on Close()
// reflects only this run, not totals accumulated across multiple
// in-process executions (e.g. SDK / embedded usage).
httpclientpool.ResetConnectionStats()

// If the user has asked for DAST server mode, run the live
// DAST fuzzing server.
if r.options.DASTServer {
Expand Down
2 changes: 1 addition & 1 deletion lib/sdk_private.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ func (e *NucleiEngine) init(ctx context.Context) error {
}

if e.opts.ProxyInternal && e.opts.AliveHttpProxy != "" || e.opts.AliveSocksProxy != "" {
httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{})
httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{}, "")
if err != nil {
return err
}
Expand Down
7 changes: 4 additions & 3 deletions lib/tests/sdk_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,12 @@ import (
)

var knownLeaks = []goleak.Option{
// prettyify the output and generate dependency graph and more details instead of just stack output
goleak.Pretty(),
// net/http transport maintains idle connections which are closed with cooldown
// hence they don't count as leaks
// net/http transport maintains idle keep-alive connections whose goroutines
// exit on idle timeout or explicit close - not real leaks.
goleak.IgnoreAnyFunction("net/http.(*http2ClientConn).readLoop"),
goleak.IgnoreAnyFunction("net/http.(*persistConn).readLoop"),
goleak.IgnoreAnyFunction("net/http.(*persistConn).writeLoop"),
}

func TestSimpleNuclei(t *testing.T) {
Expand Down
10 changes: 5 additions & 5 deletions pkg/protocols/common/automaticscan/automaticscan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
Expand Down Expand Up @@ -95,11 +94,12 @@ func New(opts Options) (*Service, error) {
return nil, err
}

// Wappalyzer fingerprinting is a stateless GET reused across every target.
// Disable the cookie jar to avoid retaining cross-target state and the
// associated memory growth from a long-lived shared client.
httpclient, err := httpclientpool.Get(opts.ExecuterOpts.Options, &httpclientpool.Configuration{
Connection: &httpclientpool.ConnectionConfiguration{
DisableKeepAlive: httputil.ShouldDisableKeepAlive(opts.ExecuterOpts.Options),
},
})
DisableCookie: true,
}, "")
if err != nil {
return nil, errors.Wrap(err, "could not get http client")
}
Expand Down
16 changes: 13 additions & 3 deletions pkg/protocols/common/protocolstate/state.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,10 @@ func initDialers(options *types.Options) error {
networkPolicy, _ := networkpolicy.New(*npOptions)

httpClientPool := mapsutil.NewSyncLockMap(
// evicts inactive httpclientpool entries after 24 hours
// of inactivity (long running instances)
mapsutil.WithEviction[string, *retryablehttp.Client](24*time.Hour, 12*time.Hour),
// Per-host HTTP clients are evicted after 90 seconds of inactivity.
// Combined with IdleConnTimeout on each transport, this ensures
// connections to already-scanned hosts are cleaned up promptly.
mapsutil.WithEviction[string, *retryablehttp.Client](90*time.Second, 30*time.Second),
)

dialersInstance := &Dialers{
Expand Down Expand Up @@ -279,6 +280,15 @@ func Close(executionId string) {
}

if dialersInstance != nil {
// Close idle keep-alive connections on all cached HTTP clients
// to avoid lingering transport goroutines after shutdown.
_ = dialersInstance.HTTPClientPool.Iterate(func(_ string, client *retryablehttp.Client) error {
if client != nil && client.HTTPClient != nil {
client.HTTPClient.CloseIdleConnections()
}
return nil
})
dialersInstance.HTTPClientPool.Clear()
dialersInstance.Fastdialer.Close()
}

Expand Down
7 changes: 4 additions & 3 deletions pkg/protocols/http/build_request.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
"github.com/projectdiscovery/rawhttp"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/utils/errkit"
Expand Down Expand Up @@ -485,8 +484,10 @@ func (r *requestGenerator) fillRequest(req *retryablehttp.Request, values map[st
}
}

// In case of multiple threads the underlying connection should remain open to allow reuse
if r.request.Threads <= 0 && req.Header.Get("Connection") == "" && r.options.Options.ScanStrategy != scanstrategy.HostSpray.String() {
// Per-host clients always have keep-alive enabled for connection reuse.
// Only force-close connections when a template explicitly disables keep-alive.
if r.request.connConfiguration != nil && r.request.connConfiguration.Connection != nil &&
r.request.connConfiguration.Connection.DisableKeepAlive && req.Header.Get("Connection") == "" {
req.Close = true
}

Expand Down
21 changes: 5 additions & 16 deletions pkg/protocols/http/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,8 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
"github.com/projectdiscovery/rawhttp"
"github.com/projectdiscovery/retryablehttp-go"
fileutil "github.com/projectdiscovery/utils/file"
)

Expand Down Expand Up @@ -143,10 +141,9 @@ type Request struct {
options *protocols.ExecutorOptions
connConfiguration *httpclientpool.Configuration
totalRequests int
customHeaders map[string]string
generator *generators.PayloadGenerator // optional, only enabled when using payloads
httpClient *retryablehttp.Client
rawhttpClient *rawhttp.Client
customHeaders map[string]string
generator *generators.PayloadGenerator // optional, only enabled when using payloads
rawhttpClient *rawhttp.Client
dialer *fastdialer.Dialer

// description: |
Expand Down Expand Up @@ -310,10 +307,8 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
MaxRedirects: request.MaxRedirects,
NoTimeout: false,
DisableCookie: request.DisableCookie,
Connection: &httpclientpool.ConnectionConfiguration{
DisableKeepAlive: httputil.ShouldDisableKeepAlive(options.Options),
},
RedirectFlow: httpclientpool.DontFollowRedirect,
Connection: &httpclientpool.ConnectionConfiguration{},
RedirectFlow: httpclientpool.DontFollowRedirect,
}
var customTimeout int
if request.Analyzer != nil && request.Analyzer.Name == "time_delay" {
Expand Down Expand Up @@ -345,13 +340,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
}
}
request.connConfiguration = connectionConfiguration

client, err := httpclientpool.Get(options.Options, connectionConfiguration)
if err != nil {
return errors.Wrap(err, "could not get dns client")
}
request.customHeaders = make(map[string]string)
request.httpClient = client

dialer, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{
CustomDialer: options.CustomFastdialer,
Expand Down
Loading
Loading