diff --git a/.gitignore b/.gitignore index 36a0b45..fa4ac83 100755 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ main TrackMe .DS_Store requests-db/ +data/ +*.parquet config.json .idea adminKey diff --git a/README.md b/README.md index 527f9a4..6fefad6 100755 --- a/README.md +++ b/README.md @@ -112,6 +112,29 @@ Param: `?by=` Returns the most seen other identifiers (user-agent, h2, JA3) that were seen together with this identifier. Only works when connected to a database. +## Logging to Parquet + +TrackMe can persist the fingerprints it captures to [Apache Parquet](https://parquet.apache.org/) files for later analysis. Enable it in `config.json`: + +```json +{ + "log_to_parquet": true, + "parquet_dir": "data", + "parquet_log_ips": false +} +``` + +- `log_to_parquet`: turns the Parquet sink on/off. +- `parquet_dir`: directory the files are written to (created if missing, defaults to `data`). +- `parquet_log_ips`: when `false` (default) the client IP column is left empty. + +Each captured request is written as a row with the timestamp, HTTP version, method, path, user-agent and the JA3 / JA4 / PeetPrint / Akamai fingerprints (and their hashes). Rows are buffered and flushed to a new `trackme-.parquet` file every 1,000,000 rows or once a day, whichever comes first — so the directory becomes a Parquet dataset you can query with DuckDB, pandas, Spark, etc.: + +```sql +-- DuckDB +SELECT ja3_hash, count(*) FROM 'data/*.parquet' GROUP BY ja3_hash ORDER BY 2 DESC; +``` + ## Docker You can also run the server in a docker container using docker-compose. diff --git a/cmd/main.go b/cmd/main.go index b3d2614..e067d74 100755 --- a/cmd/main.go +++ b/cmd/main.go @@ -7,8 +7,10 @@ import ( "net" "net/http" "os" + "os/signal" "runtime" "strconv" + "syscall" "time" "github.com/pagpeter/quic-go" @@ -57,6 +59,8 @@ func init() { if err := srv.GetConfig().LoadFromFile(); err != nil { log.Fatal(err) } + + srv.InitParquet() } func redirect(w http.ResponseWriter, r *http.Request) { @@ -123,10 +127,21 @@ func main() { defer func() { if r := recover(); r != nil { logCrash(r) + srv.CloseParquet() os.Exit(1) } }() + // Flush buffered data to disk on Ctrl+C / SIGTERM before exiting. + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + go func() { + sig := <-sigs + log.Println("Received signal, shutting down:", sig) + srv.CloseParquet() + os.Exit(0) + }() + log.Println("Starting server...") log.Println("Listening on " + srv.GetConfig().Host + ":" + srv.GetConfig().TLSPort) diff --git a/config.example.json b/config.example.json index f3300cc..31329a5 100644 --- a/config.example.json +++ b/config.example.json @@ -1,5 +1,4 @@ { - "log_to_db": false, "tls_port": "443", "http_port": "80", "cert_file": "certs/chain.pem", @@ -8,5 +7,8 @@ "http_redirect": "https://tls.peet.ws", "device": "eth0", "cors_key": "X-CORS", - "enable_quic": true + "enable_quic": true, + "log_to_parquet": false, + "parquet_dir": "data", + "parquet_log_ips": false } diff --git a/go.mod b/go.mod index 56a8a6d..6b97b3e 100755 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/pagpeter/trackme -go 1.24 +go 1.24.9 require ( github.com/google/gopacket v1.1.19 @@ -13,13 +13,20 @@ require ( require ( github.com/andybalholm/brotli v1.1.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/klauspost/compress v1.17.11 // indirect + github.com/parquet-go/bitpack v1.0.0 // indirect + github.com/parquet-go/jsonlite v1.0.0 // indirect + github.com/parquet-go/parquet-go v0.30.1 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/refraction-networking/utls v1.1.2 // indirect + github.com/twpayne/go-geom v1.6.1 // indirect golang.org/x/crypto v0.41.0 // indirect golang.org/x/mod v0.27.0 // indirect golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.35.0 // indirect + golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.28.0 // indirect golang.org/x/tools v0.36.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/go.sum b/go.sum index 136f02b..8c11925 100755 --- a/go.sum +++ b/go.sum @@ -7,11 +7,21 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/pagpeter/quic-go v0.0.0-20260120153640-0de4e3b8377b h1:Ty5nepKkVdrdTetjXskX+v+3XP2GNqmmcqvl0BX4/qQ= github.com/pagpeter/quic-go v0.0.0-20260120153640-0de4e3b8377b/go.mod h1:EJQW9gTvp3XGR6qPANdXlU55yxrA8rww5atbR2LVI9U= +github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA= +github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs= +github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU= +github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0= +github.com/parquet-go/parquet-go v0.30.1 h1:Oy6ganNrAdFiVwy7wNmWagfPTWA2X9Z3tVHBc7JtuX8= +github.com/parquet-go/parquet-go v0.30.1/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= @@ -20,6 +30,8 @@ github.com/refraction-networking/utls v1.1.2 h1:a7GQauRt72VG+wtNm0lnrAaCGlyX47gE github.com/refraction-networking/utls v1.1.2/go.mod h1:+D89TUtA8+NKVFj1IXWr0p3tSdX1+SqUB7rL0QnGqyg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4= +github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028= github.com/wwhtrbbtt/utls v0.0.0-20220918194152-45ee2a20799c h1:eDUKT2sHyNTpZTawrungpwOZgaEvcbTldzrmEmBO0pY= github.com/wwhtrbbtt/utls v0.0.0-20220918194152-45ee2a20799c/go.mod h1:cE/NJeUKssh/0XGO4KVBXZH0u7/BqRDqGs1Ij8hgy0w= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= @@ -51,6 +63,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -62,5 +76,7 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/parquet/logger.go b/pkg/parquet/logger.go new file mode 100644 index 0000000..23a5081 --- /dev/null +++ b/pkg/parquet/logger.go @@ -0,0 +1,318 @@ +// Package parquet persists captured request fingerprints to Parquet files. +// +// The logger is intentionally non-blocking: request handlers hand off rows over +// a buffered channel and a single background goroutine batches them into rolling +// Parquet files. Each flush produces a new file (data/trackme-.parquet), +// which keeps the writer simple (Parquet files are immutable once their footer is +// written) and yields a directory of files that analytics tools read as one +// dataset. +package parquet + +import ( + "fmt" + "log" + "os" + "path/filepath" + "sync" + "time" + + "github.com/pagpeter/trackme/pkg/types" + pq "github.com/parquet-go/parquet-go" +) + +// Row is the schema written to Parquet. It captures the useful fingerprints +// rather than the full nested Response (whose TLS extension list does not map +// cleanly onto a columnar schema). +type Row struct { + Timestamp int64 `parquet:"timestamp"` + IP string `parquet:"ip"` + HTTPVersion string `parquet:"http_version"` + Method string `parquet:"method"` + Path string `parquet:"path"` + JA3 string `parquet:"ja3"` + JA4 string `parquet:"ja4"` + JA4R string `parquet:"ja4_r"` + PeetPrint string `parquet:"peetprint"` + Akamai string `parquet:"akamai"` + // Headers holds the request headers in wire order, each as "name: value". + Headers []string `parquet:"headers"` + // TCPIP holds the full TCP/IP fingerprint, when one was captured for the IP. + TCPIP TCPIP `parquet:"tcpip"` +} + +// TCPIP mirrors types.TCPIPDetails with Parquet-friendly column names. +type TCPIP struct { + CapLen int32 `parquet:"cap_length"` + DstPort int32 `parquet:"dst_port"` + SrcPort int32 `parquet:"src_port"` + HeaderLen int32 `parquet:"header_length"` + TS []int32 `parquet:"ts"` + IP IP `parquet:"ip"` + TCP TCP `parquet:"tcp"` +} + +// IP mirrors types.IPDetails. +type IP struct { + DF int32 `parquet:"df"` + HDRLength int32 `parquet:"hdr_length"` + ID int32 `parquet:"id"` + MF int32 `parquet:"mf"` + NXT int32 `parquet:"nxt"` + OFF int32 `parquet:"off"` + PLEN int32 `parquet:"plen"` + Protocol int32 `parquet:"protocol"` + RF int32 `parquet:"rf"` + TOS int32 `parquet:"tos"` + TotalLength int32 `parquet:"total_length"` + TTL int32 `parquet:"ttl"` + IPVersion int32 `parquet:"ip_version"` + DstIp string `parquet:"dst_ip"` + SrcIP string `parquet:"src_ip"` +} + +// TCP mirrors types.TCPDetails. +type TCP struct { + Ack int32 `parquet:"ack"` + Checksum int32 `parquet:"checksum"` + Flags int32 `parquet:"flags"` + HeaderLength int32 `parquet:"header_length"` + MSS int32 `parquet:"mss"` + OFF int32 `parquet:"off"` + Options string `parquet:"options"` + OptionsOrder string `parquet:"options_order"` + Seq int32 `parquet:"seq"` + Timestamp int32 `parquet:"timestamp"` + TimestampEchoReply int32 `parquet:"timestamp_echo_reply"` + URP int32 `parquet:"urp"` + Window int32 `parquet:"window"` +} + +const ( + // channelBuffer is how many rows can queue before logging starts dropping + // them. Dropping (rather than blocking) keeps request handling fast even if + // disk writes stall. + channelBuffer = 4096 + // batchSize triggers a row-group flush once this many rows have accumulated. + batchSize = 500 + // flushInterval forces a row-group flush even when batchSize is not reached, + // so low-traffic data lands on disk without waiting for shutdown. + flushInterval = 3 * time.Minute +) + +// Logger persists Rows to a single Parquet file per run. Row groups are +// flushed periodically so data reaches disk without waiting for shutdown, but +// the file footer (needed for readers) is only written on Close. +type Logger struct { + dir string + logIPs bool + + ch chan Row + done chan struct{} + wg sync.WaitGroup +} + +// New creates and starts a Logger writing to dir. The directory is created if it +// does not exist. +func New(dir string, logIPs bool) (*Logger, error) { + if dir == "" { + dir = "data" + } + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, fmt.Errorf("failed to create parquet dir %q: %w", dir, err) + } + + l := &Logger{ + dir: dir, + logIPs: logIPs, + ch: make(chan Row, channelBuffer), + done: make(chan struct{}), + } + l.wg.Add(1) + go l.run() + return l, nil +} + +// Log enqueues a captured Response. It never blocks: if the buffer is full the +// row is dropped and a warning is logged. +func (l *Logger) Log(res types.Response) { + if l == nil { + return + } + row := rowFromResponse(res, l.logIPs) + select { + case l.ch <- row: + default: + log.Println("parquet: buffer full, dropping row") + } +} + +// Close flushes any buffered rows and stops the background goroutine. +func (l *Logger) Close() { + if l == nil { + return + } + close(l.done) + l.wg.Wait() +} + +func rowFromResponse(res types.Response, logIPs bool) Row { + row := Row{ + Timestamp: time.Now().UnixNano(), + HTTPVersion: res.HTTPVersion, + Method: res.Method, + Path: res.Path, + } + if logIPs { + row.IP = res.IP + } + if res.TLS != nil { + row.JA3 = res.TLS.JA3 + row.JA4 = res.TLS.JA4 + row.JA4R = res.TLS.JA4_r + row.PeetPrint = res.TLS.PeetPrint + } + if res.HTTPVersion == "h2" && res.Http2 != nil { + row.Akamai = res.Http2.AkamaiFingerprint + } else if res.HTTPVersion == "h3" && res.Http3 != nil { + row.Akamai = res.Http3.AkamaiFingerprint + } + row.Headers = orderedHeaders(res) + row.TCPIP = tcpipFromResponse(res.TCPIP) + return row +} + +// orderedHeaders returns the request headers in the order they were sent, as +// "name: value" strings, regardless of the protocol version. +func orderedHeaders(res types.Response) []string { + switch res.HTTPVersion { + case "h2": + if res.Http2 != nil { + for _, f := range res.Http2.SendFrames { + if f.Type == "HEADERS" && len(f.Headers) > 0 { + return f.Headers + } + } + } + case "h3": + if res.Http3 != nil { + return res.Http3.Headers + } + default: + if res.Http1 != nil { + return res.Http1.Headers + } + } + return nil +} + +func tcpipFromResponse(d types.TCPIPDetails) TCPIP { + ts := make([]int32, len(d.TS)) + for i, v := range d.TS { + ts[i] = int32(v) + } + return TCPIP{ + CapLen: int32(d.CapLen), + DstPort: int32(d.DstPort), + SrcPort: int32(d.SrcPort), + HeaderLen: int32(d.HeaderLen), + TS: ts, + IP: IP{ + DF: int32(d.IP.DF), + HDRLength: int32(d.IP.HDRLength), + ID: int32(d.IP.ID), + MF: int32(d.IP.MF), + NXT: int32(d.IP.NXT), + OFF: int32(d.IP.OFF), + PLEN: int32(d.IP.PLEN), + Protocol: int32(d.IP.Protocol), + RF: int32(d.IP.RF), + TOS: int32(d.IP.TOS), + TotalLength: int32(d.IP.TotalLength), + TTL: int32(d.IP.TTL), + IPVersion: int32(d.IP.IPVersion), + DstIp: d.IP.DstIp, + SrcIP: d.IP.SrcIP, + }, + TCP: TCP{ + Ack: int32(d.TCP.Ack), + Checksum: int32(d.TCP.Checksum), + Flags: int32(d.TCP.Flags), + HeaderLength: int32(d.TCP.HeaderLength), + MSS: int32(d.TCP.MSS), + OFF: int32(d.TCP.OFF), + Options: d.TCP.Options, + OptionsOrder: d.TCP.OptionsOrder, + Seq: int32(d.TCP.Seq), + Timestamp: int32(d.TCP.Timestamp), + TimestampEchoReply: int32(d.TCP.TimestampEchoReply), + URP: int32(d.TCP.URP), + Window: int32(d.TCP.Window), + }, + } +} + +func (l *Logger) run() { + defer l.wg.Done() + + ticker := time.NewTicker(flushInterval) + defer ticker.Stop() + + buf := make([]Row, 0, batchSize) + + // Open a single file for this run; close it on exit. + name := fmt.Sprintf("trackme-%d.parquet", time.Now().UnixNano()) + path := filepath.Join(l.dir, name) + f, err := os.Create(path) + if err != nil { + log.Println("parquet: failed to create file:", err) + return + } + w := pq.NewGenericWriter[Row](f, pq.Compression(&pq.Snappy)) + + closeWriter := func() { + if err := w.Close(); err != nil { + log.Println("parquet: failed to close writer:", err) + } + if err := f.Close(); err != nil { + log.Println("parquet: failed to close file:", err) + } + } + defer closeWriter() + + // flushRowGroup writes buffered rows as a row group and clears the buffer. + // The file footer is not written until closeWriter, so the file is not yet + // readable by external tools — but the row-group bytes are on disk. + flushRowGroup := func() { + if len(buf) == 0 { + return + } + if _, err := w.Write(buf); err != nil { + log.Println("parquet: failed to write rows:", err) + } else if err := w.Flush(); err != nil { + log.Println("parquet: failed to flush row group:", err) + } + buf = buf[:0] + } + + for { + select { + case row := <-l.ch: + buf = append(buf, row) + if len(buf) >= batchSize { + flushRowGroup() + } + case <-ticker.C: + flushRowGroup() + case <-l.done: + for { + select { + case row := <-l.ch: + buf = append(buf, row) + default: + flushRowGroup() + return + } + } + } + } +} diff --git a/pkg/server/router.go b/pkg/server/router.go index 47132b9..71b9434 100755 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -41,6 +41,8 @@ func Router(path string, res types.Response, srv *Server) ([]byte, string, error Log(fmt.Sprintf("%v %v %v %v %v", cleanIP(res.IP), res.Method, res.HTTPVersion, res.Path, "-")) } + srv.LogParquet(res) + u, err := url.Parse("https://tls.peet.ws" + path) var m map[string][]string if err != nil || u == nil { diff --git a/pkg/server/server.go b/pkg/server/server.go index 853de92..0f99c1c 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -1,9 +1,11 @@ package server import ( + "log" "strings" "sync" + "github.com/pagpeter/trackme/pkg/parquet" "github.com/pagpeter/trackme/pkg/types" ) @@ -12,6 +14,7 @@ type State struct { Config *types.Config TCPFingerprints sync.Map Local bool + Parquet *parquet.Logger } // Server provides access to shared state and functionality @@ -34,6 +37,31 @@ func (s *Server) GetConfig() *types.Config { return s.State.Config } +// InitParquet starts the Parquet logger if enabled in the config. It must be +// called after the config has been loaded. +func (s *Server) InitParquet() { + if !s.State.Config.LogToParquet { + return + } + l, err := parquet.New(s.State.Config.ParquetDir, s.State.Config.ParquetLogIPs) + if err != nil { + log.Println("Failed to start parquet logger:", err) + return + } + s.State.Parquet = l + log.Println("Logging captured fingerprints to parquet dir:", s.State.Config.ParquetDir) +} + +// LogParquet persists a captured response if the Parquet logger is enabled. +func (s *Server) LogParquet(res types.Response) { + s.State.Parquet.Log(res) +} + +// CloseParquet flushes and stops the Parquet logger. +func (s *Server) CloseParquet() { + s.State.Parquet.Close() +} + // GetTCPFingerprints returns the TCP fingerprints map func (s *Server) GetTCPFingerprints() *sync.Map { return &s.State.TCPFingerprints diff --git a/pkg/types/structs.go b/pkg/types/structs.go index 95d896f..58c4db2 100755 --- a/pkg/types/structs.go +++ b/pkg/types/structs.go @@ -185,6 +185,13 @@ type Config struct { Device string `json:"device"` CorsKey string `json:"cors_key"` EnableQUIC bool `json:"enable_quic"` + + // LogToParquet enables persisting captured fingerprints to Parquet files. + LogToParquet bool `json:"log_to_parquet"` + // ParquetDir is the directory the Parquet files are written to. + ParquetDir string `json:"parquet_dir"` + // ParquetLogIPs controls whether the client IP is stored in the Parquet rows. + ParquetLogIPs bool `json:"parquet_log_ips"` } func (c *Config) LoadFromFile() error { @@ -209,6 +216,12 @@ func (c *Config) LoadFromFile() error { c.Device = tmp.Device c.CorsKey = tmp.CorsKey c.EnableQUIC = tmp.EnableQUIC + c.LogToParquet = tmp.LogToParquet + c.ParquetDir = tmp.ParquetDir + c.ParquetLogIPs = tmp.ParquetLogIPs + if c.ParquetDir == "" { + c.ParquetDir = "data" + } return nil } @@ -232,4 +245,7 @@ func (c *Config) MakeDefault() { c.HTTPRedirect = "https://tls.peet.ws" c.CorsKey = "X-CORS" c.EnableQUIC = true + c.LogToParquet = false + c.ParquetDir = "data" + c.ParquetLogIPs = false }