feat: add Parquet logging for captured fingerprints

.gitignore

@@ -7,6 +7,8 @@ main
 TrackMe
 .DS_Store
 requests-db/
+data/
+*.parquet
 config.json
 .idea
 adminKey

README.md

@@ -112,6 +112,29 @@ Param: `?by=<peetprint>`
 
 Returns the most seen other identifiers (user-agent, h2, JA3) that were seen together with this identifier. Only works when connected to a database.
 
+## Logging to Parquet
+
+TrackMe can persist the fingerprints it captures to [Apache Parquet](https://parquet.apache.org/) files for later analysis. Enable it in `config.json`:
+
+```json
+{
+  "log_to_parquet": true,
+  "parquet_dir": "data",
+  "parquet_log_ips": false
+}
+```
+
+- `log_to_parquet`: turns the Parquet sink on/off.
+- `parquet_dir`: directory the files are written to (created if missing, defaults to `data`).
+- `parquet_log_ips`: when `false` (default) the client IP column is left empty.
+
+Each captured request is written as a row with the timestamp, HTTP version, method, path, user-agent and the JA3 / JA4 / PeetPrint / Akamai fingerprints (and their hashes). Rows are buffered and flushed to a new `trackme-<timestamp>.parquet` file every 1,000,000 rows or once a day, whichever comes first — so the directory becomes a Parquet dataset you can query with DuckDB, pandas, Spark, etc.:
+
+```sql
+-- DuckDB
+SELECT ja3_hash, count(*) FROM 'data/*.parquet' GROUP BY ja3_hash ORDER BY 2 DESC;
+```
+
 ## Docker
 
 You can also run the server in a docker container using docker-compose.

cmd/main.go

@@ -7,8 +7,10 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"os/signal"
 	"runtime"
 	"strconv"
+	"syscall"
 	"time"
 
 	"github.com/pagpeter/quic-go"
@@ -57,6 +59,8 @@ func init() {
 	if err := srv.GetConfig().LoadFromFile(); err != nil {
 		log.Fatal(err)
 	}
+
+	srv.InitParquet()
 }
 
 func redirect(w http.ResponseWriter, r *http.Request) {
@@ -123,10 +127,21 @@ func main() {
 	defer func() {
 		if r := recover(); r != nil {
 			logCrash(r)
+			srv.CloseParquet()
 			os.Exit(1)
 		}
 	}()
 
+	// Flush buffered data to disk on Ctrl+C / SIGTERM before exiting.
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		sig := <-sigs
+		log.Println("Received signal, shutting down:", sig)
+		srv.CloseParquet()
+		os.Exit(0)
+	}()
+
 	log.Println("Starting server...")
 	log.Println("Listening on " + srv.GetConfig().Host + ":" + srv.GetConfig().TLSPort)
 

config.example.json

@@ -1,5 +1,4 @@
 {
-  "log_to_db": false,
   "tls_port": "443",
   "http_port": "80",
   "cert_file": "certs/chain.pem",
@@ -8,5 +7,8 @@
   "http_redirect": "https://tls.peet.ws",
   "device": "eth0",
   "cors_key": "X-CORS",
-  "enable_quic": true
+  "enable_quic": true,
+  "log_to_parquet": false,
+  "parquet_dir": "data",
+  "parquet_log_ips": false
 }

go.mod

@@ -1,6 +1,6 @@
 module github.com/pagpeter/trackme
 
-go 1.24
+go 1.24.9
 
 require (
 	github.com/google/gopacket v1.1.19
@@ -13,13 +13,20 @@ require (
 
 require (
 	github.com/andybalholm/brotli v1.1.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/parquet-go/bitpack v1.0.0 // indirect
+	github.com/parquet-go/jsonlite v1.0.0 // indirect
+	github.com/parquet-go/parquet-go v0.30.1 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/quic-go/qpack v0.5.1 // indirect
 	github.com/refraction-networking/utls v1.1.2 // indirect
+	github.com/twpayne/go-geom v1.6.1 // indirect
 	golang.org/x/crypto v0.41.0 // indirect
 	golang.org/x/mod v0.27.0 // indirect
 	golang.org/x/sync v0.16.0 // indirect
-	golang.org/x/sys v0.35.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
 	golang.org/x/text v0.28.0 // indirect
 	golang.org/x/tools v0.36.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 )

go.sum

@@ -7,11 +7,21 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
 github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/pagpeter/quic-go v0.0.0-20260120153640-0de4e3b8377b h1:Ty5nepKkVdrdTetjXskX+v+3XP2GNqmmcqvl0BX4/qQ=
 github.com/pagpeter/quic-go v0.0.0-20260120153640-0de4e3b8377b/go.mod h1:EJQW9gTvp3XGR6qPANdXlU55yxrA8rww5atbR2LVI9U=
+github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
+github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
+github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU=
+github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0=
+github.com/parquet-go/parquet-go v0.30.1 h1:Oy6ganNrAdFiVwy7wNmWagfPTWA2X9Z3tVHBc7JtuX8=
+github.com/parquet-go/parquet-go v0.30.1/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
@@ -20,6 +30,8 @@ github.com/refraction-networking/utls v1.1.2 h1:a7GQauRt72VG+wtNm0lnrAaCGlyX47gE
 github.com/refraction-networking/utls v1.1.2/go.mod h1:+D89TUtA8+NKVFj1IXWr0p3tSdX1+SqUB7rL0QnGqyg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4=
+github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028=
 github.com/wwhtrbbtt/utls v0.0.0-20220918194152-45ee2a20799c h1:eDUKT2sHyNTpZTawrungpwOZgaEvcbTldzrmEmBO0pY=
 github.com/wwhtrbbtt/utls v0.0.0-20220918194152-45ee2a20799c/go.mod h1:cE/NJeUKssh/0XGO4KVBXZH0u7/BqRDqGs1Ij8hgy0w=
 github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
@@ -51,6 +63,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
 golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -62,5 +76,7 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
 golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=