prometheus: export NPOL flow decision counters

calico-vpp-agent/prometheus/prometheus.go

@@ -94,6 +94,13 @@ func cleanVppSessionStatName(vppStatName string) string {
 	return config.GetCalicoVppInitialConfig().PrometheusStatsPrefix + vppStatName
 }
 
+func cleanVppNpolStatName(vppStatName string) string {
+	vppStatName = strings.TrimPrefix(vppStatName, "/net/")
+	vppStatName = strings.ReplaceAll(vppStatName, "-", "_")
+	vppStatName = strings.ReplaceAll(vppStatName, "/", "_")
+	return config.GetCalicoVppInitialConfig().PrometheusStatsPrefix + vppStatName
+}
+
 const (
 	UnitPackets = "packets"
 	UnitBytes   = "bytes"
@@ -102,7 +109,7 @@ const (
 func (p *PrometheusServer) exportMetrics() error {
 	ifStats, err := p.statsclient.DumpStats("/if/")
 	if err != nil {
-		p.log.Errorf("Error running statsclient.DumpStats for Interface stats %v", err)
+		p.log.Warnf("Error running statsclient.DumpStats for Interface stats %v", err)
 		return nil
 	}
 	var ifNames adapter.NameStat
@@ -127,10 +134,23 @@ func (p *PrometheusServer) exportMetrics() error {
 		}
 	}
 
+	// Export NPOL stats
+	npolStats, err := p.statsclient.DumpStats("/net/npol")
+	if err != nil {
+		p.log.Warnf("Error running statsclient.DumpStats for NPOL stats %v", err)
+		return nil
+	}
+	for _, vppStat := range npolStats {
+		switch values := vppStat.Data.(type) {
+		case adapter.SimpleCounterStat:
+			p.exportNpolSimpleCounterStat(string(vppStat.Name), ifNames, values)
+		}
+	}
+
 	// Export TCP stats
 	tcpStats, err := p.statsclient.DumpStats("/sys/tcp")
 	if err != nil {
-		p.log.Errorf("Error running statsclient.DumpStats for TCP stats %v", err)
+		p.log.Warnf("Error running statsclient.DumpStats for TCP stats %v", err)
 		return nil
 	}
 	for _, vppStat := range tcpStats {
@@ -143,7 +163,7 @@ func (p *PrometheusServer) exportMetrics() error {
 	// Export TCP4 error stats
 	tcp4ErrStats, err := p.statsclient.DumpStats("/err/tcp4")
 	if err != nil {
-		p.log.Errorf("Error running statsclient.DumpStats for TCP4 error stats %v", err)
+		p.log.Warnf("Error running statsclient.DumpStats for TCP4 error stats %v", err)
 		return nil
 	}
 	for _, vppStat := range tcp4ErrStats {
@@ -156,7 +176,7 @@ func (p *PrometheusServer) exportMetrics() error {
 	// Export TCP6 error stats
 	tcp6ErrStats, err := p.statsclient.DumpStats("/err/tcp6")
 	if err != nil {
-		p.log.Errorf("Error running statsclient.DumpStats for TCP6 error stats %v", err)
+		p.log.Warnf("Error running statsclient.DumpStats for TCP6 error stats %v", err)
 		return nil
 	}
 	for _, vppStat := range tcp6ErrStats {
@@ -169,7 +189,7 @@ func (p *PrometheusServer) exportMetrics() error {
 	// Export Session stats
 	sessionStats, err := p.statsclient.DumpStats("/sys/session")
 	if err != nil {
-		p.log.Errorf("Error running statsclient.DumpStats for Session stats %v", err)
+		p.log.Warnf("Error running statsclient.DumpStats for Session stats %v", err)
 		return nil
 	}
 	for _, vppStat := range sessionStats {
@@ -292,6 +312,57 @@ func (p *PrometheusServer) exportInterfaceSimpleCounterStat(name string, ifNames
 	}
 }
 
+func (p *PrometheusServer) exportNpolSimpleCounterStat(name string, ifNames adapter.NameStat, values adapter.SimpleCounterStat) {
+	metric := &metricspb.Metric{
+		MetricDescriptor: &metricspb.MetricDescriptor{
+			Name:        cleanVppNpolStatName(name),
+			Description: getVppNpolStatDescription(name),
+			Type:        metricspb.MetricDescriptor_CUMULATIVE_DOUBLE,
+			LabelKeys: []*metricspb.LabelKey{
+				{Key: "worker", Description: "VPP worker index"},
+				{Key: "namespace", Description: "Kubernetes namespace of the pod"},
+				{Key: "podName", Description: "Name of the pod"},
+				{Key: "podInterfaceName", Description: "Name of interface in the pod"},
+				{Key: "vppInterfaceName", Description: "Name of interface in VPP"},
+			},
+		},
+	}
+	for worker, perWorkerValues := range values {
+		for swIfIndex, counter := range perWorkerValues {
+			pod := p.podInterfacesDetailsBySwifIndex[uint32(swIfIndex)]
+			vppIfName := ""
+			if swIfIndex < len(ifNames) {
+				vppIfName = string(ifNames[swIfIndex])
+			}
+			metric.Timeseries = append(metric.Timeseries, &metricspb.TimeSeries{
+				LabelValues: []*metricspb.LabelValue{
+					{Value: strconv.Itoa(worker)},
+					{Value: pod.podNamespace},
+					{Value: pod.podName},
+					{Value: pod.interfaceName},
+					{Value: vppIfName},
+				},
+				Points: []*metricspb.Point{
+					{
+						Value: &metricspb.Point_DoubleValue{
+							DoubleValue: float64(counter),
+						},
+					},
+				},
+			})
+		}
+	}
+	err := p.exporter.ExportMetric(
+		context.Background(),
+		nil, /* node */
+		nil, /* resource */
+		metric,
+	)
+	if err != nil {
+		p.log.Errorf("Error prometheus exporter.ExportMetric for NPOL %v", err)
+	}
+}
+
 func (p *PrometheusServer) exportTCPSimpleCounterStat(name string, values adapter.SimpleCounterStat) {
 	metric := &metricspb.Metric{
 		MetricDescriptor: &metricspb.MetricDescriptor{

calico-vpp-agent/prometheus/prometheus_test.go

@@ -36,6 +36,7 @@ import (
 	"github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/testutils"
 	agentConf "github.com/projectcalico/vpp-dataplane/v3/config"
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink"
+	"github.com/projectcalico/vpp-dataplane/v3/vpplink/types"
 )
 
 // Names of integration tests arguments
@@ -235,6 +236,34 @@ var _ = Describe("Prometheus exporter functionality", func() {
 				Expect(len(sessionMetrics)).To(Equal(0), "Should not contain session statistics")
 				fmt.Printf("=== Success! session statistics not found as expected ===\n")
 			})
+
+			It("should export NPOL flow decision counters", func() {
+				By("Configuring NPOL on the uplink interface")
+				err := vpp.ConfigurePolicies(uplinkSwIfIndex, types.NewInterfaceConfig(), 0)
+				Expect(err).ToNot(HaveOccurred())
+
+				By("Fetching NPOL metrics from prometheus endpoint")
+				metrics, err := fetchMetricsUntilPresent("http://localhost:9090/metrics", "npol_rx_flows_allow", 5*time.Second)
+				Expect(err).ToNot(HaveOccurred())
+
+				fmt.Printf("=== Verify NPOL flow decision counters ===\n")
+				metricEntries := parseMetrics(metrics, "npol_rx_flows_allow")
+				Expect(len(metricEntries)).To(BeNumerically(">", 0), "Should have NPOL flow decision metrics")
+
+				foundTestNamespace1 := false
+				for _, metric := range metricEntries {
+					if metric.Labels["namespace"] == "test-namespace-1" &&
+						metric.Labels["podName"] == "test-pod-1" &&
+						metric.Labels["podInterfaceName"] == "eth0" &&
+						metric.Labels["vppInterfaceName"] == "tap0" {
+						foundTestNamespace1 = true
+						fmt.Printf("Found NPOL metric: %+v\n", metric)
+					}
+				}
+
+				Expect(foundTestNamespace1).To(BeTrue(), "Should find NPOL metric for test-namespace-1 pod")
+				fmt.Printf("=== Success! NPOL flow decision counters found ===\n")
+			})
 		})
 
 		Context("When pod events occur", func() {

calico-vpp-agent/prometheus/stats_description.go

@@ -15,6 +15,8 @@
 
 package prometheus
 
+import "strings"
+
 func getVppIfStatDescription(vppStatName string) string {
 	switch cleanVppIfStatName(vppStatName) {
 	case "drops":
@@ -1492,3 +1494,18 @@ func getVppSessionStatDescription(vppStatName string) string {
 		return vppStatName
 	}
 }
+
+func getVppNpolStatDescription(vppStatName string) string {
+	switch strings.TrimPrefix(vppStatName, "/net/npol/") {
+	case "rx/flows/allow":
+		return "number of new RX flows allowed by NPOL"
+	case "rx/flows/deny":
+		return "number of new RX flows denied by NPOL"
+	case "tx/flows/allow":
+		return "number of new TX flows allowed by NPOL"
+	case "tx/flows/deny":
+		return "number of new TX flows denied by NPOL"
+	default:
+		return vppStatName
+	}
+}

docs/metrics/README.md

@@ -1,9 +1,15 @@
 # CalicoVPP metrics
 
-CalicoVPP exposes can expose metrics with a prometheus
-http endpoint.
+CalicoVPP exposes metrics with a prometheus http endpoint. It is not enabled
+by default and can be enabled by setting `prometheusEnabled` to `true` in
+`CALICOVPP_FEATURE_GATES`. There are additional configuration parameters under
+`CALICOVPP_INITIAL_CONFIG`:
 
-````yaml
+- `prometheusStatsPrefix` - prefix string (default: `cni_projectcalico_vpp_`)
+- `prometheusListenEndpoint` - http endpoint port (default: `8888`)
+- `prometheusRecordMetricInterval` - metric interval in seconds (default: `5`)
+
+```yaml
 ---
 kind: ConfigMap
 apiVersion: v1
@@ -13,20 +19,19 @@ metadata:
 data:
   CALICOVPP_FEATURE_GATES: |-
     { 
-      "prometheusEnabled": "true"
+      "prometheusEnabled": true
     }
 
   CALICOVPP_INITIAL_CONFIG: |-
     { 
       "prometheusStatsPrefix": "cni_projectcalico_vpp_",
       "prometheusListenEndpoint": ":8888",
-      "prometheusRecordMetricInterval": "5s"
+      "prometheusRecordMetricInterval": 5
     }
-````
+```
 
-Every metrics is prefixed by the value specified in
-``prometheusStatsPrefix``. Keeping in mind that all non
-alphanumeric characters are replaced by underscores.
+Every metrics is prefixed by the value specified in `prometheusStatsPrefix`.
+Keep in mind that all non alphanumeric characters are replaced by underscores.
 
 You can find the full specification for the environment variables
 in [config/config.go](https://github.com/projectcalico/vpp-dataplane/blob/master/config/config.go)

docs/metrics/metrics.md

@@ -76,6 +76,27 @@ RX is what comes in VPP, TX is what goes out of VPP
 - `cni_projectcalico_vpp_rx_no_buf`
 - `cni_projectcalico_vpp_drops`
 
+## VPP NPOL flow decision counters
+
+Every NPOL flow decision counter is tagged with the same labels as VPP
+interface counters:
+
+- `worker` - ID of the worker the stats is reported for
+- `namespace` - k8s namespace of the pod this stats is reported for.
+- `podName` - k8s name of the pod this stats is reported for.
+- `podInterfaceName` - linux netdev name of the interface in the pod this
+stats is reported for. Typically `eth0`
+- `vppInterfaceName` - vpp interface name of the interface in the pod this
+stats is reported for. Typically `tun1`
+
+These counters report NPOL allow and deny decisions for new CNAT flows. They
+are not packet or byte counters.
+
+- `cni_projectcalico_vpp_npol_rx_flows_allow`
+- `cni_projectcalico_vpp_npol_rx_flows_deny`
+- `cni_projectcalico_vpp_npol_tx_flows_allow`
+- `cni_projectcalico_vpp_npol_tx_flows_deny`
+
 ## Hoststack counters
 
 Every host stack counter is tagged with