diff --git a/Makefile b/Makefile
index 9e83ac44dc..8b2c0630ee 100644
--- a/Makefile
+++ b/Makefile
@@ -87,17 +87,20 @@ cockpit/devel: cockpit/devel-uninstall cockpit/build cockpit/devel-install
 #
 
 RPM_SPEC=cockpit/$(PACKAGE_NAME).spec
+NODE_MODULES_TEST=package-lock.json
 TARFILE=$(PACKAGE_NAME)-$(VERSION).tar.gz
 
+$(RPM_SPEC): $(RPM_SPEC) $(NODE_MODULES_TEST)
+	provides=$$(npm ls --omit dev --package-lock-only --depth=Infinity | grep -Eo '[^[:space:]]+@[^[:space:]]+' | sort -u | sed 's/^/Provides: bundled(npm(/; s/\(.*\)@/\1)) = /'); \
+	awk -v p="$$provides" '{gsub(/%{VERSION}/, "$(VERSION)"); $(SUB_NODE_ENV) gsub(/%{NPM_PROVIDES}/, p)}1' $< > $@
+
 $(TARFILE): export NODE_ENV ?= production
-$(TARFILE): cockpit/download
-	npm ci --include=dev
+$(TARFILE): cockpit/build
 	touch -r package.json package-lock.json
 	touch cockpit/public/*
-	tar czf $(TARFILE) --transform 'flags=r;s,^,$(PACKAGE_NAME)/,' \
-		$$(git ls-files) \
-		node_modules \
-		pkg
+	tar czf $(TARFILE) --transform 's,^,$(PACKAGE_NAME)/,' \
+		--exclude node_modules \
+		$$(git ls-files) $(RPM_SPEC) $(NODE_MODULES_TEST) cockpit/public/ cockpit/README.md cockpit/tmpfiles.d
 	realpath $(TARFILE)
 
 dist: $(TARFILE)
diff --git a/cockpit/cockpit-image-builder.spec b/cockpit/cockpit-image-builder.spec
index 2b636d920c..3b71fe8312 100644
--- a/cockpit/cockpit-image-builder.spec
+++ b/cockpit/cockpit-image-builder.spec
@@ -15,7 +15,6 @@ BuildRequires:  gettext
 BuildRequires:  libappstream-glib
 BuildRequires:  make
 BuildRequires:  /usr/bin/node
-BuildRequires:  npm
 # for _tmpfilesdir macro
 BuildRequires:  systemd-rpm-macros
 
@@ -34,7 +33,7 @@ as a frontend for osbuild.
 %setup -q -n %{name}
 
 %build
-NODE_ENV=production npm run build:cockpit
+# Nothing to build
 
 %install
 %make_install PREFIX=/usr
diff --git a/packit.yaml b/packit.yaml
index f5f98ce50e..695bf2056e 100644
--- a/packit.yaml
+++ b/packit.yaml
@@ -8,6 +8,7 @@ upstream_tag_template: v{version}
 
 actions:
   create-archive:
+    - npm ci
     - make dist
 
 srpm_build_deps:
@@ -55,6 +56,7 @@ jobs:
     targets: *build_targets
     actions:
       create-archive:
+        - npm ci
         - make dist
 
   - job: propose_downstream