From c40add7d702ab3b00613e34e66c6b2a00069f93c Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 3 Apr 2026 16:18:53 +0000 Subject: [PATCH 001/104] feat(database): Initial database POC implementation Using drizzle --- .gitignore | 5 +- drizzle.config.ts | 13 + package-lock.json | 998 +++++++++++++++++- package.json | 5 +- patches/kysely-node-native-sqlite+1.1.0.patch | 60 ++ src/backend/common/database/Database.ts | 44 + .../common/database/drizzle/drizzleUtils.ts | 81 ++ .../migration.sql | 35 + .../snapshot.json | 340 ++++++ .../drizzle/schema/drizzlePlaysTable.ts | 117 ++ src/backend/common/errors/MSErrors.ts | 14 +- .../scrobblers/AbstractScrobbleClient.ts | 8 + src/backend/tests/database/drizzle.test.ts | 94 ++ .../tests/scrobbler/scrobblers.test.ts | 2 +- src/backend/utils/FSUtils.ts | 18 + src/core/PlayMarshalUtils.ts | 82 ++ src/core/tests/utils/fixtures.ts | 80 +- src/stories/ActivityTimeline.stories.tsx | 3 +- src/stories/List.stories.tsx | 3 +- src/stories/PlayInfo.stories.tsx | 2 +- src/stories/TransformSteps.stories.tsx | 3 +- tsconfig.json | 3 +- 22 files changed, 1888 insertions(+), 122 deletions(-) create mode 100644 drizzle.config.ts create mode 100644 patches/kysely-node-native-sqlite+1.1.0.patch create mode 100644 src/backend/common/database/Database.ts create mode 100644 src/backend/common/database/drizzle/drizzleUtils.ts create mode 100644 src/backend/common/database/drizzle/migrations/20260422185819_white_wilson_fisk/migration.sql create mode 100644 src/backend/common/database/drizzle/migrations/20260422185819_white_wilson_fisk/snapshot.json create mode 100644 src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts create mode 100644 src/backend/tests/database/drizzle.test.ts create mode 100644 src/core/PlayMarshalUtils.ts diff --git a/.gitignore b/.gitignore index 2b5fa3b0d..ae6e1cbb2 100644 --- a/.gitignore +++ b/.gitignore @@ -151,7 +151,10 @@ tmp-* *storybook.log storybook-static -lib + +*.db +*.db.* +*.db-*lib *.db *.db.* *.db-*lib \ No newline at end of file diff --git a/drizzle.config.ts b/drizzle.config.ts new file mode 100644 index 000000000..973eac00b --- /dev/null +++ b/drizzle.config.ts @@ -0,0 +1,13 @@ +import 'dotenv/config'; +import { defineConfig } from 'drizzle-kit'; +import { configDir, projectDir } from './src/backend/common/index.js'; +import * as path from 'path'; + +export default defineConfig({ + schema: path.resolve(projectDir, 'src/backend/common/database/drizzle/schema'), + out: path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations'), + dialect: 'sqlite', + dbCredentials: { + url: path.resolve(configDir, process.env.DB_FILE_NAME! ?? 'ms.db'), + }, +}); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 28d54273d..7e8227283 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "@kenyip/backoff-strategies": "^1.0.4", "@keyv/valkey": "^1.0.8", "@lukehagar/plexjs": "^0.39.0", + "@platformatic/job-queue": "^0.5.0", "@supercharge/promise-pool": "^3.0.0", "@svrooij/sonos": "^2.5.0", "@xhayper/discord-rpc": "^1.3.0", @@ -47,6 +48,8 @@ "dbus-ts": "^0.0.7", "discord.js": "^14.26.0", "dotenv": "^10.0.0", + "drizzle-kit": "^1.0.0-beta.22", + "drizzle-orm": "^1.0.0-beta.22", "express": "^5.2.1", "express-session": "^1.19.0", "fast-equals": "^6.0.0", @@ -129,7 +132,7 @@ "@types/jest": "^27.5.2", "@types/jscodeshift": "^0.11.6", "@types/mocha": "^9.1.0", - "@types/node": "^20.19.2", + "@types/node": "^24.12.2", "@types/object-hash": "^3.0.0", "@types/passport": "^1.0.12", "@types/react": "^18.2.18", @@ -1055,6 +1058,12 @@ "version": "1.1.3", "license": "ISC" }, + "node_modules/@drizzle-team/brocli": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@drizzle-team/brocli/-/brocli-0.11.0.tgz", + "integrity": "sha512-hD3pekGiPg0WPCCGAZmusBBJsDqGUR66Y452YgQsZOnkdQ7ViEPKuyP4huUGEZQefp8g34RRodXYmJ2TbCH+tg==", + "license": "Apache-2.0" + }, "node_modules/@dword-design/chdir": { "version": "4.0.0", "dev": true, @@ -2018,6 +2027,13 @@ "@swc/helpers": "^0.5.0" } }, + "node_modules/@ioredis/commands": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.5.1.tgz", + "integrity": "sha512-JH8ZL/ywcJyR9MmJ5BNqZllXNZQqQbnVZOqpPQqE1vHiFgAw4NHbvE0FOduNU8IX9babitBT46571OnPTT0Zcw==", + "license": "MIT", + "optional": true + }, "node_modules/@iovalkey/commands": { "version": "0.1.0", "license": "MIT" @@ -2174,6 +2190,18 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@js-temporal/polyfill": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/@js-temporal/polyfill/-/polyfill-0.5.1.tgz", + "integrity": "sha512-hloP58zRVCRSpgDxmqCWJNlizAlUgJFqG2ypq79DCvyv9tHjRYMDOcPFjzfl/A1/YxDvRCZz8wvZvmapQnKwFQ==", + "license": "ISC", + "dependencies": { + "jsbi": "^4.3.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/@kenyip/backoff-strategies": { "version": "1.0.4", "license": "MIT" @@ -2605,6 +2633,12 @@ "url": "https://opencollective.com/parcel" } }, + "node_modules/@pinojs/redact": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", + "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", + "license": "MIT" + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "license": "MIT", @@ -2613,6 +2647,103 @@ "node": ">=14" } }, + "node_modules/@platformatic/job-queue": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@platformatic/job-queue/-/job-queue-0.5.0.tgz", + "integrity": "sha512-vAD4HmVy3IdDhv74+282j6YS9N8wbziWT/T1IPBQNsQO+ObKiZNfDQFeakAIEUp0w80vFJyrgy42knxJ97ACgQ==", + "license": "Apache-2.0", + "dependencies": { + "pino": "^10.3.1" + }, + "engines": { + "node": ">=22.19.0" + }, + "optionalDependencies": { + "fast-write-atomic": "^0.4.0", + "iovalkey": "^0.2.0", + "pg": "^8.20.0" + } + }, + "node_modules/@platformatic/job-queue/node_modules/iovalkey": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/iovalkey/-/iovalkey-0.2.2.tgz", + "integrity": "sha512-7eVmLOYV2UamZ/YPXuUwTu/4zBDxXcfjj/wmOwlKBBhU2qjg60Th0Y/cqfED3OxNAhc6hUV2Ft4eQMCKi2EMpQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=18.12.0" + } + }, + "node_modules/@platformatic/job-queue/node_modules/pino": { + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz", + "integrity": "sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==", + "license": "MIT", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^3.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^4.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/@platformatic/job-queue/node_modules/pino-abstract-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz", + "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/@platformatic/job-queue/node_modules/process-warning": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", + "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, + "node_modules/@platformatic/job-queue/node_modules/thread-stream": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.0.0.tgz", + "integrity": "sha512-4iMVL6HAINXWf1ZKZjIPcz5wYaOdPhtO8ATvZ+Xqp3BTdaqtAwQkNmKORqcIo5YkQqGXq5cwfswDwMqqQNrpJA==", + "license": "MIT", + "dependencies": { + "real-require": "^0.2.0" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "license": "BSD-3-Clause" @@ -4288,10 +4419,12 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "20.19.37", + "version": "24.12.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.2.tgz", + "integrity": "sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g==", "license": "MIT", "dependencies": { - "undici-types": "~6.21.0" + "undici-types": "~7.16.0" } }, "node_modules/@types/object-hash": { @@ -7205,40 +7338,675 @@ }, "engines": { "node": ">=18" - }, - "funding": { - "url": "https://github.com/discordjs/discord.js?sponsor" + }, + "funding": { + "url": "https://github.com/discordjs/discord.js?sponsor" + } + }, + "node_modules/discord.js/node_modules/undici": { + "version": "6.24.1", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, + "node_modules/doctrine": { + "version": "3.0.0", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/dom-accessibility-api": { + "version": "0.5.16", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/dotenv": { + "version": "10.0.0", + "license": "BSD-2-Clause", + "engines": { + "node": ">=10" + } + }, + "node_modules/drizzle-kit": { + "version": "1.0.0-beta.22", + "resolved": "https://registry.npmjs.org/drizzle-kit/-/drizzle-kit-1.0.0-beta.22.tgz", + "integrity": "sha512-9HTZuQRljQKTgCx4UhiGn8KYYfHGk4+B/bRR1714W67kz0qgJvdrG527i8rQD8uUyET9UTGR1u8syySJD4znGw==", + "license": "MIT", + "dependencies": { + "@drizzle-team/brocli": "^0.11.0", + "@js-temporal/polyfill": "^0.5.1", + "esbuild": "^0.25.10", + "get-tsconfig": "^4.13.6", + "jiti": "^2.6.1" + }, + "bin": { + "drizzle-kit": "bin.cjs" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" } }, - "node_modules/discord.js/node_modules/undici": { - "version": "6.24.1", + "node_modules/drizzle-kit/node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], "engines": { - "node": ">=18.17" + "node": ">=18" } }, - "node_modules/doctrine": { - "version": "3.0.0", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "esutils": "^2.0.2" - }, + "node_modules/drizzle-kit/node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], "engines": { - "node": ">=6.0.0" + "node": ">=18" } }, - "node_modules/dom-accessibility-api": { - "version": "0.5.16", - "dev": true, + "node_modules/drizzle-kit/node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], "license": "MIT", - "peer": true + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } }, - "node_modules/dotenv": { - "version": "10.0.0", - "license": "BSD-2-Clause", + "node_modules/drizzle-kit/node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], "engines": { - "node": ">=10" + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/drizzle-orm": { + "version": "1.0.0-beta.22", + "resolved": "https://registry.npmjs.org/drizzle-orm/-/drizzle-orm-1.0.0-beta.22.tgz", + "integrity": "sha512-F+DZyVIvH0oVKa/w08Cle1xfoH+pc+htIXHG/frnMLG72aby9NYYr9oc+9XvghnoO4umxFItduz0OMmQJMnenw==", + "license": "Apache-2.0", + "peerDependencies": { + "@aws-sdk/client-rds-data": ">=3", + "@cloudflare/workers-types": ">=4", + "@effect/sql": "^0.48.5", + "@effect/sql-pg": "^0.49.7", + "@electric-sql/pglite": ">=0.2.0", + "@libsql/client": ">=0.10.0", + "@libsql/client-wasm": ">=0.10.0", + "@neondatabase/serverless": ">=0.10.0", + "@op-engineering/op-sqlite": ">=2", + "@opentelemetry/api": "^1.4.1", + "@planetscale/database": ">=1.13", + "@sinclair/typebox": ">=0.34.8", + "@sqlitecloud/drivers": ">=1.0.653", + "@tidbcloud/serverless": "*", + "@tursodatabase/database": ">=0.2.1", + "@tursodatabase/database-common": ">=0.2.1", + "@tursodatabase/database-wasm": ">=0.2.1", + "@types/better-sqlite3": "*", + "@types/mssql": "^9.1.4", + "@types/pg": "*", + "@types/sql.js": "*", + "@upstash/redis": ">=1.34.7", + "@vercel/postgres": ">=0.8.0", + "@xata.io/client": "*", + "arktype": ">=2.0.0", + "better-sqlite3": ">=9.3.0", + "bun-types": "*", + "expo-sqlite": ">=14.0.0", + "gel": ">=2", + "mssql": "^11.0.1", + "mysql2": ">=2", + "pg": ">=8", + "postgres": ">=3", + "sql.js": ">=1", + "sqlite3": ">=5", + "typebox": ">=1.0.0", + "valibot": ">=1.0.0-beta.7", + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "@aws-sdk/client-rds-data": { + "optional": true + }, + "@cloudflare/workers-types": { + "optional": true + }, + "@effect/sql": { + "optional": true + }, + "@effect/sql-pg": { + "optional": true + }, + "@electric-sql/pglite": { + "optional": true + }, + "@libsql/client": { + "optional": true + }, + "@libsql/client-wasm": { + "optional": true + }, + "@neondatabase/serverless": { + "optional": true + }, + "@op-engineering/op-sqlite": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@planetscale/database": { + "optional": true + }, + "@sinclair/typebox": { + "optional": true + }, + "@sqlitecloud/drivers": { + "optional": true + }, + "@tidbcloud/serverless": { + "optional": true + }, + "@tursodatabase/database": { + "optional": true + }, + "@tursodatabase/database-common": { + "optional": true + }, + "@tursodatabase/database-wasm": { + "optional": true + }, + "@types/better-sqlite3": { + "optional": true + }, + "@types/mssql": { + "optional": true + }, + "@types/pg": { + "optional": true + }, + "@types/sql.js": { + "optional": true + }, + "@upstash/redis": { + "optional": true + }, + "@vercel/postgres": { + "optional": true + }, + "@xata.io/client": { + "optional": true + }, + "arktype": { + "optional": true + }, + "better-sqlite3": { + "optional": true + }, + "bun-types": { + "optional": true + }, + "expo-sqlite": { + "optional": true + }, + "gel": { + "optional": true + }, + "mssql": { + "optional": true + }, + "mysql2": { + "optional": true + }, + "pg": { + "optional": true + }, + "postgres": { + "optional": true + }, + "sql.js": { + "optional": true + }, + "sqlite3": { + "optional": true + }, + "typebox": { + "optional": true + }, + "valibot": { + "optional": true + }, + "zod": { + "optional": true + } } }, "node_modules/dunder-proto": { @@ -8160,6 +8928,16 @@ "version": "3.0.3", "license": "BSD-3-Clause" }, + "node_modules/fast-write-atomic": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/fast-write-atomic/-/fast-write-atomic-0.4.0.tgz", + "integrity": "sha512-O0AGqOsuOgsVXFiF/vIm7c+NHRZB6c6IDMBVPWc8rLGWVW7FYaBBfzhqxKlKyGKVF5tPR4drUOxE02lDiWjbSg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=20" + } + }, "node_modules/fast-xml-parser": { "version": "3.19.0", "license": "MIT", @@ -8564,7 +9342,9 @@ } }, "node_modules/get-tsconfig": { - "version": "4.8.1", + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz", + "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==", "license": "MIT", "dependencies": { "resolve-pkg-maps": "^1.0.0" @@ -9561,7 +10341,6 @@ "version": "2.6.1", "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz", "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", - "dev": true, "license": "MIT", "bin": { "jiti": "lib/jiti-cli.mjs" @@ -9597,6 +10376,12 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsbi": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/jsbi/-/jsbi-4.3.2.tgz", + "integrity": "sha512-9fqMSQbhJykSeii05nxKl4m6Eqn2P6rOlYiS+C5Dr/HPIU/7yZxu5qzbs40tgaFORiw2Amd0mirjxatXYMkIew==", + "license": "Apache-2.0" + }, "node_modules/jsesc": { "version": "3.1.0", "dev": true, @@ -11532,6 +12317,102 @@ "dev": true, "license": "MIT" }, + "node_modules/pg": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz", + "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==", + "license": "MIT", + "optional": true, + "dependencies": { + "pg-connection-string": "^2.12.0", + "pg-pool": "^3.13.0", + "pg-protocol": "^1.13.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.3.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz", + "integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.12.0.tgz", + "integrity": "sha512-U7qg+bpswf3Cs5xLzRqbXbQl85ng0mfSV/J0nnA31MCLgvEaAo7CIhmeyrmJpOr7o+zm0rXK+hNnT5l9RHkCkQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "optional": true, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.13.0.tgz", + "integrity": "sha512-gB+R+Xud1gLFuRD/QgOIgGOBE2KCQPaPwkzBBGC9oG69pHTkhQeIuejVIk3/cnDyX39av2AxomQiyPT13WKHQA==", + "license": "MIT", + "optional": true, + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.13.0.tgz", + "integrity": "sha512-zzdvXfS6v89r6v7OcFCHfHlyG/wvry1ALxZo4LqgUoy7W9xhBDMaqOuMiF3qEV45VqsN6rdlcehHrfDtlCPc8w==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "optional": true, + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "optional": true, + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/phin": { "version": "3.7.1", "license": "MIT", @@ -11736,6 +12617,49 @@ "dev": true, "license": "MIT" }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "dev": true, @@ -13947,7 +14871,9 @@ } }, "node_modules/undici-types": { - "version": "6.21.0", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", "license": "MIT" }, "node_modules/unicorn-magic": { @@ -14927,6 +15853,16 @@ "node": ">=4.0" } }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "dev": true, @@ -15089,7 +16025,9 @@ } }, "node_modules/zod": { - "version": "3.23.8", + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" @@ -15097,6 +16035,8 @@ }, "node_modules/zod-validation-error": { "version": "2.1.0", + "resolved": "https://registry.npmjs.org/zod-validation-error/-/zod-validation-error-2.1.0.tgz", + "integrity": "sha512-VJh93e2wb4c3tWtGgTa0OF/dTt/zoPCPzXq4V11ZjxmEAFaPi/Zss1xIZdEB5RD8GD00U0/iVXgqkF77RV7pdQ==", "license": "MIT", "engines": { "node": ">=18.0.0" diff --git a/package.json b/package.json index 02066d3f8..c43404715 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "@kenyip/backoff-strategies": "^1.0.4", "@keyv/valkey": "^1.0.8", "@lukehagar/plexjs": "^0.39.0", + "@platformatic/job-queue": "^0.5.0", "@supercharge/promise-pool": "^3.0.0", "@svrooij/sonos": "^2.5.0", "@xhayper/discord-rpc": "^1.3.0", @@ -85,6 +86,8 @@ "dbus-ts": "^0.0.7", "discord.js": "^14.26.0", "dotenv": "^10.0.0", + "drizzle-kit": "^1.0.0-beta.22", + "drizzle-orm": "^1.0.0-beta.22", "express": "^5.2.1", "express-session": "^1.19.0", "fast-equals": "^6.0.0", @@ -167,7 +170,7 @@ "@types/jest": "^27.5.2", "@types/jscodeshift": "^0.11.6", "@types/mocha": "^9.1.0", - "@types/node": "^20.19.2", + "@types/node": "^24.12.2", "@types/object-hash": "^3.0.0", "@types/passport": "^1.0.12", "@types/react": "^18.2.18", diff --git a/patches/kysely-node-native-sqlite+1.1.0.patch b/patches/kysely-node-native-sqlite+1.1.0.patch new file mode 100644 index 000000000..2f1a6462f --- /dev/null +++ b/patches/kysely-node-native-sqlite+1.1.0.patch @@ -0,0 +1,60 @@ +diff --git a/node_modules/kysely-node-native-sqlite/dist/index.cjs b/node_modules/kysely-node-native-sqlite/dist/index.cjs +index 1015da7..0679a95 100644 +--- a/node_modules/kysely-node-native-sqlite/dist/index.cjs ++++ b/node_modules/kysely-node-native-sqlite/dist/index.cjs +@@ -55,7 +55,11 @@ var import_node_sqlite = require("node:sqlite"); + var NodeNativeSqliteConnection = class { + #db; + constructor(...args) { +- this.#db = new import_node_sqlite.DatabaseSync(...args); ++ if (args[0] instanceof import_node_sqlite.DatabaseSync) { ++ this.#db = args[0]; ++ } else { ++ this.#db = new import_node_sqlite.DatabaseSync(...args); ++ } + } + [Symbol.dispose]() { + this.#db.close(); +diff --git a/node_modules/kysely-node-native-sqlite/dist/index.d.cts b/node_modules/kysely-node-native-sqlite/dist/index.d.cts +index 72366bb..32e8464 100644 +--- a/node_modules/kysely-node-native-sqlite/dist/index.d.cts ++++ b/node_modules/kysely-node-native-sqlite/dist/index.d.cts +@@ -3,7 +3,7 @@ import { Dialect, SqliteAdapter, Driver, Kysely, DatabaseIntrospector, SqliteQue + + declare class NodeNativeSqliteDialect implements Dialect { + #private; +- constructor(...args: ConstructorParameters); ++ constructor(...args: ConstructorParameters | [DatabaseSync]); + createAdapter(): SqliteAdapter; + createDriver(): Driver; + createIntrospector(db: Kysely): DatabaseIntrospector; +diff --git a/node_modules/kysely-node-native-sqlite/dist/index.d.ts b/node_modules/kysely-node-native-sqlite/dist/index.d.ts +index 72366bb..32e8464 100644 +--- a/node_modules/kysely-node-native-sqlite/dist/index.d.ts ++++ b/node_modules/kysely-node-native-sqlite/dist/index.d.ts +@@ -3,7 +3,7 @@ import { Dialect, SqliteAdapter, Driver, Kysely, DatabaseIntrospector, SqliteQue + + declare class NodeNativeSqliteDialect implements Dialect { + #private; +- constructor(...args: ConstructorParameters); ++ constructor(...args: ConstructorParameters | [DatabaseSync]); + createAdapter(): SqliteAdapter; + createDriver(): Driver; + createIntrospector(db: Kysely): DatabaseIntrospector; +diff --git a/node_modules/kysely-node-native-sqlite/dist/index.js b/node_modules/kysely-node-native-sqlite/dist/index.js +index 8c2b8cf..17b7eb1 100644 +--- a/node_modules/kysely-node-native-sqlite/dist/index.js ++++ b/node_modules/kysely-node-native-sqlite/dist/index.js +@@ -32,7 +32,11 @@ import { DatabaseSync } from "node:sqlite"; + var NodeNativeSqliteConnection = class { + #db; + constructor(...args) { +- this.#db = new DatabaseSync(...args); ++ if (args[0] instanceof DatabaseSync) { ++ this.#db = args[0]; ++ } else { ++ this.#db = new DatabaseSync(...args); ++ } + } + [Symbol.dispose]() { + this.#db.close(); diff --git a/src/backend/common/database/Database.ts b/src/backend/common/database/Database.ts new file mode 100644 index 000000000..9b54a4681 --- /dev/null +++ b/src/backend/common/database/Database.ts @@ -0,0 +1,44 @@ +import { DatabaseSync } from 'node:sqlite'; +import { configDir } from '../index.js'; +import * as path from 'path'; +import { promises as fs } from 'fs' +import { childLogger, Logger } from '@foxxmd/logging'; +import { loggerNoop } from '../MaybeLogger.js'; +import { fileExists, fileOrDirectoryIsWriteable } from '../../utils/FSUtils.js'; + +export const MEMORY_DB_NAME = ':memory:'; +export const isMemoryDb = (name: string): boolean => name === MEMORY_DB_NAME; + +export const getDbPath = (name: string = 'ms', workingDirectory?: string): string => { + if(isMemoryDb(name)) { + return MEMORY_DB_NAME; + } + return path.resolve(workingDirectory ?? configDir, `${name}.db`); +} + +export const backupDb = async (dbName: string, parentLogger: Logger = loggerNoop): Promise => { + + const logger = childLogger(parentLogger, 'Migrations'); + + const dbPath = getDbPath(dbName); + let newDb = false; + + if(dbPath !== MEMORY_DB_NAME) { + if(!fileExists(dbPath)) { + logger.info(`Database at ${dbPath} does not exist, will create it.`); + newDb = true; + } + try { + fileOrDirectoryIsWriteable(dbPath); + } catch (e) { + throw new Error('Cannot access database path for migrations', {cause: e}); + } + } + + if(dbPath !== MEMORY_DB_NAME && !newDb) { + const backupPath = `${getDbPath(`${Date.now()}-${dbName}`)}.bak`; + logger.info(`Backing up database before migrating => ${backupPath}`); + await fs.copyFile(dbPath, backupPath) + logger.info('Backed up!'); + } +} \ No newline at end of file diff --git a/src/backend/common/database/drizzle/drizzleUtils.ts b/src/backend/common/database/drizzle/drizzleUtils.ts new file mode 100644 index 000000000..f2350cc3b --- /dev/null +++ b/src/backend/common/database/drizzle/drizzleUtils.ts @@ -0,0 +1,81 @@ +import { drizzle } from 'drizzle-orm/node-sqlite'; +import { migrate } from 'drizzle-orm/node-sqlite/migrator'; +import { sql as dsl } from 'drizzle-orm'; +import * as fs from 'fs/promises'; +import * as path from 'path'; +import { getDbPath } from '../Database.js'; +import { fileExists } from '../../../utils/FSUtils.js'; +import { childLogger, Logger } from '@foxxmd/logging'; +import { loggerNoop } from '../../MaybeLogger.js'; +import { projectDir } from '../../index.js'; +import { relations } from './schema/drizzlePlaysTable.js'; + +export async function shouldBackupDb(dbPath: string, parentLogger: Logger = loggerNoop): Promise { + + const logger = childLogger(parentLogger, 'Migrations'); + + logger.info(`Checking database at ${dbPath}`); + if (!fileExists(dbPath)) { + logger.info(`No database exists!`); + return false; + } + + const db = drizzle(dbPath); + + try { + // Ensure the migrations table exists + // https://github.com/drizzle-team/drizzle-orm/issues/1953 + const res = db.all(dsl` + SELECT count(*) FROM sqlite_master WHERE type='table' AND name='__drizzle_migrations'; + `); + + if (res[0]['count(*)'] === 0) { + logger.info(`Database exists but there is no __drizzle_migrations table??`); + return false; + } + + const dbMigrations = await db.all(dsl`SELECT id, hash, created_at, name, applied_at FROM "__drizzle_migrations" ORDER BY created_at DESC`); + const appliedMigrations = new Set(dbMigrations.map((m: any) => m.name)); + + const allFiles = await fs.readdir(path.resolve(projectDir, 'src/backend/common/database/drizzleMigrations')); + const migrationFiles = allFiles + .sort(); + + const pendingMigrations = migrationFiles.filter(file => { + return !appliedMigrations.has(file); + }); + + //console.log('Applied migrations:', Array.from(appliedMigrations)); + if (pendingMigrations.length > 0) { + logger.info(`${pendingMigrations.length} pending migrations:\n${pendingMigrations.join('\n')}`); + return true; + } else { + logger.info('No pending migrations.'); + return false; + } + } catch (error) { + logger.error(new Error('Failed to get pending migrations', { cause: error })); + return true; + } +} + +export const getDb = (dbName: string = 'ms', opts: { logger?: Logger, workingDirectory?: string } = {}) => { + const { + workingDirectory, + logger = loggerNoop + } = opts; + const dbPath = getDbPath(dbName, workingDirectory); + logger.info(`Using database at ${dbPath}`); + return drizzle(dbPath, {relations: relations}); +} + +export const migrateDb = async (db: ReturnType, parentLogger: Logger = loggerNoop) => { + const logger = childLogger(parentLogger, 'Migrations'); + + try { + await migrate(db, { migrationsFolder: path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') }); + logger.info('Migrations complete'); + } catch (e) { + throw new Error('Failed to migrate database', { cause: e }); + } +} \ No newline at end of file diff --git a/src/backend/common/database/drizzle/migrations/20260422185819_white_wilson_fisk/migration.sql b/src/backend/common/database/drizzle/migrations/20260422185819_white_wilson_fisk/migration.sql new file mode 100644 index 000000000..c467d9386 --- /dev/null +++ b/src/backend/common/database/drizzle/migrations/20260422185819_white_wilson_fisk/migration.sql @@ -0,0 +1,35 @@ +CREATE TABLE `play_inputs` ( + `id` integer PRIMARY KEY, + `playId` text(30), + `data` text, + `play` text, + `createdAt` integer, + CONSTRAINT `fk_play_inputs_playId_plays_id_fk` FOREIGN KEY (`playId`) REFERENCES `plays`(`id`) ON UPDATE CASCADE ON DELETE CASCADE +); +--> statement-breakpoint +CREATE TABLE `plays` ( + `id` text(30) PRIMARY KEY, + `componentType` text(50) NOT NULL, + `componentName` text(200) NOT NULL, + `error` text, + `playedAt` integer, + `seenAt` integer, + `play` text NOT NULL, + `parentId` text(30) +); +--> statement-breakpoint +CREATE TABLE `play_queue_state` ( + `id` integer PRIMARY KEY, + `playId` text, + `queueName` text(200), + `queueStatus` text(30), + `error` text, + `createdAt` integer, + CONSTRAINT `fk_play_queue_state_playId_plays_id_fk` FOREIGN KEY (`playId`) REFERENCES `plays`(`id`) ON UPDATE CASCADE ON DELETE CASCADE +); +--> statement-breakpoint +CREATE INDEX `play_input_id_idx` ON `play_inputs` (`playId`);--> statement-breakpoint +CREATE INDEX `play_parent_id_idx` ON `plays` (`parentId`);--> statement-breakpoint +CREATE INDEX `play_playedAt_idx` ON `plays` (`playedAt`);--> statement-breakpoint +CREATE INDEX `play_seenAt_idx` ON `plays` (`seenAt`);--> statement-breakpoint +CREATE INDEX `play_queue_state_id_idx` ON `play_queue_state` (`playId`); \ No newline at end of file diff --git a/src/backend/common/database/drizzle/migrations/20260422185819_white_wilson_fisk/snapshot.json b/src/backend/common/database/drizzle/migrations/20260422185819_white_wilson_fisk/snapshot.json new file mode 100644 index 000000000..cbd6b3ec4 --- /dev/null +++ b/src/backend/common/database/drizzle/migrations/20260422185819_white_wilson_fisk/snapshot.json @@ -0,0 +1,340 @@ +{ + "version": "7", + "dialect": "sqlite", + "id": "96a2050a-7d06-4db1-b989-3082b6374e1f", + "prevIds": [ + "00000000-0000-0000-0000-000000000000" + ], + "ddl": [ + { + "name": "play_inputs", + "entityType": "tables" + }, + { + "name": "plays", + "entityType": "tables" + }, + { + "name": "play_queue_state", + "entityType": "tables" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "text(30)", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "playId", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "data", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "play", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "createdAt", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "text(30)", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text(50)", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "componentType", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text(200)", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "componentName", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "error", + "entityType": "columns", + "table": "plays" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "playedAt", + "entityType": "columns", + "table": "plays" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "seenAt", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "play", + "entityType": "columns", + "table": "plays" + }, + { + "type": "text(30)", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "parentId", + "entityType": "columns", + "table": "plays" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "play_queue_state" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "playId", + "entityType": "columns", + "table": "play_queue_state" + }, + { + "type": "text(200)", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "queueName", + "entityType": "columns", + "table": "play_queue_state" + }, + { + "type": "text(30)", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "queueStatus", + "entityType": "columns", + "table": "play_queue_state" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "error", + "entityType": "columns", + "table": "play_queue_state" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "createdAt", + "entityType": "columns", + "table": "play_queue_state" + }, + { + "columns": [ + "playId" + ], + "tableTo": "plays", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_play_inputs_playId_plays_id_fk", + "entityType": "fks", + "table": "play_inputs" + }, + { + "columns": [ + "playId" + ], + "tableTo": "plays", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_play_queue_state_playId_plays_id_fk", + "entityType": "fks", + "table": "play_queue_state" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "play_inputs_pk", + "table": "play_inputs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "plays_pk", + "table": "plays", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "play_queue_state_pk", + "table": "play_queue_state", + "entityType": "pks" + }, + { + "columns": [ + { + "value": "playId", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_input_id_idx", + "entityType": "indexes", + "table": "play_inputs" + }, + { + "columns": [ + { + "value": "parentId", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_parent_id_idx", + "entityType": "indexes", + "table": "plays" + }, + { + "columns": [ + { + "value": "playedAt", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_playedAt_idx", + "entityType": "indexes", + "table": "plays" + }, + { + "columns": [ + { + "value": "seenAt", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_seenAt_idx", + "entityType": "indexes", + "table": "plays" + }, + { + "columns": [ + { + "value": "playId", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_queue_state_id_idx", + "entityType": "indexes", + "table": "play_queue_state" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts b/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts new file mode 100644 index 000000000..6d3fc8150 --- /dev/null +++ b/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts @@ -0,0 +1,117 @@ +import { integer, sqliteTable, text, index, customType } from "drizzle-orm/sqlite-core"; +import { defineRelations } from 'drizzle-orm'; +import dayjs, { Dayjs } from "dayjs"; + +const DayjsTimestamp = customType< + { + data: Dayjs; + driverData: number; + } +>({ + dataType() { + return 'number' + }, + toDriver(value: Dayjs): number { + return value.valueOf(); + }, + fromDriver(value: number): Dayjs { + return dayjs(value); + }, +}); + +export const plays = sqliteTable("plays", { + id: text({ length: 30 }).primaryKey(), + componentType: text({ length: 50 }).notNull(), + componentName: text({ length: 200 }).notNull(), + error: text({ mode: 'json' }), + playedAt: DayjsTimestamp('playedAt'), // integer({ mode: 'timestamp_ms' }), + seenAt: DayjsTimestamp('seenAt'), // integer({ mode: 'timestamp_ms' }), + play: text({ mode: 'json' }).notNull(), + // TODO can this have a reference with cascade? + parentId: text({ length: 30 }) +}, (table) => [ + index("play_parent_id_idx").on(table.parentId), + index("play_playedAt_idx").on(table.playedAt), + index("play_seenAt_idx").on(table.seenAt), +]); + +export type NewPlay = typeof plays.$inferInsert; +export type SelectPlay = typeof plays.$inferSelect; + +export const playInputs = sqliteTable("play_inputs", { + id: integer({ mode: 'number' }).primaryKey(), + playId: text({ length: 30 }).references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + data: text({ mode: 'json' }), + play: text({ mode: 'json' }), + createdAt: DayjsTimestamp('createdAt').$defaultFn(() => dayjs()) // integer({ mode: 'timestamp_ms' }) +}, (table) => [ + index('play_input_id_idx').on(table.playId) +]); + +// export const playParentRelations = defineRelations({plays}, (r) => ({ +// plays: { +// parent: r.one.plays({ +// from: r.plays.parentId, +// to: r.plays.id +// }), +// children: r.many.plays() +// } +// })) + + +// export const playInputRelations = defineRelations({ plays, playInputs }, (r) => ({ +// plays: { +// input: r.one.playInputs({ +// from: r.plays.id, +// to: r.playInputs.playId, +// optional: false, +// }) +// } +// })); + +export const queueStates = sqliteTable("play_queue_state", { + id: integer({ mode: 'number' }).primaryKey(), + playId: text().references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + queueName: text({length: 200}), + queueStatus: text({length: 30}), + error: text({ mode: 'json' }), + createdAt: DayjsTimestamp('createdAt').$defaultFn(() => dayjs()) // integer({ mode: 'timestamp_ms' }) +}, (table) => [ + index('play_queue_state_id_idx').on(table.playId) +]); + +// export const playQueueRelations = defineRelations({ plays, queueStates }, (r) => ({ +// plays: { +// queueStates: r.many.queueStates() +// }, +// queueStates: { +// play: r.one.plays({ +// from: r.queueStates.playId, +// to: r.plays.id +// }) +// } +// })); + +export const playRelations = defineRelations({ plays, queueStates, playInputs }, (r) => ({ + plays: { + queueStates: r.many.queueStates(), + input: r.one.playInputs({ + from: r.plays.id, + to: r.playInputs.playId, + optional: false, + }), + parent: r.one.plays({ + from: r.plays.parentId, + to: r.plays.id + }), + children: r.many.plays() + }, + queueStates: { + play: r.one.plays({ + from: r.queueStates.playId, + to: r.plays.id + }) + } +})); + +export const relations = playRelations; \ No newline at end of file diff --git a/src/backend/common/errors/MSErrors.ts b/src/backend/common/errors/MSErrors.ts index dcfdc3828..4b15cae40 100644 --- a/src/backend/common/errors/MSErrors.ts +++ b/src/backend/common/errors/MSErrors.ts @@ -43,12 +43,14 @@ export class SimpleError extends Error implements HasSimpleError { stackShortened: boolean = false; shortenStack() { - const atIndex = parseRegexSingle(STACK_AT_REGEX,this.stack); - if(atIndex !== undefined) { - const firstn = this.stack.indexOf('\n', atIndex.index + atIndex.match.length); - if(firstn !== -1) { - this.stack = this.stack.slice(0, firstn); - this.stackShortened = true; + if(this.stack !== undefined) { + const atIndex = parseRegexSingle(STACK_AT_REGEX, this.stack); + if(atIndex !== undefined) { + const firstn = this.stack.indexOf('\n', atIndex.index + atIndex.match.length); + if(firstn !== -1) { + this.stack = this.stack.slice(0, firstn); + this.stackShortened = true; + } } } } diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index fdb39c70b..fdb4cca26 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -68,12 +68,14 @@ import { generateLoggableAbortReason, ScrobbleSubmitError, SimpleError } from ". import {serializeError} from 'serialize-error'; import { DEFAULT_NEW_PADDING, groupPlaysToTimeRanges } from "../utils/ListenFetchUtils.js"; import { spawn, catchAbortError, isAbortError, rethrowAbortError, delay, forever, AbortError, throwIfAborted } from 'abort-controller-x'; +import { Queue, MemoryStorage } from '@platformatic/job-queue' type PlatformMappedPlays = Map; type NowPlayingQueue = Map; const platformTruncate = truncateStringToLength(10); + export default abstract class AbstractScrobbleClient extends AbstractComponent implements Authenticatable { name: string; @@ -116,6 +118,8 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i dupeLogger: Logger; deadLogger: Logger; + scrobbleQueue: Queue<{payload: QueuedScrobble}, {scrobbled: ScrobbledPlayObject}>; + existingScrobble: (playObjPre: PlayObject, existingScrobbles: PlayObject[], log?: boolean) => Promise declare config: CommonClientConfig; @@ -145,6 +149,10 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i this.notifier = notifier; this.emitter = emitter; this.scrobbledPlayObjs = new FixedSizeList(this.MAX_STORED_SCROBBLES); + this.scrobbleQueue = new Queue<{payload: QueuedScrobble}, {scrobbled: ScrobbledPlayObject}>({ + storage: new MemoryStorage(), + concurrency: 1 + }); const { options: { diff --git a/src/backend/tests/database/drizzle.test.ts b/src/backend/tests/database/drizzle.test.ts new file mode 100644 index 000000000..c22760deb --- /dev/null +++ b/src/backend/tests/database/drizzle.test.ts @@ -0,0 +1,94 @@ +import chai, { assert, expect } from 'chai'; +import asPromised from 'chai-as-promised'; +import { getDb, migrateDb, shouldBackupDb } from '../../common/database/drizzle/drizzleUtils.js'; +import withLocalTmpDir from 'with-local-tmp-dir'; +import { playInputs, plays, queueStates } from '../../common/database/drizzle/schema/drizzlePlaysTable.js'; +import { nanoid } from 'nanoid'; +import dayjs from 'dayjs'; +import { generatePlay } from '../../../core/PlayTestUtils.js'; +import { getDbPath } from '../../common/database/Database.js'; + +// it('Detects pending migrations', async function () { + +// const res = await shouldBackupDb(getDbPath(undefined, process.cwd())); + +// expect(res).to.be.undefined; + +// }); + +describe('Basic DB Operations', function () { + + it('Should create a play', async function () { + + withLocalTmpDir(async () => { + + const db = getDb(':memory:', { workingDirectory: process.cwd() }); + await migrateDb(db); + + const playRow = await db.insert(plays).values({ + id: nanoid(), + componentName: 'mySpot', + componentType: 'spotify', + playedAt: dayjs(), + seenAt: dayjs(), + play: generatePlay() + }); + + expect(playRow.changes).eq(1); + + }, { unsafeCleanup: true }); + }); + + it('Should create a play with relations', async function () { + + withLocalTmpDir(async () => { + + const db = getDb(':memory:', { workingDirectory: process.cwd() }); + await migrateDb(db); + + const id = nanoid(); + const playRow = await db.insert(plays).values({ + id, + componentName: 'mySpot', + componentType: 'spotify', + playedAt: dayjs(), + seenAt: dayjs(), + play: generatePlay() + }).returning(); + + const input = await db.insert(playInputs).values({ + playId: playRow[0].id, + play: playRow[0].play, + data: { anything: 'foo' } + }).returning(); + + const twoQueues = await db.insert(queueStates).values([ + { + playId: id, + queueName: 'foo', + queueStatus: 'queued' + }, + { + playId: id, + queueName: 'bar', + queueStatus: 'completed' + } + ]); + + const fullPlay = await db.query.plays.findFirst({ + with: { + input: true, + queueStates: true, + }, + }); + + expect(fullPlay.queueStates).to.not.be.undefined; + expect(fullPlay.queueStates).length(2); + + expect(fullPlay.input).to.not.be.undefined; + + }, { unsafeCleanup: true }); + }); + +}); + diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts index a49d17a09..ec7115569 100644 --- a/src/backend/tests/scrobbler/scrobblers.test.ts +++ b/src/backend/tests/scrobbler/scrobblers.test.ts @@ -20,7 +20,7 @@ import { PaginatedTimeRangeOptions, PlayPlatformId, REFRESH_STALE_DEFAULT } from import { defaultLifecycle } from '../../utils/PlayTransformUtils.js'; import { shuffleArray } from '../../utils/DataUtils.js'; import { DEFAULT_CONSOLIDATE_DURATION, DEFAULT_GROUP_DURATION, groupPlaysToTimeRanges } from '../../utils/ListenFetchUtils.js'; -import { asPlay } from '../../../core/tests/utils/fixtures.js'; +import { asPlay } from '../../../core/PlayMarshalUtils.js'; import { nanoid } from 'nanoid'; import { getRoot } from '../../ioc.js'; import { transientCache } from '../utils/CacheTestUtils.js'; diff --git a/src/backend/utils/FSUtils.ts b/src/backend/utils/FSUtils.ts index dca0d5174..8ce631e46 100644 --- a/src/backend/utils/FSUtils.ts +++ b/src/backend/utils/FSUtils.ts @@ -33,6 +33,7 @@ export async function readText(path: any) { // }); // }); } + export const fileOrDirectoryIsWriteable = (location: string) => { const pathInfo = pathUtil.parse(location); const isDir = pathInfo.ext === ''; @@ -63,3 +64,20 @@ export const fileOrDirectoryIsWriteable = (location: string) => { } }; +export const fileExists = (location: string) => { + const pathInfo = pathUtil.parse(location); + const isDir = pathInfo.ext === ''; + try { + accessSync(location, constants.R_OK); + return true; + } catch (err: any) { + const { code } = err; + if (code === 'ENOENT') { + return false; + } else if (code === 'EACCES') { + throw new Error(`${isDir ? 'Directory' : 'File'} exists at ${location} but application does not have permission to write to it.`); + } else { + throw new Error(`${isDir ? 'Directory' : 'File'} exists at ${location} but application is unable to access it due to a system error`, { cause: err }); + } + } +}; \ No newline at end of file diff --git a/src/core/PlayMarshalUtils.ts b/src/core/PlayMarshalUtils.ts new file mode 100644 index 000000000..420a47a12 --- /dev/null +++ b/src/core/PlayMarshalUtils.ts @@ -0,0 +1,82 @@ +import clone from 'clone'; +import dayjs from 'dayjs'; +import { Traverse, TraverseContext } from 'neotraverse/modern'; +import { ListenRange } from '../backend/sources/PlayerState/ListenRange.js'; +import { AmbPlayObject, JsonPlayObject, PlayObject, PlayProgressAmb, REGEX_ISO8601_LOOSE } from './Atomic.js'; +import { ListenProgressPositional, ListenProgressTS } from '../backend/sources/PlayerState/ListenProgress.js'; + +interface BlockPath { key: string, parent: string }; +type BlockPaths = BlockPath[]; + +/** We know some nodes will never have data that needs to be transformed + * and these nodes can have lots of data so we can optimize them away by not (recursively) traversing them + */ +const blockedKeys: PropertyKey[] = ['patch', 'inputs', 'payload', 'response', 'error']; +/** We know some paths/nodes will never have data that needs to be transformed + * and these nodes can have lots of data so we can optimize them away by not (recursively) traversing them + */ +const blockedPaths: BlockPaths = [ + { + parent: 'data', + key: 'meta' + }, + { + parent: 'lifecycle', + key: 'input' + } +]; + +export const shouldBlock = (ctx: TraverseContext): boolean => { + if (blockedKeys.includes(ctx.key)) { + return true; + } + return blockedPaths.some((x) => { + let blocked = x.key === ctx.key; + if (blocked && x.parent !== undefined) { + blocked = ctx.parent !== undefined && ctx.parent.key === x.parent; + } + return blocked; + }); +}; + +export const asJsonPlayObject = (play: AmbPlayObject): JsonPlayObject => { + const cloned = clone(play); + new Traverse(cloned).forEach((ctx, x) => { + if (shouldBlock(ctx)) { + ctx.block(); + return; + } + + if (dayjs.isDayjs(x)) { + ctx.update(x.toISOString()); + } else if (x instanceof ListenRange) { + ctx.update(x.toJSON(), true); + } + }); + return cloned as unknown as JsonPlayObject; +}; + +export const asPlay = (data: JsonPlayObject): PlayObject => { + const cloned = clone(data); + new Traverse(cloned).forEach((ctx, x) => { + if (shouldBlock(ctx)) { + ctx.block(); + return; + } + + if (typeof x === 'string' && REGEX_ISO8601_LOOSE.test(x)) { + ctx.update(dayjs(x), true); + } else if (ctx.key === 'listenRanges') { + const ranges = x[0].map((y: PlayProgressAmb) => { + if (y.positionPercent === undefined) { + return new ListenProgressPositional({ timestamp: dayjs(y.timestamp), position: y.position }); + } else { + return new ListenProgressTS({ timestamp: dayjs(y.timestamp), positionPercent: y.positionPercent }); + } + }); + ctx.update(ranges, true); + } + }); + return cloned as unknown as PlayObject; +}; + diff --git a/src/core/tests/utils/fixtures.ts b/src/core/tests/utils/fixtures.ts index 6ec28152d..bd21c5eac 100644 --- a/src/core/tests/utils/fixtures.ts +++ b/src/core/tests/utils/fixtures.ts @@ -1,9 +1,6 @@ import { Traverse, TraverseContext } from 'neotraverse/modern'; import { faker } from '@faker-js/faker'; -import dayjs from 'dayjs'; -import { AmbPlayObject, JsonPlayObject, LifecycleInput, LifecycleStep, ObjectPlayData, PlayMeta, PlayObject, PlayProgressAmb, REGEX_ISO8601_LOOSE, ScrobbleResult } from '../../Atomic.js'; -import { ListenRange } from '../../../backend/sources/PlayerState/ListenRange.js'; -import { ListenProgressPositional, ListenProgressTS } from '../../../backend/sources/PlayerState/ListenProgress.js'; +import { LifecycleInput, LifecycleStep, ObjectPlayData, PlayMeta, PlayObject, ScrobbleResult } from '../../Atomic.js'; import { MarkOptional } from 'ts-essentials'; import { generateBrainz, generateMbid, generatePlay, GeneratePlayOpts, generatePlays } from '../../PlayTestUtils.js'; import { lifecyclelessInvariantTransform } from '../../PlayUtils.js'; @@ -14,81 +11,6 @@ import { UpstreamError } from '../../../backend/common/errors/UpstreamError.js'; import { playToListenPayload } from '../../../backend/common/vendor/listenbrainz/lzUtils.js'; import { mergeSimpleError, SimpleError, SkipTransformStageError, StagePrerequisiteError } from '../../../backend/common/errors/MSErrors.js'; -interface BlockPath { key: string, parent: string }; -type BlockPaths = BlockPath[]; - -/** We know some nodes will never have data that needs to be transformed - * and these nodes can have lots of data so we can optimize them away by not (recursively) traversing them - */ -const blockedKeys: PropertyKey[] = ['patch', 'inputs', 'payload', 'response', 'error']; -/** We know some paths/nodes will never have data that needs to be transformed - * and these nodes can have lots of data so we can optimize them away by not (recursively) traversing them - */ -const blockedPaths: BlockPaths = [ - { - parent: 'data', - key: 'meta' - }, - { - parent: 'lifecycle', - key: 'input' - } -] - -const shouldBlock = (ctx: TraverseContext): boolean => { - if (blockedKeys.includes(ctx.key)) { - return true; - } - return blockedPaths.some((x) => { - let blocked = x.key === ctx.key; - if (blocked && x.parent !== undefined) { - blocked = ctx.parent !== undefined && ctx.parent.key === x.parent; - } - return blocked; - }) -} - -export const asJsonPlayObject = (play: AmbPlayObject): JsonPlayObject => { - const cloned = clone(play); - new Traverse(cloned).forEach((ctx, x) => { - if (shouldBlock(ctx)) { - ctx.block(); - return; - } - - if (dayjs.isDayjs(x)) { - ctx.update(x.toISOString()); - } else if (x instanceof ListenRange) { - ctx.update(x.toJSON(), true); - } - }); - return cloned as unknown as JsonPlayObject; -} - -export const asPlay = (data: JsonPlayObject): PlayObject => { - const cloned = clone(data); - new Traverse(cloned).forEach((ctx, x) => { - if (shouldBlock(ctx)) { - ctx.block(); - return; - } - - if (typeof x === 'string' && REGEX_ISO8601_LOOSE.test(x)) { - ctx.update(dayjs(x), true); - } else if (ctx.key === 'listenRanges') { - const ranges = x[0].map((y: PlayProgressAmb) => { - if (y.positionPercent === undefined) { - return new ListenProgressPositional({ timestamp: dayjs(y.timestamp), position: y.position }); - } else { - return new ListenProgressTS({ timestamp: dayjs(y.timestamp), positionPercent: y.positionPercent }); - } - }) - ctx.update(ranges, true); - } - }); - return cloned as unknown as PlayObject; -} - export interface ScrobbleMatchOptions { match?: boolean warnings?: boolean diff --git a/src/stories/ActivityTimeline.stories.tsx b/src/stories/ActivityTimeline.stories.tsx index 7f3a00207..185b23b8a 100644 --- a/src/stories/ActivityTimeline.stories.tsx +++ b/src/stories/ActivityTimeline.stories.tsx @@ -8,7 +8,8 @@ import {Provider} from "../client/components/Provider"; import { generateJsonPlays } from "../core/PlayTestUtils.js"; import { ErrorLike, JsonPlayObject, PlayLifecycle } from "../core/Atomic.js"; import { examplePlay, lastfmErrorExample } from "./storyUtils.js"; -import { asJsonPlayObject, generatePlayWithLifecycle, playWithLifecycleScrobble } from "../core/tests/utils/fixtures.js"; +import { generatePlayWithLifecycle, playWithLifecycleScrobble } from "../core/tests/utils/fixtures.js"; +import { asJsonPlayObject } from '../core/PlayMarshalUtils.js'; // More on how to set up stories at: https://storybook.js.org/docs/writing-stories#default-export const meta = preview.meta({ diff --git a/src/stories/List.stories.tsx b/src/stories/List.stories.tsx index 0f3e10862..f13ea231c 100644 --- a/src/stories/List.stories.tsx +++ b/src/stories/List.stories.tsx @@ -8,9 +8,10 @@ import {Provider} from "../client/components/Provider"; import { generateJsonPlays, normalizePlays } from "../core/PlayTestUtils.js"; import { ErrorLike, JsonPlayObject } from "../core/Atomic.js"; import {examplePlay, lastfmErrorExample} from './storyUtils.js'; -import {playWithLifecycleScrobble, generatePlayWithLifecycle, asJsonPlayObject} from '../core/tests/utils/fixtures' +import {playWithLifecycleScrobble, generatePlayWithLifecycle} from '../core/tests/utils/fixtures' import { generateArray } from "../core/DataUtils.js"; import dayjs from "dayjs"; +import { asJsonPlayObject } from "../core/PlayMarshalUtils.js"; const stack = "Scrobble Submit Error: Failed to submit to Listenbrainz (listen_type single)\n at ListenbrainzApiClient.submitListen (/app/src/backend/common/vendor/ListenbrainzApiClient.ts:246:19)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async ListenbrainzScrobbler.doScrobble (/app/src/backend/scrobblers/ListenbrainzScrobbler.ts:87:28)\n at async ListenbrainzScrobbler.scrobble (/app/src/backend/scrobblers/AbstractScrobbleClient.ts:679:28)\n at async ListenbrainzScrobbler.processDeadLetterScrobble (/app/src/backend/scrobblers/AbstractScrobbleClient.ts:920:39)\n at async ListenbrainzScrobbler.processDeadLetterQueue (/app/src/backend/scrobblers/AbstractScrobbleClient.ts:894:43)\n at async PromisePoolExecutor.handler (/app/src/backend/tasks/heartbeatClients.ts:35:21)\n at async PromisePoolExecutor.waitForActiveTaskToFinish (/app/node_modules/@supercharge/promise-pool/dist/promise-pool-executor.js:375:9)\n at async PromisePoolExecutor.waitForProcessingSlot (/app/node_modules/@supercharge/promise-pool/dist/promise-pool-executor.js:368:13)\n at async PromisePoolExecutor.process (/app/node_modules/@supercharge/promise-pool/dist/promise-pool-executor.js:354:13)"; diff --git a/src/stories/PlayInfo.stories.tsx b/src/stories/PlayInfo.stories.tsx index fde02b31f..d38f88c99 100644 --- a/src/stories/PlayInfo.stories.tsx +++ b/src/stories/PlayInfo.stories.tsx @@ -7,7 +7,7 @@ import {Provider} from "../client/components/Provider"; import { Container } from '@chakra-ui/react'; import { generateArtists, generateJsonPlay, generatePlay, withBrainz } from "../core/PlayTestUtils.js" import clone from "clone"; -import { asJsonPlayObject } from "../core/tests/utils/fixtures.js"; +import { asJsonPlayObject } from '../core/PlayMarshalUtils.js'; type PropsAndCustomArgs = React.ComponentProps & { includeAlbumArtists?: boolean; diff --git a/src/stories/TransformSteps.stories.tsx b/src/stories/TransformSteps.stories.tsx index b0c989a05..aafa70e30 100644 --- a/src/stories/TransformSteps.stories.tsx +++ b/src/stories/TransformSteps.stories.tsx @@ -8,7 +8,8 @@ import {Provider} from "../client/components/Provider"; import { generateJsonPlays } from "../core/PlayTestUtils.js"; import { ErrorLike, JsonPlayObject, PlayLifecycle } from "../core/Atomic.js"; import { examplePlay, lastfmErrorExample } from "./storyUtils.js"; -import {asJsonPlayObject, generatePlayWithLifecycle} from '../core/tests/utils/fixtures' +import {generatePlayWithLifecycle} from '../core/tests/utils/fixtures' +import { asJsonPlayObject } from "../core/PlayMarshalUtils.js"; // More on how to set up stories at: https://storybook.js.org/docs/writing-stories#default-export const meta = preview.meta({ diff --git a/tsconfig.json b/tsconfig.json index 109f70a5a..4c6cfbc05 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -19,6 +19,7 @@ "sourceMap": false, }, "include": [ - "src" + "src", + "drizzle.config.ts" ], } From 83f501df0db24b4fff86ba5848ca5390088252ae Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 23 Apr 2026 01:12:35 +0000 Subject: [PATCH 002/104] add FK For parent play --- .../migration.sql | 11 ++++---- .../snapshot.json | 25 +++++++++++++++---- .../drizzle/schema/drizzlePlaysTable.ts | 6 ++--- 3 files changed, 29 insertions(+), 13 deletions(-) rename src/backend/common/database/drizzle/migrations/{20260422185819_white_wilson_fisk => 20260423011123_blue_the_santerians}/migration.sql (85%) rename src/backend/common/database/drizzle/migrations/{20260422185819_white_wilson_fisk => 20260423011123_blue_the_santerians}/snapshot.json (93%) diff --git a/src/backend/common/database/drizzle/migrations/20260422185819_white_wilson_fisk/migration.sql b/src/backend/common/database/drizzle/migrations/20260423011123_blue_the_santerians/migration.sql similarity index 85% rename from src/backend/common/database/drizzle/migrations/20260422185819_white_wilson_fisk/migration.sql rename to src/backend/common/database/drizzle/migrations/20260423011123_blue_the_santerians/migration.sql index c467d9386..2be6b12c2 100644 --- a/src/backend/common/database/drizzle/migrations/20260422185819_white_wilson_fisk/migration.sql +++ b/src/backend/common/database/drizzle/migrations/20260423011123_blue_the_santerians/migration.sql @@ -3,7 +3,7 @@ CREATE TABLE `play_inputs` ( `playId` text(30), `data` text, `play` text, - `createdAt` integer, + `createdAt` number, CONSTRAINT `fk_play_inputs_playId_plays_id_fk` FOREIGN KEY (`playId`) REFERENCES `plays`(`id`) ON UPDATE CASCADE ON DELETE CASCADE ); --> statement-breakpoint @@ -12,10 +12,11 @@ CREATE TABLE `plays` ( `componentType` text(50) NOT NULL, `componentName` text(200) NOT NULL, `error` text, - `playedAt` integer, - `seenAt` integer, + `playedAt` number, + `seenAt` number, `play` text NOT NULL, - `parentId` text(30) + `parentId` text(30), + CONSTRAINT `fk_plays_parentId_plays_id_fk` FOREIGN KEY (`parentId`) REFERENCES `plays`(`id`) ); --> statement-breakpoint CREATE TABLE `play_queue_state` ( @@ -24,7 +25,7 @@ CREATE TABLE `play_queue_state` ( `queueName` text(200), `queueStatus` text(30), `error` text, - `createdAt` integer, + `createdAt` number, CONSTRAINT `fk_play_queue_state_playId_plays_id_fk` FOREIGN KEY (`playId`) REFERENCES `plays`(`id`) ON UPDATE CASCADE ON DELETE CASCADE ); --> statement-breakpoint diff --git a/src/backend/common/database/drizzle/migrations/20260422185819_white_wilson_fisk/snapshot.json b/src/backend/common/database/drizzle/migrations/20260423011123_blue_the_santerians/snapshot.json similarity index 93% rename from src/backend/common/database/drizzle/migrations/20260422185819_white_wilson_fisk/snapshot.json rename to src/backend/common/database/drizzle/migrations/20260423011123_blue_the_santerians/snapshot.json index cbd6b3ec4..0ffe0b492 100644 --- a/src/backend/common/database/drizzle/migrations/20260422185819_white_wilson_fisk/snapshot.json +++ b/src/backend/common/database/drizzle/migrations/20260423011123_blue_the_santerians/snapshot.json @@ -1,7 +1,7 @@ { "version": "7", "dialect": "sqlite", - "id": "96a2050a-7d06-4db1-b989-3082b6374e1f", + "id": "7c59deda-bffc-4163-9eeb-4c9ba4e9dcd3", "prevIds": [ "00000000-0000-0000-0000-000000000000" ], @@ -59,7 +59,7 @@ "table": "play_inputs" }, { - "type": "integer", + "type": "number", "notNull": false, "autoincrement": false, "default": null, @@ -109,7 +109,7 @@ "table": "plays" }, { - "type": "integer", + "type": "number", "notNull": false, "autoincrement": false, "default": null, @@ -119,7 +119,7 @@ "table": "plays" }, { - "type": "integer", + "type": "number", "notNull": false, "autoincrement": false, "default": null, @@ -199,7 +199,7 @@ "table": "play_queue_state" }, { - "type": "integer", + "type": "number", "notNull": false, "autoincrement": false, "default": null, @@ -223,6 +223,21 @@ "entityType": "fks", "table": "play_inputs" }, + { + "columns": [ + "parentId" + ], + "tableTo": "plays", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "NO ACTION", + "nameExplicit": false, + "name": "fk_plays_parentId_plays_id_fk", + "entityType": "fks", + "table": "plays" + }, { "columns": [ "playId" diff --git a/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts b/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts index 6d3fc8150..15daf1ff8 100644 --- a/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts +++ b/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts @@ -1,4 +1,4 @@ -import { integer, sqliteTable, text, index, customType } from "drizzle-orm/sqlite-core"; +import { integer, sqliteTable, text, index, customType, AnySQLiteColumn } from "drizzle-orm/sqlite-core"; import { defineRelations } from 'drizzle-orm'; import dayjs, { Dayjs } from "dayjs"; @@ -27,8 +27,8 @@ export const plays = sqliteTable("plays", { playedAt: DayjsTimestamp('playedAt'), // integer({ mode: 'timestamp_ms' }), seenAt: DayjsTimestamp('seenAt'), // integer({ mode: 'timestamp_ms' }), play: text({ mode: 'json' }).notNull(), - // TODO can this have a reference with cascade? - parentId: text({ length: 30 }) + // https://orm.drizzle.team/docs/indexes-constraints#foreign-key + parentId: text({ length: 30 }).references((): AnySQLiteColumn => plays.id) }, (table) => [ index("play_parent_id_idx").on(table.parentId), index("play_playedAt_idx").on(table.playedAt), From fec39b2e2e6a39596904c1bd7459117c8ebfbd00 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 23 Apr 2026 02:01:43 +0000 Subject: [PATCH 003/104] test(database): Craete pending migration and no-db test cases --- package-lock.json | 51 ++++++++++++++++-- package.json | 3 +- .../common/database/drizzle/drizzleUtils.ts | 33 +++++++----- src/backend/tests/database/drizzle.test.ts | 53 +++++++++++++++++-- 4 files changed, 117 insertions(+), 23 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7e8227283..ba9af630e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,6 @@ "dbus-ts": "^0.0.7", "discord.js": "^14.26.0", "dotenv": "^10.0.0", - "drizzle-kit": "^1.0.0-beta.22", "drizzle-orm": "^1.0.0-beta.22", "express": "^5.2.1", "express-session": "^1.19.0", @@ -147,6 +146,7 @@ "chai": "^4.3.6", "chai-as-promised": "^8.0.2", "clsx": "^2.1.1", + "drizzle-kit": "^1.0.0-beta.23", "eslint": "^8.56.0", "eslint-plugin-prefer-arrow-functions": "^3.2.4", "eslint-plugin-storybook": "10.1.11", @@ -171,6 +171,7 @@ "sinon": "^21.0.2", "storybook": "10.2.0", "tailwindcss": "^4.2.2", + "tinyexec": "^1.1.1", "ts-essentials": "^10.1.1", "typescript": "^5.9.3", "typescript-eslint": "^7.0.1", @@ -1062,6 +1063,7 @@ "version": "0.11.0", "resolved": "https://registry.npmjs.org/@drizzle-team/brocli/-/brocli-0.11.0.tgz", "integrity": "sha512-hD3pekGiPg0WPCCGAZmusBBJsDqGUR66Y452YgQsZOnkdQ7ViEPKuyP4huUGEZQefp8g34RRodXYmJ2TbCH+tg==", + "dev": true, "license": "Apache-2.0" }, "node_modules/@dword-design/chdir": { @@ -2194,6 +2196,7 @@ "version": "0.5.1", "resolved": "https://registry.npmjs.org/@js-temporal/polyfill/-/polyfill-0.5.1.tgz", "integrity": "sha512-hloP58zRVCRSpgDxmqCWJNlizAlUgJFqG2ypq79DCvyv9tHjRYMDOcPFjzfl/A1/YxDvRCZz8wvZvmapQnKwFQ==", + "dev": true, "license": "ISC", "dependencies": { "jsbi": "^4.3.0" @@ -7375,9 +7378,10 @@ } }, "node_modules/drizzle-kit": { - "version": "1.0.0-beta.22", - "resolved": "https://registry.npmjs.org/drizzle-kit/-/drizzle-kit-1.0.0-beta.22.tgz", - "integrity": "sha512-9HTZuQRljQKTgCx4UhiGn8KYYfHGk4+B/bRR1714W67kz0qgJvdrG527i8rQD8uUyET9UTGR1u8syySJD4znGw==", + "version": "1.0.0-beta.23", + "resolved": "https://registry.npmjs.org/drizzle-kit/-/drizzle-kit-1.0.0-beta.23.tgz", + "integrity": "sha512-AXVAYTItegF3h1JQECt4s4z9RwipiBoK99A3IY/thGZ0xdSPlsXZr/SU9sh6JvQi5bq08ZYXmKcdIwJpNxITig==", + "dev": true, "license": "MIT", "dependencies": { "@drizzle-team/brocli": "^0.11.0", @@ -7397,6 +7401,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7413,6 +7418,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7429,6 +7435,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7445,6 +7452,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7461,6 +7469,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7477,6 +7486,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7493,6 +7503,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7509,6 +7520,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7525,6 +7537,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7541,6 +7554,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7557,6 +7571,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7573,6 +7588,7 @@ "cpu": [ "loong64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7589,6 +7605,7 @@ "cpu": [ "mips64el" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7605,6 +7622,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7621,6 +7639,7 @@ "cpu": [ "riscv64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7637,6 +7656,7 @@ "cpu": [ "s390x" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7653,6 +7673,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7669,6 +7690,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7685,6 +7707,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7701,6 +7724,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7717,6 +7741,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7733,6 +7758,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7749,6 +7775,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7765,6 +7792,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7781,6 +7809,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7797,6 +7826,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7810,6 +7840,7 @@ "version": "0.25.12", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, "hasInstallScript": true, "license": "MIT", "bin": { @@ -10341,6 +10372,7 @@ "version": "2.6.1", "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz", "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", + "dev": true, "license": "MIT", "bin": { "jiti": "lib/jiti-cli.mjs" @@ -10380,6 +10412,7 @@ "version": "4.3.2", "resolved": "https://registry.npmjs.org/jsbi/-/jsbi-4.3.2.tgz", "integrity": "sha512-9fqMSQbhJykSeii05nxKl4m6Eqn2P6rOlYiS+C5Dr/HPIU/7yZxu5qzbs40tgaFORiw2Amd0mirjxatXYMkIew==", + "dev": true, "license": "Apache-2.0" }, "node_modules/jsesc": { @@ -14354,6 +14387,16 @@ "dev": true, "license": "MIT" }, + "node_modules/tinyexec": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.1.1.tgz", + "integrity": "sha512-VKS/ZaQhhkKFMANmAOhhXVoIfBXblQxGX1myCQ2faQrfmobMftXeJPcZGp0gS07ocvGJWDLZGyOZDadDBqYIJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/tinyglobby": { "version": "0.2.15", "dev": true, diff --git a/package.json b/package.json index c43404715..560f6649b 100644 --- a/package.json +++ b/package.json @@ -86,7 +86,6 @@ "dbus-ts": "^0.0.7", "discord.js": "^14.26.0", "dotenv": "^10.0.0", - "drizzle-kit": "^1.0.0-beta.22", "drizzle-orm": "^1.0.0-beta.22", "express": "^5.2.1", "express-session": "^1.19.0", @@ -185,6 +184,7 @@ "chai": "^4.3.6", "chai-as-promised": "^8.0.2", "clsx": "^2.1.1", + "drizzle-kit": "^1.0.0-beta.23", "eslint": "^8.56.0", "eslint-plugin-prefer-arrow-functions": "^3.2.4", "eslint-plugin-storybook": "10.1.11", @@ -209,6 +209,7 @@ "sinon": "^21.0.2", "storybook": "10.2.0", "tailwindcss": "^4.2.2", + "tinyexec": "^1.1.1", "ts-essentials": "^10.1.1", "typescript": "^5.9.3", "typescript-eslint": "^7.0.1", diff --git a/src/backend/common/database/drizzle/drizzleUtils.ts b/src/backend/common/database/drizzle/drizzleUtils.ts index f2350cc3b..3b0e64ec6 100644 --- a/src/backend/common/database/drizzle/drizzleUtils.ts +++ b/src/backend/common/database/drizzle/drizzleUtils.ts @@ -3,21 +3,24 @@ import { migrate } from 'drizzle-orm/node-sqlite/migrator'; import { sql as dsl } from 'drizzle-orm'; import * as fs from 'fs/promises'; import * as path from 'path'; -import { getDbPath } from '../Database.js'; +import { getDbPath, MEMORY_DB_NAME } from '../Database.js'; import { fileExists } from '../../../utils/FSUtils.js'; import { childLogger, Logger } from '@foxxmd/logging'; import { loggerNoop } from '../../MaybeLogger.js'; import { projectDir } from '../../index.js'; import { relations } from './schema/drizzlePlaysTable.js'; -export async function shouldBackupDb(dbPath: string, parentLogger: Logger = loggerNoop): Promise { - +export async function shouldBackupDb(dbPath: string, opts: {parentLogger?: Logger, migrationsFolder?: string} = {}): Promise<[boolean, string[]]> { + const { + parentLogger = loggerNoop, + migrationsFolder = path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') + } = opts; const logger = childLogger(parentLogger, 'Migrations'); - + logger.info(`Checking database at ${dbPath}`); - if (!fileExists(dbPath)) { + if (dbPath !== MEMORY_DB_NAME && !fileExists(dbPath)) { logger.info(`No database exists!`); - return false; + return [false, []]; } const db = drizzle(dbPath); @@ -31,13 +34,13 @@ export async function shouldBackupDb(dbPath: string, parentLogger: Logger = logg if (res[0]['count(*)'] === 0) { logger.info(`Database exists but there is no __drizzle_migrations table??`); - return false; + return [true, []]; } const dbMigrations = await db.all(dsl`SELECT id, hash, created_at, name, applied_at FROM "__drizzle_migrations" ORDER BY created_at DESC`); const appliedMigrations = new Set(dbMigrations.map((m: any) => m.name)); - const allFiles = await fs.readdir(path.resolve(projectDir, 'src/backend/common/database/drizzleMigrations')); + const allFiles = await fs.readdir(migrationsFolder); const migrationFiles = allFiles .sort(); @@ -48,14 +51,14 @@ export async function shouldBackupDb(dbPath: string, parentLogger: Logger = logg //console.log('Applied migrations:', Array.from(appliedMigrations)); if (pendingMigrations.length > 0) { logger.info(`${pendingMigrations.length} pending migrations:\n${pendingMigrations.join('\n')}`); - return true; + return [true, pendingMigrations]; } else { logger.info('No pending migrations.'); - return false; + return [false, []]; } } catch (error) { logger.error(new Error('Failed to get pending migrations', { cause: error })); - return true; + return [true, []]; } } @@ -69,11 +72,15 @@ export const getDb = (dbName: string = 'ms', opts: { logger?: Logger, workingDir return drizzle(dbPath, {relations: relations}); } -export const migrateDb = async (db: ReturnType, parentLogger: Logger = loggerNoop) => { +export const migrateDb = async (db: ReturnType, opts: {parentLogger?: Logger, migrationsFolder?: string} = {}) => { + const { + migrationsFolder, + parentLogger = loggerNoop + } = opts; const logger = childLogger(parentLogger, 'Migrations'); try { - await migrate(db, { migrationsFolder: path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') }); + await migrate(db, { migrationsFolder: migrationsFolder ?? path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') }); logger.info('Migrations complete'); } catch (e) { throw new Error('Failed to migrate database', { cause: e }); diff --git a/src/backend/tests/database/drizzle.test.ts b/src/backend/tests/database/drizzle.test.ts index c22760deb..b45be7170 100644 --- a/src/backend/tests/database/drizzle.test.ts +++ b/src/backend/tests/database/drizzle.test.ts @@ -7,14 +7,57 @@ import { nanoid } from 'nanoid'; import dayjs from 'dayjs'; import { generatePlay } from '../../../core/PlayTestUtils.js'; import { getDbPath } from '../../common/database/Database.js'; +import { x } from 'tinyexec'; +import * as path from 'path'; +import * as fs from 'fs/promises'; +import { projectDir } from '../../common/index.js'; + +it('Detects pending migrations', async function () { + + const allFiles = await fs.readdir(path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations')); + const migrationFiles = allFiles + .sort(); + + await withLocalTmpDir(async () => { + + // copy first migration + await fs.mkdir('migrations'); + try { + await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true }); + const mf = path.resolve('./migrations'); + const db = getDb('ms', { workingDirectory: process.cwd() }); + await migrateDb(db, { migrationsFolder: mf }); + const res = await x('drizzle-kit', [ + 'generate', + '--name', + 'newMigration', + '--out', + `${mf}`, + '--custom', + '--schema', + path.resolve(projectDir, 'src/backend/common/database/drizzle/schema'), + '--dialect', + 'sqlite' + ]); + const [shouldBackup, pending] = await shouldBackupDb(getDbPath('ms', process.cwd()), { migrationsFolder: mf }); + expect(shouldBackup).is.true; + expect(pending).length(1); + expect(pending[0]).includes('newMigration'); + } catch (e) { + throw e; + } + }, { unsafeCleanup: true }); +}); -// it('Detects pending migrations', async function () { - -// const res = await shouldBackupDb(getDbPath(undefined, process.cwd())); +it('Detects non-existent db', async function () { -// expect(res).to.be.undefined; + await withLocalTmpDir(async () => { + const [shouldBackup, pending] = await shouldBackupDb(getDbPath('notreal', process.cwd())); + expect(shouldBackup).is.false; + expect(pending).length(0); + }); -// }); +}); describe('Basic DB Operations', function () { From 2df8bfe6aac516e17e95b3f76b772a99f908b113 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 23 Apr 2026 02:11:20 +0000 Subject: [PATCH 004/104] test(database): More migration tests --- src/backend/tests/database/drizzle.test.ts | 127 ++++++++++++++------- 1 file changed, 85 insertions(+), 42 deletions(-) diff --git a/src/backend/tests/database/drizzle.test.ts b/src/backend/tests/database/drizzle.test.ts index b45be7170..a6bfeb204 100644 --- a/src/backend/tests/database/drizzle.test.ts +++ b/src/backend/tests/database/drizzle.test.ts @@ -11,50 +11,93 @@ import { x } from 'tinyexec'; import * as path from 'path'; import * as fs from 'fs/promises'; import { projectDir } from '../../common/index.js'; +import { DatabaseSync } from 'node:sqlite'; -it('Detects pending migrations', async function () { - - const allFiles = await fs.readdir(path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations')); - const migrationFiles = allFiles - .sort(); - - await withLocalTmpDir(async () => { - - // copy first migration - await fs.mkdir('migrations'); - try { - await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true }); - const mf = path.resolve('./migrations'); - const db = getDb('ms', { workingDirectory: process.cwd() }); - await migrateDb(db, { migrationsFolder: mf }); - const res = await x('drizzle-kit', [ - 'generate', - '--name', - 'newMigration', - '--out', - `${mf}`, - '--custom', - '--schema', - path.resolve(projectDir, 'src/backend/common/database/drizzle/schema'), - '--dialect', - 'sqlite' - ]); - const [shouldBackup, pending] = await shouldBackupDb(getDbPath('ms', process.cwd()), { migrationsFolder: mf }); +describe('Migrations', function () { + + it('Detects non-existent db', async function () { + + await withLocalTmpDir(async () => { + const [shouldBackup, pending] = await shouldBackupDb(getDbPath('notreal', process.cwd())); + expect(shouldBackup).is.false; + expect(pending).length(0); + }); + + }); + + it('Detects abnormal db', async function () { + + withLocalTmpDir(async () => { + const otherdb = new DatabaseSync(path.resolve('./', 'other.db')); + const [shouldBackup, pending] = await shouldBackupDb(getDbPath('other', process.cwd())); expect(shouldBackup).is.true; - expect(pending).length(1); - expect(pending[0]).includes('newMigration'); - } catch (e) { - throw e; - } - }, { unsafeCleanup: true }); -}); + expect(pending).length(0); + otherdb.close(); + }, { unsafeCleanup: true }); + + }); -it('Detects non-existent db', async function () { + it('Detects pending migrations', async function () { + + const allFiles = await fs.readdir(path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations')); + const migrationFiles = allFiles + .sort(); + + await withLocalTmpDir(async () => { + + // copy first migration + await fs.mkdir('migrations'); + try { + await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true }); + const mf = path.resolve('./migrations'); + const db = getDb('ms', { workingDirectory: process.cwd() }); + await migrateDb(db, { migrationsFolder: mf }); + const res = await x('drizzle-kit', [ + 'generate', + '--name', + 'newMigration', + '--out', + `${mf}`, + '--custom', + '--schema', + path.resolve(projectDir, 'src/backend/common/database/drizzle/schema'), + '--dialect', + 'sqlite' + ]); + const [shouldBackup, pending] = await shouldBackupDb(getDbPath('ms', process.cwd()), { migrationsFolder: mf }); + expect(shouldBackup).is.true; + expect(pending).length(1); + expect(pending[0]).includes('newMigration'); + db.$client.close(); + } catch (e) { + throw e; + } + }, { unsafeCleanup: true }); + }); - await withLocalTmpDir(async () => { - const [shouldBackup, pending] = await shouldBackupDb(getDbPath('notreal', process.cwd())); - expect(shouldBackup).is.false; - expect(pending).length(0); + it('Detects no pending migrations correctly', async function () { + + const allFiles = await fs.readdir(path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations')); + const migrationFiles = allFiles + .sort(); + + await withLocalTmpDir(async () => { + + // copy first migration + await fs.mkdir('migrations'); + try { + await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true }); + const mf = path.resolve('./migrations'); + const db = getDb('ms', { workingDirectory: process.cwd() }); + await migrateDb(db, { migrationsFolder: mf }); + const [shouldBackup, pending] = await shouldBackupDb(getDbPath('ms', process.cwd()), { migrationsFolder: mf }); + expect(shouldBackup).is.false; + expect(pending).length(0); + db.$client.close(); + } catch (e) { + throw e; + } + }, { unsafeCleanup: true }); }); }); @@ -78,7 +121,7 @@ describe('Basic DB Operations', function () { }); expect(playRow.changes).eq(1); - + db.$client.close(); }, { unsafeCleanup: true }); }); @@ -129,7 +172,7 @@ describe('Basic DB Operations', function () { expect(fullPlay.queueStates).length(2); expect(fullPlay.input).to.not.be.undefined; - + db.$client.close(); }, { unsafeCleanup: true }); }); From 40234e64b3d8f2e61badc096d563cd8e97e97968 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 23 Apr 2026 13:33:59 +0000 Subject: [PATCH 005/104] refactor(database): Use integer for primary key and make string id as vanity id --- .../migration.sql | 15 ++-- .../snapshot.json | 66 ++++++++++++-- .../drizzle/schema/drizzlePlaysTable.ts | 25 +++--- src/backend/tests/database/drizzle.test.ts | 88 +++++++++++-------- 4 files changed, 135 insertions(+), 59 deletions(-) rename src/backend/common/database/drizzle/migrations/{20260423011123_blue_the_santerians => 20260423133243_calm_stepford_cuckoos}/migration.sql (76%) rename src/backend/common/database/drizzle/migrations/{20260423011123_blue_the_santerians => 20260423133243_calm_stepford_cuckoos}/snapshot.json (85%) diff --git a/src/backend/common/database/drizzle/migrations/20260423011123_blue_the_santerians/migration.sql b/src/backend/common/database/drizzle/migrations/20260423133243_calm_stepford_cuckoos/migration.sql similarity index 76% rename from src/backend/common/database/drizzle/migrations/20260423011123_blue_the_santerians/migration.sql rename to src/backend/common/database/drizzle/migrations/20260423133243_calm_stepford_cuckoos/migration.sql index 2be6b12c2..4d0d88e29 100644 --- a/src/backend/common/database/drizzle/migrations/20260423011123_blue_the_santerians/migration.sql +++ b/src/backend/common/database/drizzle/migrations/20260423133243_calm_stepford_cuckoos/migration.sql @@ -1,6 +1,6 @@ CREATE TABLE `play_inputs` ( `id` integer PRIMARY KEY, - `playId` text(30), + `playId` integer, `data` text, `play` text, `createdAt` number, @@ -8,29 +8,34 @@ CREATE TABLE `play_inputs` ( ); --> statement-breakpoint CREATE TABLE `plays` ( - `id` text(30) PRIMARY KEY, + `id` integer PRIMARY KEY, + `uid` text(30) NOT NULL, `componentType` text(50) NOT NULL, `componentName` text(200) NOT NULL, `error` text, `playedAt` number, `seenAt` number, `play` text NOT NULL, - `parentId` text(30), + `state` text NOT NULL, + `parentId` integer, CONSTRAINT `fk_plays_parentId_plays_id_fk` FOREIGN KEY (`parentId`) REFERENCES `plays`(`id`) ); --> statement-breakpoint CREATE TABLE `play_queue_state` ( `id` integer PRIMARY KEY, - `playId` text, + `playId` integer, `queueName` text(200), `queueStatus` text(30), + `retries` integer DEFAULT 0 NOT NULL, `error` text, `createdAt` number, + `updatedAt` number, CONSTRAINT `fk_play_queue_state_playId_plays_id_fk` FOREIGN KEY (`playId`) REFERENCES `plays`(`id`) ON UPDATE CASCADE ON DELETE CASCADE ); --> statement-breakpoint -CREATE INDEX `play_input_id_idx` ON `play_inputs` (`playId`);--> statement-breakpoint +CREATE UNIQUE INDEX `play_input_id_idx` ON `play_inputs` (`playId`);--> statement-breakpoint CREATE INDEX `play_parent_id_idx` ON `plays` (`parentId`);--> statement-breakpoint +CREATE UNIQUE INDEX `play_uid_idx` ON `plays` (`uid`);--> statement-breakpoint CREATE INDEX `play_playedAt_idx` ON `plays` (`playedAt`);--> statement-breakpoint CREATE INDEX `play_seenAt_idx` ON `plays` (`seenAt`);--> statement-breakpoint CREATE INDEX `play_queue_state_id_idx` ON `play_queue_state` (`playId`); \ No newline at end of file diff --git a/src/backend/common/database/drizzle/migrations/20260423011123_blue_the_santerians/snapshot.json b/src/backend/common/database/drizzle/migrations/20260423133243_calm_stepford_cuckoos/snapshot.json similarity index 85% rename from src/backend/common/database/drizzle/migrations/20260423011123_blue_the_santerians/snapshot.json rename to src/backend/common/database/drizzle/migrations/20260423133243_calm_stepford_cuckoos/snapshot.json index 0ffe0b492..4ca0458df 100644 --- a/src/backend/common/database/drizzle/migrations/20260423011123_blue_the_santerians/snapshot.json +++ b/src/backend/common/database/drizzle/migrations/20260423133243_calm_stepford_cuckoos/snapshot.json @@ -1,7 +1,7 @@ { "version": "7", "dialect": "sqlite", - "id": "7c59deda-bffc-4163-9eeb-4c9ba4e9dcd3", + "id": "8f540406-ea02-467b-bb53-3eee43a37acd", "prevIds": [ "00000000-0000-0000-0000-000000000000" ], @@ -29,7 +29,7 @@ "table": "play_inputs" }, { - "type": "text(30)", + "type": "integer", "notNull": false, "autoincrement": false, "default": null, @@ -69,7 +69,7 @@ "table": "play_inputs" }, { - "type": "text(30)", + "type": "integer", "notNull": false, "autoincrement": false, "default": null, @@ -78,6 +78,16 @@ "entityType": "columns", "table": "plays" }, + { + "type": "text(30)", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "uid", + "entityType": "columns", + "table": "plays" + }, { "type": "text(50)", "notNull": true, @@ -139,7 +149,17 @@ "table": "plays" }, { - "type": "text(30)", + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "state", + "entityType": "columns", + "table": "plays" + }, + { + "type": "integer", "notNull": false, "autoincrement": false, "default": null, @@ -159,7 +179,7 @@ "table": "play_queue_state" }, { - "type": "text", + "type": "integer", "notNull": false, "autoincrement": false, "default": null, @@ -188,6 +208,16 @@ "entityType": "columns", "table": "play_queue_state" }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "retries", + "entityType": "columns", + "table": "play_queue_state" + }, { "type": "text", "notNull": false, @@ -208,6 +238,16 @@ "entityType": "columns", "table": "play_queue_state" }, + { + "type": "number", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updatedAt", + "entityType": "columns", + "table": "play_queue_state" + }, { "columns": [ "playId" @@ -287,7 +327,7 @@ "isExpression": false } ], - "isUnique": false, + "isUnique": true, "where": null, "origin": "manual", "name": "play_input_id_idx", @@ -308,6 +348,20 @@ "entityType": "indexes", "table": "plays" }, + { + "columns": [ + { + "value": "uid", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "play_uid_idx", + "entityType": "indexes", + "table": "plays" + }, { "columns": [ { diff --git a/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts b/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts index 15daf1ff8..93efe50ab 100644 --- a/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts +++ b/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts @@ -1,4 +1,4 @@ -import { integer, sqliteTable, text, index, customType, AnySQLiteColumn } from "drizzle-orm/sqlite-core"; +import { integer, sqliteTable, text, index, uniqueIndex, customType, AnySQLiteColumn } from "drizzle-orm/sqlite-core"; import { defineRelations } from 'drizzle-orm'; import dayjs, { Dayjs } from "dayjs"; @@ -20,17 +20,20 @@ const DayjsTimestamp = customType< }); export const plays = sqliteTable("plays", { - id: text({ length: 30 }).primaryKey(), + id: integer().primaryKey(), + uid: text({ length: 30 }).notNull().unique(), componentType: text({ length: 50 }).notNull(), componentName: text({ length: 200 }).notNull(), error: text({ mode: 'json' }), - playedAt: DayjsTimestamp('playedAt'), // integer({ mode: 'timestamp_ms' }), - seenAt: DayjsTimestamp('seenAt'), // integer({ mode: 'timestamp_ms' }), + playedAt: DayjsTimestamp('playedAt'), + seenAt: DayjsTimestamp('seenAt'), play: text({ mode: 'json' }).notNull(), + state: text({enum: ['queued','discovered','scrobbled','failed','duped']}).notNull(), // https://orm.drizzle.team/docs/indexes-constraints#foreign-key - parentId: text({ length: 30 }).references((): AnySQLiteColumn => plays.id) + parentId: integer().references((): AnySQLiteColumn => plays.id) }, (table) => [ index("play_parent_id_idx").on(table.parentId), + uniqueIndex("play_uid_idx").on(table.uid), index("play_playedAt_idx").on(table.playedAt), index("play_seenAt_idx").on(table.seenAt), ]); @@ -40,12 +43,12 @@ export type SelectPlay = typeof plays.$inferSelect; export const playInputs = sqliteTable("play_inputs", { id: integer({ mode: 'number' }).primaryKey(), - playId: text({ length: 30 }).references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + playId: integer().references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}), data: text({ mode: 'json' }), play: text({ mode: 'json' }), - createdAt: DayjsTimestamp('createdAt').$defaultFn(() => dayjs()) // integer({ mode: 'timestamp_ms' }) + createdAt: DayjsTimestamp('createdAt').$defaultFn(() => dayjs()) }, (table) => [ - index('play_input_id_idx').on(table.playId) + uniqueIndex('play_input_id_idx').on(table.playId) ]); // export const playParentRelations = defineRelations({plays}, (r) => ({ @@ -71,11 +74,13 @@ export const playInputs = sqliteTable("play_inputs", { export const queueStates = sqliteTable("play_queue_state", { id: integer({ mode: 'number' }).primaryKey(), - playId: text().references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + playId: integer().references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}), queueName: text({length: 200}), queueStatus: text({length: 30}), + retries: integer().notNull().default(0), error: text({ mode: 'json' }), - createdAt: DayjsTimestamp('createdAt').$defaultFn(() => dayjs()) // integer({ mode: 'timestamp_ms' }) + createdAt: DayjsTimestamp('createdAt').$defaultFn(() => dayjs()), + updatedAt: DayjsTimestamp('updatedAt').$defaultFn(() => dayjs()) }, (table) => [ index('play_queue_state_id_idx').on(table.playId) ]); diff --git a/src/backend/tests/database/drizzle.test.ts b/src/backend/tests/database/drizzle.test.ts index a6bfeb204..e7d5eff17 100644 --- a/src/backend/tests/database/drizzle.test.ts +++ b/src/backend/tests/database/drizzle.test.ts @@ -13,6 +13,9 @@ import * as fs from 'fs/promises'; import { projectDir } from '../../common/index.js'; import { DatabaseSync } from 'node:sqlite'; +// would be great to push migrations directly from schema but doesn't seem supported in newest beta +// https://github.com/drizzle-team/drizzle-orm/discussions/4373 + describe('Migrations', function () { it('Detects non-existent db', async function () { @@ -132,46 +135,55 @@ describe('Basic DB Operations', function () { const db = getDb(':memory:', { workingDirectory: process.cwd() }); await migrateDb(db); - const id = nanoid(); - const playRow = await db.insert(plays).values({ - id, - componentName: 'mySpot', - componentType: 'spotify', - playedAt: dayjs(), - seenAt: dayjs(), - play: generatePlay() - }).returning(); - - const input = await db.insert(playInputs).values({ - playId: playRow[0].id, - play: playRow[0].play, - data: { anything: 'foo' } - }).returning(); - - const twoQueues = await db.insert(queueStates).values([ - { - playId: id, - queueName: 'foo', - queueStatus: 'queued' - }, - { - playId: id, - queueName: 'bar', - queueStatus: 'completed' - } - ]); - - const fullPlay = await db.query.plays.findFirst({ - with: { - input: true, - queueStates: true, - }, - }); + const uid = nanoid(); + try { - expect(fullPlay.queueStates).to.not.be.undefined; - expect(fullPlay.queueStates).length(2); - expect(fullPlay.input).to.not.be.undefined; + const playRow = await db.insert(plays).values({ + uid, + componentName: 'mySpot', + componentType: 'spotify', + state: 'queued', + playedAt: dayjs(), + seenAt: dayjs(), + play: generatePlay() + }).returning(); + + const input = await db.insert(playInputs).values({ + playId: playRow[0].id, + play: playRow[0].play, + data: { anything: 'foo' } + }).returning(); + + const twoQueues = await db.insert(queueStates).values([ + { + playId: playRow[0].id, + queueName: 'foo', + queueStatus: 'queued' + }, + { + playId: playRow[0].id, + queueName: 'bar', + queueStatus: 'completed' + } + ]); + + const fullPlay = await db.query.plays.findFirst({ + with: { + input: true, + queueStates: true, + }, + }); + + + expect(fullPlay.queueStates).to.not.be.undefined; + expect(fullPlay.queueStates).length(2); + + expect(fullPlay.input).to.not.be.undefined; + + } catch (e) { + throw e; + } db.$client.close(); }, { unsafeCleanup: true }); }); From 3c51149d9aab11fcef7f7769adc0a2ab73e15fe3 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 23 Apr 2026 15:06:59 +0000 Subject: [PATCH 006/104] feat(database): More database implementation * add components table * improve typing for json columns * fix non-nullable columns * implement basic entity generation functions --- .../common/database/drizzle/entityUtils.ts | 39 +++ .../migration.sql | 41 ---- .../20260423150230_messy_toad/migration.sql | 55 +++++ .../snapshot.json | 222 +++++++++++++++--- .../common/database/drizzle/repository.ts | 0 .../drizzle/schema/drizzlePlaysTable.ts | 80 +++++-- src/backend/tests/database/drizzle.test.ts | 34 ++- src/backend/tests/utils/databaseFixtures.ts | 35 +++ 8 files changed, 388 insertions(+), 118 deletions(-) create mode 100644 src/backend/common/database/drizzle/entityUtils.ts delete mode 100644 src/backend/common/database/drizzle/migrations/20260423133243_calm_stepford_cuckoos/migration.sql create mode 100644 src/backend/common/database/drizzle/migrations/20260423150230_messy_toad/migration.sql rename src/backend/common/database/drizzle/migrations/{20260423133243_calm_stepford_cuckoos => 20260423150230_messy_toad}/snapshot.json (66%) create mode 100644 src/backend/common/database/drizzle/repository.ts create mode 100644 src/backend/tests/utils/databaseFixtures.ts diff --git a/src/backend/common/database/drizzle/entityUtils.ts b/src/backend/common/database/drizzle/entityUtils.ts new file mode 100644 index 000000000..2bb468fea --- /dev/null +++ b/src/backend/common/database/drizzle/entityUtils.ts @@ -0,0 +1,39 @@ +import assert from "node:assert"; +import { ComponentNew, PlayInputNew, PlayNew, QueueStateNew } from "./schema/drizzlePlaysTable.js"; +import { MarkOptional } from "ts-essentials"; +import { ErrorLike, PlayObject } from "../../../../core/Atomic.js"; +import dayjs, { Dayjs } from "dayjs"; + +export const generateComponentEntity = (data: MarkOptional): ComponentNew => { + assert(data.name !== undefined, 'Must provide name'); + return { + ...data, + uid: data.uid ?? data.name + }; +} + +export type PlayEntityOpts = Partial> & { error?: ErrorLike }; + +export const generatePlayEntity = (play: PlayObject, opts: PlayEntityOpts = {}): PlayNew => { + const { + seenAt = dayjs(), + state = 'queued', + playedAt = play.data.playDate, + ...restOpts + } = opts; + return { + play, + state, + playedAt, + seenAt, + ...restOpts + } +} + +export const generateInputEntity = (data: PlayInputNew): PlayInputNew => { + return data; +} + +export const generateQueueStateEntity = (data: QueueStateNew): QueueStateNew => { + return data; +} \ No newline at end of file diff --git a/src/backend/common/database/drizzle/migrations/20260423133243_calm_stepford_cuckoos/migration.sql b/src/backend/common/database/drizzle/migrations/20260423133243_calm_stepford_cuckoos/migration.sql deleted file mode 100644 index 4d0d88e29..000000000 --- a/src/backend/common/database/drizzle/migrations/20260423133243_calm_stepford_cuckoos/migration.sql +++ /dev/null @@ -1,41 +0,0 @@ -CREATE TABLE `play_inputs` ( - `id` integer PRIMARY KEY, - `playId` integer, - `data` text, - `play` text, - `createdAt` number, - CONSTRAINT `fk_play_inputs_playId_plays_id_fk` FOREIGN KEY (`playId`) REFERENCES `plays`(`id`) ON UPDATE CASCADE ON DELETE CASCADE -); ---> statement-breakpoint -CREATE TABLE `plays` ( - `id` integer PRIMARY KEY, - `uid` text(30) NOT NULL, - `componentType` text(50) NOT NULL, - `componentName` text(200) NOT NULL, - `error` text, - `playedAt` number, - `seenAt` number, - `play` text NOT NULL, - `state` text NOT NULL, - `parentId` integer, - CONSTRAINT `fk_plays_parentId_plays_id_fk` FOREIGN KEY (`parentId`) REFERENCES `plays`(`id`) -); ---> statement-breakpoint -CREATE TABLE `play_queue_state` ( - `id` integer PRIMARY KEY, - `playId` integer, - `queueName` text(200), - `queueStatus` text(30), - `retries` integer DEFAULT 0 NOT NULL, - `error` text, - `createdAt` number, - `updatedAt` number, - CONSTRAINT `fk_play_queue_state_playId_plays_id_fk` FOREIGN KEY (`playId`) REFERENCES `plays`(`id`) ON UPDATE CASCADE ON DELETE CASCADE -); ---> statement-breakpoint -CREATE UNIQUE INDEX `play_input_id_idx` ON `play_inputs` (`playId`);--> statement-breakpoint -CREATE INDEX `play_parent_id_idx` ON `plays` (`parentId`);--> statement-breakpoint -CREATE UNIQUE INDEX `play_uid_idx` ON `plays` (`uid`);--> statement-breakpoint -CREATE INDEX `play_playedAt_idx` ON `plays` (`playedAt`);--> statement-breakpoint -CREATE INDEX `play_seenAt_idx` ON `plays` (`seenAt`);--> statement-breakpoint -CREATE INDEX `play_queue_state_id_idx` ON `play_queue_state` (`playId`); \ No newline at end of file diff --git a/src/backend/common/database/drizzle/migrations/20260423150230_messy_toad/migration.sql b/src/backend/common/database/drizzle/migrations/20260423150230_messy_toad/migration.sql new file mode 100644 index 000000000..3c1b93f6a --- /dev/null +++ b/src/backend/common/database/drizzle/migrations/20260423150230_messy_toad/migration.sql @@ -0,0 +1,55 @@ +CREATE TABLE `components` ( + `id` integer PRIMARY KEY, + `uid` text(200) NOT NULL UNIQUE, + `mode` text NOT NULL, + `type` text(50) NOT NULL, + `name` text NOT NULL, + `countLive` integer DEFAULT 0 NOT NULL, + `countNonLive` integer DEFAULT 0 NOT NULL, + `createdAt` number +); +--> statement-breakpoint +CREATE TABLE `play_inputs` ( + `id` integer PRIMARY KEY, + `playId` integer NOT NULL, + `data` text, + `play` text NOT NULL, + `createdAt` number, + CONSTRAINT `fk_play_inputs_playId_plays_id_fk` FOREIGN KEY (`playId`) REFERENCES `plays`(`id`) ON UPDATE CASCADE ON DELETE CASCADE +); +--> statement-breakpoint +CREATE TABLE `plays` ( + `id` integer PRIMARY KEY, + `uid` text(30) NOT NULL, + `componentId` integer, + `error` text, + `playedAt` number, + `seenAt` number, + `play` text NOT NULL, + `state` text NOT NULL, + `parentId` integer, + CONSTRAINT `fk_plays_componentId_components_id_fk` FOREIGN KEY (`componentId`) REFERENCES `components`(`id`) ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT `fk_plays_parentId_plays_id_fk` FOREIGN KEY (`parentId`) REFERENCES `plays`(`id`) +); +--> statement-breakpoint +CREATE TABLE `play_queue_states` ( + `id` integer PRIMARY KEY, + `playId` integer NOT NULL, + `componentId` integer NOT NULL, + `queueName` text(50) NOT NULL, + `queueStatus` text DEFAULT 'queued' NOT NULL, + `retries` integer DEFAULT 0 NOT NULL, + `error` text, + `createdAt` number NOT NULL, + `updatedAt` number NOT NULL, + CONSTRAINT `fk_play_queue_states_playId_plays_id_fk` FOREIGN KEY (`playId`) REFERENCES `plays`(`id`) ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT `fk_play_queue_states_componentId_components_id_fk` FOREIGN KEY (`componentId`) REFERENCES `components`(`id`) ON UPDATE CASCADE ON DELETE CASCADE +); +--> statement-breakpoint +CREATE UNIQUE INDEX `play_input_id_idx` ON `play_inputs` (`playId`);--> statement-breakpoint +CREATE INDEX `play_parent_id_idx` ON `plays` (`parentId`);--> statement-breakpoint +CREATE INDEX `play_component_id_idx` ON `plays` (`componentId`);--> statement-breakpoint +CREATE UNIQUE INDEX `play_uid_idx` ON `plays` (`uid`);--> statement-breakpoint +CREATE INDEX `play_playedAt_idx` ON `plays` (`playedAt`);--> statement-breakpoint +CREATE INDEX `play_seenAt_idx` ON `plays` (`seenAt`);--> statement-breakpoint +CREATE INDEX `play_queue_state_id_idx` ON `play_queue_states` (`playId`); \ No newline at end of file diff --git a/src/backend/common/database/drizzle/migrations/20260423133243_calm_stepford_cuckoos/snapshot.json b/src/backend/common/database/drizzle/migrations/20260423150230_messy_toad/snapshot.json similarity index 66% rename from src/backend/common/database/drizzle/migrations/20260423133243_calm_stepford_cuckoos/snapshot.json rename to src/backend/common/database/drizzle/migrations/20260423150230_messy_toad/snapshot.json index 4ca0458df..9192be117 100644 --- a/src/backend/common/database/drizzle/migrations/20260423133243_calm_stepford_cuckoos/snapshot.json +++ b/src/backend/common/database/drizzle/migrations/20260423150230_messy_toad/snapshot.json @@ -1,11 +1,15 @@ { "version": "7", "dialect": "sqlite", - "id": "8f540406-ea02-467b-bb53-3eee43a37acd", + "id": "3afdf43c-1cbb-4f51-a776-9a432e102fc6", "prevIds": [ "00000000-0000-0000-0000-000000000000" ], "ddl": [ + { + "name": "components", + "entityType": "tables" + }, { "name": "play_inputs", "entityType": "tables" @@ -15,7 +19,7 @@ "entityType": "tables" }, { - "name": "play_queue_state", + "name": "play_queue_states", "entityType": "tables" }, { @@ -26,14 +30,94 @@ "generated": null, "name": "id", "entityType": "columns", - "table": "play_inputs" + "table": "components" + }, + { + "type": "text(200)", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "uid", + "entityType": "columns", + "table": "components" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "mode", + "entityType": "columns", + "table": "components" + }, + { + "type": "text(50)", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "type", + "entityType": "columns", + "table": "components" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "components" }, { "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "countLive", + "entityType": "columns", + "table": "components" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "countNonLive", + "entityType": "columns", + "table": "components" + }, + { + "type": "number", "notNull": false, "autoincrement": false, "default": null, "generated": null, + "name": "createdAt", + "entityType": "columns", + "table": "components" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "play_inputs" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, "name": "playId", "entityType": "columns", "table": "play_inputs" @@ -50,7 +134,7 @@ }, { "type": "text", - "notNull": false, + "notNull": true, "autoincrement": false, "default": null, "generated": null, @@ -89,22 +173,12 @@ "table": "plays" }, { - "type": "text(50)", - "notNull": true, - "autoincrement": false, - "default": null, - "generated": null, - "name": "componentType", - "entityType": "columns", - "table": "plays" - }, - { - "type": "text(200)", - "notNull": true, + "type": "integer", + "notNull": false, "autoincrement": false, "default": null, "generated": null, - "name": "componentName", + "name": "componentId", "entityType": "columns", "table": "plays" }, @@ -176,37 +250,47 @@ "generated": null, "name": "id", "entityType": "columns", - "table": "play_queue_state" + "table": "play_queue_states" }, { "type": "integer", - "notNull": false, + "notNull": true, "autoincrement": false, "default": null, "generated": null, "name": "playId", "entityType": "columns", - "table": "play_queue_state" + "table": "play_queue_states" }, { - "type": "text(200)", - "notNull": false, + "type": "integer", + "notNull": true, "autoincrement": false, "default": null, "generated": null, - "name": "queueName", + "name": "componentId", "entityType": "columns", - "table": "play_queue_state" + "table": "play_queue_states" }, { - "type": "text(30)", - "notNull": false, + "type": "text(50)", + "notNull": true, "autoincrement": false, "default": null, "generated": null, + "name": "queueName", + "entityType": "columns", + "table": "play_queue_states" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": "'queued'", + "generated": null, "name": "queueStatus", "entityType": "columns", - "table": "play_queue_state" + "table": "play_queue_states" }, { "type": "integer", @@ -216,7 +300,7 @@ "generated": null, "name": "retries", "entityType": "columns", - "table": "play_queue_state" + "table": "play_queue_states" }, { "type": "text", @@ -226,27 +310,27 @@ "generated": null, "name": "error", "entityType": "columns", - "table": "play_queue_state" + "table": "play_queue_states" }, { "type": "number", - "notNull": false, + "notNull": true, "autoincrement": false, "default": null, "generated": null, "name": "createdAt", "entityType": "columns", - "table": "play_queue_state" + "table": "play_queue_states" }, { "type": "number", - "notNull": false, + "notNull": true, "autoincrement": false, "default": null, "generated": null, "name": "updatedAt", "entityType": "columns", - "table": "play_queue_state" + "table": "play_queue_states" }, { "columns": [ @@ -263,6 +347,21 @@ "entityType": "fks", "table": "play_inputs" }, + { + "columns": [ + "componentId" + ], + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_plays_componentId_components_id_fk", + "entityType": "fks", + "table": "plays" + }, { "columns": [ "parentId" @@ -289,9 +388,33 @@ "onUpdate": "CASCADE", "onDelete": "CASCADE", "nameExplicit": false, - "name": "fk_play_queue_state_playId_plays_id_fk", + "name": "fk_play_queue_states_playId_plays_id_fk", + "entityType": "fks", + "table": "play_queue_states" + }, + { + "columns": [ + "componentId" + ], + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_play_queue_states_componentId_components_id_fk", "entityType": "fks", - "table": "play_queue_state" + "table": "play_queue_states" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "components_pk", + "table": "components", + "entityType": "pks" }, { "columns": [ @@ -316,8 +439,8 @@ "id" ], "nameExplicit": false, - "name": "play_queue_state_pk", - "table": "play_queue_state", + "name": "play_queue_states_pk", + "table": "play_queue_states", "entityType": "pks" }, { @@ -348,6 +471,20 @@ "entityType": "indexes", "table": "plays" }, + { + "columns": [ + { + "value": "componentId", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "play_component_id_idx", + "entityType": "indexes", + "table": "plays" + }, { "columns": [ { @@ -402,7 +539,16 @@ "origin": "manual", "name": "play_queue_state_id_idx", "entityType": "indexes", - "table": "play_queue_state" + "table": "play_queue_states" + }, + { + "columns": [ + "uid" + ], + "nameExplicit": false, + "name": "components_uid_unique", + "entityType": "uniques", + "table": "components" } ], "renames": [] diff --git a/src/backend/common/database/drizzle/repository.ts b/src/backend/common/database/drizzle/repository.ts new file mode 100644 index 000000000..e69de29bb diff --git a/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts b/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts index 93efe50ab..e11b2e3d3 100644 --- a/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts +++ b/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts @@ -1,6 +1,8 @@ import { integer, sqliteTable, text, index, uniqueIndex, customType, AnySQLiteColumn } from "drizzle-orm/sqlite-core"; import { defineRelations } from 'drizzle-orm'; import dayjs, { Dayjs } from "dayjs"; +import { nanoid } from "nanoid"; +import { ErrorLike, PlayObject } from "../../../../../core/Atomic.js"; const DayjsTimestamp = customType< { @@ -21,36 +23,39 @@ const DayjsTimestamp = customType< export const plays = sqliteTable("plays", { id: integer().primaryKey(), - uid: text({ length: 30 }).notNull().unique(), - componentType: text({ length: 50 }).notNull(), - componentName: text({ length: 200 }).notNull(), - error: text({ mode: 'json' }), + uid: text({ length: 30 }).notNull().unique().$defaultFn(() => nanoid(20)), + componentId: integer().references(() => components.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + error: text({ mode: 'json' }).$type(), playedAt: DayjsTimestamp('playedAt'), seenAt: DayjsTimestamp('seenAt'), - play: text({ mode: 'json' }).notNull(), + play: text({ mode: 'json' }).notNull().$type(), state: text({enum: ['queued','discovered','scrobbled','failed','duped']}).notNull(), // https://orm.drizzle.team/docs/indexes-constraints#foreign-key parentId: integer().references((): AnySQLiteColumn => plays.id) }, (table) => [ index("play_parent_id_idx").on(table.parentId), + index("play_component_id_idx").on(table.componentId), uniqueIndex("play_uid_idx").on(table.uid), index("play_playedAt_idx").on(table.playedAt), index("play_seenAt_idx").on(table.seenAt), ]); -export type NewPlay = typeof plays.$inferInsert; -export type SelectPlay = typeof plays.$inferSelect; +export type PlayNew = typeof plays.$inferInsert; +export type PlaySelect = typeof plays.$inferSelect; export const playInputs = sqliteTable("play_inputs", { id: integer({ mode: 'number' }).primaryKey(), - playId: integer().references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}), - data: text({ mode: 'json' }), - play: text({ mode: 'json' }), + playId: integer().notNull().references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + data: text({ mode: 'json' }).$type(), + play: text({ mode: 'json' }).notNull().$type(), createdAt: DayjsTimestamp('createdAt').$defaultFn(() => dayjs()) }, (table) => [ uniqueIndex('play_input_id_idx').on(table.playId) ]); +export type PlayInputNew = typeof playInputs.$inferInsert; +export type PlayInputSelect = typeof playInputs.$inferSelect; + // export const playParentRelations = defineRelations({plays}, (r) => ({ // plays: { // parent: r.one.plays({ @@ -72,19 +77,23 @@ export const playInputs = sqliteTable("play_inputs", { // } // })); -export const queueStates = sqliteTable("play_queue_state", { +export const queueStates = sqliteTable("play_queue_states", { id: integer({ mode: 'number' }).primaryKey(), - playId: integer().references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}), - queueName: text({length: 200}), - queueStatus: text({length: 30}), + playId: integer().notNull().references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + componentId: integer().notNull().references(() => components.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + queueName: text({length: 50}).notNull(), + queueStatus: text({enum: ['queued','completed','failed']}).notNull().default('queued'), retries: integer().notNull().default(0), - error: text({ mode: 'json' }), - createdAt: DayjsTimestamp('createdAt').$defaultFn(() => dayjs()), - updatedAt: DayjsTimestamp('updatedAt').$defaultFn(() => dayjs()) + error: text({ mode: 'json' }).$type(), + createdAt: DayjsTimestamp('createdAt').notNull().$defaultFn(() => dayjs()), + updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()) }, (table) => [ index('play_queue_state_id_idx').on(table.playId) ]); +export type QueueStateNew = typeof queueStates.$inferInsert; +export type QueueStateSelect = typeof queueStates.$inferSelect; + // export const playQueueRelations = defineRelations({ plays, queueStates }, (r) => ({ // plays: { // queueStates: r.many.queueStates() @@ -97,7 +106,27 @@ export const queueStates = sqliteTable("play_queue_state", { // } // })); -export const playRelations = defineRelations({ plays, queueStates, playInputs }, (r) => ({ +export const components = sqliteTable("components", { + id: integer({ mode: 'number' }).primaryKey(), + // user-provided id + uid: text({ length: 200 }).notNull().unique(), + mode: text({enum: ['source','client']}).notNull(), + // spotify, lastfm, etc... + type: text({length: 50}).notNull(), + // vanity display name + // used as uid if no user-provided id + name: text().notNull(), + // number of discovered/scrobbled plays found in real time + countLive: integer().notNull().default(0), + // number of discovered/scrobbled plays from backlog/jobs + countNonLive: integer().notNull().default(0), + createdAt: DayjsTimestamp('createdAt').$defaultFn(() => dayjs()) +}); + +export type ComponentNew = typeof components.$inferInsert; +export type ComponentSelect = typeof components.$inferSelect; + +export const playRelations = defineRelations({ plays, queueStates, playInputs, components }, (r) => ({ plays: { queueStates: r.many.queueStates(), input: r.one.playInputs({ @@ -109,13 +138,26 @@ export const playRelations = defineRelations({ plays, queueStates, playInputs }, from: r.plays.parentId, to: r.plays.id }), - children: r.many.plays() + children: r.many.plays(), + component: r.one.components({ + from: r.plays.componentId, + to: r.components.id, + optional: true + }) }, queueStates: { play: r.one.plays({ from: r.queueStates.playId, to: r.plays.id + }), + component: r.one.components({ + from: r.queueStates.componentId, + to: r.components.id }) + }, + components: { + plays: r.many.plays(), + queueStates: r.many.queueStates(), } })); diff --git a/src/backend/tests/database/drizzle.test.ts b/src/backend/tests/database/drizzle.test.ts index e7d5eff17..ccfc09c53 100644 --- a/src/backend/tests/database/drizzle.test.ts +++ b/src/backend/tests/database/drizzle.test.ts @@ -2,7 +2,7 @@ import chai, { assert, expect } from 'chai'; import asPromised from 'chai-as-promised'; import { getDb, migrateDb, shouldBackupDb } from '../../common/database/drizzle/drizzleUtils.js'; import withLocalTmpDir from 'with-local-tmp-dir'; -import { playInputs, plays, queueStates } from '../../common/database/drizzle/schema/drizzlePlaysTable.js'; +import { components, playInputs, plays, queueStates } from '../../common/database/drizzle/schema/drizzlePlaysTable.js'; import { nanoid } from 'nanoid'; import dayjs from 'dayjs'; import { generatePlay } from '../../../core/PlayTestUtils.js'; @@ -12,6 +12,7 @@ import * as path from 'path'; import * as fs from 'fs/promises'; import { projectDir } from '../../common/index.js'; import { DatabaseSync } from 'node:sqlite'; +import { fixtureCreateComponent, fixtureCreateInput, fixtureCreatePlay } from '../utils/databaseFixtures.js'; // would be great to push migrations directly from schema but doesn't seem supported in newest beta // https://github.com/drizzle-team/drizzle-orm/discussions/4373 @@ -114,10 +115,11 @@ describe('Basic DB Operations', function () { const db = getDb(':memory:', { workingDirectory: process.cwd() }); await migrateDb(db); + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + const playRow = await db.insert(plays).values({ - id: nanoid(), - componentName: 'mySpot', - componentType: 'spotify', + componentId: component[0].id, + state: 'queued', playedAt: dayjs(), seenAt: dayjs(), play: generatePlay() @@ -135,34 +137,26 @@ describe('Basic DB Operations', function () { const db = getDb(':memory:', { workingDirectory: process.cwd() }); await migrateDb(db); - const uid = nanoid(); try { + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); - const playRow = await db.insert(plays).values({ - uid, - componentName: 'mySpot', - componentType: 'spotify', - state: 'queued', - playedAt: dayjs(), - seenAt: dayjs(), - play: generatePlay() - }).returning(); + const playRow = await db.insert(plays).values(fixtureCreatePlay({componentId: component[0].id})).returning(); - const input = await db.insert(playInputs).values({ + const input = await db.insert(playInputs).values(fixtureCreateInput({ playId: playRow[0].id, - play: playRow[0].play, - data: { anything: 'foo' } - }).returning(); + play: playRow[0].play + })).returning(); const twoQueues = await db.insert(queueStates).values([ { playId: playRow[0].id, - queueName: 'foo', - queueStatus: 'queued' + componentId: component[0].id, + queueName: 'foo' }, { playId: playRow[0].id, + componentId: component[0].id, queueName: 'bar', queueStatus: 'completed' } diff --git a/src/backend/tests/utils/databaseFixtures.ts b/src/backend/tests/utils/databaseFixtures.ts new file mode 100644 index 000000000..4a7a39ce0 --- /dev/null +++ b/src/backend/tests/utils/databaseFixtures.ts @@ -0,0 +1,35 @@ +import { generatePlay } from "../../../core/PlayTestUtils.js"; +import { generateRandomObj } from "../../../core/tests/utils/fixtures.js"; +import { generateComponentEntity, generateInputEntity, generatePlayEntity } from "../../common/database/drizzle/entityUtils.js"; +import { ComponentNew, PlayInputNew, PlayNew } from "../../common/database/drizzle/schema/drizzlePlaysTable.js"; + +export const fixtureCreateComponent = (data: Partial = {}): ComponentNew => { + return generateComponentEntity( + { + uid: 'test', + mode: 'source', + type: 'jellyfin', + name: 'myJelly', + ...data + }); +} + +export const fixtureCreatePlay = (data: Partial = {}): PlayNew => { + const { + play = generatePlay(), + ...rest + } = data; + return generatePlayEntity(play, {seenAt: play.data.playDate, ...rest}); +} + +export const fixtureCreateInput = (data: PlayInputNew & { data?: object | false }): PlayInputNew => { + const { + data: inputData = generateRandomObj(), + ...rest + } = data; + let realData: undefined; + if(inputData !== false) { + realData = inputData; + } + return generateInputEntity({...rest, data: realData}); +} \ No newline at end of file From d59cb2668d1373e6239879241baf8e3e45dcd22b Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 23 Apr 2026 16:54:01 +0000 Subject: [PATCH 007/104] feat(database): Add basic repository and transaction support --- .../common/database/drizzle/drizzleUtils.ts | 43 +++++++++++- .../common/database/drizzle/repository.ts | 65 +++++++++++++++++++ src/backend/tests/database/drizzle.test.ts | 40 +++++++++++- src/core/DataUtils.ts | 2 +- src/core/tests/utils/fixtures.ts | 3 +- 5 files changed, 149 insertions(+), 4 deletions(-) diff --git a/src/backend/common/database/drizzle/drizzleUtils.ts b/src/backend/common/database/drizzle/drizzleUtils.ts index 3b0e64ec6..2cef680ab 100644 --- a/src/backend/common/database/drizzle/drizzleUtils.ts +++ b/src/backend/common/database/drizzle/drizzleUtils.ts @@ -1,5 +1,6 @@ import { drizzle } from 'drizzle-orm/node-sqlite'; import { migrate } from 'drizzle-orm/node-sqlite/migrator'; +import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core"; import { sql as dsl } from 'drizzle-orm'; import * as fs from 'fs/promises'; import * as path from 'path'; @@ -85,4 +86,44 @@ export const migrateDb = async (db: ReturnType, opts: {parentLog } catch (e) { throw new Error('Failed to migrate database', { cause: e }); } -} \ No newline at end of file +} + + + +// cannot really use transactions right now because async isn't supporting for sqlite +// https://github.com/drizzle-team/drizzle-orm/issues/1472 +// https://github.com/drizzle-team/drizzle-orm/issues/2275 +// so use this workaround for now +// https://github.com/drizzle-team/drizzle-orm/issues/2275#issuecomment-2496503801 +let currentTransaction: null | Promise = null; +export const runTransaction = async < + T, + TQueryResult, + TSchema extends Record = Record +>( + db: BaseSQLiteDatabase<"sync", TQueryResult, TSchema>, + executor: () => Promise +) => { + while (currentTransaction !== null) { + await currentTransaction; + } + let resolve!: () => void; + currentTransaction = new Promise(_resolve => { + resolve = _resolve; + }); + try { + db.run(dsl.raw(`BEGIN`)) + + try { + const result = await executor(); + await db.run(dsl.raw(`COMMIT`)); + return result; + } catch (error) { + await db.run(dsl.raw(`ROLLBACK`)); + throw error; + } + } finally { + resolve(); + currentTransaction = null; + } +}; \ No newline at end of file diff --git a/src/backend/common/database/drizzle/repository.ts b/src/backend/common/database/drizzle/repository.ts index e69de29bb..920ef1b9d 100644 --- a/src/backend/common/database/drizzle/repository.ts +++ b/src/backend/common/database/drizzle/repository.ts @@ -0,0 +1,65 @@ +import { Logger, LoggerAppExtras } from "@foxxmd/logging"; +import { getDb, runTransaction } from "./drizzleUtils.js"; +import { loggerNoop } from "../../MaybeLogger.js"; +import { PlayObject } from "../../../../core/Atomic.js"; +import { generateInputEntity, generatePlayEntity, PlayEntityOpts } from "./entityUtils.js"; +import { PlayInputNew, playInputs, PlayNew, plays, PlaySelect } from "./schema/drizzlePlaysTable.js"; +import { MarkOptional, MarkRequired } from "ts-essentials"; +import { nanoid } from "nanoid"; + +export interface DrizzleRepositoryOpts { + logger?: Logger +} + +export type RepositoryCreatePlayOpts = PlayEntityOpts + & { + input: MarkOptional + } + & MarkRequired, 'componentId'>; +export class DrizzleRepository { + + logger: Logger; + db: ReturnType; + + constructor(db: ReturnType, opts: DrizzleRepositoryOpts = {}) { + this.db = db; + this.logger = opts.logger ?? loggerNoop; + } + + createPlays = async (entitiesOpts: RepositoryCreatePlayOpts[]) => { + + let playRows: PlaySelect[]; + + await runTransaction(this.db, async () => { + + const entitiesData = entitiesOpts.map((data) => { + const { + play, + input, + ...rest + } = data; + return generatePlayEntity(play, { ...rest}); + }); + + playRows = await this.db.insert(plays).values(entitiesData).returning(); + + const inputDatas = playRows.map((x, index) => { + const { + play, + input, + } = entitiesOpts[index]; + const { + play: inputPlay = play, + ...restInput + } = input; + + return generateInputEntity({ play: inputPlay, playId: x.id, ...restInput }); + }); + + const inputRow = await this.db.insert(playInputs).values(inputDatas); + + }); + + return playRows; + } +} \ No newline at end of file diff --git a/src/backend/tests/database/drizzle.test.ts b/src/backend/tests/database/drizzle.test.ts index ccfc09c53..18a9a70f7 100644 --- a/src/backend/tests/database/drizzle.test.ts +++ b/src/backend/tests/database/drizzle.test.ts @@ -13,6 +13,10 @@ import * as fs from 'fs/promises'; import { projectDir } from '../../common/index.js'; import { DatabaseSync } from 'node:sqlite'; import { fixtureCreateComponent, fixtureCreateInput, fixtureCreatePlay } from '../utils/databaseFixtures.js'; +import { DrizzleRepository, RepositoryCreatePlayOpts } from '../../common/database/drizzle/repository.js'; +import { generateRandomObj } from '../../../core/tests/utils/fixtures.js'; +import { generateArray } from '../../../core/DataUtils.js'; +import { objectsEqual } from '../../utils/DataUtils.js'; // would be great to push migrations directly from schema but doesn't seem supported in newest beta // https://github.com/drizzle-team/drizzle-orm/discussions/4373 @@ -141,7 +145,7 @@ describe('Basic DB Operations', function () { const component = await db.insert(components).values(fixtureCreateComponent()).returning(); - const playRow = await db.insert(plays).values(fixtureCreatePlay({componentId: component[0].id})).returning(); + const playRow = await db.insert(plays).values(fixtureCreatePlay({ componentId: component[0].id })).returning(); const input = await db.insert(playInputs).values(fixtureCreateInput({ playId: playRow[0].id, @@ -184,3 +188,37 @@ describe('Basic DB Operations', function () { }); +describe('Repository Operations', function () { + + it('creates Plays and inputs', async function () { + + const db = getDb(':memory:'); + await migrateDb(db); + + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + + const repo = new DrizzleRepository(db); + + const numPlays = 3; + + const playData = generateArray(numPlays, () => ({ ...fixtureCreatePlay(), componentId: component[0].id, state: 'queued', input: { data: generateRandomObj(undefined, {allowUndefined: false}) } })) + + const rows = await repo.createPlays(playData); + expect(rows).length(numPlays); + const fullPlays = await db.query.plays.findMany({ + with: { + input: true + } + }); + fullPlays.forEach((play, index) => { + const ref = playData[index]; + + expect(play.play.data.track).eq(ref.play.data.track); + expect(play.input).to.not.undefined; + expect(objectsEqual(play.input.data, ref.input.data)).is.true; + }) + + }); + +}); + diff --git a/src/core/DataUtils.ts b/src/core/DataUtils.ts index 66ab88e66..64c4049dd 100644 --- a/src/core/DataUtils.ts +++ b/src/core/DataUtils.ts @@ -82,7 +82,7 @@ export const formatNumber = (val: number | string, options?: numberFormatOptions return `${prefixStr}${localeString}${suffix}`; }; -export const generateArray = (size: number, gen: (index: number) => any) => { +export const generateArray = (size: number, gen: (index: number) => T): T[] => { return Array.from(Array(size), (v,k) => gen(k)); } diff --git a/src/core/tests/utils/fixtures.ts b/src/core/tests/utils/fixtures.ts index bd21c5eac..6545b9f3f 100644 --- a/src/core/tests/utils/fixtures.ts +++ b/src/core/tests/utils/fixtures.ts @@ -244,13 +244,14 @@ export interface RandomObjOptions { maxKeyLength?: number keyCount?: number maxDepth?: number + allowUndefined?: boolean } const generateRandomVal = (depth: number = 0, opt: RandomObjOptions = {}, typeId?: number) => { const i = typeId ?? faker.number.int({ min: 1, max: depth > (opt.maxDepth ?? 3) ? 6 : 8 }); switch (i) { case 1: - return undefined; + return (opt.allowUndefined ?? true) ? undefined : null; case 2: return faker.datatype.boolean(); case 3: From e9d1c813806cf7c6e411971806c672eebc261ff2 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 23 Apr 2026 19:30:23 +0000 Subject: [PATCH 008/104] chore: Add ts pretty errors to devcontainer extensions --- .devcontainer/devcontainer.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6a95043dd..b24e7a908 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -30,7 +30,8 @@ "dbaeumer.vscode-eslint", "unifiedjs.vscode-mdx", "bradlc.vscode-tailwindcss", - "TakumiI.markdowntable" + "TakumiI.markdowntable", + "YoavBls.pretty-ts-errors" ] } }, From 5771fbf49601f3557bdc0c0855e38a1fb09a3be5 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 23 Apr 2026 19:31:40 +0000 Subject: [PATCH 009/104] feat(database): Implement repository querying --- .../common/database/drizzle/drizzleTypes.ts | 31 +++++ .../common/database/drizzle/drizzleUtils.ts | 4 +- .../common/database/drizzle/entityUtils.ts | 5 +- .../drizzle/repositories/PlayRepository.ts | 114 ++++++++++++++++++ .../common/database/drizzle/repository.ts | 65 ---------- .../{drizzlePlaysTable.ts => schema.ts} | 14 +-- src/backend/tests/database/drizzle.test.ts | 38 +++++- src/backend/tests/utils/databaseFixtures.ts | 4 +- 8 files changed, 191 insertions(+), 84 deletions(-) create mode 100644 src/backend/common/database/drizzle/drizzleTypes.ts create mode 100644 src/backend/common/database/drizzle/repositories/PlayRepository.ts delete mode 100644 src/backend/common/database/drizzle/repository.ts rename src/backend/common/database/drizzle/schema/{drizzlePlaysTable.ts => schema.ts} (89%) diff --git a/src/backend/common/database/drizzle/drizzleTypes.ts b/src/backend/common/database/drizzle/drizzleTypes.ts new file mode 100644 index 000000000..931b9c8de --- /dev/null +++ b/src/backend/common/database/drizzle/drizzleTypes.ts @@ -0,0 +1,31 @@ +import { DBQueryConfig, ExtractTablesFromSchema, KnownKeysOnly } from "drizzle-orm"; +import { components, playInputs, plays, queueStates, relations } from "./schema/schema.js"; +import * as schema from "./schema/schema.js"; + + +export type ComponentNew = typeof components.$inferInsert; +export type ComponentSelect = typeof components.$inferSelect; + +export type QueueStateNew = typeof queueStates.$inferInsert; +export type QueueStateSelect = typeof queueStates.$inferSelect; + +export type PlayInputNew = typeof playInputs.$inferInsert; +export type PlayInputSelect = typeof playInputs.$inferSelect; + +export type PlaySelect = typeof plays.$inferSelect; +export type PlayNew = typeof plays.$inferInsert; + + +// useful references for building types +// https://github.com/drizzle-team/drizzle-orm/discussions/2596 +// https://github.com/drizzle-team/drizzle-orm/discussions/1539 +// https://gist.github.com/ikupenov/10bc89d92d92eaba8cc5569013e04069 +// https://github.com/drizzle-team/drizzle-orm/issues/695 most examples +// https://github.com/drizzle-team/drizzle-orm/discussions/2316 relation focused +type TSchema = typeof relations; +type Schema = typeof schema; +type TableName = keyof TSchema; +export type QueryConfig = DBQueryConfig<"many", TSchema, TSchema[T]>; +export type FindMany = Pick, DBQueryConfig<"many", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'> +export type FindOne = Pick, DBQueryConfig<"one", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'> +export type FindWhere = QueryConfig['where']; \ No newline at end of file diff --git a/src/backend/common/database/drizzle/drizzleUtils.ts b/src/backend/common/database/drizzle/drizzleUtils.ts index 2cef680ab..e62561c07 100644 --- a/src/backend/common/database/drizzle/drizzleUtils.ts +++ b/src/backend/common/database/drizzle/drizzleUtils.ts @@ -9,7 +9,7 @@ import { fileExists } from '../../../utils/FSUtils.js'; import { childLogger, Logger } from '@foxxmd/logging'; import { loggerNoop } from '../../MaybeLogger.js'; import { projectDir } from '../../index.js'; -import { relations } from './schema/drizzlePlaysTable.js'; +import { relations } from './schema/schema.js'; export async function shouldBackupDb(dbPath: string, opts: {parentLogger?: Logger, migrationsFolder?: string} = {}): Promise<[boolean, string[]]> { const { @@ -73,6 +73,8 @@ export const getDb = (dbName: string = 'ms', opts: { logger?: Logger, workingDir return drizzle(dbPath, {relations: relations}); } +export type DbConcrete = ReturnType; + export const migrateDb = async (db: ReturnType, opts: {parentLogger?: Logger, migrationsFolder?: string} = {}) => { const { migrationsFolder, diff --git a/src/backend/common/database/drizzle/entityUtils.ts b/src/backend/common/database/drizzle/entityUtils.ts index 2bb468fea..ad401f819 100644 --- a/src/backend/common/database/drizzle/entityUtils.ts +++ b/src/backend/common/database/drizzle/entityUtils.ts @@ -1,5 +1,8 @@ import assert from "node:assert"; -import { ComponentNew, PlayInputNew, PlayNew, QueueStateNew } from "./schema/drizzlePlaysTable.js"; +import { PlayNew } from "./drizzleTypes.js"; +import { PlayInputNew } from "./drizzleTypes.js"; +import { QueueStateNew } from "./drizzleTypes.js"; +import { ComponentNew } from "./drizzleTypes.js"; import { MarkOptional } from "ts-essentials"; import { ErrorLike, PlayObject } from "../../../../core/Atomic.js"; import dayjs, { Dayjs } from "dayjs"; diff --git a/src/backend/common/database/drizzle/repositories/PlayRepository.ts b/src/backend/common/database/drizzle/repositories/PlayRepository.ts new file mode 100644 index 000000000..77d9b7b2b --- /dev/null +++ b/src/backend/common/database/drizzle/repositories/PlayRepository.ts @@ -0,0 +1,114 @@ +import { Logger, LoggerAppExtras } from "@foxxmd/logging"; +import { DbConcrete, getDb, runTransaction } from "../drizzleUtils.js"; +import { loggerNoop } from "../../../MaybeLogger.js"; +import { PlayObject } from "../../../../../core/Atomic.js"; +import { generateInputEntity, generatePlayEntity, PlayEntityOpts } from "../entityUtils.js"; +import { playInputs, plays, relations } from "../schema/schema.js"; +import { PlayNew, PlaySelect, PlayInputNew, FindWhere, FindMany } from "../drizzleTypes.js";; +import { MarkOptional, MarkRequired, PathValue } from "ts-essentials"; +import { removeUndefinedKeys } from "../../../../utils.js"; + +// https://github.com/drizzle-team/drizzle-orm/issues/695 may be useful for typing models with relations? + +export interface DrizzleRepositoryOpts { + logger?: Logger +} + +export interface PlayWhereOpts { + state?: PlaySelect['state'][] + componentId?: number +} + +export interface QueryPlaysOpts extends PlayWhereOpts { + sort?: 'seenAt' | 'playedAt' + order?: 'asc' | 'desc' + limit?: number + offset?: number +} + +export type RepositoryCreatePlayOpts = PlayEntityOpts + & { + input: MarkOptional + } + & MarkRequired, 'componentId'>; +export class DrizzlePlayRepository { + + logger: Logger; + db: ReturnType; + + constructor(db: ReturnType, opts: DrizzleRepositoryOpts = {}) { + this.db = db; + this.logger = opts.logger ?? loggerNoop; + } + + createPlays = async (entitiesOpts: RepositoryCreatePlayOpts[]) => { + + let playRows: PlaySelect[]; + + await runTransaction(this.db, async () => { + + const entitiesData = entitiesOpts.map((data) => { + const { + play, + input, + ...rest + } = data; + return generatePlayEntity(play, { ...rest }); + }); + + playRows = await this.db.insert(plays).values(entitiesData).returning(); + + const inputDatas = playRows.map((x, index) => { + const { + play, + input, + } = entitiesOpts[index]; + const { + play: inputPlay = play, + ...restInput + } = input; + + return generateInputEntity({ play: inputPlay, playId: x.id, ...restInput }); + }); + + const inputRow = await this.db.insert(playInputs).values(inputDatas); + + }); + + return playRows; + } + + findPlays = async (args: QueryPlaysOpts): Promise => { + //let oldQuery: Parameters[0] = {}; + let query: FindMany<'plays'> = { + limit: args.limit, + offset: args.offset + }; + + query.where = buildPlayWhere(args); + + if (args.sort !== undefined) { + query.orderBy = { + [args.sort]: args.order ?? 'desc' + } + } + query = removeUndefinedKeys(query); + const results = await this.db.query.plays.findMany(query); + return results; + } +} + +export const buildPlayWhere = (args: PlayWhereOpts): FindWhere<'plays'> => { + // old way + // let where: Parameters<(ReturnType)['query']['plays']['findMany']>[0]['where'] = { + // }; + let where: FindWhere<'plays'> = { + componentId: args.componentId + }; + if (args.state !== undefined) { + where.state = { + in: args.state + } + } + return where; +} \ No newline at end of file diff --git a/src/backend/common/database/drizzle/repository.ts b/src/backend/common/database/drizzle/repository.ts deleted file mode 100644 index 920ef1b9d..000000000 --- a/src/backend/common/database/drizzle/repository.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { Logger, LoggerAppExtras } from "@foxxmd/logging"; -import { getDb, runTransaction } from "./drizzleUtils.js"; -import { loggerNoop } from "../../MaybeLogger.js"; -import { PlayObject } from "../../../../core/Atomic.js"; -import { generateInputEntity, generatePlayEntity, PlayEntityOpts } from "./entityUtils.js"; -import { PlayInputNew, playInputs, PlayNew, plays, PlaySelect } from "./schema/drizzlePlaysTable.js"; -import { MarkOptional, MarkRequired } from "ts-essentials"; -import { nanoid } from "nanoid"; - -export interface DrizzleRepositoryOpts { - logger?: Logger -} - -export type RepositoryCreatePlayOpts = PlayEntityOpts - & { - input: MarkOptional - } - & MarkRequired, 'componentId'>; -export class DrizzleRepository { - - logger: Logger; - db: ReturnType; - - constructor(db: ReturnType, opts: DrizzleRepositoryOpts = {}) { - this.db = db; - this.logger = opts.logger ?? loggerNoop; - } - - createPlays = async (entitiesOpts: RepositoryCreatePlayOpts[]) => { - - let playRows: PlaySelect[]; - - await runTransaction(this.db, async () => { - - const entitiesData = entitiesOpts.map((data) => { - const { - play, - input, - ...rest - } = data; - return generatePlayEntity(play, { ...rest}); - }); - - playRows = await this.db.insert(plays).values(entitiesData).returning(); - - const inputDatas = playRows.map((x, index) => { - const { - play, - input, - } = entitiesOpts[index]; - const { - play: inputPlay = play, - ...restInput - } = input; - - return generateInputEntity({ play: inputPlay, playId: x.id, ...restInput }); - }); - - const inputRow = await this.db.insert(playInputs).values(inputDatas); - - }); - - return playRows; - } -} \ No newline at end of file diff --git a/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts b/src/backend/common/database/drizzle/schema/schema.ts similarity index 89% rename from src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts rename to src/backend/common/database/drizzle/schema/schema.ts index e11b2e3d3..0447c7dad 100644 --- a/src/backend/common/database/drizzle/schema/drizzlePlaysTable.ts +++ b/src/backend/common/database/drizzle/schema/schema.ts @@ -40,9 +40,6 @@ export const plays = sqliteTable("plays", { index("play_seenAt_idx").on(table.seenAt), ]); -export type PlayNew = typeof plays.$inferInsert; -export type PlaySelect = typeof plays.$inferSelect; - export const playInputs = sqliteTable("play_inputs", { id: integer({ mode: 'number' }).primaryKey(), playId: integer().notNull().references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}), @@ -53,9 +50,6 @@ export const playInputs = sqliteTable("play_inputs", { uniqueIndex('play_input_id_idx').on(table.playId) ]); -export type PlayInputNew = typeof playInputs.$inferInsert; -export type PlayInputSelect = typeof playInputs.$inferSelect; - // export const playParentRelations = defineRelations({plays}, (r) => ({ // plays: { // parent: r.one.plays({ @@ -91,9 +85,6 @@ export const queueStates = sqliteTable("play_queue_states", { index('play_queue_state_id_idx').on(table.playId) ]); -export type QueueStateNew = typeof queueStates.$inferInsert; -export type QueueStateSelect = typeof queueStates.$inferSelect; - // export const playQueueRelations = defineRelations({ plays, queueStates }, (r) => ({ // plays: { // queueStates: r.many.queueStates() @@ -123,10 +114,7 @@ export const components = sqliteTable("components", { createdAt: DayjsTimestamp('createdAt').$defaultFn(() => dayjs()) }); -export type ComponentNew = typeof components.$inferInsert; -export type ComponentSelect = typeof components.$inferSelect; - -export const playRelations = defineRelations({ plays, queueStates, playInputs, components }, (r) => ({ +const playRelations = defineRelations({ plays, queueStates, playInputs, components }, (r) => ({ plays: { queueStates: r.many.queueStates(), input: r.one.playInputs({ diff --git a/src/backend/tests/database/drizzle.test.ts b/src/backend/tests/database/drizzle.test.ts index 18a9a70f7..2f8daafb3 100644 --- a/src/backend/tests/database/drizzle.test.ts +++ b/src/backend/tests/database/drizzle.test.ts @@ -2,7 +2,7 @@ import chai, { assert, expect } from 'chai'; import asPromised from 'chai-as-promised'; import { getDb, migrateDb, shouldBackupDb } from '../../common/database/drizzle/drizzleUtils.js'; import withLocalTmpDir from 'with-local-tmp-dir'; -import { components, playInputs, plays, queueStates } from '../../common/database/drizzle/schema/drizzlePlaysTable.js'; +import { components, playInputs, plays, queueStates } from '../../common/database/drizzle/schema/schema.js'; import { nanoid } from 'nanoid'; import dayjs from 'dayjs'; import { generatePlay } from '../../../core/PlayTestUtils.js'; @@ -13,7 +13,7 @@ import * as fs from 'fs/promises'; import { projectDir } from '../../common/index.js'; import { DatabaseSync } from 'node:sqlite'; import { fixtureCreateComponent, fixtureCreateInput, fixtureCreatePlay } from '../utils/databaseFixtures.js'; -import { DrizzleRepository, RepositoryCreatePlayOpts } from '../../common/database/drizzle/repository.js'; +import { DrizzlePlayRepository, RepositoryCreatePlayOpts } from '../../common/database/drizzle/repositories/PlayRepository.js'; import { generateRandomObj } from '../../../core/tests/utils/fixtures.js'; import { generateArray } from '../../../core/DataUtils.js'; import { objectsEqual } from '../../utils/DataUtils.js'; @@ -197,7 +197,7 @@ describe('Repository Operations', function () { const component = await db.insert(components).values(fixtureCreateComponent()).returning(); - const repo = new DrizzleRepository(db); + const repo = new DrizzlePlayRepository(db); const numPlays = 3; @@ -220,5 +220,37 @@ describe('Repository Operations', function () { }); + it('finds Plays', async function () { + + const db = getDb(':memory:'); + await migrateDb(db); + + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + + const repo = new DrizzlePlayRepository(db); + + const numPlays = 3; + + const playData = generateArray(numPlays, () => ({ + ...fixtureCreatePlay(), + componentId: component[0].id, + state: 'queued', + input: { data: generateRandomObj(undefined, {allowUndefined: false}) } + })); + const discovered = { + ...fixtureCreatePlay(), + componentId: component[0].id, + state: 'discovered' as 'discovered', + input: { data: generateRandomObj(undefined, {allowUndefined: false}) } + }; + playData.push(discovered) + + await repo.createPlays(playData); + + const plays = await repo.findPlays({state: ['discovered']}); + expect(plays).length(1); + expect(plays[0].play.data.track).eq(discovered.play.data.track); + }); + }); diff --git a/src/backend/tests/utils/databaseFixtures.ts b/src/backend/tests/utils/databaseFixtures.ts index 4a7a39ce0..606302f36 100644 --- a/src/backend/tests/utils/databaseFixtures.ts +++ b/src/backend/tests/utils/databaseFixtures.ts @@ -1,7 +1,9 @@ import { generatePlay } from "../../../core/PlayTestUtils.js"; import { generateRandomObj } from "../../../core/tests/utils/fixtures.js"; import { generateComponentEntity, generateInputEntity, generatePlayEntity } from "../../common/database/drizzle/entityUtils.js"; -import { ComponentNew, PlayInputNew, PlayNew } from "../../common/database/drizzle/schema/drizzlePlaysTable.js"; +import { PlayNew } from "../../common/database/drizzle/drizzleTypes.js"; +import { PlayInputNew } from "../../common/database/drizzle/drizzleTypes.js"; +import { ComponentNew } from "../../common/database/drizzle/drizzleTypes.js"; export const fixtureCreateComponent = (data: Partial = {}): ComponentNew => { return generateComponentEntity( From dda73cd96a905cd6c4a2f66e4e4a9b7aed66de74 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 23 Apr 2026 20:17:13 +0000 Subject: [PATCH 010/104] feat(database): Implement query plays by date range --- .../common/database/drizzle/drizzleTypes.ts | 8 +- .../drizzle/repositories/PlayRepository.ts | 46 ++++- src/backend/tests/database/drizzle.test.ts | 183 +++++++++++------- src/backend/tests/utils/databaseFixtures.ts | 1 + 4 files changed, 167 insertions(+), 71 deletions(-) diff --git a/src/backend/common/database/drizzle/drizzleTypes.ts b/src/backend/common/database/drizzle/drizzleTypes.ts index 931b9c8de..e7faef3ba 100644 --- a/src/backend/common/database/drizzle/drizzleTypes.ts +++ b/src/backend/common/database/drizzle/drizzleTypes.ts @@ -1,4 +1,4 @@ -import { DBQueryConfig, ExtractTablesFromSchema, KnownKeysOnly } from "drizzle-orm"; +import { DBQueryConfig, ExtractTablesFromSchema, KnownKeysOnly, RelationFieldsFilterInternals } from "drizzle-orm"; import { components, playInputs, plays, queueStates, relations } from "./schema/schema.js"; import * as schema from "./schema/schema.js"; @@ -22,10 +22,14 @@ export type PlayNew = typeof plays.$inferInsert; // https://gist.github.com/ikupenov/10bc89d92d92eaba8cc5569013e04069 // https://github.com/drizzle-team/drizzle-orm/issues/695 most examples // https://github.com/drizzle-team/drizzle-orm/discussions/2316 relation focused +// https://github.com/drizzle-team/drizzle-orm/issues/1319 type TSchema = typeof relations; type Schema = typeof schema; type TableName = keyof TSchema; export type QueryConfig = DBQueryConfig<"many", TSchema, TSchema[T]>; export type FindMany = Pick, DBQueryConfig<"many", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'> export type FindOne = Pick, DBQueryConfig<"one", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'> -export type FindWhere = QueryConfig['where']; \ No newline at end of file +export type FindWhere = QueryConfig['where']; + +export type CompareOp = Pick, 'gt' | 'gte' | 'eq' | 'lt' | 'lte' | 'ne'> +export type CompareOpKey = keyof CompareOp; \ No newline at end of file diff --git a/src/backend/common/database/drizzle/repositories/PlayRepository.ts b/src/backend/common/database/drizzle/repositories/PlayRepository.ts index 77d9b7b2b..449668233 100644 --- a/src/backend/common/database/drizzle/repositories/PlayRepository.ts +++ b/src/backend/common/database/drizzle/repositories/PlayRepository.ts @@ -4,9 +4,11 @@ import { loggerNoop } from "../../../MaybeLogger.js"; import { PlayObject } from "../../../../../core/Atomic.js"; import { generateInputEntity, generatePlayEntity, PlayEntityOpts } from "../entityUtils.js"; import { playInputs, plays, relations } from "../schema/schema.js"; -import { PlayNew, PlaySelect, PlayInputNew, FindWhere, FindMany } from "../drizzleTypes.js";; +import { PlayNew, PlaySelect, PlayInputNew, FindWhere, FindMany, CompareOpKey } from "../drizzleTypes.js";; import { MarkOptional, MarkRequired, PathValue } from "ts-essentials"; import { removeUndefinedKeys } from "../../../../utils.js"; +import dayjs, { Dayjs } from "dayjs"; +import { RelationsFieldFilter } from "drizzle-orm"; // https://github.com/drizzle-team/drizzle-orm/issues/695 may be useful for typing models with relations? @@ -14,9 +16,20 @@ export interface DrizzleRepositoryOpts { logger?: Logger } +type CompareDateOp = { + type: CompareOpKey + date: Dayjs +} | { + type: 'between', + range: [Dayjs, Dayjs], + inclusive?: boolean +} + export interface PlayWhereOpts { state?: PlaySelect['state'][] componentId?: number + seenAt?: CompareDateOp + playedAt?: CompareDateOp } export interface QueryPlaysOpts extends PlayWhereOpts { @@ -91,6 +104,10 @@ export class DrizzlePlayRepository { query.orderBy = { [args.sort]: args.order ?? 'desc' } + } else { + query.orderBy = { + id: 'asc' + } } query = removeUndefinedKeys(query); const results = await this.db.query.plays.findMany(query); @@ -110,5 +127,32 @@ export const buildPlayWhere = (args: PlayWhereOpts): FindWhere<'plays'> => { in: args.state } } + if (args.seenAt !== undefined) { + where.seenAt = buildDateCompare(args.seenAt); + } + if(args.playedAt !== undefined) { + where.playedAt = buildDateCompare(args.playedAt); + } return where; +} + +const buildDateCompare = (data: CompareDateOp): RelationsFieldFilter => { + let q: RelationsFieldFilter = {}; + if (data.type !== 'between') { + q = { + [data.type]: data.date + } + } else { + q = { + AND: [ + { + [data.inclusive ?? true ? 'gte' : 'gt']: data.range[0] + }, + { + [data.inclusive ?? true ? 'lte' : 'lt']: data.range[1] + }, + ] + } + } + return q; } \ No newline at end of file diff --git a/src/backend/tests/database/drizzle.test.ts b/src/backend/tests/database/drizzle.test.ts index 2f8daafb3..bfc984083 100644 --- a/src/backend/tests/database/drizzle.test.ts +++ b/src/backend/tests/database/drizzle.test.ts @@ -114,76 +114,70 @@ describe('Basic DB Operations', function () { it('Should create a play', async function () { - withLocalTmpDir(async () => { - - const db = getDb(':memory:', { workingDirectory: process.cwd() }); - await migrateDb(db); + const db = getDb(':memory:', { workingDirectory: process.cwd() }); + await migrateDb(db); - const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); - const playRow = await db.insert(plays).values({ - componentId: component[0].id, - state: 'queued', - playedAt: dayjs(), - seenAt: dayjs(), - play: generatePlay() - }); + const playRow = await db.insert(plays).values({ + componentId: component[0].id, + state: 'queued', + playedAt: dayjs(), + seenAt: dayjs(), + play: generatePlay() + }); - expect(playRow.changes).eq(1); - db.$client.close(); - }, { unsafeCleanup: true }); + expect(playRow.changes).eq(1); + db.$client.close(); }); it('Should create a play with relations', async function () { - withLocalTmpDir(async () => { + const db = getDb(':memory:', { workingDirectory: process.cwd() }); + await migrateDb(db); - const db = getDb(':memory:', { workingDirectory: process.cwd() }); - await migrateDb(db); + try { - try { + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); - const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + const playRow = await db.insert(plays).values(fixtureCreatePlay({ componentId: component[0].id })).returning(); - const playRow = await db.insert(plays).values(fixtureCreatePlay({ componentId: component[0].id })).returning(); + const input = await db.insert(playInputs).values(fixtureCreateInput({ + playId: playRow[0].id, + play: playRow[0].play + })).returning(); - const input = await db.insert(playInputs).values(fixtureCreateInput({ + const twoQueues = await db.insert(queueStates).values([ + { playId: playRow[0].id, - play: playRow[0].play - })).returning(); - - const twoQueues = await db.insert(queueStates).values([ - { - playId: playRow[0].id, - componentId: component[0].id, - queueName: 'foo' - }, - { - playId: playRow[0].id, - componentId: component[0].id, - queueName: 'bar', - queueStatus: 'completed' - } - ]); - - const fullPlay = await db.query.plays.findFirst({ - with: { - input: true, - queueStates: true, - }, - }); + componentId: component[0].id, + queueName: 'foo' + }, + { + playId: playRow[0].id, + componentId: component[0].id, + queueName: 'bar', + queueStatus: 'completed' + } + ]); + + const fullPlay = await db.query.plays.findFirst({ + with: { + input: true, + queueStates: true, + }, + }); - expect(fullPlay.queueStates).to.not.be.undefined; - expect(fullPlay.queueStates).length(2); + expect(fullPlay.queueStates).to.not.be.undefined; + expect(fullPlay.queueStates).length(2); - expect(fullPlay.input).to.not.be.undefined; + expect(fullPlay.input).to.not.be.undefined; - } catch (e) { - throw e; - } - db.$client.close(); - }, { unsafeCleanup: true }); + } catch (e) { + throw e; + } + db.$client.close(); }); }); @@ -201,7 +195,7 @@ describe('Repository Operations', function () { const numPlays = 3; - const playData = generateArray(numPlays, () => ({ ...fixtureCreatePlay(), componentId: component[0].id, state: 'queued', input: { data: generateRandomObj(undefined, {allowUndefined: false}) } })) + const playData = generateArray(numPlays, () => ({ ...fixtureCreatePlay(), componentId: component[0].id, state: 'queued', input: { data: generateRandomObj(undefined, { allowUndefined: false }) } })) const rows = await repo.createPlays(playData); expect(rows).length(numPlays); @@ -217,10 +211,10 @@ describe('Repository Operations', function () { expect(play.input).to.not.undefined; expect(objectsEqual(play.input.data, ref.input.data)).is.true; }) - + }); - it('finds Plays', async function () { + it('finds Plays by state', async function () { const db = getDb(':memory:'); await migrateDb(db); @@ -231,26 +225,79 @@ describe('Repository Operations', function () { const numPlays = 3; - const playData = generateArray(numPlays, () => ({ - ...fixtureCreatePlay(), - componentId: component[0].id, - state: 'queued', - input: { data: generateRandomObj(undefined, {allowUndefined: false}) } + const playData = generateArray(numPlays, () => ({ + ...fixtureCreatePlay(), + componentId: component[0].id, + state: 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } })); - const discovered = { - ...fixtureCreatePlay(), - componentId: component[0].id, - state: 'discovered' as 'discovered', - input: { data: generateRandomObj(undefined, {allowUndefined: false}) } + const discovered = { + ...fixtureCreatePlay(), + componentId: component[0].id, + state: 'discovered' as 'discovered', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } }; playData.push(discovered) await repo.createPlays(playData); - - const plays = await repo.findPlays({state: ['discovered']}); + + const plays = await repo.findPlays({ state: ['discovered'] }); expect(plays).length(1); expect(plays[0].play.data.track).eq(discovered.play.data.track); }); + it('finds Plays by date range', async function () { + + const db = getDb(':memory:'); + await migrateDb(db); + + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + + const repo = new DrizzlePlayRepository(db); + + const playData: RepositoryCreatePlayOpts[] = [ + { + ...fixtureCreatePlay({ play: generatePlay({ playDate: dayjs().subtract(2, 'm') }) }), + componentId: component[0].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay({ play: generatePlay({ playDate: dayjs().subtract(6, 'm') }) }), + componentId: component[0].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay({ play: generatePlay({ playDate: dayjs().subtract(8, 'm') }) }), + componentId: component[0].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay({ play: generatePlay({ playDate: dayjs().subtract(10, 'm') }) }), + componentId: component[0].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + ] + + await repo.createPlays(playData); + + const newerPlays = await repo.findPlays({ playedAt: { type: 'gt', date: dayjs().subtract(3, 'm') } }); + expect(newerPlays).length(1); + expect(newerPlays[0].play.data.track).eq(playData[0].play.data.track); + + const olderPlays = await repo.findPlays({ playedAt: { type: 'lt', date: dayjs().subtract(6, 'm').subtract(5, 's') } }); + expect(olderPlays).length(2); + expect(olderPlays[0].play.data.track).eq(playData[2].play.data.track); + expect(olderPlays[1].play.data.track).eq(playData[3].play.data.track); + + const bwPlays = await repo.findPlays({ playedAt: { type: 'between', range: [dayjs().subtract(9, 'm'), dayjs().subtract(3, 'm')] } }); + expect(bwPlays).length(2); + expect(bwPlays[0].play.data.track).eq(playData[1].play.data.track); + expect(bwPlays[1].play.data.track).eq(playData[2].play.data.track); + }); + }); diff --git a/src/backend/tests/utils/databaseFixtures.ts b/src/backend/tests/utils/databaseFixtures.ts index 606302f36..9184a19d0 100644 --- a/src/backend/tests/utils/databaseFixtures.ts +++ b/src/backend/tests/utils/databaseFixtures.ts @@ -4,6 +4,7 @@ import { generateComponentEntity, generateInputEntity, generatePlayEntity } from import { PlayNew } from "../../common/database/drizzle/drizzleTypes.js"; import { PlayInputNew } from "../../common/database/drizzle/drizzleTypes.js"; import { ComponentNew } from "../../common/database/drizzle/drizzleTypes.js"; +import { ObjectPlayData } from "../../../core/Atomic.js"; export const fixtureCreateComponent = (data: Partial = {}): ComponentNew => { return generateComponentEntity( From cda33ba04f3d6000a47031a558f3d7d20f6a8eac Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 23 Apr 2026 20:27:09 +0000 Subject: [PATCH 011/104] test(database): Add component query test --- src/backend/tests/database/drizzle.test.ts | 47 ++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/backend/tests/database/drizzle.test.ts b/src/backend/tests/database/drizzle.test.ts index bfc984083..4280b1ba4 100644 --- a/src/backend/tests/database/drizzle.test.ts +++ b/src/backend/tests/database/drizzle.test.ts @@ -299,5 +299,52 @@ describe('Repository Operations', function () { expect(bwPlays[1].play.data.track).eq(playData[2].play.data.track); }); + it('finds Plays by component', async function () { + + const db = getDb(':memory:'); + await migrateDb(db); + + const component1 = await db.insert(components).values(fixtureCreateComponent()).returning(); + const component2 = await db.insert(components).values(fixtureCreateComponent({uid: 'test2', name: 'jelly2'})).returning(); + const component3 = await db.insert(components).values(fixtureCreateComponent({uid: 'test3', name: 'jelly3'})).returning(); + + const repo = new DrizzlePlayRepository(db); + + const playData: RepositoryCreatePlayOpts[] = [ + { + ...fixtureCreatePlay(), + componentId: component1[0].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay(), + componentId: component3[0].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay(), + componentId: component3[0].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + } + ] + + await repo.createPlays(playData); + + const plays = await repo.findPlays({ componentId: component3[0].id }); + expect(plays).length(2); + expect(plays[0].play.data.track).eq(playData[1].play.data.track); + expect(plays[1].play.data.track).eq(playData[2].play.data.track); + + const plays1 = await repo.findPlays({ componentId: component1[0].id }); + expect(plays1).length(1); + expect(plays1[0].play.data.track).eq(playData[0].play.data.track); + + const noPlays = await repo.findPlays({ componentId: component2[0].id }); + expect(noPlays).length(0); + }); + }); From 99ebb53fc6da8fd8547aca48b10aee992704fa85 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 23 Apr 2026 20:28:41 +0000 Subject: [PATCH 012/104] chore: Remove unusued patch for kysely --- patches/kysely-node-native-sqlite+1.1.0.patch | 60 ------------------- 1 file changed, 60 deletions(-) delete mode 100644 patches/kysely-node-native-sqlite+1.1.0.patch diff --git a/patches/kysely-node-native-sqlite+1.1.0.patch b/patches/kysely-node-native-sqlite+1.1.0.patch deleted file mode 100644 index 2f1a6462f..000000000 --- a/patches/kysely-node-native-sqlite+1.1.0.patch +++ /dev/null @@ -1,60 +0,0 @@ -diff --git a/node_modules/kysely-node-native-sqlite/dist/index.cjs b/node_modules/kysely-node-native-sqlite/dist/index.cjs -index 1015da7..0679a95 100644 ---- a/node_modules/kysely-node-native-sqlite/dist/index.cjs -+++ b/node_modules/kysely-node-native-sqlite/dist/index.cjs -@@ -55,7 +55,11 @@ var import_node_sqlite = require("node:sqlite"); - var NodeNativeSqliteConnection = class { - #db; - constructor(...args) { -- this.#db = new import_node_sqlite.DatabaseSync(...args); -+ if (args[0] instanceof import_node_sqlite.DatabaseSync) { -+ this.#db = args[0]; -+ } else { -+ this.#db = new import_node_sqlite.DatabaseSync(...args); -+ } - } - [Symbol.dispose]() { - this.#db.close(); -diff --git a/node_modules/kysely-node-native-sqlite/dist/index.d.cts b/node_modules/kysely-node-native-sqlite/dist/index.d.cts -index 72366bb..32e8464 100644 ---- a/node_modules/kysely-node-native-sqlite/dist/index.d.cts -+++ b/node_modules/kysely-node-native-sqlite/dist/index.d.cts -@@ -3,7 +3,7 @@ import { Dialect, SqliteAdapter, Driver, Kysely, DatabaseIntrospector, SqliteQue - - declare class NodeNativeSqliteDialect implements Dialect { - #private; -- constructor(...args: ConstructorParameters); -+ constructor(...args: ConstructorParameters | [DatabaseSync]); - createAdapter(): SqliteAdapter; - createDriver(): Driver; - createIntrospector(db: Kysely): DatabaseIntrospector; -diff --git a/node_modules/kysely-node-native-sqlite/dist/index.d.ts b/node_modules/kysely-node-native-sqlite/dist/index.d.ts -index 72366bb..32e8464 100644 ---- a/node_modules/kysely-node-native-sqlite/dist/index.d.ts -+++ b/node_modules/kysely-node-native-sqlite/dist/index.d.ts -@@ -3,7 +3,7 @@ import { Dialect, SqliteAdapter, Driver, Kysely, DatabaseIntrospector, SqliteQue - - declare class NodeNativeSqliteDialect implements Dialect { - #private; -- constructor(...args: ConstructorParameters); -+ constructor(...args: ConstructorParameters | [DatabaseSync]); - createAdapter(): SqliteAdapter; - createDriver(): Driver; - createIntrospector(db: Kysely): DatabaseIntrospector; -diff --git a/node_modules/kysely-node-native-sqlite/dist/index.js b/node_modules/kysely-node-native-sqlite/dist/index.js -index 8c2b8cf..17b7eb1 100644 ---- a/node_modules/kysely-node-native-sqlite/dist/index.js -+++ b/node_modules/kysely-node-native-sqlite/dist/index.js -@@ -32,7 +32,11 @@ import { DatabaseSync } from "node:sqlite"; - var NodeNativeSqliteConnection = class { - #db; - constructor(...args) { -- this.#db = new DatabaseSync(...args); -+ if (args[0] instanceof DatabaseSync) { -+ this.#db = args[0]; -+ } else { -+ this.#db = new DatabaseSync(...args); -+ } - } - [Symbol.dispose]() { - this.#db.close(); From 84b43d7c3946ee372af32af4920247a3f585d3d6 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 24 Apr 2026 12:38:26 +0000 Subject: [PATCH 013/104] feat(database): Implement play deletes * Implement repository method * Test that deletes cascade to relations --- .../drizzle/repositories/PlayRepository.ts | 7 +- src/backend/tests/database/drizzle.test.ts | 72 +++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/src/backend/common/database/drizzle/repositories/PlayRepository.ts b/src/backend/common/database/drizzle/repositories/PlayRepository.ts index 449668233..d1583b7f1 100644 --- a/src/backend/common/database/drizzle/repositories/PlayRepository.ts +++ b/src/backend/common/database/drizzle/repositories/PlayRepository.ts @@ -8,7 +8,7 @@ import { PlayNew, PlaySelect, PlayInputNew, FindWhere, FindMany, CompareOpKey } import { MarkOptional, MarkRequired, PathValue } from "ts-essentials"; import { removeUndefinedKeys } from "../../../../utils.js"; import dayjs, { Dayjs } from "dayjs"; -import { RelationsFieldFilter } from "drizzle-orm"; +import { RelationsFieldFilter, eq, inArray } from "drizzle-orm"; // https://github.com/drizzle-team/drizzle-orm/issues/695 may be useful for typing models with relations? @@ -113,6 +113,11 @@ export class DrizzlePlayRepository { const results = await this.db.query.plays.findMany(query); return results; } + + deletePlays = async (playsData: (Pick | number)[]) => { + const ids = playsData.map(x => typeof x === 'number' ? x : x.id); + await this.db.delete(plays).where(inArray(plays.id, ids)); + } } export const buildPlayWhere = (args: PlayWhereOpts): FindWhere<'plays'> => { diff --git a/src/backend/tests/database/drizzle.test.ts b/src/backend/tests/database/drizzle.test.ts index 4280b1ba4..5169fea44 100644 --- a/src/backend/tests/database/drizzle.test.ts +++ b/src/backend/tests/database/drizzle.test.ts @@ -17,6 +17,7 @@ import { DrizzlePlayRepository, RepositoryCreatePlayOpts } from '../../common/da import { generateRandomObj } from '../../../core/tests/utils/fixtures.js'; import { generateArray } from '../../../core/DataUtils.js'; import { objectsEqual } from '../../utils/DataUtils.js'; +import { eq } from 'drizzle-orm'; // would be great to push migrations directly from schema but doesn't seem supported in newest beta // https://github.com/drizzle-team/drizzle-orm/discussions/4373 @@ -180,6 +181,77 @@ describe('Basic DB Operations', function () { db.$client.close(); }); + it('deletes all dependent relations when a Play is deleted', async function () { + + const db = getDb(':memory:', { workingDirectory: process.cwd() }); + await migrateDb(db); + + try { + + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + + const playRow = await db.insert(plays).values(fixtureCreatePlay({ componentId: component[0].id })).returning(); + + const input = await db.insert(playInputs).values(fixtureCreateInput({ + playId: playRow[0].id, + play: playRow[0].play + })).returning(); + + const twoQueues = await db.insert(queueStates).values([ + { + playId: playRow[0].id, + componentId: component[0].id, + queueName: 'foo' + }, + { + playId: playRow[0].id, + componentId: component[0].id, + queueName: 'bar', + queueStatus: 'completed' + } + ]).returning(); + + const fullPlay = await db.query.plays.findFirst({ + with: { + input: true, + queueStates: true, + }, + }); + + + expect(fullPlay.queueStates).to.not.be.undefined; + expect(fullPlay.queueStates).length(2); + expect(fullPlay.input).to.not.be.undefined; + + await db.delete(plays).where(eq(plays.id, fullPlay.id)); + const deletedPlay = await db.query.plays.findFirst({ + where: { + id: fullPlay.id + } + }); + expect(deletedPlay).to.be.undefined; + + const deletedInput = await db.query.playInputs.findFirst({ + where: { + id: input[0].id + } + }); + expect(deletedInput).to.be.undefined; + + const deletedQueues = await db.query.queueStates.findMany({ + where: { + id: { + in: [twoQueues[0].id, twoQueues[1].id] + } + } + }); + expect(deletedQueues).length(0); + } catch (e) { + throw e; + } + db.$client.close(); + }); + }); describe('Repository Operations', function () { From b388ca6106ac86fd24c6551e45d7b2d96dc68529 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 24 Apr 2026 12:50:14 +0000 Subject: [PATCH 014/104] feat(database): Cascade parent id to null on play deletes --- .../migration.sql | 2 +- .../snapshot.json | 6 +++--- src/backend/common/database/drizzle/schema/schema.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename src/backend/common/database/drizzle/migrations/{20260423150230_messy_toad => 20260424124953_flippant_lifeguard}/migration.sql (97%) rename src/backend/common/database/drizzle/migrations/{20260423150230_messy_toad => 20260424124953_flippant_lifeguard}/snapshot.json (99%) diff --git a/src/backend/common/database/drizzle/migrations/20260423150230_messy_toad/migration.sql b/src/backend/common/database/drizzle/migrations/20260424124953_flippant_lifeguard/migration.sql similarity index 97% rename from src/backend/common/database/drizzle/migrations/20260423150230_messy_toad/migration.sql rename to src/backend/common/database/drizzle/migrations/20260424124953_flippant_lifeguard/migration.sql index 3c1b93f6a..eacab3bf4 100644 --- a/src/backend/common/database/drizzle/migrations/20260423150230_messy_toad/migration.sql +++ b/src/backend/common/database/drizzle/migrations/20260424124953_flippant_lifeguard/migration.sql @@ -29,7 +29,7 @@ CREATE TABLE `plays` ( `state` text NOT NULL, `parentId` integer, CONSTRAINT `fk_plays_componentId_components_id_fk` FOREIGN KEY (`componentId`) REFERENCES `components`(`id`) ON UPDATE CASCADE ON DELETE CASCADE, - CONSTRAINT `fk_plays_parentId_plays_id_fk` FOREIGN KEY (`parentId`) REFERENCES `plays`(`id`) + CONSTRAINT `fk_plays_parentId_plays_id_fk` FOREIGN KEY (`parentId`) REFERENCES `plays`(`id`) ON UPDATE CASCADE ON DELETE SET NULL ); --> statement-breakpoint CREATE TABLE `play_queue_states` ( diff --git a/src/backend/common/database/drizzle/migrations/20260423150230_messy_toad/snapshot.json b/src/backend/common/database/drizzle/migrations/20260424124953_flippant_lifeguard/snapshot.json similarity index 99% rename from src/backend/common/database/drizzle/migrations/20260423150230_messy_toad/snapshot.json rename to src/backend/common/database/drizzle/migrations/20260424124953_flippant_lifeguard/snapshot.json index 9192be117..3d9c50b47 100644 --- a/src/backend/common/database/drizzle/migrations/20260423150230_messy_toad/snapshot.json +++ b/src/backend/common/database/drizzle/migrations/20260424124953_flippant_lifeguard/snapshot.json @@ -1,7 +1,7 @@ { "version": "7", "dialect": "sqlite", - "id": "3afdf43c-1cbb-4f51-a776-9a432e102fc6", + "id": "c2744e30-39a9-4d83-8b7c-229ab78fe6e5", "prevIds": [ "00000000-0000-0000-0000-000000000000" ], @@ -370,8 +370,8 @@ "columnsTo": [ "id" ], - "onUpdate": "NO ACTION", - "onDelete": "NO ACTION", + "onUpdate": "CASCADE", + "onDelete": "SET NULL", "nameExplicit": false, "name": "fk_plays_parentId_plays_id_fk", "entityType": "fks", diff --git a/src/backend/common/database/drizzle/schema/schema.ts b/src/backend/common/database/drizzle/schema/schema.ts index 0447c7dad..57833e810 100644 --- a/src/backend/common/database/drizzle/schema/schema.ts +++ b/src/backend/common/database/drizzle/schema/schema.ts @@ -31,7 +31,7 @@ export const plays = sqliteTable("plays", { play: text({ mode: 'json' }).notNull().$type(), state: text({enum: ['queued','discovered','scrobbled','failed','duped']}).notNull(), // https://orm.drizzle.team/docs/indexes-constraints#foreign-key - parentId: integer().references((): AnySQLiteColumn => plays.id) + parentId: integer().references((): AnySQLiteColumn => plays.id, {onDelete: 'set null', onUpdate: 'cascade'}) }, (table) => [ index("play_parent_id_idx").on(table.parentId), index("play_component_id_idx").on(table.componentId), From 92e027bfc4001703e19a92ee8583698b09069964 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 24 Apr 2026 15:08:04 +0000 Subject: [PATCH 015/104] feat(database): Implement finding purgable play ids --- .../drizzle/repositories/PlayRepository.ts | 44 ++++++++++- src/backend/tests/database/drizzle.test.ts | 77 ++++++++++++++++++- 2 files changed, 117 insertions(+), 4 deletions(-) diff --git a/src/backend/common/database/drizzle/repositories/PlayRepository.ts b/src/backend/common/database/drizzle/repositories/PlayRepository.ts index d1583b7f1..f0a726ad4 100644 --- a/src/backend/common/database/drizzle/repositories/PlayRepository.ts +++ b/src/backend/common/database/drizzle/repositories/PlayRepository.ts @@ -118,6 +118,48 @@ export class DrizzlePlayRepository { const ids = playsData.map(x => typeof x === 'number' ? x : x.id); await this.db.delete(plays).where(inArray(plays.id, ids)); } + + findPurgablePlayIds = async (componentId: number, olderThanDate: Dayjs, opts: { countOnly?: boolean, states?: PlaySelect['state'][] } = {}) => { + + const { + countOnly = false, + states + } = opts; + + let where: FindWhere<'plays'> = { + component: { + id: componentId + }, + seenAt: { + lte: olderThanDate + }, + NOT: { + children: {} + } + }; + + if (states !== undefined) { + where.state = { + in: states + } + } + + const rows = await this.db.query.plays.findMany({ + columns: { + id: true + }, + where, + orderBy: { + id: 'asc' + } + }); + + if (countOnly) { + return rows.length; + } + + return rows.map(x => x.id); + } } export const buildPlayWhere = (args: PlayWhereOpts): FindWhere<'plays'> => { @@ -135,7 +177,7 @@ export const buildPlayWhere = (args: PlayWhereOpts): FindWhere<'plays'> => { if (args.seenAt !== undefined) { where.seenAt = buildDateCompare(args.seenAt); } - if(args.playedAt !== undefined) { + if (args.playedAt !== undefined) { where.playedAt = buildDateCompare(args.playedAt); } return where; diff --git a/src/backend/tests/database/drizzle.test.ts b/src/backend/tests/database/drizzle.test.ts index 5169fea44..311b7f78d 100644 --- a/src/backend/tests/database/drizzle.test.ts +++ b/src/backend/tests/database/drizzle.test.ts @@ -371,14 +371,14 @@ describe('Repository Operations', function () { expect(bwPlays[1].play.data.track).eq(playData[2].play.data.track); }); - it('finds Plays by component', async function () { + it('finds Plays by component', async function () { const db = getDb(':memory:'); await migrateDb(db); const component1 = await db.insert(components).values(fixtureCreateComponent()).returning(); - const component2 = await db.insert(components).values(fixtureCreateComponent({uid: 'test2', name: 'jelly2'})).returning(); - const component3 = await db.insert(components).values(fixtureCreateComponent({uid: 'test3', name: 'jelly3'})).returning(); + const component2 = await db.insert(components).values(fixtureCreateComponent({ uid: 'test2', name: 'jelly2' })).returning(); + const component3 = await db.insert(components).values(fixtureCreateComponent({ uid: 'test3', name: 'jelly3' })).returning(); const repo = new DrizzlePlayRepository(db); @@ -418,5 +418,76 @@ describe('Repository Operations', function () { expect(noPlays).length(0); }); + it('finds purgable Plays', async function () { + + const db = getDb(':memory:'); + await migrateDb(db); + + const component1 = await db.insert(components).values(fixtureCreateComponent()).returning(); + const component2 = await db.insert(components).values(fixtureCreateComponent({ uid: 'test2', name: 'jelly2' })).returning(); + + const repo = new DrizzlePlayRepository(db); + + const playData: RepositoryCreatePlayOpts[] = [ + { + ...fixtureCreatePlay(), + componentId: component1[0].id, + seenAt: dayjs().subtract(25, 'h'), + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay(), + componentId: component1[0].id, + seenAt: dayjs().subtract(26, 'h'), + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay(), + componentId: component2[0].id, + seenAt: dayjs().subtract(26, 'h'), + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay(), + componentId: component1[0].id, + seenAt: dayjs().subtract(25, 'h').subtract(1, 'm'), + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + ] + + const initialPlays = await repo.createPlays(playData); + + const childPlays = await repo.createPlays([ + { + ...fixtureCreatePlay(), + componentId: component2[0].id, + seenAt: dayjs().subtract(25, 'h'), + parentId: initialPlays[1].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + ]) + + // does not return newer plays + expect((await repo.findPurgablePlayIds(1, dayjs().subtract(27, 'h')))).length(0); + + const pPlays = await repo.findPurgablePlayIds(1, dayjs().subtract(24, 'h')); + // only returns plays that do not have children + expect(pPlays).length(2); + expect(pPlays[0]).to.eq(initialPlays[0].id); + expect(pPlays[1]).to.eq(initialPlays[3].id); + + // only finds plays by component + // and allows purging if they have parent id + const p2Plays = await repo.findPurgablePlayIds(2, dayjs().subtract(23, 'h')); + expect(p2Plays).length(2); + expect(p2Plays[0]).to.eq(initialPlays[2].id); + expect(p2Plays[1]).to.eq(childPlays[0].id); + }); + }); From 60a6ddbc415d3fe19f2e9321dc2ee20013b8c81c Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 24 Apr 2026 15:27:37 +0000 Subject: [PATCH 016/104] feat(database): Implement drizzle logger --- .../common/database/drizzle/drizzleUtils.ts | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/backend/common/database/drizzle/drizzleUtils.ts b/src/backend/common/database/drizzle/drizzleUtils.ts index e62561c07..f84b616b0 100644 --- a/src/backend/common/database/drizzle/drizzleUtils.ts +++ b/src/backend/common/database/drizzle/drizzleUtils.ts @@ -1,12 +1,12 @@ import { drizzle } from 'drizzle-orm/node-sqlite'; import { migrate } from 'drizzle-orm/node-sqlite/migrator'; import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core"; -import { sql as dsl } from 'drizzle-orm'; +import { sql as dsl, LogWriter, Logger as DrizzleLogger } from 'drizzle-orm'; import * as fs from 'fs/promises'; import * as path from 'path'; import { getDbPath, MEMORY_DB_NAME } from '../Database.js'; import { fileExists } from '../../../utils/FSUtils.js'; -import { childLogger, Logger } from '@foxxmd/logging'; +import { childLogger, Logger, LogLevel } from '@foxxmd/logging'; import { loggerNoop } from '../../MaybeLogger.js'; import { projectDir } from '../../index.js'; import { relations } from './schema/schema.js'; @@ -70,7 +70,7 @@ export const getDb = (dbName: string = 'ms', opts: { logger?: Logger, workingDir } = opts; const dbPath = getDbPath(dbName, workingDirectory); logger.info(`Using database at ${dbPath}`); - return drizzle(dbPath, {relations: relations}); + return drizzle(dbPath, {relations: relations, logger: createDrizzleLogger(logger)}); } export type DbConcrete = ReturnType; @@ -90,6 +90,26 @@ export const migrateDb = async (db: ReturnType, opts: {parentLog } } +export const createDrizzleLogger = (parentLogger: Logger, opts: {level?: LogLevel, query?: boolean} = {}): LogWriter & DrizzleLogger => { + const { + level = 'trace', + query = false, + } = opts; + + const logger = childLogger(parentLogger, 'Drizzle'); + + let queryFunc: (query: string, params: unknown[]) => void = (_, __) => {}; + if(query) { + queryFunc = (query: string, params: unknown[]) => logger[level]({params}, `SQL Query => ${query}`); + } + + return { + write(message: string) { + logger[level](message); + }, + logQuery: queryFunc + } +} // cannot really use transactions right now because async isn't supporting for sqlite From c7361a897673f7fbc776555a6a6ce88b79e2500d Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 24 Apr 2026 16:20:16 +0000 Subject: [PATCH 017/104] feat(database): Add full backup-and-migrate logic --- src/backend/common/database/Database.ts | 14 ++++-- .../common/database/drizzle/drizzleUtils.ts | 25 ++++++++-- src/backend/tests/database/drizzle.test.ts | 47 +++++++++++++++++-- 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/src/backend/common/database/Database.ts b/src/backend/common/database/Database.ts index 9b54a4681..473700b91 100644 --- a/src/backend/common/database/Database.ts +++ b/src/backend/common/database/Database.ts @@ -1,4 +1,3 @@ -import { DatabaseSync } from 'node:sqlite'; import { configDir } from '../index.js'; import * as path from 'path'; import { promises as fs } from 'fs' @@ -16,11 +15,16 @@ export const getDbPath = (name: string = 'ms', workingDirectory?: string): strin return path.resolve(workingDirectory ?? configDir, `${name}.db`); } -export const backupDb = async (dbName: string, parentLogger: Logger = loggerNoop): Promise => { +export const backupDb = async (dbName: string, opts: { logger?: Logger, workingDirectory?: string } = {}): Promise => { + + const { + logger: parentLogger = loggerNoop, + workingDirectory + } = opts; const logger = childLogger(parentLogger, 'Migrations'); - const dbPath = getDbPath(dbName); + const dbPath = getDbPath(dbName, workingDirectory); let newDb = false; if(dbPath !== MEMORY_DB_NAME) { @@ -31,12 +35,12 @@ export const backupDb = async (dbName: string, parentLogger: Logger = loggerNoop try { fileOrDirectoryIsWriteable(dbPath); } catch (e) { - throw new Error('Cannot access database path for migrations', {cause: e}); + throw new Error('Database path/folder is not writeable, cannot backup database', {cause: e}); } } if(dbPath !== MEMORY_DB_NAME && !newDb) { - const backupPath = `${getDbPath(`${Date.now()}-${dbName}`)}.bak`; + const backupPath = `${getDbPath(`${Date.now()}-${dbName}`, workingDirectory)}.bak`; logger.info(`Backing up database before migrating => ${backupPath}`); await fs.copyFile(dbPath, backupPath) logger.info('Backed up!'); diff --git a/src/backend/common/database/drizzle/drizzleUtils.ts b/src/backend/common/database/drizzle/drizzleUtils.ts index f84b616b0..54906dcad 100644 --- a/src/backend/common/database/drizzle/drizzleUtils.ts +++ b/src/backend/common/database/drizzle/drizzleUtils.ts @@ -4,16 +4,16 @@ import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core"; import { sql as dsl, LogWriter, Logger as DrizzleLogger } from 'drizzle-orm'; import * as fs from 'fs/promises'; import * as path from 'path'; -import { getDbPath, MEMORY_DB_NAME } from '../Database.js'; +import { backupDb, getDbPath, MEMORY_DB_NAME } from '../Database.js'; import { fileExists } from '../../../utils/FSUtils.js'; import { childLogger, Logger, LogLevel } from '@foxxmd/logging'; import { loggerNoop } from '../../MaybeLogger.js'; import { projectDir } from '../../index.js'; import { relations } from './schema/schema.js'; -export async function shouldBackupDb(dbPath: string, opts: {parentLogger?: Logger, migrationsFolder?: string} = {}): Promise<[boolean, string[]]> { +export async function shouldBackupDb(dbPath: string, opts: {logger?: Logger, migrationsFolder?: string} = {}): Promise<[boolean, string[]]> { const { - parentLogger = loggerNoop, + logger: parentLogger = loggerNoop, migrationsFolder = path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') } = opts; const logger = childLogger(parentLogger, 'Migrations'); @@ -60,6 +60,10 @@ export async function shouldBackupDb(dbPath: string, opts: {parentLogger?: Logge } catch (error) { logger.error(new Error('Failed to get pending migrations', { cause: error })); return [true, []]; + } finally { + if(db.$client.isOpen) { + db.$client.close(); + } } } @@ -75,10 +79,10 @@ export const getDb = (dbName: string = 'ms', opts: { logger?: Logger, workingDir export type DbConcrete = ReturnType; -export const migrateDb = async (db: ReturnType, opts: {parentLogger?: Logger, migrationsFolder?: string} = {}) => { +export const migrateDb = async (db: ReturnType, opts: {logger?: Logger, migrationsFolder?: string} = {}) => { const { migrationsFolder, - parentLogger = loggerNoop + logger: parentLogger = loggerNoop } = opts; const logger = childLogger(parentLogger, 'Migrations'); @@ -90,6 +94,17 @@ export const migrateDb = async (db: ReturnType, opts: {parentLog } } +export const performDbMigrationWithBackup = async (dbName: string = 'ms', opts: { logger?: Logger, workingDirectory?: string, migrationsFolder?: string } = {}) => { + const dbPath = getDbPath(dbName, opts.workingDirectory); + + const [shouldBackup, pendingMigrations] = await shouldBackupDb(dbPath, opts); + if(shouldBackup) { + await backupDb(dbName, opts); + } + const db = getDb(dbName, opts); + await migrateDb(db, opts); +} + export const createDrizzleLogger = (parentLogger: Logger, opts: {level?: LogLevel, query?: boolean} = {}): LogWriter & DrizzleLogger => { const { level = 'trace', diff --git a/src/backend/tests/database/drizzle.test.ts b/src/backend/tests/database/drizzle.test.ts index 311b7f78d..6086c8206 100644 --- a/src/backend/tests/database/drizzle.test.ts +++ b/src/backend/tests/database/drizzle.test.ts @@ -1,9 +1,8 @@ import chai, { assert, expect } from 'chai'; import asPromised from 'chai-as-promised'; -import { getDb, migrateDb, shouldBackupDb } from '../../common/database/drizzle/drizzleUtils.js'; +import { getDb, migrateDb, performDbMigrationWithBackup, shouldBackupDb } from '../../common/database/drizzle/drizzleUtils.js'; import withLocalTmpDir from 'with-local-tmp-dir'; import { components, playInputs, plays, queueStates } from '../../common/database/drizzle/schema/schema.js'; -import { nanoid } from 'nanoid'; import dayjs from 'dayjs'; import { generatePlay } from '../../../core/PlayTestUtils.js'; import { getDbPath } from '../../common/database/Database.js'; @@ -36,7 +35,7 @@ describe('Migrations', function () { it('Detects abnormal db', async function () { - withLocalTmpDir(async () => { + await withLocalTmpDir(async () => { const otherdb = new DatabaseSync(path.resolve('./', 'other.db')); const [shouldBackup, pending] = await shouldBackupDb(getDbPath('other', process.cwd())); expect(shouldBackup).is.true; @@ -109,6 +108,48 @@ describe('Migrations', function () { }, { unsafeCleanup: true }); }); + it('Backs up database when migrations are pending', async function () { + + const allFiles = await fs.readdir(path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations')); + const migrationFiles = allFiles + .sort(); + + await withLocalTmpDir(async () => { + + // copy first migration + await fs.mkdir('migrations'); + try { + await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true }); + const mf = path.resolve('./migrations'); + const db = getDb('ms', { workingDirectory: process.cwd() }); + await migrateDb(db, { migrationsFolder: mf }); + const res = await x('drizzle-kit', [ + 'generate', + '--name', + 'newMigration', + '--out', + `${mf}`, + '--custom', + '--schema', + path.resolve(projectDir, 'src/backend/common/database/drizzle/schema'), + '--dialect', + 'sqlite' + ]); + db.$client.close(); + + // add dummy data to migration so migrate() doesn't fail + const newMigrationFolder = (await fs.readdir(path.resolve('./migrations/'))).find(x => x.includes('newMigration')); + await fs.appendFile(path.resolve('./migrations/',newMigrationFolder, 'migration.sql'),`\nselect count(*) from plays;`); + + await performDbMigrationWithBackup('ms', {workingDirectory: process.cwd(), migrationsFolder: mf}); + const contents = await fs.readdir(path.resolve('./')); + expect(contents.some(x => x.includes('ms.db.bak'))); + } catch (e) { + throw e; + } + }, { unsafeCleanup: true }); + }); + }); describe('Basic DB Operations', function () { From 3b5468499a6ccae4579d16069dde01140f459fe5 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 24 Apr 2026 18:21:09 +0000 Subject: [PATCH 018/104] feat(database): Use asyncstorage for query logging --- .../common/database/drizzle/drizzleUtils.ts | 28 +++----- .../common/database/drizzle/logContext.ts | 71 +++++++++++++++++++ 2 files changed, 79 insertions(+), 20 deletions(-) create mode 100644 src/backend/common/database/drizzle/logContext.ts diff --git a/src/backend/common/database/drizzle/drizzleUtils.ts b/src/backend/common/database/drizzle/drizzleUtils.ts index 54906dcad..f8975a8b8 100644 --- a/src/backend/common/database/drizzle/drizzleUtils.ts +++ b/src/backend/common/database/drizzle/drizzleUtils.ts @@ -10,6 +10,7 @@ import { childLogger, Logger, LogLevel } from '@foxxmd/logging'; import { loggerNoop } from '../../MaybeLogger.js'; import { projectDir } from '../../index.js'; import { relations } from './schema/schema.js'; +import { addToContext, executeQuery } from './logContext.js'; export async function shouldBackupDb(dbPath: string, opts: {logger?: Logger, migrationsFolder?: string} = {}): Promise<[boolean, string[]]> { const { @@ -70,10 +71,9 @@ export async function shouldBackupDb(dbPath: string, opts: {logger?: Logger, mig export const getDb = (dbName: string = 'ms', opts: { logger?: Logger, workingDirectory?: string } = {}) => { const { workingDirectory, - logger = loggerNoop + logger = loggerNoop, } = opts; const dbPath = getDbPath(dbName, workingDirectory); - logger.info(`Using database at ${dbPath}`); return drizzle(dbPath, {relations: relations, logger: createDrizzleLogger(logger)}); } @@ -87,7 +87,8 @@ export const migrateDb = async (db: ReturnType, opts: {logger?: const logger = childLogger(parentLogger, 'Migrations'); try { - await migrate(db, { migrationsFolder: migrationsFolder ?? path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') }); + logger.info('Starting migrations...'); + await executeQuery('migrations', async () => migrate(db, { migrationsFolder: migrationsFolder ?? path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') }), logger, process.env.LOG_MIGRATION === 'true' ? true : 'error'); logger.info('Migrations complete'); } catch (e) { throw new Error('Failed to migrate database', { cause: e }); @@ -105,24 +106,11 @@ export const performDbMigrationWithBackup = async (dbName: string = 'ms', opts: await migrateDb(db, opts); } -export const createDrizzleLogger = (parentLogger: Logger, opts: {level?: LogLevel, query?: boolean} = {}): LogWriter & DrizzleLogger => { - const { - level = 'trace', - query = false, - } = opts; - - const logger = childLogger(parentLogger, 'Drizzle'); - - let queryFunc: (query: string, params: unknown[]) => void = (_, __) => {}; - if(query) { - queryFunc = (query: string, params: unknown[]) => logger[level]({params}, `SQL Query => ${query}`); - } - +export const createDrizzleLogger = (parentLogger: Logger, opts: {level?: LogLevel} = {}): DrizzleLogger => { return { - write(message: string) { - logger[level](message); - }, - logQuery: queryFunc + logQuery: (query: string, params: unknown[]) => { + addToContext({sql: query, params}) + } } } diff --git a/src/backend/common/database/drizzle/logContext.ts b/src/backend/common/database/drizzle/logContext.ts new file mode 100644 index 000000000..3901f6c00 --- /dev/null +++ b/src/backend/common/database/drizzle/logContext.ts @@ -0,0 +1,71 @@ +import { Logger } from '@foxxmd/logging' +import { AsyncLocalStorage } from 'async_hooks' + +// based on https://numeric.substack.com/p/upgrading-drizzleorm-logging-with +interface QueryContext { + queryKey: string + startTime: number + queries: { sql?: string, params?: unknown[] }[] +} + +const queryStorage = new AsyncLocalStorage() + +function wrapQuery(queryKey: string, fn: () => Promise): Promise { + return queryStorage.run( + { + queryKey, + startTime: Date.now(), + queries: [] + }, + fn + ) +} + +function getContext(): QueryContext | undefined { + return queryStorage.getStore() +} + +export function addToContext(data: { sql?: string, params?: unknown[] }): void { + const context = getContext() + if (context) { + context.queries.push(data); + } +} + +/** + * Log all queries made by drizzle during the execution of a promise + * + * use second parameter to configure when logging occurs + * * true => log everything (default) + * * false => log nothing, skips asyncstorage entirely + * * 'error' => only log if promise throws + * + */ +export async function executeQuery(queryKey: string, queryPromise: () => Promise, logger: Logger, when: boolean | 'error' = true) { + if(when === false) { + try { + return await queryPromise(); + } catch (e) { + throw e; + } + } + return wrapQuery(queryKey, async () => { + try { + const results = await queryPromise() + + if (when !== 'error') { + // Query is done - grab everything from context + const context = getContext() + const executionTime = context ? Date.now() - context.startTime : 0 + logger.info({ labels: ['DB Query', queryKey], queries: context?.queries }, `Execution Complete in ${executionTime}ms`); + } + + return results + } catch (error) { + const context = getContext() + const executionTime = context ? Date.now() - context.startTime : 0; + logger.warn({ labels: ['DB Query', queryKey], queries: context?.queries }, `Execution failed in ${executionTime}ms`); + throw error + } + }) +} \ No newline at end of file From cb1ff7d030041d754915539891f9369f2d723443 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 24 Apr 2026 18:21:38 +0000 Subject: [PATCH 019/104] feat(database): Init and migrate database on startup --- src/backend/index.ts | 25 ++++++++++++++++--------- src/backend/ioc.ts | 4 ++++ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/backend/index.ts b/src/backend/index.ts index 37a4dae72..69dcfd026 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -20,10 +20,11 @@ import { createHeartbeatClientsTask } from "./tasks/heartbeatClients.js"; import { createHeartbeatSourcesTask } from "./tasks/heartbeatSources.js"; import { isDebugMode, parseBool, retry } from "./utils.js"; import { readJson } from './utils/DataUtils.js'; -//import { createVegaGenerator } from './utils/SchemaUtils.js'; import ScrobbleClients from './scrobblers/ScrobbleClients.js'; import ScrobbleSources from './sources/ScrobbleSources.js'; import { Notifiers } from './notifier/Notifiers.js'; +import { getDb, performDbMigrationWithBackup } from './common/database/drizzle/drizzleUtils.js'; +import { getDbPath } from './common/database/Database.js'; dayjs.extend(utc) dayjs.extend(isBetween); @@ -88,17 +89,23 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) initLogger.info(`Debug Mode: ${isDebugMode() ? 'YES' : 'NO'}`); - await parseVersion(); + const version = await parseVersion(); + + initLogger.info(`Version: ${version}`); const [aLogger, appLoggerStream] = await appLogger(logging) logger = childLogger(aLogger, 'App'); - - const root = getRoot({...config, logger, loggingConfig: logging, loggerStream: appLoggerStream}); - initLogger.info(`Version: ${root.get('version')}`); - - //initLogger.info('Generating schema definitions...'); - //createVegaGenerator() - //initLogger.info('Schema definitions generated'); + + logger.info(`Using database at ${getDbPath('ms')}`); + await performDbMigrationWithBackup('ms', {logger}); + + const root = getRoot({ + ...config, + logger, + loggingConfig: logging, + loggerStream: appLoggerStream, + db: getDb('ms', {logger}) + }); const internalConfigOptional = { localUrl: root.get('localUrl'), diff --git a/src/backend/ioc.ts b/src/backend/ioc.ts index 7d299c4af..2311f2fe8 100644 --- a/src/backend/ioc.ts +++ b/src/backend/ioc.ts @@ -15,6 +15,7 @@ import prom, { Counter, Gauge } from 'prom-client'; import { CoverArtApiClient } from "./common/vendor/musicbrainz/CoverArtApiClient.js"; import { version } from "./version.js"; import { StaggerOptions } from "./utils/AsyncUtils.js"; +import { DbConcrete } from "./common/database/drizzle/drizzleUtils.js"; let root: ReturnType; export interface RootOptions { @@ -27,6 +28,7 @@ export interface RootOptions { cache?: CacheConfigOptions | MSCache | (() => MSCache) mbMap?: MusicBrainzSingletonMap | (() => MusicBrainzSingletonMap) transformers?: TransformerCommonConfig[] + db?: DbConcrete } const discovered = new prom.Counter({ @@ -60,6 +62,7 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => { logger, cache, mbMap, + db, transformers = [] } = options || {}; const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); @@ -148,6 +151,7 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => { cache: () => maybeSingletonCache !== undefined ? () => maybeSingletonCache : cacheFunc, mbMap: () => maybeSingletonMb !== undefined ? () => maybeSingletonMb : mbFunc, coverArtApi, + db: db as DbConcrete }).add((items) => { const localUrl = generateBaseURL(baseUrl, items.port) return { From 6e66cb06a94568e9f9ea3c56249583042b8ac1ee Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 24 Apr 2026 20:16:33 +0000 Subject: [PATCH 020/104] refactor: Remove duplicated regex functions --- .../common/vendor/koito/KoitoApiClient.ts | 4 +- src/backend/sources/EndpointLastfmSource.ts | 6 +-- .../sources/EndpointListenbrainzSource.ts | 8 ++-- src/backend/utils.ts | 40 ------------------- src/backend/utils/NetworkUtils.ts | 4 +- src/backend/utils/StringUtils.ts | 9 +++-- 6 files changed, 16 insertions(+), 55 deletions(-) diff --git a/src/backend/common/vendor/koito/KoitoApiClient.ts b/src/backend/common/vendor/koito/KoitoApiClient.ts index 72999b0d8..eb451e4a3 100644 --- a/src/backend/common/vendor/koito/KoitoApiClient.ts +++ b/src/backend/common/vendor/koito/KoitoApiClient.ts @@ -9,10 +9,10 @@ import { UpstreamError } from "../../errors/UpstreamError.js"; import { playToListenPayload } from '../listenbrainz/lzUtils.js'; import { SubmitPayload } from '../listenbrainz/interfaces.js'; import { ListenType } from '../listenbrainz/interfaces.js'; -import { parseRegexSingleOrFail } from "../../../utils.js"; import { baseFormatPlayObj } from "../../../utils/PlayTransformUtils.js"; import { ScrobbleSubmitError } from "../../errors/MSErrors.js"; import { tryApiCall } from "../../../utils/RequestUtils.js"; +import { parseRegexSingle } from "@foxxmd/regex-buddy-core"; interface SubmitOptions { log?: boolean @@ -36,7 +36,7 @@ export class KoitoApiClient extends AbstractApiClient implements PaginatedTimeRa const u = normalizeWebAddress(url); if(u.url.pathname === '/') { this.url = u; - } else if(parseRegexSingleOrFail(KOITO_LZ_PATH, u.url.pathname) !== undefined) { + } else if(parseRegexSingle(KOITO_LZ_PATH, u.url.pathname) !== undefined) { this.logger.verbose('Detected Koito Server URL path only contains listenbrainz prefix. Removing this for API calls so non-listenbrainz paths work correctly.'); this.url = normalizeWebAddress(getBaseFromUrl(u.url).toString()); } else { diff --git a/src/backend/sources/EndpointLastfmSource.ts b/src/backend/sources/EndpointLastfmSource.ts index 42a20e38f..ed4575345 100644 --- a/src/backend/sources/EndpointLastfmSource.ts +++ b/src/backend/sources/EndpointLastfmSource.ts @@ -11,13 +11,13 @@ import { REPORTED_PLAYER_STATUSES, ReportedPlayerStatus } from "../common/infrastructure/Atomic.js"; -import { parseRegexSingleOrFail } from "../utils.js"; import MemorySource from "./MemorySource.js"; import { LastFMEndpointSourceConfig } from "../common/infrastructure/config/source/endpointlfm.js"; import { LastFMScrobbleRequestPayload, scrobblePayloadToPlay } from "../common/vendor/LastfmApiClient.js"; import { Logger } from "@foxxmd/logging"; import { PlayerStateOptions } from "./PlayerState/AbstractPlayerState.js"; import { NowPlayingPlayerState } from "./PlayerState/NowPlayingPlayerState.js"; +import { parseRegexSingle } from "@foxxmd/regex-buddy-core"; const noSlugMatch = new RegExp(/(?:\/api\/lastfm\/?)$|(?:\/1\/?|\/2.0\/?)$/i); const slugMatch = new RegExp(/\/api\/lastfm\/([^\/]+)$/i); @@ -97,11 +97,11 @@ export const playStateFromRequest = (obj: LastFMScrobbleRequestPayload): PlayerS } export const parseSlugFromString = (path: string): string | false | undefined => { - const noSlug = parseRegexSingleOrFail(noSlugMatch, path); + const noSlug = parseRegexSingle(noSlugMatch, path); if (noSlug !== undefined) { return undefined; } - const slugResult = parseRegexSingleOrFail(slugMatch, path); + const slugResult = parseRegexSingle(slugMatch, path); if (slugResult !== undefined) { return slugResult.groups[0]; } diff --git a/src/backend/sources/EndpointListenbrainzSource.ts b/src/backend/sources/EndpointListenbrainzSource.ts index 77053e16d..a3eb9becb 100644 --- a/src/backend/sources/EndpointListenbrainzSource.ts +++ b/src/backend/sources/EndpointListenbrainzSource.ts @@ -15,11 +15,11 @@ import { ListenbrainzEndpointSourceConfig } from "../common/infrastructure/confi import { ListenbrainzApiClient, listenPayloadToPlay } from "../common/vendor/ListenbrainzApiClient.js"; import { SubmitPayload } from '../common/vendor/listenbrainz/interfaces.js'; import { ListenPayload } from '../common/vendor/listenbrainz/interfaces.js'; -import { parseRegexSingleOrFail } from "../utils.js"; import MemorySource from "./MemorySource.js"; import { NowPlayingPlayerState } from "./PlayerState/NowPlayingPlayerState.js"; import { Logger } from "@foxxmd/logging"; import { PlayerStateOptions } from "./PlayerState/AbstractPlayerState.js"; +import { parseRegexSingle } from "@foxxmd/regex-buddy-core"; const noSlugMatch = new RegExp(/(?:\/api\/listenbrainz\/?)$|(?:\/1\/?|\/1\/submit-listens\/?\/1\/validate-token\/|)$/i); const slugMatch = new RegExp(/\/api\/listenbrainz\/([^\/]+)$/i); @@ -131,7 +131,7 @@ export const listenTypeAsPlayerStatus = (event: string): ReportedPlayerStatus => } export const parseTokenFromString = (str: string): string | undefined => { - const tokenMatch = parseRegexSingleOrFail(authHeaderRegex, str); + const tokenMatch = parseRegexSingle(authHeaderRegex, str); if(tokenMatch !== undefined) { return tokenMatch.groups[0]; } @@ -151,11 +151,11 @@ export const parseTokenFromRequest = (req: ExpressRequest): string | false | und } export const parseSlugFromString = (path: string): string | false | undefined => { - const noSlug = parseRegexSingleOrFail(noSlugMatch, path); + const noSlug = parseRegexSingle(noSlugMatch, path); if (noSlug !== undefined) { return undefined; } - const slugResult = parseRegexSingleOrFail(slugMatch, path); + const slugResult = parseRegexSingle(slugMatch, path); if (slugResult !== undefined) { return slugResult.groups[0]; } diff --git a/src/backend/utils.ts b/src/backend/utils.ts index e1085fed7..861da8da7 100644 --- a/src/backend/utils.ts +++ b/src/backend/utils.ts @@ -338,46 +338,6 @@ export const pollingBackoff = (attempt: number, scaleFactor: number = 1): number return Math.round(backoffStrat(attempt + 1) / 1000); } -export const parseRegex = (reg: RegExp, val: string): RegExResult[] | undefined => { - - if (reg.global) { - const g = Array.from(val.matchAll(reg)); - if (g.length === 0) { - return undefined; - } - return g.map(x => { - return { - match: x[0], - index: x.index, - groups: x.slice(1), - named: x.groups || {}, - } as RegExResult; - }); - } - - const m = val.match(reg) - if (m === null) { - return undefined; - } - return [{ - match: m[0], - index: m.index as number, - groups: m.slice(1), - named: m.groups || {} - }]; -} - -export const parseRegexSingleOrFail = (reg: RegExp, val: string): RegExResult | undefined => { - const results = parseRegex(reg, val); - if (results !== undefined) { - if (results.length > 1) { - throw new Error(`Expected Regex to match once but got ${results.length} results. Either Regex must NOT be global (using 'g' flag) or parsed value must only match regex once. Given: ${val} || Regex: ${reg.toString()}`); - } - return results[0]; - } - return undefined; -} - export const intersect = (a: Array, b: Array) => { const setA = new Set(a); const setB = new Set(b); diff --git a/src/backend/utils/NetworkUtils.ts b/src/backend/utils/NetworkUtils.ts index 3e50bceb0..410ea225a 100644 --- a/src/backend/utils/NetworkUtils.ts +++ b/src/backend/utils/NetworkUtils.ts @@ -4,7 +4,7 @@ import address from "address"; import net from 'node:net'; import normalizeUrl from "normalize-url"; import { join as joinPath } from "path"; -import { getFirstNonEmptyVal, isDebugMode, parseRegexSingleOrFail } from "../utils.js"; +import { getFirstNonEmptyVal, isDebugMode} from "../utils.js"; import { URLData } from "../../core/Atomic.js"; import { CloseEvent, ErrorEvent, RetryEvent } from 'iso-websocket' import { WEBSOCKET_CLOSE_CODE_REASONS } from "../common/infrastructure/Atomic.js"; @@ -225,7 +225,7 @@ export const getAddress = (host = '0.0.0.0', logger?: Logger): { v4?: string, v6 } const IPV4_REGEX = new RegExp(/^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}$/); export const isIPv4 = (address: string): boolean => { - return parseRegexSingleOrFail(IPV4_REGEX, address) !== undefined; + return parseRegexSingle(IPV4_REGEX, address) !== undefined; } export const formatWebsocketClose = (e: CloseEvent): string => { diff --git a/src/backend/utils/StringUtils.ts b/src/backend/utils/StringUtils.ts index 7fe7e3ddd..15557bf50 100644 --- a/src/backend/utils/StringUtils.ts +++ b/src/backend/utils/StringUtils.ts @@ -2,9 +2,10 @@ import { strategies, stringSameness, StringSamenessResult } from "@foxxmd/string import { hasher } from 'node-object-hash'; import { PlayObject } from "../../core/Atomic.js"; import { asPlayerStateData, DELIMITERS, DELIMITERS_NO_AMP, PlayerStateDataMaybePlay } from "../common/infrastructure/Atomic.js"; -import { getPlatformIdFromData, intersect, parseBool, parseBoolStrict, parseRegexSingleOrFail } from "../utils.js"; +import { getPlatformIdFromData, intersect, parseBool, parseBoolStrict } from "../utils.js"; import { genGroupIdStr } from '../../core/PlayUtils.js'; import { buildTrackString } from "../../core/StringUtils.js"; +import { parseRegexSingle } from "@foxxmd/regex-buddy-core"; const {levenStrategy, diceStrategy} = strategies; @@ -104,7 +105,7 @@ export const parseCredits = (str: string, delimiters?: boolean | string[]): Play let primary: string | undefined; let secondary: string[] = []; let suffix: string | undefined; - const results = parseRegexSingleOrFail(PRIMARY_SECONDARY_SECTIONS_REGEX, str); + const results = parseRegexSingle(PRIMARY_SECONDARY_SECTIONS_REGEX, str); if(results !== undefined) { let delims: string[] | undefined; @@ -116,7 +117,7 @@ export const parseCredits = (str: string, delimiters?: boolean | string[]): Play primary = results.named.primary.trim(); for(const strat of SECONDARY_REGEX_STRATS) { - const secCredits = parseRegexSingleOrFail(strat, results.named.secondary); + const secCredits = parseRegexSingle(strat, results.named.secondary); if(secCredits !== undefined) { secondary = parseContextAwareStringList(secCredits.named.credits as string, delims) suffix = secCredits.named.creditsSuffix; @@ -437,7 +438,7 @@ export const buildStatePlayerPlayIdententifyingInfo = (data: PlayObject | Player export const LZ_VERSION_PATH: RegExp = new RegExp(/\/?1\/?$/); export const normalizeListenbrainzUrl = (urlVal: string): string | undefined => { - if (parseRegexSingleOrFail(LZ_VERSION_PATH, urlVal)) { + if (parseRegexSingle(LZ_VERSION_PATH, urlVal)) { return urlVal.replace(LZ_VERSION_PATH, ''); } return undefined; From c0384317b14c22aeced108dbce568075bf103764 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 24 Apr 2026 21:21:42 +0000 Subject: [PATCH 021/104] feat(database): Implement retention options and configuration parsing --- src/backend/common/AbstractComponent.ts | 4 ++ src/backend/common/database/Database.ts | 72 +++++++++++++++++-- src/backend/common/errors/MSErrors.ts | 19 +++++ src/backend/common/infrastructure/Atomic.ts | 14 +++- .../common/infrastructure/config/aioConfig.ts | 7 +- .../infrastructure/config/client/index.ts | 4 ++ .../common/infrastructure/config/database.ts | 18 +++++ .../infrastructure/config/source/index.ts | 4 ++ src/backend/scrobblers/ScrobbleClients.ts | 25 ++++--- src/backend/sources/ScrobbleSources.ts | 6 +- src/backend/utils/TimeUtils.ts | 59 +++++++++++++++ 11 files changed, 213 insertions(+), 19 deletions(-) create mode 100644 src/backend/common/infrastructure/config/database.ts diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index bab2f1229..eff45e8f3 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -27,6 +27,8 @@ import { diffObjects, diffObjectsConsoleOutput, patchObject } from "../../core/D import clone from "clone"; import { loggerNoop } from "./MaybeLogger.js"; import { objectsEqual } from "../utils/DataUtils.js"; +import { RetentionOptionsFull } from "./infrastructure/config/database.js"; +import { parseRetentionOptions } from "./database/Database.js"; export type AbstractComponentConfig = (CommonClientConfig | CommonSourceConfig) & { transformManager?: TransformerManager }; @@ -38,11 +40,13 @@ export default abstract class AbstractComponent extends AbstractInitializable { regexCache!: ReturnType; protected transformManager: TransformerManager; protected cache: MSCache; + protected retentionOpts: RetentionOptionsFull; protected constructor(config: AbstractComponentConfig) { super(config); this.transformManager = config.transformManager ?? getRoot().items.transformerManager; this.cache = getRoot().items.cache(); + this.retentionOpts = parseRetentionOptions(config.options.retention); } protected postCache(): Promise { diff --git a/src/backend/common/database/Database.ts b/src/backend/common/database/Database.ts index 473700b91..2da1cca0a 100644 --- a/src/backend/common/database/Database.ts +++ b/src/backend/common/database/Database.ts @@ -4,12 +4,17 @@ import { promises as fs } from 'fs' import { childLogger, Logger } from '@foxxmd/logging'; import { loggerNoop } from '../MaybeLogger.js'; import { fileExists, fileOrDirectoryIsWriteable } from '../../utils/FSUtils.js'; +import { DEFAULT_RETENTION_DELETE_AFTER, RententionGranular, RetentionOptions, RetentionOptionsFull } from '../infrastructure/config/database.js'; +import { DurationValue } from '../infrastructure/Atomic.js'; +import { Duration } from 'dayjs/plugin/duration.js'; +import dayjs from 'dayjs'; +import { parseDurationFromDurationValue } from '../../utils/TimeUtils.js'; export const MEMORY_DB_NAME = ':memory:'; export const isMemoryDb = (name: string): boolean => name === MEMORY_DB_NAME; export const getDbPath = (name: string = 'ms', workingDirectory?: string): string => { - if(isMemoryDb(name)) { + if (isMemoryDb(name)) { return MEMORY_DB_NAME; } return path.resolve(workingDirectory ?? configDir, `${name}.db`); @@ -27,22 +32,79 @@ export const backupDb = async (dbName: string, opts: { logger?: Logger, workingD const dbPath = getDbPath(dbName, workingDirectory); let newDb = false; - if(dbPath !== MEMORY_DB_NAME) { - if(!fileExists(dbPath)) { + if (dbPath !== MEMORY_DB_NAME) { + if (!fileExists(dbPath)) { logger.info(`Database at ${dbPath} does not exist, will create it.`); newDb = true; } try { fileOrDirectoryIsWriteable(dbPath); } catch (e) { - throw new Error('Database path/folder is not writeable, cannot backup database', {cause: e}); + throw new Error('Database path/folder is not writeable, cannot backup database', { cause: e }); } } - if(dbPath !== MEMORY_DB_NAME && !newDb) { + if (dbPath !== MEMORY_DB_NAME && !newDb) { const backupPath = `${getDbPath(`${Date.now()}-${dbName}`, workingDirectory)}.bak`; logger.info(`Backing up database before migrating => ${backupPath}`); await fs.copyFile(dbPath, backupPath) logger.info('Backed up!'); } +} + +const parseRetentionFromEnv = (): Required> => { + const deleteAfterEnv = process.env.RETENTION_DELETE_AFTER ?? DEFAULT_RETENTION_DELETE_AFTER, + deleteCompletedEnv = process.env.RETENTION_DELETE_COMPLETED_AFTER ?? deleteAfterEnv, + deleteFailedEnv = process.env.RETENTION_DELETE_FAILED_AFTER ?? deleteAfterEnv, + deleteDupedEnv = process.env.RETENTION_DELETE_DUPED_AFTER ?? deleteAfterEnv; + + return { + completed: parseDurationFromDurationValue(deleteCompletedEnv), + failed: parseDurationFromDurationValue(deleteFailedEnv), + duped: parseDurationFromDurationValue(deleteDupedEnv) + } +} + +let retentionFromEnv: Required>; +const getRetentionFromEnv = () => { + if (retentionFromEnv === undefined) { + retentionFromEnv = parseRetentionFromEnv(); + } + return retentionFromEnv; +} + +export const parseRetentionOptions = (opts: RetentionOptions = {}): RetentionOptionsFull => { + if (typeof opts.deleteAfter === 'number' || typeof opts.deleteAfter === 'string') { + const dur = parseDurationFromDurationValue(opts.deleteAfter); + return { + deleteAfter: { + completed: dur, + duped: dur, + failed: dur + } + } + } + + const fromEnv = getRetentionFromEnv(); + if (opts.deleteAfter === undefined) { + return { + deleteAfter: fromEnv + } + } + + const { + deleteAfter: { + completed = fromEnv.completed, + failed = fromEnv.failed, + duped = fromEnv.duped + } = {} + } = opts; + + return { + deleteAfter: { + completed: dayjs.isDuration(completed) ? completed : parseDurationFromDurationValue(completed), + failed: dayjs.isDuration(failed) ? failed : parseDurationFromDurationValue(failed), + duped: dayjs.isDuration(duped) ? duped : parseDurationFromDurationValue(duped), + } + } } \ No newline at end of file diff --git a/src/backend/common/errors/MSErrors.ts b/src/backend/common/errors/MSErrors.ts index 4b15cae40..06aba95f2 100644 --- a/src/backend/common/errors/MSErrors.ts +++ b/src/backend/common/errors/MSErrors.ts @@ -130,4 +130,23 @@ export const generateLoggableAbortReason = (msg: string, signal: AbortSignal): A } Error.captureStackTrace(err, generateLoggableAbortReason); return err; +} + +export class InvalidRegexError extends SimpleError { + constructor(regex: RegExp | RegExp[], val?: string, url?: string, message?: string) { + const msgParts = [ + message ?? 'Regex(es) did not match the value given.', + ]; + let regArr = Array.isArray(regex) ? regex : [regex]; + for(const r of regArr) { + msgParts.push(`Regex: ${r}`) + } + if (val !== undefined) { + msgParts.push(`Value: ${val}`); + } + if (url !== undefined) { + msgParts.push(`Sample regex: ${url}`); + } + super(msgParts.join('\r\n')); + } } \ No newline at end of file diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index d53a1a623..2624d7847 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -414,4 +414,16 @@ export interface ScrobbleRangeResult { fetchedAt: Dayjs } -export const REFRESH_STALE_DEFAULT = 60; \ No newline at end of file +export const REFRESH_STALE_DEFAULT = 60; + +/** + * A duration of time + * + * May be either: + * + * * a `number` of seconds + * * a `string` containing a number and a unit of time compatible with dayjs + * + * @example [60, 3600, "1 hour", "4 days"] + */ +export type DurationValue = number | string; \ No newline at end of file diff --git a/src/backend/common/infrastructure/config/aioConfig.ts b/src/backend/common/infrastructure/config/aioConfig.ts index be1241e5b..ef3dd3eeb 100644 --- a/src/backend/common/infrastructure/config/aioConfig.ts +++ b/src/backend/common/infrastructure/config/aioConfig.ts @@ -5,8 +5,9 @@ import { RequestRetryOptions } from "./common.js"; import { WebhookConfig } from "./health/webhooks.js"; import { CommonSourceOptions, SourceRetryOptions } from "./source/index.js"; import { SourceAIOConfig } from "./source/sources.js"; -import { CacheConfigOptions } from "../Atomic.js"; +import { CacheConfigOptions, DurationValue } from "../Atomic.js"; import { TransformerCommonConfig } from "../../../../core/Atomic.js"; +import { RetentionOptions } from "./database.js"; export interface SourceDefaults extends CommonSourceOptions { @@ -69,6 +70,10 @@ export interface AIOConfig { cache?: CacheConfigOptions transformers?: TransformerCommonConfig[] + + database?: { + retention?: RetentionOptions + } } export interface AIOClientConfig { diff --git a/src/backend/common/infrastructure/config/client/index.ts b/src/backend/common/infrastructure/config/client/index.ts index 0a2803033..a4d80cec7 100644 --- a/src/backend/common/infrastructure/config/client/index.ts +++ b/src/backend/common/infrastructure/config/client/index.ts @@ -1,5 +1,7 @@ +import { DurationValue } from "../../Atomic.js"; import { PlayTransformConfig, PlayTransformOptions } from "../../Transform.js"; import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js"; +import { RetentionOptions } from "../database.js"; /** * Scrobble matching (between new source track and existing client scrobbles) logging options. Used for debugging. @@ -105,6 +107,8 @@ export interface CommonClientOptions extends RequestRetryOptions, UpstreamRefres deadLetterRetries?: number playTransform?: PlayTransformOptions + + retention?: RetentionOptions } export interface CommonClientConfig extends CommonConfig { diff --git a/src/backend/common/infrastructure/config/database.ts b/src/backend/common/infrastructure/config/database.ts new file mode 100644 index 000000000..66379ae2d --- /dev/null +++ b/src/backend/common/infrastructure/config/database.ts @@ -0,0 +1,18 @@ +import { Duration } from "dayjs/plugin/duration.js"; +import { DurationValue } from "../Atomic.js"; + +export interface RententionGranular { + failed?: T + completed?: T + duped?: T +} + +export interface RetentionOptions { + deleteAfter?: T | RententionGranular +} + +export interface RetentionOptionsFull { + deleteAfter: RententionGranular +} + +export const DEFAULT_RETENTION_DELETE_AFTER = 604800; // 7 days \ No newline at end of file diff --git a/src/backend/common/infrastructure/config/source/index.ts b/src/backend/common/infrastructure/config/source/index.ts index b184bc689..aa6d1196a 100644 --- a/src/backend/common/infrastructure/config/source/index.ts +++ b/src/backend/common/infrastructure/config/source/index.ts @@ -2,6 +2,8 @@ import { FileLogOptions, LogLevel } from "@foxxmd/logging"; import { PlayTransformConfig, PlayTransformOptions } from "../../Transform.js"; import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js"; +import { RetentionOptions } from "../database.js"; +import { DurationValue } from "../../Atomic.js"; export interface SourceRetryOptions extends RequestRetryOptions { /** @@ -108,6 +110,8 @@ export interface CommonSourceOptions extends SourceRetryOptions { scrobbleBacklogCount?: number playTransform?: PlayTransformOptions + + retention?: RetentionOptions } export interface ManualListeningOptions { diff --git a/src/backend/scrobblers/ScrobbleClients.ts b/src/backend/scrobblers/ScrobbleClients.ts index 4ce8277f1..af6d8d756 100644 --- a/src/backend/scrobblers/ScrobbleClients.ts +++ b/src/backend/scrobblers/ScrobbleClients.ts @@ -130,8 +130,11 @@ export default class ScrobbleClients { const { clients: mainConfigClientConfigs = [], clientDefaults: cd = {}, + database: { + retention + } = {}, } = aioConfig; - clientDefaults = cd; + clientDefaults = {retention, ...cd}; for (const [index, c] of mainConfigClientConfigs.entries()) { const {name = 'unnamed'} = c; if(c.type === undefined) { @@ -414,7 +417,7 @@ ${sources.join('\n')}`); if (isValidConfig !== true) { throw new Error(`Config object from ${clientConfig.source || 'unknown'} with name [${clientConfig.name || 'unnamed'}] of type [${clientConfig.type || 'unknown'}] has errors: ${isValidConfig.join(' | ')}`) }*/ - const {type, name, enable = true, source, data: d = {}} = clientConfig; + const {type, name, enable = true, source, data: d = {}, options = {}} = clientConfig; if(enable === false) { this.logger.warn({labels: [`${type} - ${name}`]}, `Client from ${source} was disabled by config`); @@ -422,41 +425,41 @@ ${sources.join('\n')}`); } // add defaults - const data = {...defaults, ...d}; + const compositeOptions = {...defaults, ...options}; let newClient; this.logger.debug({labels: [`${type} - ${name}`]}, `Constructing Client from ${source}`); switch (type) { case 'maloja': const MalojaScrobbler = (await import('./MalojaScrobbler.js')).default; - newClient = new MalojaScrobbler(name, ({...clientConfig, data} as unknown as MalojaClientConfig), notifier, this.emitter, this.logger); + newClient = new MalojaScrobbler(name, ({...clientConfig, data: d, options: compositeOptions} as unknown as MalojaClientConfig), notifier, this.emitter, this.logger); break; case 'lastfm': const LastfmScrobbler = (await import('./LastfmScrobbler.js')).default; - newClient = new LastfmScrobbler(name, {...clientConfig, data } as unknown as LastfmClientConfig, this.internalConfig, notifier, this.emitter, this.logger); + newClient = new LastfmScrobbler(name, {...clientConfig, data: d, options: compositeOptions } as unknown as LastfmClientConfig, this.internalConfig, notifier, this.emitter, this.logger); break; case 'librefm': const LibrefmScrobbler = (await import('./LibrefmScrobbler.js')).default; - newClient = new LibrefmScrobbler(name, {...clientConfig, data } as unknown as LibrefmClientConfig, this.internalConfig, notifier, this.emitter, this.logger); + newClient = new LibrefmScrobbler(name, {...clientConfig, data: d, options: compositeOptions } as unknown as LibrefmClientConfig, this.internalConfig, notifier, this.emitter, this.logger); break; case 'listenbrainz': const ListenbrainzScrobbler = (await import('./ListenbrainzScrobbler.js')).default; - newClient = new ListenbrainzScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...data} } as unknown as ListenBrainzClientConfig, {}, notifier, this.emitter, this.logger); + newClient = new ListenbrainzScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...d}, options: compositeOptions } as unknown as ListenBrainzClientConfig, {}, notifier, this.emitter, this.logger); break; case 'koito': const KoitoScrobbler = (await import('./KoitoScrobbler.js')).default; - newClient = new KoitoScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...data} } as unknown as KoitoClientConfig, {}, notifier, this.emitter, this.logger); + newClient = new KoitoScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...d}, options: compositeOptions } as unknown as KoitoClientConfig, {}, notifier, this.emitter, this.logger); break; case 'tealfm': const TealScrobbler = (await import('./TealfmScrobbler.js')).default; - newClient = new TealScrobbler(name, {...clientConfig, data: {...data}} as unknown as TealClientConfig, {}, notifier, this.emitter, this.logger); + newClient = new TealScrobbler(name, {...clientConfig, data: d, options: compositeOptions} as unknown as TealClientConfig, {}, notifier, this.emitter, this.logger); break; case 'rocksky': const RockskyScrobbler = (await import('./RockskyScrobbler.js')).default; - newClient = new RockskyScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...data} } as unknown as RockSkyClientConfig, {}, notifier, this.emitter, this.logger); + newClient = new RockskyScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...d}, options: compositeOptions } as unknown as RockSkyClientConfig, {}, notifier, this.emitter, this.logger); break; case 'discord': const DiscordScrobbler = (await import('./DiscordScrobbler.js')).default; - newClient = new DiscordScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...data} } as unknown as DiscordClientConfig, {}, notifier, this.emitter, this.logger); + newClient = new DiscordScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...d}, options: compositeOptions } as unknown as DiscordClientConfig, {}, notifier, this.emitter, this.logger); break; default: break; diff --git a/src/backend/sources/ScrobbleSources.ts b/src/backend/sources/ScrobbleSources.ts index b11bac508..db80b941f 100644 --- a/src/backend/sources/ScrobbleSources.ts +++ b/src/backend/sources/ScrobbleSources.ts @@ -54,6 +54,7 @@ import { ListenBrainzData } from '../common/infrastructure/config/client/listenb import { KoitoData } from '../common/infrastructure/config/client/koito.js'; import { TealData } from '../common/infrastructure/config/client/tealfm.js'; import { RockSkyData } from '../common/infrastructure/config/client/rocksky.js'; +import { DEFAULT_RETENTION_DELETE_AFTER } from '../common/infrastructure/config/database.js'; type groupedNamedConfigs = {[key: string]: ParsedConfig[]}; @@ -230,8 +231,11 @@ export default class ScrobbleSources { const { sources: mainConfigSourcesConfigs = [], sourceDefaults: sd = {}, + database: { + retention + } = {}, } = aioConfig; - sourceDefaults = this.buildSourceDefaults(sd); + sourceDefaults = this.buildSourceDefaults({retention, ...sd}); for (const [index, c] of mainConfigSourcesConfigs.entries()) { const {name = 'unnamed'} = c; if(c.type === undefined) { diff --git a/src/backend/utils/TimeUtils.ts b/src/backend/utils/TimeUtils.ts index 15a692005..566dfe741 100644 --- a/src/backend/utils/TimeUtils.ts +++ b/src/backend/utils/TimeUtils.ts @@ -25,11 +25,15 @@ import { DEFAULT_DURATION_REPEAT_PERCENT, DEFAULT_SCROBBLE_DURATION_THRESHOLD, DEFAULT_SCROBBLE_PERCENT_THRESHOLD, + DurationValue, lowGranularitySources, ScrobbleThresholdResult, } from "../common/infrastructure/Atomic.js"; import { ScrobbleThresholds } from "../common/infrastructure/config/source/index.js"; import { formatNumber } from '../../core/DataUtils.js'; +import { InvalidRegexError, SimpleError } from "../common/errors/MSErrors.js"; +import { NamedGroup, parseRegex } from "@foxxmd/regex-buddy-core"; +import { Duration } from "dayjs/plugin/duration.js"; //dayjs.extend(isToday); @@ -399,4 +403,59 @@ export const repeatDurationPlayed = (play: PlayObject, duration: number, thresho /** Convert unix timestamp in microseconds to unix timestamp in seconds */ export const usecToUnix = (usec: number): UnixTimestamp => { return Math.floor(usec / 1000); +} + +// string must only contain ISO8601 optionally wrapped by whitespace +const ISO8601_REGEX: RegExp = /^\s*((-?)P(?=\d|T\d)(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)([DW]))?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?)\s*$/; +// finds ISO8601 in any part of a string +const ISO8601_SUBSTRING_REGEX: RegExp = /((-?)P(?=\d|T\d)(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)([DW]))?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?)/g; +// string must only duration optionally wrapped by whitespace +const DURATION_REGEX: RegExp = /^\s*(?