diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6a95043dd..bc52fd1e9 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,7 +17,7 @@ // "features": {}, // Use 'forwardPorts' to make a list of ports inside the container available locally. - "forwardPorts": [9078,3000], + "forwardPorts": [9078,3000,5433], // Use 'postCreateCommand' to run commands after the container is created. "postCreateCommand": ".devcontainer/postCreate.sh", @@ -30,7 +30,9 @@ "dbaeumer.vscode-eslint", "unifiedjs.vscode-mdx", "bradlc.vscode-tailwindcss", - "TakumiI.markdowntable" + "TakumiI.markdowntable", + "YoavBls.pretty-ts-errors", + "bowlerr.sqlite-intelliview-vscode" ] } }, @@ -40,6 +42,9 @@ }, "9078": { "label": "App" + }, + "5433": { + "label": "PGLite Server" } } diff --git a/.gitignore b/.gitignore index 2b5fa3b0d..e8a2071f6 100644 --- a/.gitignore +++ b/.gitignore @@ -122,6 +122,7 @@ config/*.json config/mscache config/yti-* config/*.cache +config/msDb *.txt !robots.txt .idea/ @@ -138,6 +139,7 @@ docsite/static/schemas/*.json *.bak +*.bak.used *.p8 .flatpak-builder flatpak/generated-sources.json @@ -151,7 +153,10 @@ tmp-* *storybook.log storybook-static -lib + +*.db +*.db.* +*.db-*lib *.db *.db.* *.db-*lib \ No newline at end of file diff --git a/.mocharc.json b/.mocharc.json index c22a525b2..3d59041e9 100644 --- a/.mocharc.json +++ b/.mocharc.json @@ -6,5 +6,6 @@ "file": [ "./src/backend/tests/setup.ts" ], - "exit": true + "exit": true, + "timeout": 2500 } diff --git a/.vscode/settings.json b/.vscode/settings.json index 7094e312a..37b557082 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,5 +13,6 @@ "files.associations": { "*.css": "tailwindcss" }, - "tailwindCSS.experimental.configFile": "src/client/index.css" + "tailwindCSS.experimental.configFile": "src/client/index.css", + "pgliteExplorer.databasePaths": [] } \ No newline at end of file diff --git a/docsite/docs/configuration/configuration.mdx b/docsite/docs/configuration/configuration.mdx index 13ce858ac..9ffb49417 100644 --- a/docsite/docs/configuration/configuration.mdx +++ b/docsite/docs/configuration/configuration.mdx @@ -245,11 +245,8 @@ WARN : [App] [Sources] [spotify Secrets] Matched: None | Unmatched: SPOTIFY_SECR -## Application Options -These options affect multi-scrobbler's behavior and are not specific to any source/client. - -### Base URL +## Base URL Defines the URL that is used to generate default redirect URLs for authentication on [spotify](/configuration/sources/spotify) and [lastfm](/configuration/clients/lastfm) -- as well as some logging hints. @@ -280,185 +277,82 @@ Useful when running with [docker](../installation/installation.mdx#docker) so th -### Caching - -Multi-scrobbler caches some activities to persist important data across restarts, reduce external API calls, and make some actions faster. - -All of the activities below are **always** cached **in-memory** with an optional, configurable [**secondary** store](#secondary-caching-configuration) for persistence. - - - - -**Queued** and **Failed** Scrobbles are cached so that any un-scrobbled data you have is persisted across restarts of multi-scrobbler. - -:::tip - -By default, this data use a [Secondary](#secondary-caching-configuration) [File](./?cacheType=file#secondary-caching-configuration) store, configured for you automatically. - -If you have configured a [persisted volume/bind mount](/installation?dockerSetting=storage#recommended-settings) for configuration (`/config` is mounted in [docker compose](/quickstart#create-docker-compose-file)) then you are already done. If you are not persisting this directory then you should consider setting up [Valkey Cache](./?cacheType=valkey#secondary-caching-configuration) for this. - -::: - -##### Configuration - -Use any [Secondary Cache](#secondary-caching-configuration), the config examples below show the default values: - - - - -| Environmental Variable | Required? | Default | Description | -| :--------------------- | --------- | --------- | --------------------- | -| `CACHE_SCROBBLE` | No | `file` | The cache type to use | -| `CACHE_SCROBBLE_CONN` | No | `/config` | | +## Caching - +Multi-scrobbler implements caching to persist important data across restarts, reduce external API calls, and make some actions faster. - +A default **in-memory** cache store is used so that you always benefit from some caching. An optional, [**secondary** store](#secondary-caching) can be configured for greater caching capabilities. -```json5 title="config.json" -{ - "cache": { - "scrobble": { - "provider": "file", - "connection": "/config" - } - }, - // ... -} -``` - - + - + - Authentication sessions/tokens/etc... are cached for quicker requests and for persistence across restarts. - -:::tip - -By default, this data use a [Secondary](#secondary-caching-configuration) [File](./?cacheType=file#secondary-caching-configuration) store, configured for you automatically. - -If you have configured a [persisted volume/bind mount](/installation?dockerSetting=storage#recommended-settings) for configuration (`/config` is mounted in [docker compose](/quickstart#create-docker-compose-file)) then you are already done. If you are not persisting this directory then you should consider setting up [Valkey Cache](./?cacheType=valkey#secondary-caching-configuration) for this. - -::: - -##### Configuration - -Use any [Secondary Cache](#secondary-caching-configuration), the config examples below show the default values: - - - - -| Environmental Variable | Required? | Default | Description | -| :--------------------- | --------- | --------- | --------------------- | -| `CACHE_AUTH` | No | `file` | The cache type to use | -| `CACHE_AUTH_CONN` | No | `/config` | | - - - - -```json5 title="config.json" -{ - "cache": { - "auth": { - "provider": "file", - "connection": "/config" - } - }, - // ... -} -``` - - - + +The results of [transform rules](/configuration/transforms) are cached so that if a scrobble with identical data (track/artists/album) is identified and it has the same set of transform rules then the cached transform results can be applied. - API Calls to external (metadata) services used to [Enhance Scrobbles](/configuration/transforms), like calls to [Musicbrainz](/configuration/transforms/musicbrainz), can be cached to avoid duplicate calls and speed up scrobble transformations. - -By default, these calls are only cached in memory. If you wish for cached calls to be persisted across restarts then setup [Valkey Cache](./?cacheType=valkey#secondary-caching-configuration). - -##### Configuration - -Use any [Secondary Cache](#secondary-caching-configuration), the config examples below show the default values: - - - - -| Environmental Variable | Required? | Default | Description | -| :--------------------- | --------- | ------- | --------------------- | -| `CACHE_METADATA` | No | | The cache type to use | -| `CACHE_METADATA_CONN` | No | | | - - - - - -```json5 title="config.json" -{ - "cache": { - "metadata": { - "provider": "valkey", - "connection": "yourConnectionStringHere" - } - }, - // ... -} -``` - - + -#### Secondary Caching Configuration + -The type of cache used, and its connection properties, can be configured through **ENV** or **AIO** config. +Auth caching defaults to a **file** that is stored in the `CONFIG_DIR` directory using the pre-defined file name `ms-auth.cache`. - +This provides automatic persistence across restarts for long-lived auth data/credentials if you have configured a [persisted volume/bind mount](/installation?dockerSetting=storage#recommended-settings) for configuration (`/config` is mounted in [docker compose](/quickstart#create-docker-compose-file)). - +If you wish to use the [secondary store](#secondary-caching) for caching Auth you must explicitly configure it. This is because valkey can potentially be *ephemeral* if you do not provide a volume for its data directory. -**File** cache is stored in the `CONFIG_DIR` directory using the pre-defined file name `ms-[cacheName].cache`. +To explicitly configure auth to use the secondary store: - -Example - -| Environmental Variable | Required? | Default | Description | -| :--------------------- | --------- | --------- | ------------------------------------------------------------ | -| `CACHE_SCROBBLE` | No | `file` | The cache type to use | -| `CACHE_SCROBBLE_CONN` | No | `/config` | The directory, within the container, to store the cache file | +```yaml title="compose.yaml" +services: + multi-scrobbler: + # ... + environment: + // highlight-start + - CACHE_AUTH=valkey + // highlight-end + # ... +``` -Example - ```json5 title="config.json" { "cache": { - "scrobble": { - "provider": "file", - "connection": "/config" + "auth": { + "provider": "valkey" } }, // ... } ``` - - - - + + +### Secondary Caching Store {#secondary-caching} + +Using a secondary store enables: -[**Valkey**](https://valkey.io/) is an open-source fork of Redis. +* persistence of cached data across restarts +* a larger store (more data is saved) +* a longer time-to-live in the store (cached data is fetchable for a longer period) + +These benefits are particularly beneficial when using [transforms](/configuration/transforms) like Musicbrainz and it is **strongly recommended** for these scenarios. + +Currently, Multi-scrobbler only supports [**Valkey**](https://valkey.io/), an open-source fork of Redis, as a secondary store.
@@ -466,11 +360,13 @@ Example A valkey container can be added to the [multi-scrobbler docker compose stack](/installation?runType=docker-compose#docker): + + + ```yaml title="docker-compose.yml" services: multi-scrobbler: # ... - # adding everything below // highlight-start valkey: image: valkey/valkey @@ -483,6 +379,25 @@ volumes: // highlight-end ``` + + + +```yaml title="docker-compose.yml" +services: + multi-scrobbler: + # ... + // highlight-start + valkey: + image: valkey/valkey + volumes: + - ./valkeyData:/data + // highlight-end +``` + + + + + Use `redis://valkey:6379` as the connection string in the configurations below.
@@ -497,12 +412,16 @@ redis://HOST_IP:HOST_PORT -Example - -| Environmental Variable | Required? | Default | Description | -| :--------------------- | --------- | -------- | ------------------------------------------------------------------- | -| `CACHE_METADATA` | Yes | `valkey` | The cache type to use | -| `CACHE_METADATA_CONN` | Yes | | The host/IP and port to connect to EX: `redis://192.168.0.120:6379` | +```yaml title="compose.yaml" +services: + multi-scrobbler: + # ... + environment: + // highlight-start + - CACHE_VALKEY=redis://192.168.0.120:6379 + // highlight-end + # ... +``` @@ -513,10 +432,7 @@ Example ```json5 title="config.json" { "cache": { - "metadata": { - "provider": "valkey", - "connection": "redis://192.168.0.120:6379" - } + "valkey": "redis://192.168.0.120:6379" }, // ... } @@ -524,12 +440,9 @@ Example - - - -### Debug Mode +## Debug Mode Turning on Debug Mode will @@ -550,7 +463,7 @@ To set debug mode either add it to [AIO `config.json`](./?configType=aio#configu or set the [ENV](./?configType=env#configuration-types) `DEBUG_MODE=true` -### Disable Web +## Disable Web If you do not need the dashboard and/or ingress sources, or have security concerns about ingress and cannot control your hosting environment, the web server and API can be disabled. diff --git a/docsite/docs/configuration/sources/lastfm-endpoint.mdx b/docsite/docs/configuration/sources/lastfm-endpoint.mdx index bf99c8705..6359d922b 100644 --- a/docsite/docs/configuration/sources/lastfm-endpoint.mdx +++ b/docsite/docs/configuration/sources/lastfm-endpoint.mdx @@ -41,6 +41,6 @@ http://localhost:9078/api/lastfm/mySlug | Environmental Variable | Required? | Default | Description | | :--------------------- | :-------- | ------- | ------------------------------------------------------------------------------------------------------------------ | - | `LFMENDPOINT_ENABLE` | No | | Use LFM Endpoint as a Source without any other configuration. Only required if slug/token are not provided as ENVs | + | `LFM_ENABLE` | No | | Use LFM Endpoint as a Source without any other configuration. Only required if slug/token are not provided as ENVs | | `LFM_SLUG` | No | | (Optional) The URL suffix to use for accepting LFM scrobbles | \ No newline at end of file diff --git a/docsite/docs/configuration/sources/listenbrainz-endpoint.mdx b/docsite/docs/configuration/sources/listenbrainz-endpoint.mdx index ce44dc3c4..976b84bde 100644 --- a/docsite/docs/configuration/sources/listenbrainz-endpoint.mdx +++ b/docsite/docs/configuration/sources/listenbrainz-endpoint.mdx @@ -111,7 +111,7 @@ To troubleshoot any errors, and assuming you are using Home Assistant, view the | Environmental Variable | Required? | Default | Description | | :--------------------- | :-------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | - | `LZENDPOINT_ENABLE` | No | | Use LZ Endpoint as a Source without any other configuration. Only required if slug/token are not provided as ENVs | + | `LZE_ENABLE` | No | | Use LZ Endpoint as a Source without any other configuration. Only required if slug/token are not provided as ENVs | | `LZE_TOKEN` | No | | LZ "Authentication Token" you provided to the scrobbling application | | `LZE_SLUG` | No | | (Optional) The URL suffix to use for accepting LZ scrobbles | | `LZE_USERNAME` | No | | (Optional) A fake username that will be returned for token validation and on now-playing responses. If none is provided the Source's name is used instead. | diff --git a/docsite/docs/configuration/transforms/transforms.mdx b/docsite/docs/configuration/transforms/transforms.mdx index be1a3bcaf..b5780c691 100644 --- a/docsite/docs/configuration/transforms/transforms.mdx +++ b/docsite/docs/configuration/transforms/transforms.mdx @@ -591,7 +591,7 @@ The output shows the diff between the previous stage (or original Play) and the MS uses [caching](/configuration/#caching) to reduce the number of API calls needed for stages like [Musicbrainz](/configuration/transforms/musicbrainz) and to speed up all transforms by caching steps and results. However, the default caching strategy uses a small cache size and very short [TTLs](https://en.wikipedia.org/wiki/Time_to_live) because it is *in-memory*. -**If you are using any Transform stages you should configure [secondary caching with Valkey for Metadata](/configuration/?cacheType=valkey&cachedThings=metadata#secondary-caching-configuration)** to increase the cache size and lifetime of cached items. This will also reduce memory usage in MS. +**If you are using any Transform stages you should configure [secondary caching](/configuration#secondary-caching)** to increase the cache size and lifetime of cached items. This will also reduce memory usage in MS.
diff --git a/docsite/docs/installation/installation.mdx b/docsite/docs/installation/installation.mdx index 83f6f9bbd..93a292960 100644 --- a/docsite/docs/installation/installation.mdx +++ b/docsite/docs/installation/installation.mdx @@ -132,7 +132,7 @@ services: -**Optionally**, add a [Valkey](https://valkey.io/) service to your stack for [secondary caching](/configuration/?cacheType=valkey#secondary-caching-configuration) to take advantage of faster performance and reduced memory usage. +**Optionally**, add a [Valkey](https://valkey.io/) service to your stack for [secondary caching](/configuration#secondary-caching) to take advantage of faster performance and reduced memory usage. ```yaml title="docker-compose.yml" services: @@ -142,8 +142,7 @@ services: environment: # ... // highlight-start - CACHE_METADATA=valkey - CACHE_METADATA_CONN=redis://valkey:6379 + CACHE_VALKEY=redis://valkey:6379 // highlight-end # ... diff --git a/docsite/docs/quickstart.mdx b/docsite/docs/quickstart.mdx index 140d106e5..f38abac6a 100644 --- a/docsite/docs/quickstart.mdx +++ b/docsite/docs/quickstart.mdx @@ -336,7 +336,7 @@ Visit `http://192.168.0.100:9078` to see the dashboard where ## Next Steps * See more advanced docker options as well as other install methods in the [**Installation**](/installation#docker) docs - * Setup [secondary caching](/configuration/?cacheType=valkey#secondary-caching-configuration) with [valkey](/installation?dockerSetting=caching#recommended-settings) for increased performance and reduced memory usage + * Setup [secondary caching](/configuration#secondary-caching) with [valkey](/installation?dockerSetting=caching#recommended-settings) for increased performance and reduced memory usage * Review the [**Configuration**](/configuration) docs * Learn about how to configure multi-scrobbler using files for more complicated Source/Client scenarios * See all available [**Sources**](/configuration/sources) and [**Clients**](/configuration/clients) alongside configuration examples diff --git a/docsite/docs/updating/updating.mdx b/docsite/docs/updating/updating.mdx index 74aec3763..81b1563a9 100644 --- a/docsite/docs/updating/updating.mdx +++ b/docsite/docs/updating/updating.mdx @@ -10,11 +10,37 @@ import CodeBlock from '@theme/CodeBlock'; ## Updating -Currently, multi-scrobbler does not have any databases or dependencies that require additional interaction when updating. +The majority of Multi-scrobbler updates can be completed without any manual intervention. This is **guaranteed** for [patch version updates](#versioning). -Any **breaking changes** will be related to [configuration](/configuration) that has been deprecated/changed, or tooling that usually only affects [Local Installations.](/installation#local-installation) +Regardless, it is recommended to consult the [**Github Release Notes**](https://github.com/FoxxMD/multi-scrobbler/releases) before any upgrades. The release notes contain all changelogs as well as most **breaking changes**/notices. -These changes, and how to update configs accordingly, are detailed in [**Github Release Notes**](https://github.com/FoxxMD/multi-scrobbler/releases). It is recommended to check this page before upgrading [minor or major versions.](#versioning) +The [**Upgrade Path** docs](/updating/upgrade-path) contain information for upgrading through **required versions** or in-depth migration guides. Check this section before upgrading any [minor versions](#versioning). + +### Database + +Multi-scrobbler depends on a SQLite database (`ms.db`) that is created on first run and stored in the [`CONFIG_DIR`](/installation/?dockerSetting=storage#recommended-settings). + +If any database migrations are required for an update then your database is **automatically backed up** before migration occurs. The backup file is created in the same directory. + +You can manually make a backup of this database by making a copy of `ms.db` and any similarly named files like `ms.db-journal`. + +:::tip + +If upgrading a [minor version](#versioning) you may want to make a backup for extra safety. + +::: + +### Configuration + +[Minor versions](#versioning) may have **breaking changes** related to [configuration](/configuration). Consult the [**Github Release Notes**](https://github.com/FoxxMD/multi-scrobbler/releases) and [Upgrade Paths](/updating/upgrade-path) before upgrading. + + +## Update Instructions + +Assuming: + +* upgrade(s) are only [patch versions](#versioning) and the [Release Notes](https://github.com/FoxxMD/multi-scrobbler/releases) do not contain any other guidance or +* you have checked the release notes, upgrade paths docs, and completed all migration steps for the specific version upgrade diff --git a/docsite/docs/updating/upgrade-path/0140.mdx b/docsite/docs/updating/upgrade-path/0140.mdx new file mode 100644 index 000000000..06c98dd89 --- /dev/null +++ b/docsite/docs/updating/upgrade-path/0140.mdx @@ -0,0 +1,212 @@ +--- +sidebar_position: 1 +title: From < 0.14.0 +description: Upgrading to 0.14.0 +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import RequiredUpgrade from "@site/src/components/snippets/_upgrade-required.mdx" + + + +## Source/Client IDs + +MS `0.14.0` introduces a database for persisting Plays/Scrobbles, queues, and other data **associated with a Source/Client**. + +To make this association atomic a Source/Client **ID** is now configurable. This ID is what MS will use to identify the Source/Client in your **config** with the Source/Client in the **database**. + +This ID needs to be unique to the client or source type it is used on IE: + +* two [Koito Clients](/configuration/clients/koito/) cannot both have the ID `myKoitoID` +* it is recommended that the ID be *globally* unique among all sources/clients, but it's not required + +### Configuring IDs + + + + + +For each Source/Client in your environmental variables, add a key with the suffix `_ID` and the ID value. EX + +```yaml title="compose.yaml" +services: + multi-scrobbler: + # ... + environment: + - KOTIO_TOKEN=... + # ...more koito config + // highlight-start + - KOITO_ID=myKoitoID + // highlight-end + + - JELLYFIN_URL=... + # ...more jellyin config + // highlight-start + - JELLYFIN_ID=myJellyfinID + // highlight-end +``` + + + + +Add an `id` to the top-level for each Source/Client configuration, next to `data`: + +```json title="koito.json" +[ + { + "name": "koito-source", + "configureAs": "source", + // highlight-start + "id": "myKoitoID", + // highlight-end + "data": { + "token": "029b081ba-9156-4pe7-88e5-3be671f5ea2b", + "username": "admin", + "url": "http://192.168.0.100:4110" + } + } +] +``` + + + + +Add an `id` to the top-level for each Source/Client configuration, next to `data`: + +```json title="koito.json" +[ + { + "name": "koito-source", + "configureAs": "source", + "type": "koito", + // highlight-start + "id": "myKoitoID", + // highlight-end + "data": { + "token": "029b081ba-9156-4pe7-88e5-3be671f5ea2b", + "username": "admin", + "url": "http://192.168.0.100:4110" + } + } +] +``` + + + + +:::note[Default ID] + +If you do not add an ID then Multi-Scrobbler will automatically use the **name** of the Source/Client as the ID. The name is shown in the Dashboard. + +If you decide to add an ID later you must use the name as the ID in order to keep Plays/Scobbles associated with the same config. + +::: + +## Cached Scrobble Migration + +:::tip + +Before upgrading, if your MS dashboard shows 0 queued/failed for all [Scrobble Clients](/configuration/clients) then you **skip this step** and can safely delete `ms-scrobble.cache` before the upgrade occurs. + +::: + +MS `0.14.0` introduces a database for persisting Plays/Scrobbles, queues, and other data. Prior to `0.14.0`, queued/failed scrobbles were stored in a cache **file** inside your [`CONFIG_DIR`](/installation/?dockerSetting=storage#recommended-settings) named `ms-scrobble.cache`. + +On first run of `0.14.0`: + +* MS will make a copy of `ms-scrobble.cache` named `ms-scrobble.cache.bak` + * You may make a manual copy of this file before upgrading, for additional safety +* Scrobbles in the cache will automatically be migrated to the new database +* The now "old" cache data will be cleared so that subsequent application starts don't duplicate migrations + * the empty `ms-scrobble.cache` file will remain + +You can follow progress of this by looking for log lines starting with `Migrating cached scrobbles to database...` + +If MS does not report/log any errors during this time and you see your scrobbles processed normally then it is safe to delete `ms-scrobble.cache` the next time MS is stopped/restarted. + +## Cache Configuration + +In `0.14.0` [Cache](/configuration#caching) has been simplified with much of the required configuration being removed. + +:::tip + +This only applies to users who have `cache` in their [AIO Config](/configuration/?configType=aio#configuration-types) `config.json` or are using `CACHE_*` [ENV Config](/configuration/?configType=env#configuration-types) + +If you do not have any of the above [Cache](/configuration#caching) configuration defined then you can **skip this step.** + +::: + +#### Scrobble Caching Removed + +[Scrobble caching has removed and replaced by the new database.](#cached-scrobble-migration) + +* Remove any ENV Config starting with `CACHE_SCROBBLE` +* Remove `cache.scrobble` from the AIO Config + +#### Metadata Config Simplified + +Metadata caching remains the same but the config has been simplified. If you were using [Valkey for caching](/configuration/#secondary-caching) update your config: + + + + + +* Remove `CACHE_METADATA` +* Rename `CACHE_METADATA_CONN` to [`CACHE_VALKEY`](/configuration/#secondary-caching) + + + + +Remove `cache.metadata` and add a new string key `valkey` to the `cache` object, containing your valkey connection string: + +```diff + "cache": { ++ "valkey": "redis://valkey:6379" +- "metadata": { +- "provider": "valkey", +- "connection": "redis://valkey:6379" +- } + } +``` + + + + + +#### Auth Config Simplified + +Auth caching remains the same but the config has been simplified. The connection option for Auth is no longer configurable. You may specify provider as either `file` (default, uses `CONFIG_DIR`) or `valkey`. If you want to use Valkey for auth it will use the same config as Metadata. + + + + + +* Remove `CACHE_AUTH_CONN` +* Add [`CACHE_VALKEY`](/configuration/#secondary-caching) (if using valkey) + + + + +Remove the `connection` property from the `auth` object. If using `file` the entire `auth` object can be removed. + +```diff + "cache": { ++ "valkey": "redis://valkey:6379", + "auth": { + "provider": "valkey", +- "connection": "redis://valkey:6379" + } + } +``` + + + + +### Renamed ENV Keys + +In an effort to standard ENV prefixes some ENV keys have been renamed: + +* `LZENDPOINT_ENABLE` => `LZE_ENABLE` +* `LFMENDPOINT_ENABLE` => `LFM_ENABLE` \ No newline at end of file diff --git a/docsite/docs/updating/upgrade-path/_category_.json b/docsite/docs/updating/upgrade-path/_category_.json new file mode 100644 index 000000000..ab29ac513 --- /dev/null +++ b/docsite/docs/updating/upgrade-path/_category_.json @@ -0,0 +1,7 @@ +{ + "label": "Upgrade Path", + "link": { + "type": "doc", + "id": "updating/upgrade-path/upgrade-path" + } +} diff --git a/docsite/docs/updating/upgrade-path/upgrade-path.mdx b/docsite/docs/updating/upgrade-path/upgrade-path.mdx new file mode 100644 index 000000000..8f3505b6e --- /dev/null +++ b/docsite/docs/updating/upgrade-path/upgrade-path.mdx @@ -0,0 +1,39 @@ +--- +sidebar_position: 1 +title: Upgrade Path +description: Updating Multi-Scrobbler +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +The majority of changes are detailed in [**Github Release Notes**](https://github.com/FoxxMD/multi-scrobbler/releases), including *most* breaking changes. You should consult the release notes for each version you will be updating through/to. + +Upgrade paths for specific version, detailed here, are reserved for: + + + + + +:::warning[Required Upgrade] + +Versions you **must** upgrade to **before** upgrading to a newer version. Likely due to the codebase containing functionality to upgrade persistent data that cannot be migrating across multiple versions. + +::: + + + + + +:::note[Optional Upgrade] + +(Breaking) Changes to configuration/data that require more detail or formatting than is feasible in Github release notes. These upgrades may be skipped. + +::: + + + + + +The type of upgrade is shown at the top of each page with the same colored callout shown above. \ No newline at end of file diff --git a/docsite/src/components/AdmonitionDetails.tsx b/docsite/src/components/AdmonitionDetails.tsx index fa098547f..5f3352e97 100644 --- a/docsite/src/components/AdmonitionDetails.tsx +++ b/docsite/src/components/AdmonitionDetails.tsx @@ -5,9 +5,10 @@ import IconDanger from '@theme/Admonition/Icon/Danger'; import IconTip from '@theme/Admonition/Icon/Tip'; import IconNote from '@theme/Admonition/Icon/Note'; import { ReactElement } from 'react'; +import AdmonitionIconImportant from './ImportantIcon'; export interface DetailsAdmoProps extends DetailProps { - type?: 'warning' | 'danger' | 'note' | 'tip' + type?: 'warning' | 'danger' | 'note' | 'tip' | 'important' } const DetailsAdmo = (props: DetailsAdmoProps) => { @@ -32,6 +33,10 @@ const DetailsAdmo = (props: DetailsAdmoProps) => { cn = 'alert--success'; icon = break; + case 'important': + cn = 'alert--important'; + icon = ; + break; } const iconWrapper = icon === undefined ? null : {icon}; diff --git a/docsite/src/components/ImportantIcon.tsx b/docsite/src/components/ImportantIcon.tsx new file mode 100644 index 000000000..b37786787 --- /dev/null +++ b/docsite/src/components/ImportantIcon.tsx @@ -0,0 +1,13 @@ +import type { Props } from '@theme/Admonition/Icon/Info'; + +export default function AdmonitionIconImportant(props: Props) { + // based on https://icon-sets.iconify.design/zondicons/exclamation-outline/ + return ( + + + + ); +} \ No newline at end of file diff --git a/docsite/src/components/snippets/_upgrade-optional.mdx b/docsite/src/components/snippets/_upgrade-optional.mdx new file mode 100644 index 000000000..e69de29bb diff --git a/docsite/src/components/snippets/_upgrade-required.mdx b/docsite/src/components/snippets/_upgrade-required.mdx new file mode 100644 index 000000000..657519f96 --- /dev/null +++ b/docsite/src/components/snippets/_upgrade-required.mdx @@ -0,0 +1,15 @@ +:::warning[Required Upgrade] + +You **MUST** upgrade to this version **before** upgrading Multi-Scrobbler to a newer version. + +This version contains functionality to migrate your data from **{props.before}** that newer versions depend on. + +
+ +Upgrade Process Example + +* Upgrade Multi-Scrobbler {props.old} to {props.current} +* **After** upgrading, you can optionally upgrade to > {props.current} + +
+::: \ No newline at end of file diff --git a/docsite/src/css/custom.css b/docsite/src/css/custom.css index 25603721f..46485615b 100644 --- a/docsite/src/css/custom.css +++ b/docsite/src/css/custom.css @@ -13,6 +13,7 @@ --ifm-color-primary-light: #33925d; --ifm-color-primary-lighter: #359962; --ifm-color-primary-lightest: #3cad6e; + --ifm-color-important-contrast-background: #bb91ff; --ifm-code-font-size: 95%; --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.1); } @@ -26,6 +27,7 @@ --ifm-color-primary-light: #29d5b0; --ifm-color-primary-lighter: #32d8b4; --ifm-color-primary-lightest: #4fddbf; + --ifm-color-important-contrast-background: #5a3795; --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3); } @@ -59,6 +61,10 @@ details[class^='details_'].alert--info { --ifm-alert-background-color: var(--ifm-color-secondary-contrast-background); --ifm-alert-foreground-color: var(--ifm-color-secondary-contrast-foreground); } + &.alert--important { + --ifm-alert-background-color: var(--ifm-color-important-contrast-background); + --ifm-alert-foreground-color: var(--ifm-color-secondary-contrast-foreground); + } > summary { --docusaurus-details-decoration-color: currentColor; diff --git a/drizzle.config.ts b/drizzle.config.ts new file mode 100644 index 000000000..8f3a7112f --- /dev/null +++ b/drizzle.config.ts @@ -0,0 +1,13 @@ +import 'dotenv/config'; +import { defineConfig } from 'drizzle-kit'; +import { configDir, projectDir } from './src/backend/common/index.js'; +import * as path from 'path'; + +export default defineConfig({ + schema: path.resolve(projectDir, 'src/backend/common/database/drizzle/schema'), + out: path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations'), + dialect: 'postgresql', + dbCredentials: { + url: path.resolve(configDir, 'msDb'), + }, +}); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 28d54273d..82d80b471 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,8 @@ "@atproto/api": "^0.18.0", "@atproto/oauth-client-node": "^0.3.10", "@donedeal0/superdiff": "^1.1.1", + "@electric-sql/pglite": "^0.4.5", + "@electric-sql/pglite-socket": "^0.1.5", "@ewanc26/tid": "^1.0.2", "@foxxmd/chromecast-client": "^1.0.4", "@foxxmd/get-version": "^0.0.3", @@ -47,6 +49,7 @@ "dbus-ts": "^0.0.7", "discord.js": "^14.26.0", "dotenv": "^10.0.0", + "drizzle-orm": "^1.0.0-rc.2-c5a84d1", "express": "^5.2.1", "express-session": "^1.19.0", "fast-equals": "^6.0.0", @@ -71,6 +74,7 @@ "mpc-js": "^2.1.1", "musicbrainz-api": "^0.27.0", "nanoid": "^3.3.1", + "neotraverse": "^0.6.18", "node-object-hash": "^3.1.1", "normalize-url": "^8.0.1", "ntfy": "^1.15.2", @@ -102,6 +106,7 @@ "@chromatic-com/storybook": "^5.0.1", "@curvenote/ansi-to-react": "^7.0.0", "@dbus-types/notifications": "^0.0.5", + "@electric-sql/pglite-prepopulatedfs": "^0.0.3", "@emotion/react": "^11.14.0", "@eslint/js": "^8.56.0", "@faker-js/faker": "^9.0.1", @@ -129,7 +134,7 @@ "@types/jest": "^27.5.2", "@types/jscodeshift": "^0.11.6", "@types/mocha": "^9.1.0", - "@types/node": "^20.19.2", + "@types/node": "^24.12.2", "@types/object-hash": "^3.0.0", "@types/passport": "^1.0.12", "@types/react": "^18.2.18", @@ -144,6 +149,7 @@ "chai": "^4.3.6", "chai-as-promised": "^8.0.2", "clsx": "^2.1.1", + "drizzle-kit": "^1.0.0-rc.2-c5a84d1", "eslint": "^8.56.0", "eslint-plugin-prefer-arrow-functions": "^3.2.4", "eslint-plugin-storybook": "10.1.11", @@ -152,7 +158,6 @@ "mocha": "^10.3.0", "mockdate": "^3.0.5", "msw": "^2.12.10", - "neotraverse": "^0.6.18", "next-themes": "^0.4.6", "nodemon": "^3.0.3", "playwright": "^1.58.2", @@ -168,11 +173,12 @@ "sinon": "^21.0.2", "storybook": "10.2.0", "tailwindcss": "^4.2.2", - "ts-essentials": "^10.1.1", + "tinyexec": "^1.1.1", + "ts-essentials": "^10.2.0", "typescript": "^5.9.3", "typescript-eslint": "^7.0.1", "vite": "^5.4.21", - "with-local-tmp-dir": "^6.0.0" + "with-local-tmp-dir": "^7.0.1" }, "engines": { "node": ">=24.14.0", @@ -1055,6 +1061,13 @@ "version": "1.1.3", "license": "ISC" }, + "node_modules/@drizzle-team/brocli": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@drizzle-team/brocli/-/brocli-0.11.0.tgz", + "integrity": "sha512-hD3pekGiPg0WPCCGAZmusBBJsDqGUR66Y452YgQsZOnkdQ7ViEPKuyP4huUGEZQefp8g34RRodXYmJ2TbCH+tg==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/@dword-design/chdir": { "version": "4.0.0", "dev": true, @@ -1066,6 +1079,31 @@ "url": "https://github.com/sponsors/dword-design" } }, + "node_modules/@electric-sql/pglite": { + "version": "0.4.5", + "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.4.5.tgz", + "integrity": "sha512-aGG2zGEyZzGWKy8P+9ZoNUV0jxt1+hgbeTf+bVAYyxVZZLXg3/9aFlfLxb08AYZVAfAkQlQIysmWjhc5hwDG8g==", + "license": "Apache-2.0" + }, + "node_modules/@electric-sql/pglite-prepopulatedfs": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@electric-sql/pglite-prepopulatedfs/-/pglite-prepopulatedfs-0.0.3.tgz", + "integrity": "sha512-3MNFt+gR0P22foWi55j/HZ6DvQ82DEVIvmoKVYCdoG/gezMikGR794tO07/15CV4RcR3PnfCPoQjApPfXfD01w==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/@electric-sql/pglite-socket": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/@electric-sql/pglite-socket/-/pglite-socket-0.1.5.tgz", + "integrity": "sha512-/RAye+3EPKfO9nY4tljzxXmkT7yIpFDm0L3F+c28b+Z6uxPOjy/Zz/QEHYHXcrfuUC88/a9S72EO0+3E0j97wQ==", + "license": "Apache-2.0", + "bin": { + "pglite-server": "dist/scripts/server.js" + }, + "peerDependencies": { + "@electric-sql/pglite": "0.4.5" + } + }, "node_modules/@emotion/babel-plugin": { "version": "11.13.5", "dev": true, @@ -2174,6 +2212,19 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@js-temporal/polyfill": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/@js-temporal/polyfill/-/polyfill-0.5.1.tgz", + "integrity": "sha512-hloP58zRVCRSpgDxmqCWJNlizAlUgJFqG2ypq79DCvyv9tHjRYMDOcPFjzfl/A1/YxDvRCZz8wvZvmapQnKwFQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "jsbi": "^4.3.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/@kenyip/backoff-strategies": { "version": "1.0.4", "license": "MIT" @@ -4288,10 +4339,12 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "20.19.37", + "version": "24.12.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.2.tgz", + "integrity": "sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g==", "license": "MIT", "dependencies": { - "undici-types": "~6.21.0" + "undici-types": "~7.16.0" } }, "node_modules/@types/object-hash": { @@ -4364,11 +4417,6 @@ "@types/node": "*" } }, - "node_modules/@types/retry": { - "version": "0.12.2", - "dev": true, - "license": "MIT" - }, "node_modules/@types/send": { "version": "0.17.4", "dev": true, @@ -7241,6 +7289,665 @@ "node": ">=10" } }, + "node_modules/drizzle-kit": { + "version": "1.0.0-rc.2-c5a84d1", + "resolved": "https://registry.npmjs.org/drizzle-kit/-/drizzle-kit-1.0.0-rc.2-c5a84d1.tgz", + "integrity": "sha512-TjXBd6/Jo8aqC2uBbbgPoIVE5Qr9s0KubVtdj3Ro7bF2lz1TkGOhnDIzsOgMJPMSPn9uyhNG6WBKdGz+XzcL9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@drizzle-team/brocli": "^0.11.0", + "@js-temporal/polyfill": "^0.5.1", + "esbuild": "^0.25.10", + "get-tsconfig": "^4.13.6", + "jiti": "^2.6.1" + }, + "bin": { + "drizzle-kit": "bin.cjs" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/drizzle-orm": { + "version": "1.0.0-rc.2-c5a84d1", + "resolved": "https://registry.npmjs.org/drizzle-orm/-/drizzle-orm-1.0.0-rc.2-c5a84d1.tgz", + "integrity": "sha512-2nw0eVFNcLRU7mhs/f/UodwvRIP78ZzZXMrl/r6LpTpcJ42ZzQZZTH0ObPCR722SaEzn35kT4gvuoiam/OCK9g==", + "license": "Apache-2.0", + "peerDependencies": { + "@aws-sdk/client-rds-data": ">=3", + "@cloudflare/workers-types": ">=4", + "@effect/sql-pg": ">=4.0.0-beta.58 || >=4.0.0", + "@electric-sql/pglite": ">=0.2.0", + "@libsql/client": ">=0.10.0", + "@libsql/client-wasm": ">=0.10.0", + "@neondatabase/serverless": ">=0.10.0", + "@op-engineering/op-sqlite": ">=2", + "@opentelemetry/api": "^1.4.1", + "@planetscale/database": ">=1.13", + "@sinclair/typebox": ">=0.34.8", + "@sqlitecloud/drivers": ">=1.0.653", + "@tidbcloud/serverless": "*", + "@tursodatabase/database": ">=0.2.1", + "@tursodatabase/database-common": ">=0.2.1", + "@tursodatabase/database-wasm": ">=0.2.1", + "@types/better-sqlite3": "*", + "@types/mssql": "^9.1.4", + "@types/pg": "*", + "@types/sql.js": "*", + "@upstash/redis": ">=1.34.7", + "@vercel/postgres": ">=0.8.0", + "@xata.io/client": "*", + "arktype": ">=2.0.0", + "better-sqlite3": ">=9.3.0", + "bun-types": "*", + "effect": ">=4.0.0-beta.58 || >=4.0.0", + "expo-sqlite": ">=14.0.0", + "mssql": "^11.0.1", + "mysql2": ">=2", + "pg": ">=8", + "postgres": ">=3", + "sql.js": ">=1", + "sqlite3": ">=5", + "typebox": ">=1.0.0", + "valibot": ">=1.0.0-beta.7", + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "@aws-sdk/client-rds-data": { + "optional": true + }, + "@cloudflare/workers-types": { + "optional": true + }, + "@effect/sql-pg": { + "optional": true + }, + "@electric-sql/pglite": { + "optional": true + }, + "@libsql/client": { + "optional": true + }, + "@libsql/client-wasm": { + "optional": true + }, + "@neondatabase/serverless": { + "optional": true + }, + "@op-engineering/op-sqlite": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@planetscale/database": { + "optional": true + }, + "@sinclair/typebox": { + "optional": true + }, + "@sqlitecloud/drivers": { + "optional": true + }, + "@tidbcloud/serverless": { + "optional": true + }, + "@tursodatabase/database": { + "optional": true + }, + "@tursodatabase/database-common": { + "optional": true + }, + "@tursodatabase/database-wasm": { + "optional": true + }, + "@types/better-sqlite3": { + "optional": true + }, + "@types/mssql": { + "optional": true + }, + "@types/pg": { + "optional": true + }, + "@types/sql.js": { + "optional": true + }, + "@upstash/redis": { + "optional": true + }, + "@vercel/postgres": { + "optional": true + }, + "@xata.io/client": { + "optional": true + }, + "arktype": { + "optional": true + }, + "better-sqlite3": { + "optional": true + }, + "bun-types": { + "optional": true + }, + "effect": { + "optional": true + }, + "expo-sqlite": { + "optional": true + }, + "mssql": { + "optional": true + }, + "mysql2": { + "optional": true + }, + "pg": { + "optional": true + }, + "postgres": { + "optional": true + }, + "sql.js": { + "optional": true + }, + "sqlite3": { + "optional": true + }, + "typebox": { + "optional": true + }, + "valibot": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "license": "MIT", @@ -8564,7 +9271,9 @@ } }, "node_modules/get-tsconfig": { - "version": "4.8.1", + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz", + "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==", "license": "MIT", "dependencies": { "resolve-pkg-maps": "^1.0.0" @@ -9597,6 +10306,13 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsbi": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/jsbi/-/jsbi-4.3.2.tgz", + "integrity": "sha512-9fqMSQbhJykSeii05nxKl4m6Eqn2P6rOlYiS+C5Dr/HPIU/7yZxu5qzbs40tgaFORiw2Amd0mirjxatXYMkIew==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/jsesc": { "version": "3.1.0", "dev": true, @@ -10924,7 +11640,8 @@ }, "node_modules/neotraverse": { "version": "0.6.18", - "dev": true, + "resolved": "https://registry.npmjs.org/neotraverse/-/neotraverse-0.6.18.tgz", + "integrity": "sha512-Z4SmBUweYa09+o6pG+eASabEpP6QkQ70yHj351pQoEXIs8uHbaU2DWVmzBANKgflPa47A50PtB2+NgRpQvr7vA==", "license": "MIT", "engines": { "node": ">= 10" @@ -13430,6 +14147,16 @@ "dev": true, "license": "MIT" }, + "node_modules/tinyexec": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.1.1.tgz", + "integrity": "sha512-VKS/ZaQhhkKFMANmAOhhXVoIfBXblQxGX1myCQ2faQrfmobMftXeJPcZGp0gS07ocvGJWDLZGyOZDadDBqYIJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/tinyglobby": { "version": "0.2.15", "dev": true, @@ -13594,7 +14321,9 @@ } }, "node_modules/ts-essentials": { - "version": "10.1.1", + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/ts-essentials/-/ts-essentials-10.2.0.tgz", + "integrity": "sha512-z9FlLywg0XEV46Ws1FwYN4NZDMr9qAe38lTTtgVBqzhhyEgwrnCUkFe4MEqnvar1kY1kFEnlkp56bxn2g0V+UA==", "dev": true, "license": "MIT", "peerDependencies": { @@ -13947,7 +14676,9 @@ } }, "node_modules/undici-types": { - "version": "6.21.0", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", "license": "MIT" }, "node_modules/unicorn-magic": { @@ -14723,37 +15454,23 @@ } }, "node_modules/with-local-tmp-dir": { - "version": "6.0.0", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/with-local-tmp-dir/-/with-local-tmp-dir-7.0.1.tgz", + "integrity": "sha512-ipyC5Q9ZBbpeLnrBi4BEUElnvDxv0QM0I+ptSIGUMegR9/KVz/bd+WMMWm0Q0qnjOSwSRaxqVfqzy0kVPmaVig==", "dev": true, "license": "MIT", "dependencies": { "@dword-design/chdir": "^4.0.0", "@types/fs-extra": "^11.0.4", - "p-retry": "^6.2.1" + "p-retry": "^7.1.1" }, "engines": { - "node": ">=20" + "node": ">=22" }, "funding": { "url": "https://github.com/sponsors/dword-design" } }, - "node_modules/with-local-tmp-dir/node_modules/p-retry": { - "version": "6.2.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/retry": "0.12.2", - "is-network-error": "^1.0.0", - "retry": "^0.13.1" - }, - "engines": { - "node": ">=16.17" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/word-wrap": { "version": "1.2.5", "dev": true, @@ -15089,7 +15806,9 @@ } }, "node_modules/zod": { - "version": "3.23.8", + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" @@ -15097,6 +15816,8 @@ }, "node_modules/zod-validation-error": { "version": "2.1.0", + "resolved": "https://registry.npmjs.org/zod-validation-error/-/zod-validation-error-2.1.0.tgz", + "integrity": "sha512-VJh93e2wb4c3tWtGgTa0OF/dTt/zoPCPzXq4V11ZjxmEAFaPi/Zss1xIZdEB5RD8GD00U0/iVXgqkF77RV7pdQ==", "license": "MIT", "engines": { "node": ">=18.0.0" diff --git a/package.json b/package.json index 02066d3f8..9951bd2cc 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "build:backend": "tsc -p src/backend && npm run -s schema:app", "build": "npm run -s build:backend && npm run -s build:frontend && npm run -s docs:build", "build:parallel": "concurrently --kill-others-on-fail --names backend,frontend,docs \"npm run -s build:backend\" \"npm run -s build:frontend\" \"npm run docs:build\"", + "db:start": "pglite-server --db=./config/msDb --port=5433 --host=0.0.0.0 -m 10", "docs:install": "cd docsite && npm install --no-audit", "docs:start": "cd docsite && npm start", "docs:build": "npm run -s schema:docs && cd docsite && npm run build", @@ -53,6 +54,8 @@ "@atproto/api": "^0.18.0", "@atproto/oauth-client-node": "^0.3.10", "@donedeal0/superdiff": "^1.1.1", + "@electric-sql/pglite": "^0.4.5", + "@electric-sql/pglite-socket": "^0.1.5", "@ewanc26/tid": "^1.0.2", "@foxxmd/chromecast-client": "^1.0.4", "@foxxmd/get-version": "^0.0.3", @@ -85,6 +88,7 @@ "dbus-ts": "^0.0.7", "discord.js": "^14.26.0", "dotenv": "^10.0.0", + "drizzle-orm": "^1.0.0-rc.2-c5a84d1", "express": "^5.2.1", "express-session": "^1.19.0", "fast-equals": "^6.0.0", @@ -109,6 +113,7 @@ "mpc-js": "^2.1.1", "musicbrainz-api": "^0.27.0", "nanoid": "^3.3.1", + "neotraverse": "^0.6.18", "node-object-hash": "^3.1.1", "normalize-url": "^8.0.1", "ntfy": "^1.15.2", @@ -140,6 +145,7 @@ "@chromatic-com/storybook": "^5.0.1", "@curvenote/ansi-to-react": "^7.0.0", "@dbus-types/notifications": "^0.0.5", + "@electric-sql/pglite-prepopulatedfs": "^0.0.3", "@emotion/react": "^11.14.0", "@eslint/js": "^8.56.0", "@faker-js/faker": "^9.0.1", @@ -167,7 +173,7 @@ "@types/jest": "^27.5.2", "@types/jscodeshift": "^0.11.6", "@types/mocha": "^9.1.0", - "@types/node": "^20.19.2", + "@types/node": "^24.12.2", "@types/object-hash": "^3.0.0", "@types/passport": "^1.0.12", "@types/react": "^18.2.18", @@ -182,6 +188,7 @@ "chai": "^4.3.6", "chai-as-promised": "^8.0.2", "clsx": "^2.1.1", + "drizzle-kit": "^1.0.0-rc.2-c5a84d1", "eslint": "^8.56.0", "eslint-plugin-prefer-arrow-functions": "^3.2.4", "eslint-plugin-storybook": "10.1.11", @@ -190,7 +197,6 @@ "mocha": "^10.3.0", "mockdate": "^3.0.5", "msw": "^2.12.10", - "neotraverse": "^0.6.18", "next-themes": "^0.4.6", "nodemon": "^3.0.3", "playwright": "^1.58.2", @@ -206,11 +212,12 @@ "sinon": "^21.0.2", "storybook": "10.2.0", "tailwindcss": "^4.2.2", - "ts-essentials": "^10.1.1", + "tinyexec": "^1.1.1", + "ts-essentials": "^10.2.0", "typescript": "^5.9.3", "typescript-eslint": "^7.0.1", "vite": "^5.4.21", - "with-local-tmp-dir": "^6.0.0" + "with-local-tmp-dir": "^7.0.1" }, "browserslist": { "production": [ diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index bab2f1229..514bd6221 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -6,7 +6,7 @@ import { LifecycleStep, PlayData, PlayObject, TransformResult } from "../../core import { buildPlayHumanDiffable, buildTrackString } from "../../core/StringUtils.js"; import { CommonClientConfig } from "./infrastructure/config/client/index.js"; import { CommonSourceConfig } from "./infrastructure/config/source/index.js"; -import { mergeSimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.js"; +import { mergeSimpleError, SimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.js"; import { FLOW_CONTROL_TERM, PlayTransformRules, @@ -20,13 +20,21 @@ import { getRoot } from "../ioc.js"; import { nanoid } from "nanoid"; import { isDebugMode } from "../utils.js"; import { findCauseByReference } from "../utils/ErrorUtils.js"; -import { hashObject } from "../utils/StringUtils.js"; +import { hashObject, parseArrayFromMaybeString } from "../utils/StringUtils.js"; import { metaInvariantTransform, playContentInvariantTransform } from "../utils/PlayComparisonUtils.js"; import { MSCache } from "./Cache.js"; import { diffObjects, diffObjectsConsoleOutput, patchObject } from "../../core/DataUtils.js"; import clone from "clone"; import { loggerNoop } from "./MaybeLogger.js"; import { objectsEqual } from "../utils/DataUtils.js"; +import { RetentionOptions } from "./infrastructure/config/database.js"; +import { getRetentionCompactAfterFromEnv, getRetentionDeleteAfterFromEnv, isCompactableProperty, parseRetentionOptions, parseRetentionOptionsDurations } from "./database/Database.js"; +import { DbConcrete } from "./database/drizzle/drizzleUtils.js"; +import { ComponentSelect } from "./database/drizzle/drizzleTypes.js"; +import { DrizzlePlayRepository } from "./database/drizzle/repositories/PlayRepository.js"; +import { ClientType } from "./infrastructure/config/client/clients.js"; +import { SourceType } from "./infrastructure/config/source/sources.js"; +import { DrizzleComponentRepository } from "./database/drizzle/repositories/ComponentRepository.js"; export type AbstractComponentConfig = (CommonClientConfig | CommonSourceConfig) & { transformManager?: TransformerManager }; @@ -38,11 +46,27 @@ export default abstract class AbstractComponent extends AbstractInitializable { regexCache!: ReturnType; protected transformManager: TransformerManager; protected cache: MSCache; + protected db: DbConcrete; + protected componentRepo!: DrizzleComponentRepository; + protected dbComponent!: ComponentSelect; + protected retentionOpts: RetentionOptions; + + protected componentType: 'source' | 'client'; + type: ClientType | SourceType; protected constructor(config: AbstractComponentConfig) { super(config); this.transformManager = config.transformManager ?? getRoot().items.transformerManager; this.cache = getRoot().items.cache(); + const cProps = config.options?.retention?.compact ?? parseArrayFromMaybeString(process.env.COMPACT_PROPERTIES, {lower: true}); + if(!cProps.every(isCompactableProperty)) { + throw new SimpleError(`Compactable properties must be one of 'transform' or 'input'. Given: ${cProps.join(',')}`); + } + this.retentionOpts = { + deleteAfter: parseRetentionOptionsDurations(config.options?.retention?.deleteAfter, getRetentionDeleteAfterFromEnv()), + compactAfter: parseRetentionOptions(config.options?.retention?.compactAfter, getRetentionCompactAfterFromEnv()), + compact: cProps + }; } protected postCache(): Promise { @@ -54,6 +78,25 @@ export default abstract class AbstractComponent extends AbstractInitializable { } } + protected async doBuildDatabase(): Promise { + super.doBuildDatabase(); + + let name: string; + if('name' in this) { + name = this.name as string; + } + + this.db = await getRoot().items.db(); + this.componentRepo = new DrizzleComponentRepository(this.db, {logger: this.logger}); + this.dbComponent = await this.componentRepo.findOrInsert({ + mode: this.componentType, + type: this.type, + uid: this.config.id ?? this.config.name ?? name, + name: this.config.name ?? name + }); + return true; + } + public buildTransformRules() { this.logger.debug('Building transformer rules...'); try { @@ -154,6 +197,19 @@ export default abstract class AbstractComponent extends AbstractInitializable { } } + public retentionCleanup = async () => { + if(this.databaseOK !== true) { + this.logger.warn(`Cannot run retention cleanup because ${this.componentType} database state is not OK`); + return; + } + try { + const repo = new DrizzlePlayRepository(this.db, {logger: this.logger}); + await repo.retentionCleanup(this.componentType, this.retentionOpts); + } catch (e) { + this.logger.warn(new Error('Failed to do retention cleanup', {cause: e})); + } + } + protected transformPartToStrong(data: any) { if(data === undefined) { return undefined; diff --git a/src/backend/common/AbstractInitializable.ts b/src/backend/common/AbstractInitializable.ts index 43c688424..56afa7dcf 100644 --- a/src/backend/common/AbstractInitializable.ts +++ b/src/backend/common/AbstractInitializable.ts @@ -13,6 +13,7 @@ export default abstract class AbstractInitializable { authFailure?: boolean; buildOK?: boolean | null; + databaseOK?: boolean | null; connectionOK?: boolean | null; cacheOK?: boolean | null; @@ -41,6 +42,7 @@ export default abstract class AbstractInitializable { if(this.componentLogger === undefined) { await this.buildComponentLogger(); } + await this.buildDatabase(force); await this.buildInitData(force); await this.parseCache(force); try { @@ -96,13 +98,13 @@ export default abstract class AbstractInitializable { if(!force) { return; } - this.logger.debug('Cache OK but step was forced'); + this.logger.verbose('Cache OK but step was forced'); } try { const res = await this.doParseCache(); if(res === undefined) { this.cacheOK = null; - this.logger.debug('No cache to parse.'); + this.logger.trace('No cache to parse.'); return; } if (res === true) { @@ -139,13 +141,13 @@ export default abstract class AbstractInitializable { if(!force) { return; } - this.logger.debug('Build OK but step was forced'); + this.logger.verbose('Build OK but step was forced'); } try { const res = await this.doBuildInitData(); if(res === undefined) { this.buildOK = null; - this.logger.debug('No required data to build.'); + this.logger.trace('No required data to build.'); return; } if (res === true) { @@ -172,6 +174,58 @@ export default abstract class AbstractInitializable { return; } + public async buildDatabase(force: boolean = false) { + if(this.databaseOK) { + if(!force) { + return; + } + this.logger.verbose('Database OK but step was forced'); + } + try { + const res = await this.doBuildDatabase(); + if(res === undefined) { + this.databaseOK = null; + this.logger.trace('No required database steps.'); + return; + } + if (res === true) { + this.logger.verbose('Required database init succeeded'); + } else if (typeof res === 'string') { + this.logger.verbose(`Required database init succeeded => ${res}`); + } + this.databaseOK = true; + } catch (e) { + this.databaseOK = false; + throw new BuildDataError('Required database init failed', {cause: e}); + } + + try { + await this.postDatabase(); + } catch (e) { + if(e instanceof StageError) { + throw e; + } else { + throw new Error('Error occurred during post-database hook', {cause: e}); + } + } + } + + protected async postDatabase(): Promise { + return; + } + + /** + * Run/fetch/create any database data needed for this component to operate when ready + * + * * Return undefined if not possible or not required + * * Return TRUE if database steps succeeded + * * Return string if database steps succeeded and should log result + * * Throw error on failure + * */ + protected async doBuildDatabase(): Promise { + return; + } + public async checkConnection(force: boolean = false) { if(this.connectionOK) { if(!force) { @@ -252,12 +306,14 @@ export default abstract class AbstractInitializable { public isReady() { return (this.buildOK === null || this.buildOK === true) && + (this.databaseOK === null || this.databaseOK === true) && (this.connectionOK === null || this.connectionOK === true) && !this.authGated(); } public isUsable() { return (this.buildOK === null || this.buildOK === true) && + (this.databaseOK === null || this.databaseOK === true) && (this.connectionOK === null || this.connectionOK === true); } diff --git a/src/backend/common/Cache.ts b/src/backend/common/Cache.ts index 8483489c2..a4d7a8151 100644 --- a/src/backend/common/Cache.ts +++ b/src/backend/common/Cache.ts @@ -13,14 +13,15 @@ import { childLogger, Logger } from '@foxxmd/logging'; import { projectDir } from './index.js'; import path from 'path'; import { cacheFunctions } from "@foxxmd/regex-buddy-core"; -import { fileOrDirectoryIsWriteable } from '../utils/FSUtils.js'; -import { asCacheAuthProvider, asCacheConfig, asCacheMetadataProvider, asCacheScrobbleProvider, CacheAuthProvider, CacheConfig, CacheConfigOptions, CacheMetadataProvider, CacheProvider, CacheScrobbleProvider } from './infrastructure/Atomic.js'; +import { fileExists, fileOrDirectoryIsWriteable } from '../utils/FSUtils.js'; +import { copyFile } from 'fs/promises'; +import { asCacheConfig, CacheAuthProvider, CacheConfig, CacheConfigOptions, CacheConfigUser, CacheScrobbleProvider } from './infrastructure/Atomic.js'; import { Typeson } from 'typeson'; import { builtin } from 'typeson-registry'; import { loggerNoop } from './MaybeLogger.js'; -import { ListenProgressPositional, ListenProgressTS } from '../sources/PlayerState/ListenProgress.js'; const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); import prom, { Gauge } from 'prom-client'; +import { nonEmptyStringOrDefault } from '../../core/StringUtils.js'; dayjs.extend(utc) dayjs.extend(isBetween); @@ -37,17 +38,24 @@ typeson.register({ (x) => dayjs.isDayjs(x), (d: Dayjs) => d.toJSON(), (date) => dayjs(date) - ], - ListenProgressTS, - ListenProgressPositional + ] }); +const unsupportedEnvKeys = [ + 'CACHE_AUTH_CONN', + 'CACHE_METADATA', + 'CACHE_SCROBBLE', + 'CACHE_SCROBBLE_CONN', + 'CACHE_AUTH_CONN' +]; + export class MSCache { config: Required cacheMetadata: Cacheable; cacheScrobble: Cacheable; + cacheDb: Cacheable; cacheAuth: Cacheable; regexCache: ReturnType; cacheTransform: Cacheable; @@ -68,19 +76,19 @@ export class MSCache { const { metadata: { - provider: mProvider = (process.env.CACHE_METADATA as (CacheMetadataProvider | undefined) ?? false), - connection: mConn = process.env.CACHE_METADATA_CONN, - ...restMetadata + provider: mProvider = false, + connection: mConn, + //...restMetadata } = {}, scrobble: { - provider: sProvider = (process.env.CACHE_SCROBBLE as (CacheScrobbleProvider | undefined) ?? 'file'), - connection = (process.env.CACHE_SCROBBLE_CONN ?? configDir), - ...restScrobble + provider: sProvider = false, + connection: sConnection, + //...restScrobble } = {}, auth: { - provider: aProvider = (process.env.CACHE_AUTH as (CacheAuthProvider | undefined) ?? 'file'), - connection: aConn = (process.env.CACHE_AUTH_CONN ?? configDir), - ...restAuth + provider: aProvider = false, + connection: aConn, + //...restAuth } = {}, regex = 200, } = config; @@ -89,17 +97,14 @@ export class MSCache { metadata: { provider: mProvider, connection: mConn, - ...restMetadata, }, scrobble: { provider: sProvider, - connection, - ...restScrobble + connection: sConnection, }, auth: { provider: aProvider, connection: aConn, - ...restAuth }, regex }; @@ -114,6 +119,7 @@ export class MSCache { this.cacheAuth = inMemory; this.cacheScrobble = inMemory; this.cacheApi = inMemory; + this.cacheDb = new Cacheable({primary: initMemoryCache({lruSize: 500, ttl: '1m'})}); } init = async (enableCollectors: boolean = false) => { @@ -133,7 +139,8 @@ export class MSCache { { cache: this.cacheScrobble, name: 'queued_scrobbles' }, { cache: this.cacheTransform, name: 'transformer' }, { cache: this.cacheClientScrobbles, name: 'historical_scrobbles' }, - { cache: this.cacheApi, name: 'external_apis' } + { cache: this.cacheApi, name: 'external_apis' }, + { cache: this.cacheDb, name: 'database' } ]; this.cacheHits = new prom.Gauge({ @@ -271,10 +278,10 @@ export class MSCache { } } if (config.provider === 'file') { - logger.debug(`Building file cache from ${path.join(config.connection, `${namespace}.cache`)}`); + logger.debug(`Building file cache from ${path.join(config.connection ?? configDir, `${namespace}.cache`)}`); try { - const [keyvFile] = await initFileCache({ ...config, cacheDir: config.connection, cacheId: `${namespace}.cache` }, {ttl: config.ttl}, logger); + const [keyvFile] = await initFileCache({ ...config, cacheDir: config.connection ?? configDir, cacheId: `${namespace}.cache` }, {ttl: config.ttl}, logger); return keyvFile; } catch (e) { throw e; @@ -364,8 +371,8 @@ export const flatCacheCreate = (opts: FlatCacheOptions) => { return new FlatCache({ ttl: 0, lruSize: 2000, - cacheDir: opts.cacheDir ?? configDir, - cacheId: opts.cacheId ?? 'scrobble.cache', + cacheDir: opts.cacheDir, + cacheId: opts.cacheId ?? 'ms.cache', persistInterval: 1 * 1000 * 10, expirationInterval: 1 * 1000 * 10, // 10 seconds ...opts @@ -381,6 +388,12 @@ export const flatCacheLoad = async (flatCache: FlatCache, logger: Logger = logge throw new Error(`Unable to use path for file cache at ${cachePath}`, { cause: e }) } + if(fileExists(cachePath) && !fileExists(`${cachePath}.bak`)) { + logger.info(`Backing up ${cachePath} in preparation for migration to database...`); + await copyFile(cachePath, `${cachePath}.bak`); + logger.info(`Done! Backed up to ${cachePath}.bak\nAll data has been loaded into cache. It will be deleted (from cache) after migrating to database.\nIf there are migration issues or you wish to downgrade then overwrite ${cachePath} with the .bak backup copy`); + } + const streamPromise = new Promise((resolve, reject) => { flatCache.loadFileStream(cachePath, (progress: number, total: number) => { logger.trace(`Loading ${progress}/${total} chunks...`); @@ -497,4 +510,86 @@ const noopKeyv: KeyvStoreAdapter = { delete: (_) => undefined, clear: () => Promise.resolve(), on: (_, __) => undefined +} + +export const parseUserConfig = (config: CacheConfigUser = {}, parentLogger: Logger = loggerNoop): CacheConfigOptions => { + const logger = childLogger(parentLogger, 'Cache'); + + let valkeyEnvVal: string | undefined = nonEmptyStringOrDefault(process.env.CACHE_VALKEY); + if(valkeyEnvVal === undefined) { + valkeyEnvVal = nonEmptyStringOrDefault(process.env.CACHE_METADATA_CONN); + if(valkeyEnvVal !== undefined) { + logger.warn('ENV CACHE_METADATA_CONN is deprecated! Replace it with CACHE_VALKEY'); + } + } + + for(const key of unsupportedEnvKeys) { + if(nonEmptyStringOrDefault(process.env[key]) !== undefined) { + logger.warn(`ENV ${key} is no longer supported. Refer to the Caching docs.`); + } + } + + const { + valkey = valkeyEnvVal, + // metadata: { + // provider: mProvider = (process.env.CACHE_METADATA as (CacheMetadataProvider | undefined) ?? false), + // connection: mConn = process.env.CACHE_METADATA_CONN, + // //...restMetadata + // } = {}, + scrobble: { + provider: sProvider = (process.env.CACHE_SCROBBLE as (CacheScrobbleProvider | undefined) ?? 'file'), + connection = (process.env.CACHE_SCROBBLE_CONN ?? configDir), + ...restScrobble + } = {}, + auth: { + provider: aProvider = (process.env.CACHE_AUTH as (CacheAuthProvider | undefined) ?? 'file'), + //...restAuth + } = {}, + regex = 200, + } = config; + + if(config.metadata !== undefined) { + logger.warn('Configuring cache.metadata is no longer supported. Refer to the Caching docs.'); + } + if(config.scrobble !== undefined) { + logger.warn('Configuring cache.scrobble is no longer supported. Refer to the Caching docs.'); + } + if(config.auth?.connection !== undefined) { + logger.warn('Configuring cache.auth.connection is no longer supported. Refer to the Caching docs.'); + } + + let authConn: string, + authProvider = aProvider; + if(authProvider === 'valkey') { + if(valkey === undefined) { + logger.warn(`Auth Provider set to 'valkey' but not valkey connection string was not provided, falling back to file.`); + authConn = configDir; + authProvider = 'file'; + } else { + authConn = valkey; + } + } else { + if(authProvider !== 'file') { + logger.warn(`Unsupported provider given for auth: ${authProvider}`); + } + authConn = configDir; + authProvider = 'file'; + } + + return { + metadata: { + provider: valkey !== undefined ? 'valkey' : false, + connection: valkey, + }, + scrobble: { + provider: sProvider, + connection, + ...restScrobble + }, + auth: { + provider: authProvider, + connection: authConn, + }, + regex + }; } \ No newline at end of file diff --git a/src/backend/common/database/Database.ts b/src/backend/common/database/Database.ts new file mode 100644 index 000000000..6a64373f3 --- /dev/null +++ b/src/backend/common/database/Database.ts @@ -0,0 +1,192 @@ +import { configDir } from '../index.js'; +import * as path from 'path'; +import { promises as fs } from 'fs' +import { childLogger, Logger } from '@foxxmd/logging'; +import { loggerNoop } from '../MaybeLogger.js'; +import { fileExists, fileOrDirectoryIsWriteable } from '../../utils/FSUtils.js'; +import { COMPACTABLE, compactableProperties, CompactableProperty, DEFAULT_RETENTION_DELETE_AFTER, RententionGranular, RetentionConfig, RetentionConfigValue, RetentionOption, RetentionValue, RetentionValueUnparsed } from '../infrastructure/config/database.js'; +import { DurationValue } from '../infrastructure/Atomic.js'; +import { Duration } from 'dayjs/plugin/duration.js'; +import dayjs from 'dayjs'; +import { parseDurationFromDurationValue } from '../../utils/TimeUtils.js'; +import assert, { AssertionError } from 'node:assert'; +import { parseBoolStrict } from '../../utils.js'; +import { SimpleError } from '../errors/MSErrors.js'; + +export const MEMORY_DB_NAME = ':memory:'; +export const isMemoryDb = (name: string): boolean => name === MEMORY_DB_NAME; + +export const getDbPath = (name: string = 'msDb', workingDirectory?: string): string => { + if (isMemoryDb(name)) { + return MEMORY_DB_NAME; + } + return path.resolve(workingDirectory ?? configDir, `${name}`); +} + +export const getDbBackupPath = (dbPath: string, suffix?: string): string => { + const pathInfo = path.parse(dbPath); + const backupPath = `${path.join(pathInfo.dir, pathInfo.name)}.bak${suffix !== undefined ? `.${suffix}` : ''}`; + return backupPath; +} + +export const backupDb = async (dbName: string, opts: { logger?: Logger, workingDirectory?: string } = {}): Promise => { + + const { + logger: parentLogger = loggerNoop, + workingDirectory + } = opts; + + const logger = childLogger(parentLogger, 'Migrations'); + + const dbPath = getDbPath(dbName, workingDirectory); + let newDb = false; + + if (dbPath !== MEMORY_DB_NAME) { + if (!fileExists(dbPath)) { + logger.info(`Database at ${dbPath} does not exist, will create it.`); + newDb = true; + } + try { + fileOrDirectoryIsWriteable(dbPath); + } catch (e) { + throw new Error('Database path/folder is not writeable, cannot backup database', { cause: e }); + } + } + + if (dbPath !== MEMORY_DB_NAME && !newDb) { + const backupPath = `${getDbPath(`${Date.now()}-${dbName}`, workingDirectory)}.bak`; + logger.info(`Backing up database before migrating => ${backupPath}`); + await fs.copyFile(dbPath, backupPath) + logger.info('Backed up!'); + } +} + +const parseRetentionValue = (val: RetentionValueUnparsed): RetentionValue => { + if(typeof val === 'string' || typeof val === 'boolean') { + try { + const boolVal = parseBoolStrict(val); + assert(boolVal === false, 'retention value cannot be true'); + return boolVal; + } catch (e) { + // swallow + } + } else if(dayjs.isDuration(val)) { + return val; + } else { + return parseDurationFromDurationValue(val); + } + throw new SimpleError('retention value be of one: false, number, or string'); +} + +const parseRetentionFromEnv = (): RetentionOption => { + const deleteAfterEnv = process.env.RETENTION_DELETE_AFTER ?? DEFAULT_RETENTION_DELETE_AFTER, + deleteCompletedEnv = process.env.RETENTION_DELETE_COMPLETED_AFTER ?? deleteAfterEnv, + deleteFailedEnv = process.env.RETENTION_DELETE_FAILED_AFTER ?? deleteAfterEnv, + deleteDupedEnv = process.env.RETENTION_DELETE_DUPED_AFTER ?? deleteAfterEnv; + + return { + completed: parseRetentionValue(deleteCompletedEnv), + failed: parseRetentionValue(deleteFailedEnv), + duped: parseRetentionValue(deleteDupedEnv) + } +} + +const isRetentionOptionDurations = (val: RetentionOption): val is RetentionOption => { + return dayjs.isDuration(val.completed) + && dayjs.isDuration(val.duped) + && dayjs.isDuration(val.failed); +} + +let retentionDeleteAfterFromEnv: RetentionOption, +retentionCompactAfterFromEnv: RetentionOption; + +export const getRetentionDeleteAfterFromEnv = () => { + if (retentionDeleteAfterFromEnv === undefined) { + const deleteEnv = parseRetentionFromEnv(); + if(isRetentionOptionDurations(deleteEnv)) { + retentionDeleteAfterFromEnv = deleteEnv; + } else { + throw new SimpleError('retention deleteAfter values from env must all be one of: number or string'); + } + } + return retentionDeleteAfterFromEnv; +} +export const getRetentionCompactAfterFromEnv = () => { + if (retentionCompactAfterFromEnv === undefined) { + const compactEnv = parseRetentionFromEnv(); + retentionCompactAfterFromEnv = compactEnv; + } + return retentionCompactAfterFromEnv; +} + +export const parseRetentionOptions = (opts: RetentionConfigValue = {}, defaults: RetentionOption): RetentionOption => { + if (typeof opts === 'number' || typeof opts === 'string') { + const dur = parseDurationFromDurationValue(opts); + return { + completed: dur, + duped: dur, + failed: dur + } + } + + if (opts === undefined) { + return defaults; + } + + if(dayjs.isDuration(opts)) { + return { + completed: opts, + failed: opts, + duped: opts + } + } + + const { + completed = defaults.completed, + failed = defaults.failed, + duped = defaults.duped + } = opts; + + return { + completed: parseRetentionValue(completed), + failed: parseRetentionValue(failed), + duped: parseRetentionValue(duped), + } +} + +export const parseRetentionOptionsDurations = (opts: RetentionConfigValue> = {}, defaults: RetentionOption): RetentionOption => { + if (typeof opts === 'number' || typeof opts === 'string') { + const dur = parseDurationFromDurationValue(opts); + return { + completed: dur, + duped: dur, + failed: dur + } + } + + if(dayjs.isDuration(opts)) { + return { + completed: opts, + failed: opts, + duped: opts + } + } + + if (opts === undefined) { + return defaults; + } + + const { + completed = defaults.completed, + failed = defaults.failed, + duped = defaults.duped + } = opts; + + return { + completed: dayjs.isDuration(completed) ? completed : parseDurationFromDurationValue(completed), + failed: dayjs.isDuration(failed) ? failed : parseDurationFromDurationValue(failed), + duped: dayjs.isDuration(duped) ? duped : parseDurationFromDurationValue(duped), + } +} + +export const isCompactableProperty = (val: string): val is CompactableProperty => val === COMPACTABLE.input || val === COMPACTABLE.transform; \ No newline at end of file diff --git a/src/backend/common/database/drizzle/drizzleTypes.ts b/src/backend/common/database/drizzle/drizzleTypes.ts new file mode 100644 index 000000000..654c70120 --- /dev/null +++ b/src/backend/common/database/drizzle/drizzleTypes.ts @@ -0,0 +1,100 @@ +import { DBQueryConfig, DBQueryConfigWith, ExtractTablesFromSchema, KnownKeysOnly, RelationFieldsFilterInternals, Many, InferSelectModel, ExtractTablesWithRelations, type BuildQueryResult, RelationsFilter } from "drizzle-orm"; +import { components, playInputs, plays, queueStates, relations } from "./schema/schema.js"; +import {TSchema, TableName, Schema } from "./schema/schema.js"; +import { MarkOptional, MarkRequired } from "ts-essentials"; + + +export type ComponentNew = typeof components.$inferInsert; +export type ComponentSelect = typeof components.$inferSelect; + +export type QueueStateNew = typeof queueStates.$inferInsert; +export type QueueStateSelect = typeof queueStates.$inferSelect; + +export type PlayInputNew = typeof playInputs.$inferInsert; +export type PlayInputSelect = typeof playInputs.$inferSelect; + +export type PlaySelect = typeof plays.$inferSelect; +export type PlaySelectRel = ModelWithRelations; +export type PlaySelectWithQueueStates = GenericRelationResult<'plays', 'queueStates'>; +export type PlayNew = typeof plays.$inferInsert; + + +// useful references for building types +// https://github.com/drizzle-team/drizzle-orm/discussions/2596 +// https://github.com/drizzle-team/drizzle-orm/discussions/1539 +// https://gist.github.com/ikupenov/10bc89d92d92eaba8cc5569013e04069 +// https://github.com/drizzle-team/drizzle-orm/issues/695 most examples +// https://github.com/drizzle-team/drizzle-orm/discussions/2316 relation focused +// https://github.com/drizzle-team/drizzle-orm/issues/1319 + +//type p = TSchema['plays']['relations']; +export type FindWith = DBQueryConfigWith; +export type QueryConfig = DBQueryConfig<"many", TSchema, TSchema[T]>; +export type FindMany = Pick, DBQueryConfig<"many", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'> & {with?: FindWith} +export type FindOne = Pick, DBQueryConfig<"one", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'> & {with?: FindWith} +export type FindWhere = QueryConfig['where']; +// https://github.com/drizzle-team/drizzle-orm/issues/5218#issuecomment-4154686086 +export type WhereClause = RelationsFilter + + +export type CompareOp = Pick, 'gt' | 'gte' | 'eq' | 'lt' | 'lte' | 'ne'> +export type CompareOpKey = keyof CompareOp; + + + +/** + * Based on https://github.com/drizzle-team/drizzle-orm/issues/695#issuecomment-3133969178 + */ + +// Helper type to find the tsName corresponding to a given dbName in TSchema +type FindTsNameByDbName = { + [K in keyof TSchema]: TSchema[K] extends { + // updated dbName -> name + name: TDbNameToFind; + } + ? K + : TDbNameToFind; +}[keyof TSchema]; + +// Helper type to find the dbName corresponding to a given tsName in TSchema +type FindDbNameByTsName = { + [K in keyof Schema]: Schema[K] extends TTable ? K : never; +}[keyof Schema]; + +/** + * Utility type to infer the model type for a given table name from the schema. + * Handles nested relations recursively. + * Uses referencedTableName (dbName) and FindTsNameByDbName helper. + */ +export type ModelWithRelationsFromName< + TTableName extends keyof TSchema, +> = InferSelectModel & { + [K in keyof TSchema[TTableName]['relations']]?: TSchema[TTableName]['relations'][K] extends infer TRelation + // updated referencedTableName -> targetTableName + ? TRelation extends { targetTableName: infer TRefDbName extends string } + ? FindTsNameByDbName extends infer TRefTsName extends + keyof TSchema + ? TRelation extends Many + ? ModelWithRelationsFromName[] + : ModelWithRelationsFromName | null + : never + : never + : never; +}; + +/** + * Utility type to infer the model type for a given table from the schema. + * Handles nested relations recursively. + * Uses referencedTableName (dbName) and FindDbNameByTsName helper. + */ +export type ModelWithRelations = + FindDbNameByTsName extends infer TTableName extends keyof TSchema + ? ModelWithRelationsFromName + : never; + + +// all relations are are now fully typed and optional +//type FullPlay = ModelWithRelations; + +// https://github.com/drizzle-team/drizzle-orm/issues/695#issuecomment-4389296482 +type GenericRelationResult = BuildQueryResult }>; \ No newline at end of file diff --git a/src/backend/common/database/drizzle/drizzleUtils.ts b/src/backend/common/database/drizzle/drizzleUtils.ts new file mode 100644 index 000000000..25f0bbbf2 --- /dev/null +++ b/src/backend/common/database/drizzle/drizzleUtils.ts @@ -0,0 +1,263 @@ +import { drizzle } from 'drizzle-orm/node-sqlite'; +import { drizzle as drizzlePglite } from 'drizzle-orm/pglite'; +import { migrate } from 'drizzle-orm/node-sqlite/migrator'; +import { migrate as migratePglite } from 'drizzle-orm/pglite/migrator'; +import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core"; +import { PGlite, PGliteOptions } from '@electric-sql/pglite'; +import { sql as dsl, LogWriter, Logger as DrizzleLogger } from 'drizzle-orm'; +import * as fs from 'fs/promises'; +import * as fsSync from 'fs'; +import * as path from 'path'; +import { backupDb, getDbBackupPath, getDbPath, MEMORY_DB_NAME } from '../Database.js'; +import { fileExists, fileOrDirectoryIsWriteable } from '../../../utils/FSUtils.js'; +import { childLogger, Logger, LogLevel } from '@foxxmd/logging'; +import { loggerNoop } from '../../MaybeLogger.js'; +import { projectDir } from '../../index.js'; +import { relations } from './schema/schema.js'; +import { addToContext, executeQuery } from './logContext.js'; + +export async function shouldBackupDb(dbVal: string | DbConcrete, opts: {logger?: Logger, migrationsFolder?: string} = {}): Promise<[boolean, string[]]> { + const { + logger: parentLogger = loggerNoop, + migrationsFolder = path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') + } = opts; + const logger = childLogger(parentLogger, 'Migrations'); + + let db: DbConcrete; + + if(typeof dbVal === 'string') { + logger.info(`Checking for database at ${dbVal}`); + if (dbVal !== MEMORY_DB_NAME && !fileExists(dbVal)) { + logger.info(`No database exists, no backup needed.`); + return [false, []]; + } + db = await getDb(dbVal); + } else { + db = dbVal; + } + + + // const db = drizzlePglite(dbPath); + + try { + // Ensure the migrations table exists + // https://github.com/drizzle-team/drizzle-orm/issues/1953 + const res = await db.execute(dsl` + SELECT EXISTS ( + SELECT FROM + pg_tables + WHERE + schemaname = 'drizzle' AND + tablename = '__drizzle_migrations' + ); + `); + + // const res3 = await db.execute(dsl` + // SELECT * FROM + // pg_tables; + // `); + + if (res.rows[0].exists === false) { + logger.info(`Database exists but there is no __drizzle_migrations table??`); + return [true, []]; + } + + const dbMigrations = await db.execute(dsl`SELECT id, hash, created_at, name, applied_at FROM drizzle.__drizzle_migrations ORDER BY created_at DESC`); + // @ts-ignore + const appliedMigrations = new Set(dbMigrations.rows.map((m: any) => m.name)); + + const allFiles = await fs.readdir(migrationsFolder); + const migrationFiles = allFiles + .sort(); + + const pendingMigrations = migrationFiles.filter(file => { + return !appliedMigrations.has(file); + }); + + //console.log('Applied migrations:', Array.from(appliedMigrations)); + if (pendingMigrations.length > 0) { + logger.info(`${pendingMigrations.length} pending migrations:\n${pendingMigrations.join('\n')}`); + return [true, pendingMigrations]; + } else { + logger.info('No pending migrations.'); + return [false, []]; + } + } catch (error) { + logger.error(new Error('Failed to get pending migrations', { cause: error })); + return [true, []]; + } +} + +export const getDb = async (dbVal: string | PGlite, opts: { logger?: Logger, backupPath?: string, loadDataDir?: Promise } = {}) => { + const { + logger = loggerNoop, + backupPath, + loadDataDir + } = opts; + let client: PGlite; + + if(typeof dbVal === 'string') { + const opts: PGliteOptions = {}; + if(dbVal !== MEMORY_DB_NAME) { + opts.dataDir = dbVal; + if(backupPath !== undefined) { + opts.loadDataDir = new Blob([fsSync.readFileSync(backupPath)]); + } + } + // only load one + // but this could be for a memory db so don't put it in above if + if(loadDataDir !== undefined && backupDb === undefined) { + opts.loadDataDir = await loadDataDir + } + client = await PGlite.create(opts); + } else { + client = dbVal; + } + return drizzlePglite({relations: relations, logger: createDrizzleLogger(logger), client}); +} + +export type DbConcrete = Awaited>; + +export const migrateDb = async (db: DbConcrete, opts: {logger?: Logger, migrationsFolder?: string} = {}) => { + const { + migrationsFolder, + logger: parentLogger = loggerNoop + } = opts; + const logger = childLogger(parentLogger, 'Migrations'); + + try { + logger.info('Starting migrations...'); + await executeQuery('migrations', async () => migratePglite(db, { migrationsFolder: migrationsFolder ?? path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') }), logger, process.env.LOG_MIGRATION === 'true' ? true : 'error'); + logger.info('Migrations complete'); + } catch (e) { + throw new Error('Failed to migrate database', { cause: e }); + } +} + +export const migrateDbSync = (db: ReturnType, opts: {logger?: Logger, migrationsFolder?: string} = {}) => { + const { + migrationsFolder, + logger: parentLogger = loggerNoop + } = opts; + const logger = childLogger(parentLogger, 'Migrations'); + + try { + logger.info('Starting migrations...'); + migrate(db, { migrationsFolder: migrationsFolder ?? path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations') }); + logger.info('Migrations complete'); + } catch (e) { + throw new Error('Failed to migrate database', { cause: e }); + } +} + +export const getMigratedDb = async (dbPath: string, opts: { logger?: Logger, workingDirectory?: string, migrationsFolder?: string, backupPath?: string, loadDataDir?: Promise } = {}): Promise<[DbConcrete, boolean]> => { + const { + logger = loggerNoop + } = opts; + let db: DbConcrete, + isNew = false, + hasPendingMigrations: boolean = true; + if (dbPath !== MEMORY_DB_NAME) { + try { + fileOrDirectoryIsWriteable(dbPath); + } catch (e) { + throw new Error('Database directory is not accessible', { cause: e }); + } + + const backupPath = getDbBackupPath(dbPath); + + if (fileExists(dbPath)) { + db = await getDb(dbPath, opts); + const [shouldBackup, pendingMigrations] = await shouldBackupDb(db, opts); + if (shouldBackup) { + hasPendingMigrations = true; + await backupPgDb(db, dbPath, { logger: opts.logger }); + } + } else if(fileExists(backupPath)) { + logger.info(`Detected no database, using backup to recreate db. Backup file: ${backupPath}`); + db = await getDb(dbPath, {...opts, backupPath}); + const usedBackedPath = getDbBackupPath(dbPath, 'used'); + logger.info(`Backup loaded! Renaming backup to indicate it has already been used, new path: ${usedBackedPath}`); + await fs.rename(backupPath, usedBackedPath); + } else { + logger.info('Detected no database, creating a new one...'); + db = await getDb(dbPath, opts); + isNew = true; + } + } else { + logger.info('Detected in-memory database'); + db = await getDb(dbPath, opts); + isNew = true; + } + + if(hasPendingMigrations && dbPath !== MEMORY_DB_NAME) { + logger.info('TIP: Migrations may take some time, depending on the size of your database'); + } + await migrateDb(db, opts); + + return [db, isNew]; +} + +export const backupPgDb = async (db: DbConcrete, dbPath: string, opts: { logger?: Logger } = {}): Promise => { + + const { + logger: parentLogger = loggerNoop, + } = opts; + + const logger = childLogger(parentLogger, 'Backup'); + + const pathInfo = path.parse(dbPath); + // being extra sure there isn't a trailing slash + const backupPath = `${path.join(pathInfo.dir, pathInfo.name)}-${Date.now()}.bak`; + logger.info(`Backing up database before migrating => ${backupPath}`); + fs.writeFile(backupPath, Buffer.from(await (await db.$client.dumpDataDir()).arrayBuffer())); + //await fs.copyFile(dbPath, backupPath) + logger.info('Backed up!'); +} + +export const createDrizzleLogger = (parentLogger: Logger, opts: {level?: LogLevel} = {}): DrizzleLogger => { + return { + logQuery: (query: string, params: unknown[]) => { + addToContext({sql: query, params}) + } + } +} + + +// cannot really use transactions right now because async isn't supporting for sqlite +// https://github.com/drizzle-team/drizzle-orm/issues/1472 +// https://github.com/drizzle-team/drizzle-orm/issues/2275 +// so use this workaround for now +// https://github.com/drizzle-team/drizzle-orm/issues/2275#issuecomment-2496503801 +let currentTransaction: null | Promise = null; +export const runTransaction = async < + T, + TQueryResult, + TSchema extends Record = Record +>( + db: BaseSQLiteDatabase<"sync", TQueryResult, TSchema>, + executor: () => Promise +) => { + while (currentTransaction !== null) { + await currentTransaction; + } + let resolve!: () => void; + currentTransaction = new Promise(_resolve => { + resolve = _resolve; + }); + try { + db.run(dsl.raw(`BEGIN`)) + + try { + const result = await executor(); + await db.run(dsl.raw(`COMMIT`)); + return result; + } catch (error) { + await db.run(dsl.raw(`ROLLBACK`)); + throw error; + } + } finally { + resolve(); + currentTransaction = null; + } +}; \ No newline at end of file diff --git a/src/backend/common/database/drizzle/entityUtils.ts b/src/backend/common/database/drizzle/entityUtils.ts new file mode 100644 index 000000000..5c208922b --- /dev/null +++ b/src/backend/common/database/drizzle/entityUtils.ts @@ -0,0 +1,93 @@ +import assert from "node:assert"; +import { PlayNew, PlaySelect, PlaySelectWithQueueStates } from "./drizzleTypes.js"; +import { PlayInputNew } from "./drizzleTypes.js"; +import { QueueStateNew } from "./drizzleTypes.js"; +import { ComponentNew } from "./drizzleTypes.js"; +import { MarkOptional, MarkRequired } from "ts-essentials"; +import { CLIENT_DEAD_QUEUE, DeadLetterScrobble, ErrorLike, PlayObject } from "../../../../core/Atomic.js"; +import dayjs, { Dayjs } from "dayjs"; +import { asPlay } from "../../../../core/PlayMarshalUtils.js"; +import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../utils/PlayComparisonUtils.js"; +import { hashObject } from "../../../utils/StringUtils.js"; +import { messageWithCauses } from "../../../utils/ErrorUtils.js"; + +export const generateComponentEntity = (data: MarkOptional): ComponentNew => { + assert(data.name !== undefined, 'Must provide name'); + return { + ...data, + uid: data.uid ?? data.name + }; +} + +export type PlayEntityOpts = Partial> & { error?: ErrorLike }; + +export const generatePlayEntity = (play: PlayObject, opts: PlayEntityOpts = {}): PlayNew => { + const { + seenAt = dayjs(), + state = 'queued', + playedAt = play.data.playDate, + ...restOpts + } = opts; + let playHash: string = undefined; + try { + playHash = hashObject(playContentBasicInvariantTransform(play).data); + } catch (e) { + // swallow + } + const data: PlayNew = { + play, + playHash, + state, + playedAt, + seenAt: play.meta.seenAt ?? seenAt, + ...restOpts + }; + const mbidId = playMbidIdentifier(play); + if(mbidId !== undefined) { + data.mbidIdentifier = mbidId; + } + return data; +} + +export type PlayHydateOptions = 'asPlay' | 'id' | 'uid'; + +export const hydratePlaySelect = (select: PlaySelect, opts: PlayHydateOptions[] = ['id','uid']): PlayObject => { + if(opts.length === 0) { + return select.play; + } + + let res = select.play; + // if(opts.includes('asPlay')) { + // res = asPlay(res); + // } + if(opts.includes('uid')) { + res.uid = select.uid; + //res.meta.dbUid = select.uid; + } + if(opts.includes('id')) { + res.id = select.id; + //res.meta.dbId = select.id; + } + return res; +} + +export const playSelectToDeadScrobble = (select: PlaySelectWithQueueStates): DeadLetterScrobble => { + const deadQueue = select.queueStates.find(x => x.queueName === CLIENT_DEAD_QUEUE); + return { + play: select.play, + id: select.uid, + source: select.play.meta.source, + retries: deadQueue.retries, + lastRetry: deadQueue.updatedAt, + error: deadQueue.error as unknown as string, + status: deadQueue.queueStatus as 'queued' | 'failed' + } +} + +export const generateInputEntity = (data: PlayInputNew): PlayInputNew => { + return data; +} + +export const generateQueueStateEntity = (data: QueueStateNew): QueueStateNew => { + return data; +} \ No newline at end of file diff --git a/src/backend/common/database/drizzle/logContext.ts b/src/backend/common/database/drizzle/logContext.ts new file mode 100644 index 000000000..3901f6c00 --- /dev/null +++ b/src/backend/common/database/drizzle/logContext.ts @@ -0,0 +1,71 @@ +import { Logger } from '@foxxmd/logging' +import { AsyncLocalStorage } from 'async_hooks' + +// based on https://numeric.substack.com/p/upgrading-drizzleorm-logging-with +interface QueryContext { + queryKey: string + startTime: number + queries: { sql?: string, params?: unknown[] }[] +} + +const queryStorage = new AsyncLocalStorage() + +function wrapQuery(queryKey: string, fn: () => Promise): Promise { + return queryStorage.run( + { + queryKey, + startTime: Date.now(), + queries: [] + }, + fn + ) +} + +function getContext(): QueryContext | undefined { + return queryStorage.getStore() +} + +export function addToContext(data: { sql?: string, params?: unknown[] }): void { + const context = getContext() + if (context) { + context.queries.push(data); + } +} + +/** + * Log all queries made by drizzle during the execution of a promise + * + * use second parameter to configure when logging occurs + * * true => log everything (default) + * * false => log nothing, skips asyncstorage entirely + * * 'error' => only log if promise throws + * + */ +export async function executeQuery(queryKey: string, queryPromise: () => Promise, logger: Logger, when: boolean | 'error' = true) { + if(when === false) { + try { + return await queryPromise(); + } catch (e) { + throw e; + } + } + return wrapQuery(queryKey, async () => { + try { + const results = await queryPromise() + + if (when !== 'error') { + // Query is done - grab everything from context + const context = getContext() + const executionTime = context ? Date.now() - context.startTime : 0 + logger.info({ labels: ['DB Query', queryKey], queries: context?.queries }, `Execution Complete in ${executionTime}ms`); + } + + return results + } catch (error) { + const context = getContext() + const executionTime = context ? Date.now() - context.startTime : 0; + logger.warn({ labels: ['DB Query', queryKey], queries: context?.queries }, `Execution failed in ${executionTime}ms`); + throw error + } + }) +} \ No newline at end of file diff --git a/src/backend/common/database/drizzle/migrations/20260509011012_dizzy_naoko/migration.sql b/src/backend/common/database/drizzle/migrations/20260509011012_dizzy_naoko/migration.sql new file mode 100644 index 000000000..ef685e580 --- /dev/null +++ b/src/backend/common/database/drizzle/migrations/20260509011012_dizzy_naoko/migration.sql @@ -0,0 +1,83 @@ +CREATE TABLE "components" ( + "id" serial PRIMARY KEY, + "uid" varchar(200) NOT NULL, + "mode" varchar(15) NOT NULL, + "type" varchar(50) NOT NULL, + "name" varchar NOT NULL, + "countLive" integer DEFAULT 0 NOT NULL, + "countNonLive" integer DEFAULT 0 NOT NULL, + "createdAt" timestamp +); +--> statement-breakpoint +CREATE TABLE "jobs" ( + "id" serial PRIMARY KEY, + "componentFromId" integer NOT NULL, + "componentToId" integer NOT NULL, + "name" varchar(200) NOT NULL, + "status" varchar(20) DEFAULT 'idle' NOT NULL, + "retries" integer DEFAULT 0 NOT NULL, + "error" json, + "transformOptions" json, + "initialParameters" json, + "cursor" json, + "total" integer, + "imported" integer DEFAULT 0 NOT NULL, + "scrobbled" integer DEFAULT 0 NOT NULL, + "createdAt" timestamp NOT NULL, + "updatedAt" timestamp NOT NULL, + "completedAt" timestamp +); +--> statement-breakpoint +CREATE TABLE "play_inputs" ( + "id" serial PRIMARY KEY, + "playId" integer NOT NULL, + "data" json, + "play" jsonb NOT NULL, + "createdAt" timestamp +); +--> statement-breakpoint +CREATE TABLE "plays" ( + "id" serial PRIMARY KEY, + "uid" varchar(30) NOT NULL UNIQUE, + "componentId" integer, + "error" json, + "playedAt" timestamp, + "seenAt" timestamp, + "updatedAt" timestamp NOT NULL, + "play" jsonb NOT NULL, + "state" varchar(20) NOT NULL, + "parentId" integer, + "jobId" integer, + "playHash" varchar(100), + "mbidIdentifier" varchar(100), + "compacted" varchar(30) +); +--> statement-breakpoint +CREATE TABLE "play_queue_states" ( + "id" serial PRIMARY KEY, + "playId" integer NOT NULL, + "componentId" integer NOT NULL, + "queueName" varchar(50) NOT NULL, + "queueStatus" varchar(20) DEFAULT 'queued' NOT NULL, + "retries" integer DEFAULT 0 NOT NULL, + "error" json, + "createdAt" timestamp NOT NULL, + "updatedAt" timestamp NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX "uid_mode_type_idx" ON "components" ("uid","mode","type");--> statement-breakpoint +CREATE UNIQUE INDEX "play_input_id_idx" ON "play_inputs" ("playId");--> statement-breakpoint +CREATE INDEX "play_parent_id_idx" ON "plays" ("parentId");--> statement-breakpoint +CREATE INDEX "play_component_id_idx" ON "plays" ("componentId");--> statement-breakpoint +CREATE UNIQUE INDEX "play_uid_idx" ON "plays" ("uid");--> statement-breakpoint +CREATE INDEX "play_playedAt_idx" ON "plays" ("playedAt");--> statement-breakpoint +CREATE INDEX "play_seenAt_idx" ON "plays" ("seenAt");--> statement-breakpoint +CREATE INDEX "play_queue_state_id_idx" ON "play_queue_states" ("playId");--> statement-breakpoint +ALTER TABLE "jobs" ADD CONSTRAINT "jobs_componentFromId_components_id_fkey" FOREIGN KEY ("componentFromId") REFERENCES "components"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint +ALTER TABLE "jobs" ADD CONSTRAINT "jobs_componentToId_components_id_fkey" FOREIGN KEY ("componentToId") REFERENCES "components"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint +ALTER TABLE "play_inputs" ADD CONSTRAINT "play_inputs_playId_plays_id_fkey" FOREIGN KEY ("playId") REFERENCES "plays"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint +ALTER TABLE "plays" ADD CONSTRAINT "plays_componentId_components_id_fkey" FOREIGN KEY ("componentId") REFERENCES "components"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint +ALTER TABLE "plays" ADD CONSTRAINT "plays_parentId_plays_id_fkey" FOREIGN KEY ("parentId") REFERENCES "plays"("id") ON DELETE SET NULL ON UPDATE CASCADE;--> statement-breakpoint +ALTER TABLE "plays" ADD CONSTRAINT "plays_jobId_jobs_id_fkey" FOREIGN KEY ("jobId") REFERENCES "jobs"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint +ALTER TABLE "play_queue_states" ADD CONSTRAINT "play_queue_states_playId_plays_id_fkey" FOREIGN KEY ("playId") REFERENCES "plays"("id") ON DELETE CASCADE ON UPDATE CASCADE;--> statement-breakpoint +ALTER TABLE "play_queue_states" ADD CONSTRAINT "play_queue_states_componentId_components_id_fkey" FOREIGN KEY ("componentId") REFERENCES "components"("id") ON DELETE CASCADE ON UPDATE CASCADE; \ No newline at end of file diff --git a/src/backend/common/database/drizzle/migrations/20260509011012_dizzy_naoko/snapshot.json b/src/backend/common/database/drizzle/migrations/20260509011012_dizzy_naoko/snapshot.json new file mode 100644 index 000000000..a31bdd0c3 --- /dev/null +++ b/src/backend/common/database/drizzle/migrations/20260509011012_dizzy_naoko/snapshot.json @@ -0,0 +1,1096 @@ +{ + "version": "8", + "dialect": "postgres", + "id": "b9a8b9d5-1df4-49b2-a538-3e095bb43c32", + "prevIds": [ + "00000000-0000-0000-0000-000000000000" + ], + "ddl": [ + { + "isRlsEnabled": false, + "name": "components", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "jobs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "play_inputs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "plays", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "play_queue_states", + "entityType": "tables", + "schema": "public" + }, + { + "type": "serial", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "components" + }, + { + "type": "varchar(200)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "uid", + "entityType": "columns", + "schema": "public", + "table": "components" + }, + { + "type": "varchar(15)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "mode", + "entityType": "columns", + "schema": "public", + "table": "components" + }, + { + "type": "varchar(50)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "public", + "table": "components" + }, + { + "type": "varchar", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "components" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "countLive", + "entityType": "columns", + "schema": "public", + "table": "components" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "countNonLive", + "entityType": "columns", + "schema": "public", + "table": "components" + }, + { + "type": "timestamp", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "components" + }, + { + "type": "serial", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "jobs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "componentFromId", + "entityType": "columns", + "schema": "public", + "table": "jobs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "componentToId", + "entityType": "columns", + "schema": "public", + "table": "jobs" + }, + { + "type": "varchar(200)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "jobs" + }, + { + "type": "varchar(20)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'idle'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "jobs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "retries", + "entityType": "columns", + "schema": "public", + "table": "jobs" + }, + { + "type": "json", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "public", + "table": "jobs" + }, + { + "type": "json", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "transformOptions", + "entityType": "columns", + "schema": "public", + "table": "jobs" + }, + { + "type": "json", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "initialParameters", + "entityType": "columns", + "schema": "public", + "table": "jobs" + }, + { + "type": "json", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "cursor", + "entityType": "columns", + "schema": "public", + "table": "jobs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "total", + "entityType": "columns", + "schema": "public", + "table": "jobs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "imported", + "entityType": "columns", + "schema": "public", + "table": "jobs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "scrobbled", + "entityType": "columns", + "schema": "public", + "table": "jobs" + }, + { + "type": "timestamp", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "jobs" + }, + { + "type": "timestamp", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "jobs" + }, + { + "type": "timestamp", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "completedAt", + "entityType": "columns", + "schema": "public", + "table": "jobs" + }, + { + "type": "serial", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "play_inputs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "playId", + "entityType": "columns", + "schema": "public", + "table": "play_inputs" + }, + { + "type": "json", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "data", + "entityType": "columns", + "schema": "public", + "table": "play_inputs" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "play", + "entityType": "columns", + "schema": "public", + "table": "play_inputs" + }, + { + "type": "timestamp", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "play_inputs" + }, + { + "type": "serial", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "plays" + }, + { + "type": "varchar(30)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "uid", + "entityType": "columns", + "schema": "public", + "table": "plays" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "componentId", + "entityType": "columns", + "schema": "public", + "table": "plays" + }, + { + "type": "json", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "public", + "table": "plays" + }, + { + "type": "timestamp", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "playedAt", + "entityType": "columns", + "schema": "public", + "table": "plays" + }, + { + "type": "timestamp", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "seenAt", + "entityType": "columns", + "schema": "public", + "table": "plays" + }, + { + "type": "timestamp", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "plays" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "play", + "entityType": "columns", + "schema": "public", + "table": "plays" + }, + { + "type": "varchar(20)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "state", + "entityType": "columns", + "schema": "public", + "table": "plays" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "parentId", + "entityType": "columns", + "schema": "public", + "table": "plays" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "jobId", + "entityType": "columns", + "schema": "public", + "table": "plays" + }, + { + "type": "varchar(100)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "playHash", + "entityType": "columns", + "schema": "public", + "table": "plays" + }, + { + "type": "varchar(100)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "mbidIdentifier", + "entityType": "columns", + "schema": "public", + "table": "plays" + }, + { + "type": "varchar(30)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "compacted", + "entityType": "columns", + "schema": "public", + "table": "plays" + }, + { + "type": "serial", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "play_queue_states" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "playId", + "entityType": "columns", + "schema": "public", + "table": "play_queue_states" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "componentId", + "entityType": "columns", + "schema": "public", + "table": "play_queue_states" + }, + { + "type": "varchar(50)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "queueName", + "entityType": "columns", + "schema": "public", + "table": "play_queue_states" + }, + { + "type": "varchar(20)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'queued'", + "generated": null, + "identity": null, + "name": "queueStatus", + "entityType": "columns", + "schema": "public", + "table": "play_queue_states" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "retries", + "entityType": "columns", + "schema": "public", + "table": "play_queue_states" + }, + { + "type": "json", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "public", + "table": "play_queue_states" + }, + { + "type": "timestamp", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "play_queue_states" + }, + { + "type": "timestamp", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "play_queue_states" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "uid", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "mode", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "uid_mode_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "components" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "playId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "play_input_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "play_inputs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "parentId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "play_parent_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "plays" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "componentId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "play_component_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "plays" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "uid", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "play_uid_idx", + "entityType": "indexes", + "schema": "public", + "table": "plays" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "playedAt", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "play_playedAt_idx", + "entityType": "indexes", + "schema": "public", + "table": "plays" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "seenAt", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "play_seenAt_idx", + "entityType": "indexes", + "schema": "public", + "table": "plays" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "playId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "play_queue_state_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "play_queue_states" + }, + { + "nameExplicit": false, + "columns": [ + "componentFromId" + ], + "schemaTo": "public", + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "name": "jobs_componentFromId_components_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "jobs" + }, + { + "nameExplicit": false, + "columns": [ + "componentToId" + ], + "schemaTo": "public", + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "name": "jobs_componentToId_components_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "jobs" + }, + { + "nameExplicit": false, + "columns": [ + "playId" + ], + "schemaTo": "public", + "tableTo": "plays", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "name": "play_inputs_playId_plays_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "play_inputs" + }, + { + "nameExplicit": false, + "columns": [ + "componentId" + ], + "schemaTo": "public", + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "name": "plays_componentId_components_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "plays" + }, + { + "nameExplicit": false, + "columns": [ + "parentId" + ], + "schemaTo": "public", + "tableTo": "plays", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "SET NULL", + "name": "plays_parentId_plays_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "plays" + }, + { + "nameExplicit": false, + "columns": [ + "jobId" + ], + "schemaTo": "public", + "tableTo": "jobs", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "name": "plays_jobId_jobs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "plays" + }, + { + "nameExplicit": false, + "columns": [ + "playId" + ], + "schemaTo": "public", + "tableTo": "plays", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "name": "play_queue_states_playId_plays_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "play_queue_states" + }, + { + "nameExplicit": false, + "columns": [ + "componentId" + ], + "schemaTo": "public", + "tableTo": "components", + "columnsTo": [ + "id" + ], + "onUpdate": "CASCADE", + "onDelete": "CASCADE", + "name": "play_queue_states_componentId_components_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "play_queue_states" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "components_pkey", + "schema": "public", + "table": "components", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "jobs_pkey", + "schema": "public", + "table": "jobs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "play_inputs_pkey", + "schema": "public", + "table": "play_inputs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "plays_pkey", + "schema": "public", + "table": "plays", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "play_queue_states_pkey", + "schema": "public", + "table": "play_queue_states", + "entityType": "pks" + }, + { + "nameExplicit": false, + "columns": [ + "uid" + ], + "nullsNotDistinct": false, + "name": "plays_uid_key", + "schema": "public", + "table": "plays", + "entityType": "uniques" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/src/backend/common/database/drizzle/repositories/BaseRepository.ts b/src/backend/common/database/drizzle/repositories/BaseRepository.ts new file mode 100644 index 000000000..8c50921f3 --- /dev/null +++ b/src/backend/common/database/drizzle/repositories/BaseRepository.ts @@ -0,0 +1,111 @@ +import { childLogger, Logger } from "@foxxmd/logging"; +import { DbConcrete } from "../drizzleUtils.js"; +import { CompareOpKey } from "../drizzleTypes.js"; +import { Dayjs } from "dayjs"; +import { RelationsFieldFilter, eq, inArray } from "drizzle-orm"; +import { loggerNoop } from "../../../MaybeLogger.js"; +import { capitalize } from "../../../../../core/StringUtils.js"; +import { getConfigByTableName, relations, TableName } from "../schema/schema.js"; + +export interface DrizzleRepositoryOpts { + logger?: Logger + componentId?: number +} + +export type CompareDateOp = { + type: CompareOpKey + date: Dayjs +} | { + type: 'between', + range: [Dayjs, Dayjs], + inclusive?: boolean +} + +export interface PaginatedQueryResponse { + limit: number, + offset: number +} + +export interface PaginatedResponse { + data: T[] + meta: PaginatedQueryResponse +} + +export interface ComponentConstrainedRepoOpts { + componentId?: number +} + +export abstract class DrizzleBaseRepository { + + logger: Logger; + displayName: string; + tableName: TableName; + table: ReturnType> + db: DbConcrete; + componentId?: number + + constructor(db: DbConcrete, tableName: TableName, displayName: string, opts: DrizzleRepositoryOpts = {}) { + this.db = db; + this.displayName = displayName; + this.tableName = tableName; + this.table = getConfigByTableName(this.tableName); + this.logger = childLogger(opts.logger ?? loggerNoop, ['Database', capitalize(displayName)]); + this.componentId = opts.componentId; + } + + async deleteByIds(ids: number[]): Promise { + await this.db.delete(this.table).where(inArray(this.table.id, ids)); + } + + async updateById(id: number, data: Partial): Promise { + await this.db.update(this.table).set(data).where(eq(this.table.id, id)); + } + + async create(data: typeof this.table.$inferInsert): Promise { + const res = await this.db.insert(this.table).values([data]).returning(); + return res[0]; + } + + async createMany(data: typeof this.table.$inferInsert[]): Promise { + const res = await this.db.insert(this.table).values(data).returning(); + return res; + } + + async findById(id: number): Promise { + // const res = await this.db.query[this.tableName].findFirst({ + // where: { + // id + // } + // }); + const res = await this.db.select().from(this.table).where(eq(this.table.id, id)); + if(res.length === 0) { + return undefined; + } + return res[0]; + } +} + +export class GenericRepository extends DrizzleBaseRepository { + +} + +export const buildDateCompare = (data: CompareDateOp): RelationsFieldFilter => { + let q: RelationsFieldFilter = {}; + if (data.type !== 'between') { + q = { + [data.type]: data.date + } + } else { + q = { + AND: [ + { + [data.inclusive ?? true ? 'gte' : 'gt']: data.range[0] + }, + { + [data.inclusive ?? true ? 'lte' : 'lt']: data.range[1] + }, + ] + } + } + return q; +} \ No newline at end of file diff --git a/src/backend/common/database/drizzle/repositories/ComponentRepository.ts b/src/backend/common/database/drizzle/repositories/ComponentRepository.ts new file mode 100644 index 000000000..ac19a3251 --- /dev/null +++ b/src/backend/common/database/drizzle/repositories/ComponentRepository.ts @@ -0,0 +1,34 @@ +import { Logger } from "drizzle-orm"; +import { DrizzleBaseRepository, DrizzleRepositoryOpts } from "./BaseRepository.js"; +import { DbConcrete } from "../drizzleUtils.js"; +import { ComponentNew, ComponentSelect, FindWhere } from "../drizzleTypes.js"; +import { components } from "../schema/schema.js"; +import { generateComponentEntity } from "../entityUtils.js"; + +export class DrizzleComponentRepository extends DrizzleBaseRepository<'components'> { + + constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) { + super(db, 'components', 'Component', opts); + } + + findOrInsert = async (data: { mode: 'source' | 'client', type: string, uid?: string, name?: string }): Promise => { + const where: FindWhere<'components'> = { + mode: data.mode, + type: data.type, + uid: data.uid ?? data.name + }; + const component = await this.db.query.components.findFirst({ + where + }); + if (component !== undefined) { + return component; + } + + return (await this.db.insert(components).values(generateComponentEntity({ + uid: data.uid ?? data.name, + mode: data.mode, + type: data.type, + name: data.name + })).returning())[0]; + } +} \ No newline at end of file diff --git a/src/backend/common/database/drizzle/repositories/PlayRepository.ts b/src/backend/common/database/drizzle/repositories/PlayRepository.ts new file mode 100644 index 000000000..3764b2321 --- /dev/null +++ b/src/backend/common/database/drizzle/repositories/PlayRepository.ts @@ -0,0 +1,881 @@ +import { childLogger, Logger, LoggerAppExtras } from "@foxxmd/logging"; +import { DbConcrete } from "../drizzleUtils.js"; +import { loggerNoop } from "../../../MaybeLogger.js"; +import { ErrorLike, PlayObject, TA_CLOSE, TA_DEFAULT_ACCURACY, TA_EXACT, TemporalAccuracy } from "../../../../../core/Atomic.js"; +import { generateInputEntity, generatePlayEntity, PlayEntityOpts, hydratePlaySelect, PlayHydateOptions } from "../entityUtils.js"; +import { playInputs, plays, queueStates, relations } from "../schema/schema.js"; +import { PlayNew, PlaySelect, PlayInputNew, FindWhere, FindMany, CompareOpKey, QueueStateSelect, PlayInputSelect, PlaySelectRel, FindWith, PlaySelectWithQueueStates, WhereClause } from "../drizzleTypes.js";; +import { MarkOptional, MarkRequired, PathValue } from "ts-essentials"; +import { genGroupIdStrFromPlay, removeEmptyArrays, removeUndefinedKeys } from "../../../../utils.js"; +import dayjs, { Dayjs } from "dayjs"; +import { RelationsFieldFilter, eq, inArray, ne, notInArray, desc, asc, and, sql, Placeholder } from "drizzle-orm"; +import { CompactableProperty, RetentionOptions, retentionPlayTypes } from "../../../infrastructure/config/database.js"; +import { shortTodayAwareFormat } from "../../../../../core/TimeUtils.js"; +import { buildDateCompare, CompareDateOp, ComponentConstrainedRepoOpts, DrizzleBaseRepository, DrizzleRepositoryOpts, PaginatedQueryResponse, PaginatedResponse } from "./BaseRepository.js"; +import { asPlay } from "../../../../../core/PlayMarshalUtils.js"; +import assert, { Assert } from "node:assert"; +import { hashObject, parseArrayFromMaybeString } from "../../../../utils/StringUtils.js"; +import { playContentBasicInvariantTransform, playMbidIdentifier } from "../../../../utils/PlayComparisonUtils.js"; +import { comparePlayTemporally, getScrobbleTsSOCDate, getScrobbleTsSOCDateWithContext, getTemporalAccuracyCloseVal, hasAcceptableTemporalAccuracy } from "../../../../utils/TimeUtils.js"; +import { SourceType } from "../../../infrastructure/config/source/sources.js"; + +// https://github.com/drizzle-team/drizzle-orm/issues/695 may be useful for typing models with relations? + +export interface QueueCriteria { + queueName: string + queueStatus: QueueStateSelect['queueStatus'][] | QueueStateSelect['queueStatus'] +} + +export interface PlayWhereOpts { + state?: PlaySelect['state'][] + stateNot?: PlaySelect['state'][] + componentId?: number + seenAt?: CompareDateOp + playedAt?: CompareDateOp + queues?: QueueCriteria[] + uid?: string[] +} + +export type WithPlayRelation = 'input' | 'parent' | 'parent-input' | 'queues'; +export interface QueryPlaysOpts extends PlayWhereOpts { + sort?: 'seenAt' | 'playedAt' + order?: 'asc' | 'desc' + with?: WithPlayRelation[] + limit?: number + offset?: number +} + +export interface HydrateOpts { + hydrate?: PlayHydateOptions[] +} + +export type RepositoryCreatePlayOpts = PlayEntityOpts + & { + input: MarkOptional + } + & Pick; + +type PlayIdentifierPrimitiveMap = { + uid: string; + id: number; +}; +const identifierExtractor: { [K in keyof PlayIdentifierPrimitiveMap]: (play: {id: number, uid: string}) => PlayIdentifierPrimitiveMap[K] } = { + id: (play) => play.id, + uid: (play) => play.uid, +}; +export class DrizzlePlayRepository extends DrizzleBaseRepository<'plays'> { + + protected hasQueueNextPrepared?: ReturnType + protected getQueueNextPrepared?: ReturnType + protected getQueuedScrobbleRangePrepared?: ReturnType + + constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) { + super(db, 'plays', 'Plays', opts); + } + + findByUid = async (uid: string, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise => { + const res = await this.db.query.plays.findFirst({ + where: { + uid, + componentId: opts.componentId ?? this.componentId + }, + with: { + queueStates: true + } + }); + res.play = hydratePlaySelect(res, opts.hydrate); + return res; + } + + createPlays = async (entitiesOpts: RepositoryCreatePlayOpts[], opts: HydrateOpts = {}) => { + + const { + hydrate + } = opts; + let playRows: PlaySelect[]; + + await this.db.transaction(async (tx) => { + + const entitiesData = entitiesOpts.map((data) => { + const { + play, + input, + ...rest + } = data; + return generatePlayEntity(play, { componentId: this.componentId, ...rest }); + }); + + playRows = await tx.insert(plays).values(entitiesData).returning(); + + const inputDatas = playRows.map((x, index) => { + const { + play, + input, + } = entitiesOpts[index]; + const { + play: inputPlay = play, + ...restInput + } = input; + + return generateInputEntity({ play: inputPlay, playId: x.id, ...restInput }); + }); + + const inputRow = await tx.insert(playInputs).values(inputDatas); + + }); + + return playRows.map(x => ({...x, play: hydratePlaySelect(x, hydrate)})); + } + + findPlays = async (args: QueryPlaysOpts, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise => { + const { + hydrate, + componentId = this.componentId + } = opts; + // this does not work as type for query variable + // it erases the result type for some reason + // + // Parameters[0] + + // this does work but it is also integrated into FindWith + //let withQuery: Parameters[0]['with'] = undefined; + + let query: FindMany<'plays'> = { + limit: args.limit, + offset: args.offset + }; + + query.where = buildPlayWhere({componentId: componentId, ...args}); + + if (args.sort !== undefined) { + query.orderBy = { + [args.sort]: args.order ?? 'desc' + } + } else { + query.orderBy = { + id: 'asc' + } + } + + if(args.with !== undefined) { + query.with = {}; + for(const w of args.with) { + switch (w) { + case 'input': + query.with.input = true; + break; + case 'parent': + query.with.parent = true; + break; + case 'parent-input': + query.with.parent = { + with: { + input: true + } + }; + break; + case 'queues': + query.with.queueStates = true; + break; + default: + throw new Error(`Unknown relation ${w}`); + } + } + } + query = removeUndefinedKeys(query); + const results = await this.db.query.plays.findMany(query); + return results.map((x) => ({...x, play: hydratePlaySelect(x, hydrate)})); + } + + findPlayIds = async (args: QueryPlaysOpts, opts: ComponentConstrainedRepoOpts = {}): Promise => { + const { + componentId = this.componentId + } = opts; + + let query: FindMany<'plays'> = { + limit: args.limit, + offset: args.offset, + }; + + query.where = buildPlayWhere({componentId: componentId, ...args}); + + if (args.sort !== undefined) { + query.orderBy = { + [args.sort]: args.order ?? 'desc' + } + } else { + query.orderBy = { + id: 'asc' + } + } + + query = removeUndefinedKeys(query); + const results = await this.db.query.plays.findMany({ + limit: args.limit, + offset: args.offset, + columns: {id: true}, + orderBy: args.sort !== undefined ? {[args.sort]: args.order ?? 'desc'} : {id: 'asc'}, + }); + return results.map((x) => x.id); + } + + findPlayIdentifiers = async (args: QueryPlaysOpts, identifier: T, opts: ComponentConstrainedRepoOpts = {}): Promise => { + const { + componentId = this.componentId, + } = opts; + + const results = await this.db.query.plays.findMany({ + limit: args.limit, + offset: args.offset, + columns: {id: true, uid: true}, + orderBy: args.sort !== undefined ? {[args.sort]: args.order ?? 'desc'} : {id: 'asc'}, + where: buildPlayWhere({componentId: componentId, ...args}) + }); + + // we getting fancy now + return results.map(identifierExtractor[identifier]); + } + + findPlaysPaginated = async (args: QueryPlaysOpts, opts: HydrateOpts & ComponentConstrainedRepoOpts = {}): Promise> => { + const { + limit = 100, + offset = 0, + ...rest + } = args; + const clampedLimit = Math.min(limit, 100); + const res = await this.findPlays({limit: clampedLimit, offset, ...rest}, opts) as T[]; + return {data: res, meta: {limit: clampedLimit, offset}}; + } + + // async updateById(id: number, data: Partial): Promise { + // if(data.play !== undefined) { + // data.play = withoutDbAwareness(data.play); + // } + // super.updateById(id, data); + // } + + setStateById = async (state: PlayNew['state'], ids: number[]): Promise => { + const validIds = ids.filter(x => x !== undefined && x !== null); + assert(validIds.length > 0, `Should not pass empty array of ids, after filtering, to update state. Original ids list: ${ids}`); + await this.db.update(plays).set({state}).where(inArray(plays.id, ids)); + } + + deletePlays = async (playsData: (Pick | number)[]) => { + const ids = playsData.map(x => typeof x === 'number' ? x : x.id); + await this.db.delete(plays).where(inArray(plays.id, ids)); + } + + findPurgablePlayIds = async (olderThanDate: Dayjs, opts: { states?: PlaySelect['state'][], compacted?: string, dateComparer?: 'updatedAt' | 'seenAt' } & ComponentConstrainedRepoOpts = {}): Promise => { + + const { + states, + compacted, + componentId = this.componentId, + dateComparer = 'updatedAt' + } = opts; + + let where: FindWhere<'plays'> = { + component: { + id: componentId + }, + [dateComparer]: { + lte: olderThanDate + }, + NOT: { + children: {} + } + }; + + if (states !== undefined) { + where.state = { + in: states + } + } + + if(compacted !== undefined) { + where.compacted = { + OR: [ + { + isNull: true + }, + { + NOT: { + eq: compacted + } + } + ] + } + } + + const rows = await this.db.query.plays.findMany({ + columns: { + id: true + }, + where, + orderBy: { + id: 'asc' + } + }); + + return rows.map(x => x.id); + } + + public retentionCleanup = async (componentType: string, retentionOpts: RetentionOptions & ComponentConstrainedRepoOpts) => { + + const loggerDel = childLogger(this.logger, ['Retention', 'Delete']); + const loggerCom = childLogger(this.logger, ['Retention', 'Compact']); + let summaryDelStates: string[] = []; + let summaryCompactStates: string[] = []; + + loggerDel.debug('Starting cleanup...'); + for(const retentionType of retentionPlayTypes) { + try { + const date = dayjs().subtract(retentionOpts.deleteAfter[retentionType].asMilliseconds()); + let state: PlaySelect['state']; + if(retentionType === 'completed') { + state = componentType === 'source' ? 'discovered' : 'scrobbled'; + } else { + state = retentionType; + } + loggerDel.trace(`Finding '${retentionType}' plays older than ${shortTodayAwareFormat(date)}...`); + const ids = await this.findPurgablePlayIds(date, {states: [state], componentId: retentionOpts.componentId}); + loggerDel.trace(`Found ${ids.length} '${retentionType}' plays`); + if(ids.length === 0) { + summaryDelStates.push(`No '${retentionType}' Plays older than ${shortTodayAwareFormat(date)}`); + } else { + loggerDel.trace(`Deleting ${ids.length} '${retentionType}' plays`); + await this.deletePlays(ids); + loggerDel.trace(`'${retentionType}' plays deleted!`); + summaryDelStates.push(`${ids.length} '${retentionType}' Plays older than ${shortTodayAwareFormat(date)}`) + } + } catch (e) { + loggerDel.warn(new Error(`Failed to perform retention cleanup on '${retentionType}' type`, {cause: e})); + } + } + loggerDel.verbose(`Cleanup done! Summary:\n${summaryDelStates.join(' | ')}`); + + if(retentionOpts.compact.length === 0) { + loggerCom.debug('Compacting is disabled, skipping cleanup.'); + return; + } + + const compactTypes = retentionOpts.compact; + let compactedFlags: CompactableProperty[] = []; + if(compactTypes.includes('input')) { + compactedFlags.push('input'); + } + if(compactTypes.includes('transform')) { + compactedFlags.push('transform'); + } + + loggerCom.debug('Starting cleanup...'); + for(const retentionType of retentionPlayTypes) { + if(retentionOpts.compactAfter[retentionType] === false) { + summaryCompactStates.push(`Skipped ${retentionType}`); + continue; + } + try { + const date = dayjs().subtract(retentionOpts.compactAfter[retentionType].asMilliseconds()); + let state: PlaySelect['state']; + if(retentionType === 'completed') { + state = componentType === 'source' ? 'discovered' : 'scrobbled'; + } else { + state = retentionType; + } + loggerCom.trace(`Finding '${retentionType}' plays older than ${shortTodayAwareFormat(date)}...`); + const ids = await this.findPurgablePlayIds(date, {compacted: compactedFlags.join('-'), states: [state], dateComparer: 'seenAt', componentId: retentionOpts.componentId}); + loggerCom.trace(`Found ${ids.length} '${retentionType}' plays`); + if(ids.length === 0) { + summaryDelStates.push(`No '${retentionType}' Plays older than ${shortTodayAwareFormat(date)}`); + } else { + for(const id of ids) { + let compactedPlay: PlayObject; + if(compactTypes.includes('input')) { + await this.db.update(playInputs).set({ + data: {removedReason: 'Removed by compaction'} + }).where(eq(playInputs.playId, id)); + } + if(compactTypes.includes('transform')) { + const playRow = await this.db.query.plays.findFirst({where: {id: id}}); + if(playRow === undefined) { + // uhh shouldn't be + loggerCom.warn(`No Play found with ID ${id}, but it should have been...`); + continue; + } + + compactedPlay = playRow.play; + compactedPlay.meta.lifecycle.steps = compactedPlay.meta.lifecycle.steps.map(x => { + if(x.inputs == undefined) { + return x; + } + return {...x, inputs: x.inputs.map(y => ({type: y.type, input: 'Removed by compaction'}))}; + }); + } + + const updater = this.db.update(plays); + const vals: Parameters[0] = { + compacted: compactedFlags.join('-') + }; + if(compactedPlay !== undefined) { + vals.play = compactedPlay; + } + await this.db.update(plays).set(vals).where(eq(plays.id, id)); + } + loggerCom.trace(`Compacted ${ids.length} '${retentionType}' plays`); + summaryCompactStates.push(`${ids.length} '${retentionType}' Plays older than ${shortTodayAwareFormat(date)}`) + } + } catch (e) { + loggerCom.warn(new Error(`Failed to perform retention cleanup on '${retentionType}' type`, {cause: e})); + } + } + + loggerCom.verbose(`Cleanup done! Summary:\n${summaryDelStates.join(' | ')}`); + } + + protected prepareGetQueueNext = () => this.db.query.plays.findFirst({ + where: { + componentId: sql.placeholder('componentId'), + queueStates: { + queueName: sql.placeholder('queueName'), + queueStatus: 'queued', + retries: { + lte: sql.placeholder('retries') + } + }, + }, + with: { + queueStates: true + }, + orderBy: { + seenAt: 'asc' + }, + }).prepare() + + public getQueueNext = async (queueName: string, opts: {order?: 'asc' | 'desc', retries?: number} & ComponentConstrainedRepoOpts = {}): Promise => { + const { + retries = 0, + order = 'asc', + componentId = this.componentId + } = opts; + + // let where: FindWhere<'plays'> = { + // componentId + // } + + // if(retries !== undefined) { + // where.queueStates = { + // queueName, + // queueStatus: 'queued', + // retries: { + // lte: retries + // } + // } + // } else { + // where.queueStates = { + // queueName, + // queueStatus: 'queued' + // } + // } + + // const res = await this.db.query.plays.findFirst({ + // where: where, + // orderBy: { + // seenAt: order + // }, + // with: { + // queueStates: true + // } + // }); + + if(this.getQueueNextPrepared === undefined) { + this.getQueueNextPrepared = this.prepareGetQueueNext(); + } + + const res = await this.getQueueNextPrepared.execute({queueName, retries, componentId}); + + if(res === undefined) { + return undefined; + } + res.play = hydratePlaySelect(res); // asPlay(res.play); + return res; + } + + protected prepareHasQueueNext = () => this.db.query.plays.findFirst({ + columns: { + id: true + }, + where: { + componentId: this.componentId, + queueStates: { + queueName: sql.placeholder('queueName'), + queueStatus: 'queued', + retries: { + lte: sql.placeholder('retries') + } + } + } + }).prepare() + + public hasQueueNext = async (queueName: string, retries: number = 0): Promise => { + if(this.hasQueueNextPrepared === undefined) { + this.hasQueueNextPrepared = this.prepareHasQueueNext(); + } + const nextId = await this.hasQueueNextPrepared.execute({queueName, retries}); + return nextId !== undefined; + } + + protected prepareGetQueuedScrobbleRange = () => this.db.query.plays.findMany({ + where: { + componentId: this.componentId, + queueStates: { + queueName: sql.placeholder('queueName'), + queueStatus: 'queued', + retries: { + lte: sql.placeholder('retries') + } + }, + }, + orderBy: { + seenAt: 'asc', + }, + limit: sql.placeholder('limit') + }).prepare() + + public getQueuedScrobbleRange = async (queueName: string, opts: {retries?: number, limit?: number} = {}): Promise => { + if(this.getQueuedScrobbleRangePrepared === undefined) { + this.getQueuedScrobbleRangePrepared = this.prepareGetQueuedScrobbleRange(); + } + const res = await this.getQueuedScrobbleRangePrepared.execute({queueName, retries: opts.retries ?? 0, limit: opts.limit ?? 30}); + return res.map(x => x.play); + } + + public getQueued = async (queueName: string, opts: { + order?: 'asc' | 'desc', + limit?: number, + offset?: number, + retries?: number, + } & ComponentConstrainedRepoOpts & HydrateOpts = {} + ): Promise<{data: PlaySelect[], meta: PaginatedQueryResponse}> => { + const { + order = 'asc', + limit = 100, + offset = 0, + retries, + componentId = this.componentId, + hydrate + } = opts; + let where: FindWhere<'plays'> = { + componentId + } + if(retries !== undefined) { + where.queueStates = { + queueName, + queueStatus: 'queued', + retries: { + lte: retries + } + } + } else { + where.queueStates = { + queueName, + queueStatus: 'queued' + } + } + const res = await this.db.query.plays.findMany({ + where, + orderBy: { + seenAt: order + }, + limit, + offset + }); + return {data: res.map(x => ({...x, play: hydratePlaySelect(x, hydrate)})), meta: {limit, offset}}; + } + + public checkExisting = async (play: PlayObject, opts: {queueName?: string, states?: PlaySelect['state'][], taAccuracy?: TemporalAccuracy[]} & ComponentConstrainedRepoOpts = {}): Promise => { + const { + queueName, + componentId = this.componentId, + taAccuracy = TA_DEFAULT_ACCURACY, + states + } = opts; + const hash = hashObject(playContentBasicInvariantTransform(play).data); + + // we get all plays with a play date between playdate - (source accuracy) AND (playDateCompleted or playDate) + (source accuracy) + // which we can then use with temporal comparison to make sure we are comparing the correct dates + // + // this isn't as fast as just comparing playDate directly but its still much faster/cheaper than paginating plays and doing everything in-memory + const dateGranularity = getTemporalAccuracyCloseVal(play.meta.source as SourceType); + let endRange: Dayjs; + if(play.data.playDateCompleted !== undefined) { + // this will be present if source reports it + // or we tracked it live with MemorySource + endRange = play.data.playDateCompleted.add(dateGranularity, 's'); + } else { + endRange = play.data.playDate.add(dateGranularity, 's'); + } + let where: FindWhere<'plays'> = { + componentId, + playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play)), + }; + + if(queueName !== undefined) { + where.queueStates = { + queueName, + queueStatus: 'queued' + } + } + if(states !== undefined) { + where.state = { + in: states + } + } + + const mbidId = playMbidIdentifier(play); + if(mbidId !== undefined) { + where.AND = [ + { + OR: [ + { + playHash: hash + }, + { + mbidIdentifier: mbidId + } + ] + } + ] + } else { + where.playHash = hash; + } + + const res = await this.db.query.plays.findMany({ + where, + with: { + queueStates: true + } + }); + if(res.length === 0) { + return undefined; + } + return res.map(x => ({...x, play: hydratePlaySelect(x)})).find(x => { + const temporalComparison = comparePlayTemporally(x.play, play); + return hasAcceptableTemporalAccuracy(temporalComparison.match, taAccuracy) + }) + } + + public getTemporallyClosePlays = async (play: PlayObject, opts: {states?: PlaySelect['state'][], bufferTime?: number} & { with?: WithPlayRelation[] } & ComponentConstrainedRepoOpts = {}): Promise => { + const { + componentId = this.componentId, + bufferTime, + states, + with: qWith + } = opts; + + let query: FindMany<'plays'> = {}; + + let where: FindWhere<'plays'> = { + componentId, + playedAt: buildDateCompare(getTemporallyCloseDateCompareOp(play, {bufferTime})), + }; + if(states !== undefined) { + where.state = { + in: states + } + } + query.where = where; + + return ((await this.db.query.plays.findMany({ + where, + with: buildPlayWith(qWith) + })) as PlaySelectRel[]).map(x => ({...x, play: hydratePlaySelect(x)})); + } +} + +export const getTemporallyCloseDateCompareOp = (play: PlayObject, opts: {bufferTime?: number, useCompleted?: boolean} = {}): CompareDateOp => { + const { + // use either provided arg or default to using source granularity + bufferTime = getTemporalAccuracyCloseVal(play.meta.source as SourceType), + useCompleted = true + } = opts; + // we get all plays with a play date between playdate - (buffer) AND (playDateCompleted or playDate) + (buffer) + let endRange: Dayjs; + if(play.data.playDateCompleted !== undefined && useCompleted) { + // this will be present if source reports it + // or we tracked it live with MemorySource + endRange = play.data.playDateCompleted.add(bufferTime, 's'); + } else { + endRange = play.data.playDate.add(bufferTime, 's'); + } + return { + type: 'between', + range: [play.data.playDate.subtract(bufferTime, 's'), endRange] + } +} + +export const buildPlayWith = (args: WithPlayRelation[] | undefined): FindWith<'plays'> | undefined => { + if(args === undefined) { + return undefined; + } + const qWith: FindWith<'plays'> = {}; + for(const w of args) { + switch (w) { + case 'input': + qWith.input = true; + break; + case 'parent': + qWith.parent = true; + break; + case 'parent-input': + qWith.parent = { + with: { + input: true + } + }; + break; + case 'queues': + qWith.queueStates = true; + break; + default: + throw new Error(`Unknown relation ${w}`); + } + } + return qWith; +} + +export const buildPlayWhere = (args: PlayWhereOpts): WhereClause<'plays'> => { + // old way + // let where: Parameters<(ReturnType)['query']['plays']['findMany']>[0]['where'] = { + // }; + let where: FindWhere<'plays'> = { + componentId: args.componentId + }; + if (args.state !== undefined) { + where.state = { + in: args.state + } + } + if(args.stateNot !== undefined) { + where.state = { + NOT: { + in: args.stateNot + } + } + } + if (args.seenAt !== undefined) { + where.seenAt = buildDateCompare(args.seenAt); + } + if (args.playedAt !== undefined) { + where.playedAt = buildDateCompare(args.playedAt); + } + if(args.uid !== undefined) { + where.uid = { + in: args.uid + } + } + const { + queues = [] + } = args; + if(queues.length > 0) { + // need to do this optimistically even if we overwrite with only 1 condition later + where.queueStates = { + OR: [] + } + // so that we can use this type + // or else assigning an array to OR using only `typeof where.queueStates` causes a type error + let queueWhere: typeof where.queueStates.OR[0][] = []; + for(const q of queues) { + queueWhere.push( + { + queueName: q.queueName, + queueStatus: typeof q.queueStatus === 'string' ? q.queueStatus : { + in: q.queueStatus + } + } + ) + } + if(queueWhere.length === 1) { + where.queueStates = queueWhere[0]; + } else { + where.queueStates = { + OR: queueWhere + } + } + } + return where; +} + +export const playToRepositoryCreatePlayOpts = (data: MarkOptional): RepositoryCreatePlayOpts => { + const { + play: { + meta: { + lifecycle: { + input, + original, + ...lifecycleRest + } = {}, + ...metaRest + }, + ...playRest + }, + ...rest + } = data; + + return { + play: { + ...playRest, + meta: { + ...metaRest, + // @ts-expect-error + lifecycle: { + ...lifecycleRest + } + } + }, + ...rest, + input: { + play: original, + data: input + } + } +} + +export type RequestPlayQuery = Partial< Record, string>>; + +export const queryArgsFromRequest = (rec: RequestPlayQuery): QueryPlaysOpts => { + + const { + state, + stateNot, + uid, + with: withQuery, + seenAt, + playedAt, + limit, + sort, + order, + offset, + componentId, + queues, + ...rest + } = rec; + + let queryArgs: QueryPlaysOpts = removeEmptyArrays({ + state: parseArrayFromMaybeString(state) as PlaySelect['state'][], + stateNot: parseArrayFromMaybeString(stateNot) as PlaySelect['state'][], + uid: parseArrayFromMaybeString(uid), + with: parseArrayFromMaybeString(withQuery) as WithPlayRelation[], + sort: sort as 'playedAt' | 'seenAt', + order: order as 'asc' | 'desc', + ...rest + }); + + if(limit !== undefined) { + queryArgs.limit = Number.parseInt(limit); + } + if(offset !== undefined) { + queryArgs.offset = Number.parseInt(offset); + } + + return queryArgs; +} \ No newline at end of file diff --git a/src/backend/common/database/drizzle/repositories/QueueRepository.ts b/src/backend/common/database/drizzle/repositories/QueueRepository.ts new file mode 100644 index 000000000..54c8d379d --- /dev/null +++ b/src/backend/common/database/drizzle/repositories/QueueRepository.ts @@ -0,0 +1,41 @@ +import { eq, and, lte, inArray } from "drizzle-orm"; +import { DrizzleBaseRepository, DrizzleRepositoryOpts } from "./BaseRepository.js"; +import { DbConcrete } from "../drizzleUtils.js"; +import { QueueStateSelect } from "../drizzleTypes.js"; +import { queueStates } from "../schema/schema.js"; +import { CLIENT_DEAD_QUEUE } from "../../../../../core/Atomic.js"; +export class DrizzleQueueRepository extends DrizzleBaseRepository<'queueStates'> { + + constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) { + super(db, 'queueStates', 'Queue', opts); + } + + public deadFailedToQueue = async (componentId: number, retries: number): Promise => { + await this.db.update(queueStates).set({ + queueStatus: 'queued', + }).where(and( + eq(queueStates.componentId, componentId), + lte(queueStates.retries, retries), + eq(queueStates.queueStatus, 'failed'), + eq(queueStates.queueName, CLIENT_DEAD_QUEUE) + )); + } + + public failedQueueToCompleted = async (componentId: number): Promise => { + await this.db.update(queueStates).set({ + queueStatus: 'completed', + }).where(and( + eq(queueStates.componentId, componentId), + eq(queueStates.queueStatus, 'queued'), + eq(queueStates.queueName, CLIENT_DEAD_QUEUE) + )); + } + + public getQueueCount = async (componentId: number, queueNames: string[], queueStatus: QueueStateSelect['queueStatus'][] = ['queued']): Promise => { + return await this.db.$count(queueStates, and( + eq(queueStates.componentId, componentId), + inArray(queueStates.queueName, queueNames), + inArray(queueStates.queueStatus, queueStatus) + )); + } +} \ No newline at end of file diff --git a/src/backend/common/database/drizzle/schema/schema.ts b/src/backend/common/database/drizzle/schema/schema.ts new file mode 100644 index 000000000..b21a48460 --- /dev/null +++ b/src/backend/common/database/drizzle/schema/schema.ts @@ -0,0 +1,239 @@ +import { integer, serial as primaryInt, pgTable as table, text, varchar, json, index, uniqueIndex, customType, AnyPgColumn, timestamp } from "drizzle-orm/pg-core"; +import { defineRelations } from 'drizzle-orm'; +import dayjs, { Dayjs } from "dayjs"; +import { nanoid } from "nanoid"; +import { ErrorLike, PlayObject } from "../../../../../core/Atomic.js"; +import { asPlayCheap } from "../../../../../core/PlayMarshalUtils.js"; +import { ExternalMetadataTerm, PlayTransformPartsConfig, SearchAndReplaceTerm } from "../../../infrastructure/Transform.js"; +import { JobRangeCount, JobRangeTime } from "../../../infrastructure/Job.js"; + +const DayjsTimestamp = customType< + { + data: Dayjs; + driverData: string; + } +>({ + dataType() { + return 'timestamp' + }, + toDriver(value: Dayjs): string { + return value.toISOString(); + }, + fromDriver(value: string): Dayjs { + return dayjs(value); + }, +}); + +const PlayJson = customType< + { + data: PlayObject; + driverData: string; + } +>({ + dataType() { + return 'jsonb' + }, + toDriver(value: PlayObject): string { + const { + // meta: { + // // dbId, + // // dbUid, + // ...metaRest + // }, + id, + uid, + ...rest + } = value; + return JSON.stringify(rest); + }, + fromDriver(value: any): PlayObject { + return asPlayCheap(value); + }, +}); + + +export const plays = table("plays", { + id: primaryInt().primaryKey(), + uid: varchar({ length: 30 }).notNull().unique().$defaultFn(() => nanoid(20)), + componentId: integer().references(() => components.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + error: json().$type(), + playedAt: DayjsTimestamp('playedAt'), + seenAt: DayjsTimestamp('seenAt'), + updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()), + play: PlayJson('play').notNull(), // text({ mode: 'json' }).notNull().$type(), + state: varchar({enum: ['queued','discovered','discarded','scrobbled','failed','duped'], length: 20}).notNull(), + // https://orm.drizzle.team/docs/indexes-constraints#foreign-key + parentId: integer().references((): AnyPgColumn => plays.id, {onDelete: 'set null', onUpdate: 'cascade'}), + jobId: integer().references(() => jobs.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + playHash: varchar({length: 100}), + mbidIdentifier: varchar({length: 100}), + compacted: varchar({length: 30}) +}, (table) => [ + index("play_parent_id_idx").on(table.parentId), + index("play_component_id_idx").on(table.componentId), + uniqueIndex("play_uid_idx").on(table.uid), + index("play_playedAt_idx").on(table.playedAt), + index("play_seenAt_idx").on(table.seenAt) +]); + +export const playInputs = table("play_inputs", { + id: primaryInt().primaryKey(), + playId: integer().notNull().references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + data: json().$type(), + play: PlayJson('play').notNull(),//text({ mode: 'json' }).notNull().$type(), + createdAt: DayjsTimestamp('createdAt').$defaultFn(() => dayjs()) +}, (table) => [ + uniqueIndex('play_input_id_idx').on(table.playId) +]); + +// export const playParentRelations = defineRelations({plays}, (r) => ({ +// plays: { +// parent: r.one.plays({ +// from: r.plays.parentId, +// to: r.plays.id +// }), +// children: r.many.plays() +// } +// })) + + +// export const playInputRelations = defineRelations({ plays, playInputs }, (r) => ({ +// plays: { +// input: r.one.playInputs({ +// from: r.plays.id, +// to: r.playInputs.playId, +// optional: false, +// }) +// } +// })); + +export const queueStates = table("play_queue_states", { + id: primaryInt().primaryKey(), + playId: integer().notNull().references(() => plays.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + componentId: integer().notNull().references(() => components.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + queueName: varchar({length: 50}).notNull(), + queueStatus: varchar({enum: ['queued','completed','failed'], length: 20}).notNull().default('queued'), + retries: integer().notNull().default(0), + error: json().$type(), + createdAt: DayjsTimestamp('createdAt').notNull().$defaultFn(() => dayjs()), + updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()) +}, (table) => [ + index('play_queue_state_id_idx').on(table.playId) +]); + +// export const playQueueRelations = defineRelations({ plays, queueStates }, (r) => ({ +// plays: { +// queueStates: r.many.queueStates() +// }, +// queueStates: { +// play: r.one.plays({ +// from: r.queueStates.playId, +// to: r.plays.id +// }) +// } +// })); + +export const components = table("components", { + id: primaryInt().primaryKey(), + // user-provided id + uid: varchar({ length: 200 }).notNull(), + mode: varchar({enum: ['source','client'], length: 15}).notNull(), + // spotify, lastfm, etc... + type: varchar({length: 50}).notNull(), + // vanity display name + // used as uid if no user-provided id + name: varchar().notNull(), + // number of discovered/scrobbled plays found in real time + countLive: integer().notNull().default(0), + // number of discovered/scrobbled plays from backlog/jobs + countNonLive: integer().notNull().default(0), + createdAt: DayjsTimestamp('createdAt').$defaultFn(() => dayjs()) +}, +(table) => [ + uniqueIndex('uid_mode_type_idx').on(table.uid,table.mode,table.type) +]); + +export const jobs = table("jobs", { + id: primaryInt().primaryKey(), + componentFromId: integer().notNull().references(() => components.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + componentToId: integer().notNull().references(() => components.id, {onDelete: 'cascade', onUpdate: 'cascade'}), + name: varchar({length: 200}).notNull(), + status: varchar({enum: ['idle','completed','failed','processing'], length: 20}).notNull().default('idle'), + retries: integer().notNull().default(0), + error: json().$type(), + transformOptions: json().$type>(), + initialParameters: json().$type(), + cursor: json(), + total: integer(), + imported: integer().notNull().default(0), + scrobbled: integer().notNull().default(0), + createdAt: DayjsTimestamp('createdAt').notNull().$defaultFn(() => dayjs()), + updatedAt: DayjsTimestamp('updatedAt').notNull().$defaultFn(() => dayjs()), + completedAt: DayjsTimestamp('completedAt') +}); + +const playRelations = defineRelations({ plays, queueStates, playInputs, components, jobs }, (r) => ({ + plays: { + queueStates: r.many.queueStates(), + input: r.one.playInputs({ + from: r.plays.id, + to: r.playInputs.playId, + optional: false, + }), + parent: r.one.plays({ + from: r.plays.parentId, + to: r.plays.id + }), + children: r.many.plays(), + component: r.one.components({ + from: r.plays.componentId, + to: r.components.id, + optional: true + }), + job: r.one.jobs({ + from: r.plays.jobId, + to: r.jobs.id, + optional: true + }) + }, + queueStates: { + play: r.one.plays({ + from: r.queueStates.playId, + to: r.plays.id + }), + component: r.one.components({ + from: r.queueStates.componentId, + to: r.components.id + }) + }, + components: { + plays: r.many.plays(), + queueStates: r.many.queueStates(), + }, + jobs: { + plays: r.many.plays() + } +})); + +export const relations = playRelations; + +export const getConfigByTableName = (name: T) => { + switch(name) { + case 'plays': + return plays; + case 'components': + return components; + case 'playInputs': + return playInputs; + case 'queueStates': + return queueStates; + case 'jobs': + return jobs; + } +} + +const schema = {playInputs, plays, components, queueStates, jobs}; + +export type TSchema = typeof relations; +export type Schema = typeof schema; +export type TableName = keyof TSchema; \ No newline at end of file diff --git a/src/backend/common/errors/MSErrors.ts b/src/backend/common/errors/MSErrors.ts index dcfdc3828..06aba95f2 100644 --- a/src/backend/common/errors/MSErrors.ts +++ b/src/backend/common/errors/MSErrors.ts @@ -43,12 +43,14 @@ export class SimpleError extends Error implements HasSimpleError { stackShortened: boolean = false; shortenStack() { - const atIndex = parseRegexSingle(STACK_AT_REGEX,this.stack); - if(atIndex !== undefined) { - const firstn = this.stack.indexOf('\n', atIndex.index + atIndex.match.length); - if(firstn !== -1) { - this.stack = this.stack.slice(0, firstn); - this.stackShortened = true; + if(this.stack !== undefined) { + const atIndex = parseRegexSingle(STACK_AT_REGEX, this.stack); + if(atIndex !== undefined) { + const firstn = this.stack.indexOf('\n', atIndex.index + atIndex.match.length); + if(firstn !== -1) { + this.stack = this.stack.slice(0, firstn); + this.stackShortened = true; + } } } } @@ -128,4 +130,23 @@ export const generateLoggableAbortReason = (msg: string, signal: AbortSignal): A } Error.captureStackTrace(err, generateLoggableAbortReason); return err; +} + +export class InvalidRegexError extends SimpleError { + constructor(regex: RegExp | RegExp[], val?: string, url?: string, message?: string) { + const msgParts = [ + message ?? 'Regex(es) did not match the value given.', + ]; + let regArr = Array.isArray(regex) ? regex : [regex]; + for(const r of regArr) { + msgParts.push(`Regex: ${r}`) + } + if (val !== undefined) { + msgParts.push(`Value: ${val}`); + } + if (url !== undefined) { + msgParts.push(`Sample regex: ${url}`); + } + super(msgParts.join('\r\n')); + } } \ No newline at end of file diff --git a/src/backend/common/index.ts b/src/backend/common/index.ts index bb62500a0..234de76d6 100644 --- a/src/backend/common/index.ts +++ b/src/backend/common/index.ts @@ -5,4 +5,4 @@ import * as path from 'path'; //const __dirname = path.dirname(__filename); export const projectDir = process.cwd(); //path.resolve(__dirname, '../../../'); -export const configDir: string = path.resolve(projectDir, './config'); +export const configDir: string = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index d53a1a623..a4fe28989 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -141,9 +141,6 @@ export interface RemoteIdentityParts { agent: string | undefined } -// https://stackoverflow.com/questions/40510611/typescript-interface-require-one-of-two-properties-to-exist#comment116238286_49725198 -export type RequireAtLeastOne = Omit & { [ P in R ] : Required> & Partial> }[R]; - /** * https://www.last.fm/api/scrobbling (When is a scrobble a scrobble?) * https://github.com/krateng/maloja/blob/master/API.md#scrobbling-guideline @@ -299,6 +296,21 @@ export interface CacheConfigOptions { regex?: number } +export interface CacheConfigUser { + auth?: { + provider: 'valkey' | 'file', + [key: string]: any + }; + valkey?: string + /** Number of regex functions to cache (LRU) + * + * @default 200 + */ + regex?: number + // to allow deprecated scrobble config without having it show up in schema docs + [key: string]: any +} + export interface MusicbrainzApiConfigData { url?: string contact: string, @@ -414,4 +426,18 @@ export interface ScrobbleRangeResult { fetchedAt: Dayjs } -export const REFRESH_STALE_DEFAULT = 60; \ No newline at end of file +export const REFRESH_STALE_DEFAULT = 60; + +/** + * A duration of time + * + * May be either: + * + * * a `number` of seconds + * * a `string` containing a number and a unit of time compatible with dayjs + * + * @example [60, 3600, "1 hour", "4 days"] + */ +export type DurationValue = number | string; + +export type DbExternalMode = 'none' | 'live' | 'standalone'; \ No newline at end of file diff --git a/src/backend/common/infrastructure/Job.ts b/src/backend/common/infrastructure/Job.ts new file mode 100644 index 000000000..1dc12b783 --- /dev/null +++ b/src/backend/common/infrastructure/Job.ts @@ -0,0 +1,33 @@ +import { UnixTimestamp } from "../../../core/Atomic.js" + +export interface JobParameters { + /** maximum number of results to get for the entire job */ + fetchMax?: number + order?: 'asc' | 'desc' + /** Whether to increment targeted scrobbler client scrobble count */ + countInScrobbler?: boolean + maxRetries?: number + /** Whether to keep non-failed plays in db after scrobbling has occurred + * + * * `true` => keep all plays + * * `false` => keep no plays + * * number => max number of play to keep based on import date + */ + keepPlays?: boolean | number +} + +export interface JobRangeCount extends JobParameters { + order: 'asc' | 'desc' +} + +export interface JobRangeTime extends JobParameters { + /** Unix timestamp */ + from: UnixTimestamp + /** Unix timestamp */ + to: UnixTimestamp +} + +// TODO +export interface JobCursor { + +} \ No newline at end of file diff --git a/src/backend/common/infrastructure/config/aioConfig.ts b/src/backend/common/infrastructure/config/aioConfig.ts index be1241e5b..6b7f9dfda 100644 --- a/src/backend/common/infrastructure/config/aioConfig.ts +++ b/src/backend/common/infrastructure/config/aioConfig.ts @@ -5,8 +5,9 @@ import { RequestRetryOptions } from "./common.js"; import { WebhookConfig } from "./health/webhooks.js"; import { CommonSourceOptions, SourceRetryOptions } from "./source/index.js"; import { SourceAIOConfig } from "./source/sources.js"; -import { CacheConfigOptions } from "../Atomic.js"; +import { CacheConfigOptions, CacheConfigUser, DurationValue } from "../Atomic.js"; import { TransformerCommonConfig } from "../../../../core/Atomic.js"; +import { RetentionConfig } from "./database.js"; export interface SourceDefaults extends CommonSourceOptions { @@ -66,9 +67,13 @@ export interface AIOConfig { * */ debugMode?: boolean - cache?: CacheConfigOptions + cache?: CacheConfigUser transformers?: TransformerCommonConfig[] + + database?: { + retention?: RetentionConfig + } } export interface AIOClientConfig { diff --git a/src/backend/common/infrastructure/config/client/index.ts b/src/backend/common/infrastructure/config/client/index.ts index 0a2803033..83b13a597 100644 --- a/src/backend/common/infrastructure/config/client/index.ts +++ b/src/backend/common/infrastructure/config/client/index.ts @@ -1,5 +1,7 @@ +import { DurationValue } from "../../Atomic.js"; import { PlayTransformConfig, PlayTransformOptions } from "../../Transform.js"; import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js"; +import { RetentionConfig } from "../database.js"; /** * Scrobble matching (between new source track and existing client scrobbles) logging options. Used for debugging. @@ -105,6 +107,8 @@ export interface CommonClientOptions extends RequestRetryOptions, UpstreamRefres deadLetterRetries?: number playTransform?: PlayTransformOptions + + retention?: RetentionConfig } export interface CommonClientConfig extends CommonConfig { diff --git a/src/backend/common/infrastructure/config/common.ts b/src/backend/common/infrastructure/config/common.ts index d64a94ec4..f9f6fea6d 100644 --- a/src/backend/common/infrastructure/config/common.ts +++ b/src/backend/common/infrastructure/config/common.ts @@ -1,7 +1,20 @@ import { keyOmit } from "../Atomic.js"; +export interface CommonConfigPrimitives { + name?: string + id?: string + enable?: boolean +} + export interface CommonConfig { name?: string + /** A UNIQUE identifier for this Source/Client + * + * It should be unique for the given Source/Client type. No other Source/Client of the same type should have this ID. This ID will be used to register this Source/Client in the database so that it can be identified even if you change the name of the component. + * + * If no id is given the name of this component will be used. + */ + id?: string data?: CommonData /** * Should MS use this client/source? Defaults to true diff --git a/src/backend/common/infrastructure/config/database.ts b/src/backend/common/infrastructure/config/database.ts new file mode 100644 index 000000000..92085e1e0 --- /dev/null +++ b/src/backend/common/infrastructure/config/database.ts @@ -0,0 +1,36 @@ +import { Duration } from "dayjs/plugin/duration.js"; +import { DurationValue } from "../Atomic.js"; + +export type RetentionPlayType = 'failed' | 'completed' | 'duped'; +export const retentionPlayTypes: RetentionPlayType[] = ['failed','completed','duped']; + +export type RetentionValueUnparsed = DurationValue | Duration | false; +export type RetentionValue = Duration | false; +export interface RententionGranular { + failed?: T + completed?: T + duped?: T +} + +export type RetentionConfigValue = T | RententionGranular; +export type RetentionOption = Required>; + +export type CompactableProperty = 'transform' | 'input'; +export const COMPACTABLE = { + transform: 'transform', + input: 'input' +} as const satisfies Record; +export const compactableProperties: CompactableProperty[] = [COMPACTABLE.transform, COMPACTABLE.input]; +export interface RetentionConfig { + deleteAfter?: RetentionConfigValue + compactAfter?: RetentionConfigValue + compact?: CompactableProperty[] +} + +export interface RetentionOptions { + deleteAfter: RetentionOption + compactAfter: RetentionOption + compact: CompactableProperty[] +} + +export const DEFAULT_RETENTION_DELETE_AFTER = 604800; // 7 days \ No newline at end of file diff --git a/src/backend/common/infrastructure/config/source/index.ts b/src/backend/common/infrastructure/config/source/index.ts index b184bc689..8d87c72f7 100644 --- a/src/backend/common/infrastructure/config/source/index.ts +++ b/src/backend/common/infrastructure/config/source/index.ts @@ -2,6 +2,8 @@ import { FileLogOptions, LogLevel } from "@foxxmd/logging"; import { PlayTransformConfig, PlayTransformOptions } from "../../Transform.js"; import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js"; +import { RetentionConfig } from "../database.js"; +import { DurationValue } from "../../Atomic.js"; export interface SourceRetryOptions extends RequestRetryOptions { /** @@ -108,6 +110,8 @@ export interface CommonSourceOptions extends SourceRetryOptions { scrobbleBacklogCount?: number playTransform?: PlayTransformOptions + + retention?: RetentionConfig } export interface ManualListeningOptions { diff --git a/src/backend/common/transforms/AtomicPartsTransformer.ts b/src/backend/common/transforms/AtomicPartsTransformer.ts index 14a298c4c..8386aae89 100644 --- a/src/backend/common/transforms/AtomicPartsTransformer.ts +++ b/src/backend/common/transforms/AtomicPartsTransformer.ts @@ -1,4 +1,4 @@ -import { isPlayObject, ObjectPlayData, PlayObject, TrackMeta } from "../../../core/Atomic.js"; +import { ArtistCredit, isPlayObject, ObjectPlayData, PlayObject, TrackMeta } from "../../../core/Atomic.js"; import { AtomicStageConfig, StageConfig } from "../infrastructure/Transform.js"; import AbstractTransformer from "./AbstractTransformer.js"; @@ -135,8 +135,8 @@ export default abstract class AtomicPartsTransformer; - protected abstract handleArtists(play: PlayObject, parts: Y, transformData: T): Promise; - protected abstract handleAlbumArtists(play: PlayObject, parts: Y, transformData: T): Promise; + protected abstract handleArtists(play: PlayObject, parts: Y, transformData: T): Promise; + protected abstract handleAlbumArtists(play: PlayObject, parts: Y, transformData: T): Promise; protected abstract handleAlbum(play: PlayObject, parts: Y, transformData: T): Promise; protected async handleDuration(play: PlayObject, parts: Y, transformData: T): Promise { return play.data.duration; diff --git a/src/backend/common/transforms/MusicbrainzTransformer.ts b/src/backend/common/transforms/MusicbrainzTransformer.ts index e9749cb22..26cd66df7 100644 --- a/src/backend/common/transforms/MusicbrainzTransformer.ts +++ b/src/backend/common/transforms/MusicbrainzTransformer.ts @@ -1,4 +1,4 @@ -import { asMBReleasePrimaryGroupType, asMBReleaseSecondaryGroupType, asMBReleaseStatus, DEFAULT_MISSING_TYPES, isMBReleasePrimaryGroupType, MBReleaseGroupPrimaryType, MBReleaseGroupSecondaryType, MBReleaseStatus, MissingMbidType, PlayObject, TrackMeta, TransformerCommon, TransformOptions } from "../../../core/Atomic.js"; +import { ArtistCredit, asMBReleasePrimaryGroupType, asMBReleaseSecondaryGroupType, asMBReleaseStatus, DEFAULT_MISSING_TYPES, isMBReleasePrimaryGroupType, MBReleaseGroupPrimaryType, MBReleaseGroupSecondaryType, MBReleaseStatus, MissingMbidType, PlayObject, TrackMeta, TransformerCommon, TransformOptions } from "../../../core/Atomic.js"; import { isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.js"; import { WebhookPayload } from "../infrastructure/config/health/webhooks.js"; import { ExternalMetadataTerm, PlayTransformMetadataStage } from "../infrastructure/Transform.js"; @@ -661,7 +661,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { + protected async handleArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise { if (parts === false) { return play.data.artists; } @@ -676,7 +676,7 @@ export default class MusicbrainzTransformer extends AtomicPartsTransformer { + protected async handleAlbumArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise { if (parts === false) { return play.data.albumArtists; } diff --git a/src/backend/common/transforms/NativeTransformer.ts b/src/backend/common/transforms/NativeTransformer.ts index 4e6f20e66..34d81178f 100644 --- a/src/backend/common/transforms/NativeTransformer.ts +++ b/src/backend/common/transforms/NativeTransformer.ts @@ -1,4 +1,4 @@ -import { PlayObject, TransformerCommon } from "../../../core/Atomic.js"; +import { ArtistCredit, PlayObject, TransformerCommon } from "../../../core/Atomic.js"; import { isWhenCondition, testWhenConditions } from "../../utils/PlayTransformUtils.js"; import { WebhookPayload } from "../infrastructure/config/health/webhooks.js"; import { ExternalMetadataTerm, PlayTransformNativeStage, StageConfig } from "../infrastructure/Transform.js"; @@ -10,6 +10,7 @@ import { DELIMITERS_NO_AMP } from "../infrastructure/Atomic.js"; import { asArray } from "../../utils/DataUtils.js"; import { MaybeLogger } from '../MaybeLogger.js'; import { childLogger } from "@foxxmd/logging"; +import { artistCreditToName, artistNameToCredit } from "../../../core/StringUtils.js"; export type ArtistParseSource = 'artists' | 'title' @@ -167,7 +168,7 @@ export default class NativeTransformer extends AtomicPartsTransformer { return play.data.track; } - protected async handleArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise { + protected async handleArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise { if (parts === false) { return play.data.artists; } @@ -182,7 +183,7 @@ export default class NativeTransformer extends AtomicPartsTransformer { + protected async handleAlbumArtists(play: PlayObject, parts: ExternalMetadataTerm, _transformData: undefined): Promise { return play.data.albumArtists; } protected async handleAlbum(play: PlayObject, parts: ExternalMetadataTerm, _transformData: undefined): Promise { @@ -205,7 +206,7 @@ export const nativeParse = (play: PlayObject, options?: NativeTransformerDataStr logger = new MaybeLogger() } = options || {}; - let artists = []; + let artists: ArtistCredit[] = []; let track = play.data.track; if(artistsParseFrom.includes('artists')) { @@ -214,18 +215,18 @@ export const nativeParse = (play: PlayObject, options?: NativeTransformerDataStr for(const artist of play.data.artists) { - const matchedIgnoreArtists = ignoreArtistsRegex.map(x => ({reg: x.toString(), res: parseRegexSingle(x, artist)})).filter(x => x.res !== undefined); + const matchedIgnoreArtists = ignoreArtistsRegex.map(x => ({reg: x.toString(), res: parseRegexSingle(x, artist.name)})).filter(x => x.res !== undefined); if(matchedIgnoreArtists.length > 0) { logger.debug(`Will not parse artist because it matched an ignore regex:\n${matchedIgnoreArtists.map(x => `Reg: ${x.reg} => ${x.res.match}`).join('\n')}`); artists.push(artist); } else { - const artistCredits = parseArtistCredits(artist, delimiters); + const artistCredits = parseArtistCredits(artist.name, delimiters); if (artistCredits !== undefined) { if (artistCredits.primary !== undefined) { - artists.push(artistCredits.primary); + artists.push({name: artistCredits.primary}); } if (artistCredits.secondary !== undefined) { - artists = artists.concat(artistCredits.secondary); + artists = artists.concat(artistCredits.secondary.map(artistNameToCredit)); } } else { // couldn't parse anything from artist string, use as-is @@ -246,14 +247,14 @@ export const nativeParse = (play: PlayObject, options?: NativeTransformerDataStr if(artistsParseFrom.includes('title')) { const trackArtists = parseTrackCredits(play.data.track, delimiters); if (trackArtists !== undefined && trackArtists.secondary !== undefined) { - artists = artists.concat(trackArtists.secondary); + artists = artists.concat(trackArtists.secondary.map(artistNameToCredit)); if(titleClean) { track = trackArtists.primary; } } } - artists = uniqueNormalizedStrArr([...artists]); + artists = (uniqueNormalizedStrArr([...artists.map(artistCreditToName)])).map(artistNameToCredit); return { ...play, diff --git a/src/backend/common/transforms/UserTransformer.ts b/src/backend/common/transforms/UserTransformer.ts index 50ea477eb..c16bbc763 100644 --- a/src/backend/common/transforms/UserTransformer.ts +++ b/src/backend/common/transforms/UserTransformer.ts @@ -1,5 +1,5 @@ import { searchAndReplace } from "@foxxmd/regex-buddy-core"; -import { PlayObject } from "../../../core/Atomic.js"; +import { ArtistCredit, PlayObject } from "../../../core/Atomic.js"; import { configValToSearchReplace, isSearchAndReplaceTerm, isUserStage, testWhenConditions } from "../../utils/PlayTransformUtils.js"; import { WebhookPayload } from "../infrastructure/config/health/webhooks.js"; import { ConditionalSearchAndReplaceRegExp, PlayTransformUserStage, StageConfig } from "../infrastructure/Transform.js"; @@ -74,30 +74,30 @@ export default class UserTransformer extends AtomicPartsTransformer { + protected async handleArtists(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise { if(play.data.artists === undefined || play.data.artists.length === 0) { return play.data.artists; } const mapper = this.generateMapper(play); const transformedArtists = []; for(const artist of play.data.artists) { - const a = searchAndReplace(artist, parts.map(mapper)); + const a = searchAndReplace(artist.name, parts.map(mapper)); if(a.trim() !== '') { - transformedArtists.push(a); + transformedArtists.push({...artist, name: a}); } } return transformedArtists; } - protected async handleAlbumArtists(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise { + protected async handleAlbumArtists(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise { if(play.data.albumArtists === undefined || play.data.albumArtists.length === 0) { return play.data.albumArtists; } const mapper = this.generateMapper(play); const transformedArtists = []; for(const artist of play.data.albumArtists) { - const a = searchAndReplace(artist, parts.map(mapper)); + const a = searchAndReplace(artist.name, parts.map(mapper)); if(a.trim() !== '') { - transformedArtists.push(a); + transformedArtists.push({...artist, name: a}); } } return transformedArtists; diff --git a/src/backend/common/vendor/KodiApiClient.ts b/src/backend/common/vendor/KodiApiClient.ts index a8bac8339..ef81cdc35 100644 --- a/src/backend/common/vendor/KodiApiClient.ts +++ b/src/backend/common/vendor/KodiApiClient.ts @@ -8,6 +8,7 @@ import { AbstractApiOptions, FormatPlayObjectOptions } from "../infrastructure/A import { KodiData } from "../infrastructure/config/source/kodi.js"; import AbstractApiClient from "./AbstractApiClient.js"; import { baseFormatPlayObj } from "../../utils/PlayTransformUtils.js"; +import { artistNamesToCredits } from "../../../core/StringUtils.js"; interface KodiDuration { hours: number @@ -109,8 +110,8 @@ export class KodiApiClient extends AbstractApiClient { data: { track: title, album: album, - albumArtists: albumartist, - artists, + albumArtists: artistNamesToCredits(albumartist), + artists: artistNamesToCredits(artistVal), duration, playDate: dayjs() }, diff --git a/src/backend/common/vendor/LastfmApiClient.ts b/src/backend/common/vendor/LastfmApiClient.ts index 524a06f93..0c0f6f5f0 100644 --- a/src/backend/common/vendor/LastfmApiClient.ts +++ b/src/backend/common/vendor/LastfmApiClient.ts @@ -1,6 +1,6 @@ import dayjs, { Dayjs, ManipulateType } from "dayjs"; import { BrainzMeta, PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, UnixTimestamp, URLData, Writeable } from "../../../core/Atomic.js"; -import { nonEmptyStringOrDefault, splitByFirstFound } from "../../../core/StringUtils.js"; +import { artistNamesToCredits, artistNameToCredit, nonEmptyStringOrDefault, splitByFirstFound } from "../../../core/StringUtils.js"; import { removeUndefinedKeys, sleep } from "../../utils.js"; import { writeFile } from '../../utils/FSUtils.js'; import { objectIsEmpty, readJson } from '../../utils/DataUtils.js'; @@ -570,10 +570,10 @@ export const scrobblePayloadToPlay = (obj: LastFMScrobbleRequestPayload): PlayOb data: { track, album: nonEmptyStringOrDefault(album), - albumArtists: nonEmptyStringOrDefault(albumArtist) !== undefined ? [albumArtist] : undefined, + albumArtists: nonEmptyStringOrDefault(albumArtist) !== undefined ? [artistNameToCredit(albumArtist)] : undefined, duration: typeof duration === 'string' ? parseInt(duration, 10) : duration, playDate: ts, - artists + artists: artistNamesToCredits(artists) }, meta: { source: 'lastfm', @@ -615,7 +615,7 @@ export const playToClientPayload = (playObj: PlayObject): LastFMScrobblePayload if (artists.length === 0) { artist = ""; } else { - artist = artists[0]; + artist = artists[0].name; } const additionalRichPayload: Partial = {}; @@ -634,11 +634,11 @@ export const playToClientPayload = (playObj: PlayObject): LastFMScrobblePayload // LFM ignores scrobbles where album artist is VA // https://github.com/FoxxMD/multi-scrobbler/issues/340#issuecomment-3220774257 - const nonVaAlbumArtists = albumArtists.filter(x => x.trim().toLocaleLowerCase() !== 'va'); + const nonVaAlbumArtists = albumArtists.filter(x => x.name.trim().toLocaleLowerCase() !== 'va'); // LFM does not support multiple artists in scrobble payload // https://www.last.fm/api/show/track.scrobble if (nonVaAlbumArtists.length > 0) { - rawPayload.albumArtist = nonVaAlbumArtists[0]; + rawPayload.albumArtist = nonVaAlbumArtists[0].name; } // I don't know if its lastfm-node-client building the request params incorrectly @@ -694,7 +694,7 @@ export const formatPlayObj = (obj: LastFMTrackObject, options: FormatPlayObjectO const play: PlayObjectLifecycleless = { data: { - artists: [...new Set(artistStrings)] as string[], + artists: artistNamesToCredits([...new Set(artistStrings)] as string[]), track: title, album: al, duration, diff --git a/src/backend/common/vendor/ListenbrainzApiClient.ts b/src/backend/common/vendor/ListenbrainzApiClient.ts index 1ba6327a0..e20aa9806 100644 --- a/src/backend/common/vendor/ListenbrainzApiClient.ts +++ b/src/backend/common/vendor/ListenbrainzApiClient.ts @@ -2,7 +2,7 @@ import { stringSameness } from '@foxxmd/string-sameness'; import dayjs from "dayjs"; import request, { Request, Response } from 'superagent'; import { BrainzMeta, PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, UnixTimestamp, URLData } from "../../../core/Atomic.js"; -import { combinePartsToString, slice } from "../../../core/StringUtils.js"; +import { artistNamesToCredits, combinePartsToString, slice } from "../../../core/StringUtils.js"; import { normalizeListenbrainzUrl, normalizeStr, @@ -548,7 +548,13 @@ export const listenResponseToPlay = (listen: ListenResponse): PlayObject => { ...naivePlay, data: { ...naivePlay.data, - artists: derivedArtists + artists: derivedArtists.map(x => { + const mappedArtist = artistMappings.find(y => y.artist_credit_name === x); + if(mappedArtist !== undefined) { + return {name: mappedArtist.artist_credit_name, mbid: mappedArtist.artist_mbid} + } + return {name: x}; + }) } }; } @@ -579,7 +585,13 @@ export const listenResponseToPlay = (listen: ListenResponse): PlayObject => { data: { ...naivePlay.data, track: normalTrackName, - artists: derivedArtists + artists: derivedArtists.map(x => { + const mappedArtist = artistMappings.find(y => y.artist_credit_name === x); + if(mappedArtist !== undefined) { + return {name: mappedArtist.artist_credit_name, mbid: mappedArtist.artist_mbid} + } + return {name: x}; + }) }, meta: naivePlay.meta } @@ -641,7 +653,7 @@ export const listenToNaivePlay = (listen: ListenResponse): PlayObject => { if(artist_names.length > 0) { artists = artist_names; - } else { + } else if(artist_name !== null) { artists = [artist_name]; // since we aren't using MB mappings we should be conservative and assume artist string with & are proper names (not joiner) @@ -675,9 +687,9 @@ export const listenToNaivePlay = (listen: ListenResponse): PlayObject => { data: { playDate: dayjs.unix(listened_at), track: normalTrackName, - artists: artists, + artists: artistNamesToCredits(artists), album: release_name, - albumArtists, + albumArtists: artistNamesToCredits(albumArtists), duration: dur, isrc: isrc !== undefined ? isrc : undefined, meta: { diff --git a/src/backend/common/vendor/RockSkyApiClient.ts b/src/backend/common/vendor/RockSkyApiClient.ts index c7950c4f2..275ed6d0a 100644 --- a/src/backend/common/vendor/RockSkyApiClient.ts +++ b/src/backend/common/vendor/RockSkyApiClient.ts @@ -1,7 +1,7 @@ import dayjs from "dayjs"; import request, { Request, Response } from 'superagent'; import { PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, URLData } from "../../../core/Atomic.js"; -import { nonEmptyStringOrDefault } from "../../../core/StringUtils.js"; +import { artistNamesToCredits, nonEmptyStringOrDefault } from "../../../core/StringUtils.js"; import { UpstreamError } from "../errors/UpstreamError.js"; import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions } from "../infrastructure/Atomic.js"; import { RockSkyClientData, RockSkyData, RockSkyOptions } from "../infrastructure/config/client/rocksky.js"; @@ -221,8 +221,8 @@ export const rockskyScrobbleToPlay = (obj: RockskyScrobble): PlayObject => { const play: PlayObjectLifecycleless = { data: { track: obj.title, - artists: nonEmptyStringOrDefault(obj.artist) ? [obj.artist] : [], - albumArtists: nonEmptyStringOrDefault(obj.albumArtist) ? [obj.albumArtist] : [], + artists: artistNamesToCredits(nonEmptyStringOrDefault(obj.artist) ? [obj.artist] : []), + albumArtists: artistNamesToCredits(nonEmptyStringOrDefault(obj.albumArtist) ? [obj.albumArtist] : []), album: nonEmptyStringOrDefault(obj.album), playDate: dayjs.utc(obj.createdAt).local() }, diff --git a/src/backend/common/vendor/bluesky/AbstractBlueSkyApiClient.ts b/src/backend/common/vendor/bluesky/AbstractBlueSkyApiClient.ts index 90ae04a16..850904b8b 100644 --- a/src/backend/common/vendor/bluesky/AbstractBlueSkyApiClient.ts +++ b/src/backend/common/vendor/bluesky/AbstractBlueSkyApiClient.ts @@ -94,7 +94,7 @@ export const playToRecord = (play: PlayObject): ScrobbleRecord => { const record: ScrobbleRecord = { $type: "fm.teal.alpha.feed.play", trackName: play.data.track, - artists: play.data.artists.map(x => ({ artistName: x })), + artists: play.data.artists.map(x => ({ artistName: x.name })), duration: Math.round(play.data.duration), playedTime: getScrobbleTsSOCDateWithContext(play)[0].toISOString(), releaseName: play.data.album, @@ -124,7 +124,7 @@ export const recordToPlay = (record: ScrobbleRecord, options: RecordOptions = {} const play: PlayObjectLifecycleless = { data: { track: record.trackName, - artists: record.artists.filter(x => x.artistName !== undefined).map(x => x.artistName), + artists: record.artists.filter(x => x.artistName !== undefined).map(x => ({name: x.artistName, mbid: x.artistMbId})), duration: record.duration, playDate: dayjs(record.playedTime), album: record.releaseName, diff --git a/src/backend/common/vendor/discord/DiscordUtils.ts b/src/backend/common/vendor/discord/DiscordUtils.ts index 03c8f845e..a9ba58883 100644 --- a/src/backend/common/vendor/discord/DiscordUtils.ts +++ b/src/backend/common/vendor/discord/DiscordUtils.ts @@ -46,7 +46,7 @@ export const playStateToActivityData = (data: SourcePlayerObj, opts: { useArt?: name: activityName, details: play.data.track.padEnd(2,'\u200B'), - state: play.data.artists !== undefined && play.data.artists.length > 0 ? play.data.artists.map(x => x.padEnd(2, '\u200B')).join(' / ') : undefined, + state: play.data.artists !== undefined && play.data.artists.length > 0 ? play.data.artists.map(x => x.name.padEnd(2, '\u200B')).join(' / ') : undefined, // https://docs.discord.com/developers/events/gateway-events#activity-object-activity-assets // https://docs.discord.com/developers/events/gateway-events#activity-object-activity-asset-image assets: { diff --git a/src/backend/common/vendor/koito/KoitoApiClient.ts b/src/backend/common/vendor/koito/KoitoApiClient.ts index 72999b0d8..8dfc2020f 100644 --- a/src/backend/common/vendor/koito/KoitoApiClient.ts +++ b/src/backend/common/vendor/koito/KoitoApiClient.ts @@ -9,10 +9,11 @@ import { UpstreamError } from "../../errors/UpstreamError.js"; import { playToListenPayload } from '../listenbrainz/lzUtils.js'; import { SubmitPayload } from '../listenbrainz/interfaces.js'; import { ListenType } from '../listenbrainz/interfaces.js'; -import { parseRegexSingleOrFail } from "../../../utils.js"; import { baseFormatPlayObj } from "../../../utils/PlayTransformUtils.js"; import { ScrobbleSubmitError } from "../../errors/MSErrors.js"; import { tryApiCall } from "../../../utils/RequestUtils.js"; +import { parseRegexSingle } from "@foxxmd/regex-buddy-core"; +import { artistNamesToCredits } from "../../../../core/StringUtils.js"; interface SubmitOptions { log?: boolean @@ -36,7 +37,7 @@ export class KoitoApiClient extends AbstractApiClient implements PaginatedTimeRa const u = normalizeWebAddress(url); if(u.url.pathname === '/') { this.url = u; - } else if(parseRegexSingleOrFail(KOITO_LZ_PATH, u.url.pathname) !== undefined) { + } else if(parseRegexSingle(KOITO_LZ_PATH, u.url.pathname) !== undefined) { this.logger.verbose('Detected Koito Server URL path only contains listenbrainz prefix. Removing this for API calls so non-listenbrainz paths work correctly.'); this.url = normalizeWebAddress(getBaseFromUrl(u.url).toString()); } else { @@ -218,7 +219,7 @@ export const listenObjectResponseToPlay = (obj: ListenObjectResponse, options: { const play: PlayObjectLifecycleless = { data: { track: obj.track.title, - artists: (obj.track.artists ?? []).map(x => x.name), + artists: artistNamesToCredits((obj.track.artists ?? []).map(x => x.name)), duration: obj.track.duration, playDate: dayjs(obj.time) }, diff --git a/src/backend/common/vendor/listenbrainz/lzUtils.ts b/src/backend/common/vendor/listenbrainz/lzUtils.ts index 4226be1aa..53985214f 100644 --- a/src/backend/common/vendor/listenbrainz/lzUtils.ts +++ b/src/backend/common/vendor/listenbrainz/lzUtils.ts @@ -4,6 +4,7 @@ import { getScrobbleTsSOCDate } from "../../../utils/TimeUtils.js"; import { SubmitOptions } from "../ListenbrainzApiClient.js"; import { ListenPayload, MinimumTrack, SubmitListenAdditionalTrackInfo, SubmitPayload } from "./interfaces.js"; import {version as appVersion } from '../../../version.js'; +import { artistCreditsToNames, artistCreditToName } from "../../../../core/StringUtils.js"; export const playToListenPayload = (play: PlayObject, version?: string): ListenPayload => { const { @@ -25,10 +26,10 @@ export const playToListenPayload = (play: PlayObject, version?: string): ListenP let addInfo: SubmitListenAdditionalTrackInfo = { // primary artists - artist_names: Array.from(new Set([...artists])), + artist_names: Array.from(new Set([...artists.map(artistCreditToName)])), // primary artist - release_artist_name: albumArtists.length === 1 ? albumArtists[0] : undefined, - release_artist_names: albumArtists.length > 0 ? albumArtists : undefined, + release_artist_name: albumArtists.length === 1 ? albumArtists[0].name : undefined, + release_artist_names: albumArtists.length > 0 ? artistCreditsToNames(albumArtists) : undefined, // use data from LZ response, if this Play was originally from LZ Source media_player: mediaPlayerName ?? msAdditionalInfo.media_player, media_player_version: mediaPlayerVersion ?? msAdditionalInfo.media_player_version, diff --git a/src/backend/common/vendor/maloja/MalojaApiClient.ts b/src/backend/common/vendor/maloja/MalojaApiClient.ts index 7a90c1838..4682f1a73 100644 --- a/src/backend/common/vendor/maloja/MalojaApiClient.ts +++ b/src/backend/common/vendor/maloja/MalojaApiClient.ts @@ -12,7 +12,7 @@ import { getNonEmptyVal, parseRetryAfterSecsFromObj, removeUndefinedKeys, sleep import { UpstreamError } from "../../errors/UpstreamError.js"; import { getMalojaResponseError, isMalojaAPIErrorBody, MalojaResponseV3CommonData, MalojaScrobbleData, MalojaScrobbleRequestData, MalojaScrobbleV3RequestData, MalojaScrobbleV3ResponseData, MalojaScrobbleWarning } from "./interfaces.js"; import { getScrobbleTsSOCDate, getScrobbleTsSOCDateWithContext } from '../../../utils/TimeUtils.js'; -import { buildTrackString } from '../../../../core/StringUtils.js'; +import { artistCreditsToNames, artistNamesToCredits, buildTrackString } from '../../../../core/StringUtils.js'; import { baseFormatPlayObj } from '../../../utils/PlayTransformUtils.js'; import { ScrobbleSubmitError } from '../../errors/MSErrors.js'; import { NO_RETRY_HTTP_STATUS, tryApiCall } from '../../../utils/RequestUtils.js'; @@ -272,7 +272,7 @@ export class MalojaApiClient extends AbstractApiClient implements PaginatedTimeR } = track; scrobbleResponse.track.album = { name: album, - artists: albumArtists, + artists: artistCreditsToNames(albumArtists), ...malojaAlbum, } } @@ -412,7 +412,7 @@ export const formatPlayObj = (obj: MalojaScrobbleData, options: FormatPlayObject const urlParams = new URLSearchParams([['artist', artists[0]], ['title', title]]); const play: PlayObjectLifecycleless = { data: removeUndefinedKeys({ - artists: [...new Set(artistStrings)] as string[], + artists: artistNamesToCredits([...new Set(artistStrings)] as string[]), track: title, album, duration, @@ -446,7 +446,7 @@ export const playToScrobblePayload = (playObj: PlayObject, apiKey?: string): Mal const scrobbleData: MalojaScrobbleV3RequestData = { title: track, - artists, + artists: artistCreditsToNames(artists), album, key: apiKey, time: pd.unix(), @@ -463,7 +463,7 @@ export const playToScrobblePayload = (playObj: PlayObject, apiKey?: string): Mal // https://github.com/krateng/maloja/blob/master/maloja/web/static/js/manualscrobble.js#L136 // BUT this is not actually working! if (albumArtists.length > 0) { - scrobbleData.albumartists = albumArtists; + scrobbleData.albumartists = artistCreditsToNames(albumArtists); } // see also https://github.com/krateng/maloja/issues/96#issuecomment-1490562761 // https://github.com/FoxxMD/multi-scrobbler/issues/454#issuecomment-3806367420 diff --git a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts index e5b5f6db8..850aed400 100644 --- a/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts +++ b/src/backend/common/vendor/musicbrainz/MusicbrainzApiClient.ts @@ -1,5 +1,5 @@ import { Response } from 'superagent'; -import { PlayObject, PlayObjectLifecycleless, URLData } from "../../../../core/Atomic.js"; +import { ArtistCredit, PlayObject, PlayObjectLifecycleless, URLData } from "../../../../core/Atomic.js"; import { UpstreamError } from "../../errors/UpstreamError.js"; import { AbstractApiOptions, FormatPlayObjectOptions, MUSICBRAINZ_URL, MusicbrainzApiConfigData } from "../../infrastructure/Atomic.js"; import AbstractApiClient from "../AbstractApiClient.js"; @@ -21,6 +21,7 @@ import { SimpleError } from '../../errors/MSErrors.js'; import { baseFormatPlayObj } from '../../../utils/PlayTransformUtils.js'; import { IRecordingMSList } from '../../transforms/MusicbrainzTransformer.js'; import dayjs, { Dayjs } from 'dayjs'; +import { artistCreditsToNames } from '../../../../core/StringUtils.js'; export interface SubmitResponse { payload?: { @@ -235,7 +236,16 @@ export class MusicbrainzApiClient extends AbstractApiClient { // https://wiki.musicbrainz.org/MusicBrainz_API/Search#Recording // https://beta.musicbrainz.org/doc/MusicBrainz_API/Search const res = await this.callApi((mb) => { - const query: Record = { + const query: { + recording_mbid?: string + track_mbid?: string + release_mbid?: string + artist_mbids?: string[] + isrc?: string + recording?: string + artist?: string[] + release?: string + } = { }; if(play.data?.meta?.brainz?.recording !== undefined && using.includes('mbidrecording')) { @@ -257,7 +267,7 @@ export class MusicbrainzApiClient extends AbstractApiClient { query.recording = play.data.track; } if(play.data.artists !== undefined && play.data.artists.length > 0 && using.includes('artist')) { - query.artist = play.data.artists; + query.artist = artistCreditsToNames(play.data.artists); } if(play.data.album !== undefined && using.includes('album')) { query.release = play.data.album; @@ -380,17 +390,17 @@ export const recordingToPlay = (data: IRecording, options?: {ignoreVA?: boolean} let album: IRelease; - let albumArtists: string[]; + let albumArtists: ArtistCredit[]; let albumArtistIds: string[]; - const artists = (data["artist-credit"] ?? []).map(x => x.name); + const artists = (data["artist-credit"] ?? []).map(x => ({ name: x.name, mbid: x.artist.id})); if(data.releases !== undefined && data.releases.length > 0) { album = data.releases[0]; if(album["artist-credit"] !== undefined) { if(difference(album["artist-credit"].map(x => x.artist.id), (data["artist-credit"] ?? []).map(x => x.artist.id)).length > 0) { - albumArtists = album["artist-credit"].map(x => x.artist.name); + albumArtists = album["artist-credit"].map(x => ({name: x.artist.name, mbid: x.artist.id})); albumArtistIds = album["artist-credit"].map(x => x.artist.id); } - if(albumArtists !== undefined && ignoreVA && albumArtists.includes('Various Artists')) { + if(albumArtists !== undefined && ignoreVA && albumArtists.map(x => x.name).includes('Various Artists')) { albumArtists = undefined; albumArtistIds = undefined; } diff --git a/src/backend/index.ts b/src/backend/index.ts index 37a4dae72..a023f87a5 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -16,14 +16,18 @@ import { appLogger, initLogger as getInitLogger } from "./common/logging.js"; import { getRoot } from "./ioc.js"; import { parseVersion } from "./version.js"; import { initServer } from "./server/index.js"; -import { createHeartbeatClientsTask } from "./tasks/heartbeatClients.js"; -import { createHeartbeatSourcesTask } from "./tasks/heartbeatSources.js"; -import { isDebugMode, parseBool, retry } from "./utils.js"; +import { isDebugMode, parseBool, parseBoolStrict, retry, sleep } from "./utils.js"; import { readJson } from './utils/DataUtils.js'; -//import { createVegaGenerator } from './utils/SchemaUtils.js'; import ScrobbleClients from './scrobblers/ScrobbleClients.js'; import ScrobbleSources from './sources/ScrobbleSources.js'; import { Notifiers } from './notifier/Notifiers.js'; +import { DbConcrete, getMigratedDb } from './common/database/drizzle/drizzleUtils.js'; +import { getDbPath } from './common/database/Database.js'; +import { createRetentionCleanupTask } from './tasks/retentionCleanup.js'; +import { parseUserConfig } from './common/Cache.js'; +import { nonEmptyStringOrDefault } from '../core/StringUtils.js'; +import { DbExternalMode } from './common/infrastructure/Atomic.js'; +import { PGLiteSocketServer } from '@electric-sql/pglite-socket'; dayjs.extend(utc) dayjs.extend(isBetween); @@ -49,13 +53,48 @@ output = output.slice(0, 301); let logger: FoxLogger; -process.on('uncaughtExceptionMonitor', (err, origin) => { +let server: PGLiteSocketServer; +let db: DbConcrete; +let dbConnectionsClosed = false; + +process.on('uncaughtExceptionMonitor', async (err, origin) => { const appError = new Error(`Uncaught exception is crashing the app! :( Type: ${origin}`, {cause: err}); if(logger !== undefined) { logger.error(appError) } else { initLogger.error(appError); } + if(!dbConnectionsClosed) { + const parts = []; + if(server !== undefined) { + await server.stop(); + parts.push('PGLite Socket Server'); + } + if(db !== undefined && !db.$client.closed) { + await db.$client.close(); + parts.push('Database'); + } + if(parts.length > 0 && logger !== undefined) { + logger.info(`Closed ${parts.join(' and ')}`); + } + } +}); +process.on('SIGINT', async () => { + if(!dbConnectionsClosed) { + const parts = []; + if(server !== undefined) { + await server.stop(); + parts.push('PGLite Socket Server'); + } + if(db !== undefined && !db.$client.closed) { + await db.$client.close(); + parts.push('Database'); + } + if(parts.length > 0 && logger !== undefined) { + logger.info(`Closed ${parts.join(' and ')}`); + } + } + process.exit(0); }) const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); @@ -75,6 +114,7 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) webhooks = [], logging = {}, debugMode, + cache, } = (config || {}) as AIOConfig; if (process.env.DEBUG_MODE === undefined && debugMode !== undefined) { @@ -88,90 +128,164 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) initLogger.info(`Debug Mode: ${isDebugMode() ? 'YES' : 'NO'}`); - await parseVersion(); + const version = await parseVersion(); + + initLogger.info(`Version: ${version}`); const [aLogger, appLoggerStream] = await appLogger(logging) logger = childLogger(aLogger, 'App'); - const root = getRoot({...config, logger, loggingConfig: logging, loggerStream: appLoggerStream}); - initLogger.info(`Version: ${root.get('version')}`); - - //initLogger.info('Generating schema definitions...'); - //createVegaGenerator() - //initLogger.info('Schema definitions generated'); - - const internalConfigOptional = { - localUrl: root.get('localUrl'), - configDir: root.get('configDir'), - version: root.get('version') - }; - - const scrobbleClients = new ScrobbleClients(root.get('clientEmitter'), root.get('sourceEmitter'), internalConfigOptional, root.get('logger')); - const scrobbleSources = new ScrobbleSources(root.get('sourceEmitter'), internalConfigOptional, root.get('logger')); - - await root.items.cache().init(true); - - initServer(logger, appLoggerStream, output, scrobbleSources, scrobbleClients); - - if(process.env.IS_LOCAL === 'true') { - logger.info('multi-scrobbler can be run as a background service! See: https://docs.multi-scrobbler.app/installation/service'); + const dbModeVal: string = nonEmptyStringOrDefault(process.env.DB_MODE, undefined); + let dbMode: DbExternalMode; + if(dbModeVal !== undefined) { + if(['none','live','standalone'].includes(dbModeVal.toLocaleLowerCase())) { + dbMode = dbModeVal as typeof dbMode; + } else { + throw new Error(`DB_MODE env must be one of 'none' 'live' 'standalone', found ${dbModeVal}`); + } + } else { + dbMode = 'none'; } - - if(appConfigFail !== undefined) { - logger.warn('App config file exists but could not be parsed!'); - logger.warn(appConfigFail); + logger.info(`DB External Mode: ${dbMode}`); + + + const dbPath = getDbPath('msDb'); + logger.info(`Using database at ${getDbPath('msDb')}`); + const [migratedDb, isNew] = await getMigratedDb(dbPath, {logger: childLogger(logger, 'DB')}); + db = migratedDb; + + + + if(['live','standalone'].includes(dbMode)) { + server = new PGLiteSocketServer({ + host: '0.0.0.0', + port: 5433, + db: db.$client, + maxConnections: 10, + debug: parseBoolStrict(nonEmptyStringOrDefault(process.env.DB_DEBUG, false)) + }); + await server.start(); + logger.info('Started PGLite Socket Server'); } - const notifiers = new Notifiers(root.get('notifierEmitter'), root.get('clientEmitter'), root.get('sourceEmitter'), root.get('logger')); //root.get('notifiers'); - await notifiers.buildWebhooks(webhooks); - - await root.items.transformerManager.registerFromEnv(); - await root.items.transformerManager.registeryDefaults(); - await root.items.transformerManager.initTransformers(); - - /* - * setup clients - * */ - await scrobbleClients.buildClientsFromConfig(notifiers); - /* - * setup sources - * */ - await scrobbleSources.buildSourcesFromConfig([]); - - // check ambiguous client/source types like this for now - const lastfmSources = scrobbleSources.getByType('lastfm'); - const lastfmScrobbles = scrobbleClients.getByType('lastfm'); - - const scrobblerNames = lastfmScrobbles.map(x => x.name); - const nameColl = lastfmSources.filter(x => scrobblerNames.includes(x.name)); - if(nameColl.length > 0) { - logger.warn(`Last.FM source and clients have same names [${nameColl.map(x => x.name).join(',')}] -- this may cause issues`); - } + if (dbMode === 'standalone') { + logger.info('MS App startup stopped early due to Standalone DB Mode.'); + } else { - const clientTask = createHeartbeatClientsTask(scrobbleClients, logger); - clientTask.execute(); - try { - await retry(() => { - if(clientTask.isExecuting) { - throw new Error('Waiting') + const root = getRoot({ + ...config, + cache: parseUserConfig(cache, logger), + logger, + loggingConfig: logging, + loggerStream: appLoggerStream, + db + }); + + const internalConfigOptional = { + localUrl: root.get('localUrl'), + configDir: root.get('configDir'), + version: root.get('version') + }; + + const scrobbleClients = new ScrobbleClients(root.get('clientEmitter'), root.get('sourceEmitter'), internalConfigOptional, root.get('logger')); + const scrobbleSources = new ScrobbleSources(root.get('sourceEmitter'), internalConfigOptional, root.get('logger')); + + await root.items.cache().init(true); + + initServer(logger, appLoggerStream, output, scrobbleSources, scrobbleClients); + + if (process.env.IS_LOCAL === 'true') { + logger.info('multi-scrobbler can be run as a background service! See: https://docs.multi-scrobbler.app/installation/service'); + } + + if (appConfigFail !== undefined) { + logger.warn('App config file exists but could not be parsed!'); + logger.warn(appConfigFail); + } + + const notifiers = new Notifiers(root.get('notifierEmitter'), root.get('clientEmitter'), root.get('sourceEmitter'), root.get('logger')); //root.get('notifiers'); + await notifiers.buildWebhooks(webhooks); + + await root.items.transformerManager.registerFromEnv(); + await root.items.transformerManager.registeryDefaults(); + await root.items.transformerManager.initTransformers(); + + /* + * setup clients + * */ + await scrobbleClients.buildClientsFromConfig(notifiers); + /* + * setup sources + * */ + await scrobbleSources.buildSourcesFromConfig([]); + + // check ambiguous client/source types like this for now + const lastfmSources = scrobbleSources.getByType('lastfm'); + const lastfmScrobbles = scrobbleClients.getByType('lastfm'); + + const scrobblerNames = lastfmScrobbles.map(x => x.name); + const nameColl = lastfmSources.filter(x => scrobblerNames.includes(x.name)); + if (nameColl.length > 0) { + logger.warn(`Last.FM source and clients have same names [${nameColl.map(x => x.name).join(',')}] -- this may cause issues`); + } + const clientInitOptions = { deadDelay: nonEmptyStringOrDefault(process.env.DEBUG_DEAD_DELAY, undefined) !== undefined ? Number.parseInt(process.env.DEBUG_DEAD_DELAY) : undefined }; for (const c of scrobbleClients.clients) { + c.initTasks(clientInitOptions); + const res = await Promise.race([ + sleep(2200), + (async () => { + while (!c.isReady()) { + await sleep(400) + } + return true; + })() + ]); + if (res === undefined) { + logger.debug(`Not waiting for Client ${c.name} to finish init, moving on to the next Client...`); } - return true; - },{retries: scrobbleClients.clients.length + 1, retryIntervalMs: 2000}); - } catch (e) { - logger.warn('Waited too long for clients to start! Moving ahead with sources init...'); + } + + for (const c of scrobbleSources.sources) { + c.initTasks(); + const res = await Promise.race([ + sleep(2200), + (async () => { + while (!c.isReady()) { + await sleep(400) + } + return true; + })() + ]); + if (res === undefined) { + logger.debug(`Not waiting for Source ${c.name} to finish init, moving on to the next Source...`); + } + } + + let runRetentionNow = parseBool(process.env.RETENTION_IMMEDIATE, false); + + const retentionTask = createRetentionCleanupTask(scrobbleSources, scrobbleClients, logger); + let retentionJobAdded = false; + const addJob = () => { + retentionJobAdded = true; + scheduler.addSimpleIntervalJob(new SimpleIntervalJob({ + minutes: 60, + runImmediately: runRetentionNow + }, retentionTask, { id: 'retention', preventOverrun: true })); + logger.debug('Added Retention Cleanup task to scheduler'); + }; + logger.debug('Added Client Heartbeat task to scheduler'); + + if (runRetentionNow === false || (scrobbleClients.clients.every(x => x.isReady()) && scrobbleSources.sources.every(x => x.isReady()))) { + addJob(); + } + + logger.info('Scheduler started.'); + + if (runRetentionNow === true && !retentionJobAdded) { + logger.info('Detected that Retention Cleanup should run immediately but all sources/clients have not started yet! Delaying retention cleanup by 1 minute to allow all sources/clients to finish starting.'); + await sleep(60 * 1000); + addJob(); + } } - scheduler.addSimpleIntervalJob(new SimpleIntervalJob({ - minutes: 20, - runImmediately: false - }, clientTask, {id: 'clients_heart'})); - - const sourceTask = createHeartbeatSourcesTask(scrobbleSources, logger); - scheduler.addSimpleIntervalJob(new SimpleIntervalJob({ - minutes: 20, - runImmediately: true - }, sourceTask, {id: 'sources_heart'})); - - logger.info('Scheduler started.'); } catch (e) { const appError = new Error('Exited with uncaught error', {cause: e}); @@ -180,6 +294,20 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) } else { initLogger.error(appError); } + if(!dbConnectionsClosed) { + const parts = []; + if(server !== undefined) { + await server.stop(); + parts.push('PGLite Socket Server'); + } + if(db !== undefined && !db.$client.closed) { + await db.$client.close(); + parts.push('Database'); + } + if(parts.length > 0 && logger !== undefined) { + logger.info(`Closed ${parts.join(' and ')}`); + } + } process.exit(1); } }()); diff --git a/src/backend/ioc.ts b/src/backend/ioc.ts index 7d299c4af..bbe098f34 100644 --- a/src/backend/ioc.ts +++ b/src/backend/ioc.ts @@ -15,6 +15,7 @@ import prom, { Counter, Gauge } from 'prom-client'; import { CoverArtApiClient } from "./common/vendor/musicbrainz/CoverArtApiClient.js"; import { version } from "./version.js"; import { StaggerOptions } from "./utils/AsyncUtils.js"; +import { DbConcrete } from "./common/database/drizzle/drizzleUtils.js"; let root: ReturnType; export interface RootOptions { @@ -27,6 +28,7 @@ export interface RootOptions { cache?: CacheConfigOptions | MSCache | (() => MSCache) mbMap?: MusicBrainzSingletonMap | (() => MusicBrainzSingletonMap) transformers?: TransformerCommonConfig[] + db?: DbConcrete | (() => Promise) } const discovered = new prom.Counter({ @@ -60,6 +62,7 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => { logger, cache, mbMap, + db, transformers = [] } = options || {}; const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); @@ -89,6 +92,12 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => { maybeSingletonMb = new Map(); } + let dbFunc: () => Promise; + if(typeof db === 'function') { + dbFunc = db; + } else { + dbFunc = async () => db; + } const cEmitter = new WildcardEmitter(); // do nothing, just catch @@ -148,6 +157,7 @@ const createRoot = (options: RootOptions = {logger: loggerDebug}) => { cache: () => maybeSingletonCache !== undefined ? () => maybeSingletonCache : cacheFunc, mbMap: () => maybeSingletonMb !== undefined ? () => maybeSingletonMb : mbFunc, coverArtApi, + db: () => dbFunc }).add((items) => { const localUrl = generateBaseURL(baseUrl, items.port) return { diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index fdb39c70b..8e9023fae 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -1,9 +1,9 @@ -import { childLogger, Logger } from "@foxxmd/logging"; +import { childLogger, Logger, LogLevel } from "@foxxmd/logging"; import dayjs, { Dayjs } from "dayjs"; import EventEmitter from "events"; import { FixedSizeList } from 'fixed-size-list'; import { nanoid } from "nanoid"; -import { MarkOptional } from "ts-essentials"; +import { MarkOptional, MarkRequired } from "ts-essentials"; import { DeadLetterScrobble, NowPlayingUpdateThreshold, @@ -13,9 +13,12 @@ import { TA_FUZZY, TrackStringOptions, TA_EXACT, - SOURCE_SOT + SOURCE_SOT, + ErrorLike, + CLIENT_INGRESS_QUEUE, + CLIENT_DEAD_QUEUE } from "../../core/Atomic.js"; -import { buildTrackString, capitalize, truncateStringToLength } from "../../core/StringUtils.js"; +import { artistNamesToCredits, buildTrackString, capitalize, truncateStringToLength } from "../../core/StringUtils.js"; import AbstractComponent from "../common/AbstractComponent.js"; import { hasUpstreamError } from "../common/errors/UpstreamError.js"; import { @@ -43,12 +46,14 @@ import { parseBool, playObjDataMatch, pollingBackoff, + removeUndefinedKeys, sleep, sortByOldestPlayDate, } from "../utils.js"; import { findCauseByReference, messageWithCauses, messageWithCausesTruncatedDefault } from "../utils/ErrorUtils.js"; import { comparePlayTemporally, + getTemporalAccuracyCloseVal, hasAcceptableTemporalAccuracy, temporalAccuracyToString, temporalPlayComparisonSummary, @@ -65,27 +70,33 @@ import { lifecyclelessInvariantTransform } from "../../core/PlayUtils.js"; import { normalizeStr } from "../utils/StringUtils.js"; import prom, { Counter, Gauge } from 'prom-client'; import { generateLoggableAbortReason, ScrobbleSubmitError, SimpleError } from "../common/errors/MSErrors.js"; -import {serializeError} from 'serialize-error'; +import {isErrorLike, serializeError} from 'serialize-error'; import { DEFAULT_NEW_PADDING, groupPlaysToTimeRanges } from "../utils/ListenFetchUtils.js"; import { spawn, catchAbortError, isAbortError, rethrowAbortError, delay, forever, AbortError, throwIfAborted } from 'abort-controller-x'; +import { DrizzlePlayRepository, playToRepositoryCreatePlayOpts, QueryPlaysOpts } from "../common/database/drizzle/repositories/PlayRepository.js"; +import { PlaySelect, PlaySelectWithQueueStates, QueueStateNew, QueueStateSelect } from "../common/database/drizzle/drizzleTypes.js"; +import { asPlay } from "../../core/PlayMarshalUtils.js"; +import { DrizzleQueueRepository } from "../common/database/drizzle/repositories/QueueRepository.js"; +import { SourceType } from "../common/infrastructure/config/source/sources.js"; type PlatformMappedPlays = Map; type NowPlayingQueue = Map; const platformTruncate = truncateStringToLength(10); + export default abstract class AbstractScrobbleClient extends AbstractComponent implements Authenticatable { name: string; - type: ClientType; + declare type: ClientType; scheduler: ToadScheduler = new ToadScheduler(); + protected initDeadTimeout: NodeJS.Timeout | undefined; protected MAX_STORED_SCROBBLES = 40; protected MAX_INITIAL_SCROBBLES_FETCH = this.MAX_STORED_SCROBBLES; scrobbleSOTRanges: PaginatedTimeRangeOptions[] = []; - scrobbledPlayObjs: FixedSizeList; tracksScrobbled: number = 0; lastScrobbleAttempt: Dayjs = dayjs(0) @@ -98,12 +109,17 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i scrobbleWaitStopInterval: number = 2000; protected scrobbleQueueAbortController: AbortController | undefined; protected scrobbleQueuePromise: Promise | undefined; + protected deadQueueAbortController: AbortController | undefined; + protected deadQueuePromise: Promise | undefined; scrobbleRetries: number = 0; scrobbling: boolean = false; - queuedScrobbles: QueuedScrobble[] = []; - deadLetterScrobbles: DeadLetterScrobble[] = []; + deadQueueProcessing: boolean = false; + queuedLength: number = 0; + deadLetterLength: number = 0; + deadLetterQueued: number = 0; supportsNowPlaying: boolean = false; + nowPlayingInit: boolean = false; nowPlayingEnabled: boolean; nowPlayingFilter: (queue: NowPlayingQueue) => SourcePlayerObj | undefined; nowPlayingMinThreshold: NowPlayingUpdateThreshold = (_) => 10; @@ -131,20 +147,25 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i protected staggerOpts: Partial; protected staggerMappers = { preCompare: staggerMapper({concurrency: 2}), - existing: staggerMapper({concurrency: 2}) + existing: staggerMapper({concurrency: 2}) } + declare protected componentType: 'client'; + + protected playRepo!: DrizzlePlayRepository; + protected queueRepo!: DrizzleQueueRepository; + constructor(type: any, name: any, config: CommonClientConfig, notifier: Notifiers, emitter: EventEmitter, logger: Logger) { super(config); + this.componentType = 'client'; this.type = type; this.name = name; this.logger = childLogger(logger, this.getIdentifier()); this.npLogger = childLogger(this.logger, 'Now Playing'); this.dupeLogger = childLogger(this.logger, 'Dupe'); - this.deadLogger = childLogger(this.logger, 'Dead'); + this.deadLogger = childLogger(this.logger, CLIENT_DEAD_QUEUE); this.notifier = notifier; this.emitter = emitter; - this.scrobbledPlayObjs = new FixedSizeList(this.MAX_STORED_SCROBBLES); const { options: { @@ -213,11 +234,118 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i await this.tryStopScrobbling(); } + public initTasks(opts: {deadDelay?: number} = {}) { + if(this.scheduler.existsById('heartbeat') === false) { + this.logger.info('Adding Heartbeat Task and running immediately'); + this.scheduler.addSimpleIntervalJob(new SimpleIntervalJob({ + minutes: 20, + runImmediately: true + }, new AsyncTask( + 'Heartbeat', + (): Promise => { + return this.heartbeatTask().then(() => null).catch((err) => { + this.logger.error(err); + }); + }, + (err: Error) => { + this.logger.error(err); + } + ), {id: 'heartbeat'})); + } else { + this.logger.warn('Heartbeat task is already added to scheduler.'); + } + + this.initializeNowPlayingSchedule(); + + if(this.scheduler.existsById('dead') === false && this.initDeadTimeout === undefined) { + const deadDelay = opts.deadDelay ?? 120; + this.logger.verbose(`Delaying Dead Scrobbler Processing Task by ${deadDelay} seconds`); + this.initDeadTimeout = setTimeout(() => { + this.logger.info('Adding Dead Scrobbler Processing Task and running immediately'); + this.initDeadTimeout = undefined; + this.scheduler.addSimpleIntervalJob(new SimpleIntervalJob({ + minutes: 20, + runImmediately: true + }, new AsyncTask( + 'Dead', + (): Promise => { + if(this.isReady()) { + return this.processDeadLetterQueue().then(() => null).catch((e) => { + this.logger.error(e); + }) + } + return new Promise((resolve, reject) => resolve); + }, + (err: Error) => { + this.logger.error(err); + } + ), {id: 'dead'})); + }, deadDelay * 1000); + + } else { + if(this.initDeadTimeout !== undefined) { + this.logger.warn('Dead scrobble task timeout is already set'); + } else { + this.logger.warn('Dead scrobble task is already added to the scheduler'); + } + } + } + + protected async heartbeatTask(): Promise { + if(!this.isReady()) { + if(!this.canAuthUnattended()) { + this.logger.warn({labels: 'Heartbeat'}, 'Client is not ready but will not try to initialize because auth state is not good and cannot be corrected unattended.') + return false; + } + try { + await this.tryInitialize({force: false, notify: true, notifyTitle: 'Could not initialize automatically'}); + } catch (e) { + this.logger.error(new Error('Could not initialize automatically', {cause: e})); + return false; + } + + if(!this.canAuthUnattended()) { + this.logger.warn({label: 'Heartbeat'}, 'Should be monitoring scrobbles but will not attempt to start because auth state is not good and cannot be correct unattended.'); + return false; + } + + //await client.processDeadLetterQueue(); + if(!this.scrobbling) { + this.logger.info({labels: 'Heartbeat'}, 'Should be processing scrobbles! Attempting to restart scrobbling...'); + this.initScrobbleMonitoring().catch((e) => this.logger.error('Failed to initialize scrobbler monitoring during heartbeat')); + return true; + } + } + return true; + } + protected async postCache(): Promise { await super.postCache(); this.generateStaggerMappers(); } + protected async postDatabase(): Promise { + this.playRepo = new DrizzlePlayRepository(this.db, {logger: this.logger}); + this.queueRepo = new DrizzleQueueRepository(this.db, {logger: this.logger}); + this.playRepo.componentId = this.dbComponent.id; + this.queueRepo.componentId = this.dbComponent.id; + this.tracksScrobbled = this.dbComponent.countLive + this.dbComponent.countNonLive; + await this.updateQueueStats([CLIENT_INGRESS_QUEUE, CLIENT_DEAD_QUEUE]); + } + + protected async updateQueueStats(queueNames: string[]) { + if(queueNames.includes(CLIENT_INGRESS_QUEUE)) { + this.queuedLength = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_INGRESS_QUEUE]); + this.queuedGauge.labels(this.getPrometheusLabels()).set(this.queuedLength); + } + if(queueNames.includes(CLIENT_DEAD_QUEUE)) { + this.deadLetterLength = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_DEAD_QUEUE], ['queued', 'failed']); + this.deadLetterQueued = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_DEAD_QUEUE], ['queued']); + // TODO + this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterLength); + } + } + protected generateStaggerMappers() { const { preCompare = [], @@ -245,7 +373,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i eInits.push(t.staggerOpts?.initialInterval ?? 0); eMaxStagger.push(t.staggerOpts?.maxRandomStagger ?? 0) } - this.staggerMappers.existing = staggerMapper({initialInterval: Math.max(...eInits), maxRandomStagger: Math.max(...eMaxStagger), concurrency: 3}); + this.staggerMappers.existing = staggerMapper({initialInterval: Math.max(...eInits), maxRandomStagger: Math.max(...eMaxStagger), concurrency: 3}); } } @@ -308,7 +436,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i } this.initializeNowPlayingFilter(); - this.initializeNowPlayingSchedule(); + this.nowPlayingInit = true; } else { this.npLogger.debug('Unsupported feature, disabled.'); } @@ -316,18 +444,18 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i protected initializeNowPlayingSchedule() { - const t = new AsyncTask('Playing Now', (): Promise => { - return this.processingPlayingNow(); - }, (err: Error) => { - this.npLogger.error(new Error('Unexpected error while processing Now Playing queue', {cause: err})); - }); - - this.scheduler.removeById('pn_task'); + if(this.scheduler.existsById('pn_task') === false) { + const t = new AsyncTask('Playing Now', (): Promise => { + return this.processingPlayingNow(); + }, (err: Error) => { + this.npLogger.error(new Error('Unexpected error while processing Now Playing queue', {cause: err})); + }); - // even though we are processing every 5 seconds the interval that Now Playing is updated at, and that the queue is cleared on, - // is still set by shouldUpdatePlayingNow() - // 5 seconds makes sure our granularity for updates is decently fast *when* we do need to actually update - this.scheduler.addSimpleIntervalJob(new SimpleIntervalJob({milliseconds: this.nowPlayingTaskInterval}, t, {id: 'pn_task'})); + // even though we are processing every 5 seconds the interval that Now Playing is updated at, and that the queue is cleared on, + // is still set by shouldUpdatePlayingNow() + // 5 seconds makes sure our granularity for updates is decently fast *when* we do need to actually update + this.scheduler.addSimpleIntervalJob(new SimpleIntervalJob({milliseconds: this.nowPlayingTaskInterval}, t, {id: 'pn_task'})); + } } protected initializeNowPlayingFilter() { @@ -414,14 +542,105 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i protected async doParseCache(): Promise { const cachedQueue = (await this.cache.cacheScrobble.get(`${this.getMachineId()}-queue`) as QueuedScrobble[] ?? []); - const cachedQLength = cachedQueue.length; - this.queuedScrobbles = cachedQueue.map(x => ({...x, play: rehydratePlay(x.play)})); + if (cachedQueue.length > 0) { + this.logger.info('Migrating cached scrobbles to database...'); + let allGood = true; + for (const cachedQueuedScrobble of cachedQueue) { + const play = asPlay(cachedQueuedScrobble.play); + const { + meta: { + lifecycle, + ...metaRest + }, + } = play; + try { + const res = await this.playRepo.createPlays([ + playToRepositoryCreatePlayOpts({ + play: { + ...play, + data: { + ...play.data, + artists: play.data?.artists === undefined ? undefined : artistNamesToCredits(play.data?.artists as unknown as string[]), + albumArtists: play.data?.albumArtists === undefined ? undefined : artistNamesToCredits(play.data?.albumArtists as unknown as string[]) + }, + meta: { + ...metaRest, + lifecycle: { + steps: [] + } + } + }, + componentId: this.dbComponent.id, + state: 'queued', + parentId: play.id + }) + ]); + this.logger.verbose(`Migrated Play ${res[0].uid} => ${buildTrackString(play)}`); + } catch (e) { + allGood = false; + this.logger.verbose(new Error(`Failed to migrate Play ${buildTrackString(play)}`, {cause: e})); + } + } + this.logger[allGood ? 'info' : 'warn'](allGood ? 'Finished migrating all queued scrobbles.' : 'Migrated queued scrobbles with errors'); + await this.cache.cacheScrobble.delete(`${this.getMachineId()}-queue`); + this.logger.info('Deleted legacy cached queued scrobbles'); + } const cachedDead = (await this.cache.cacheScrobble.get(`${this.getMachineId()}-dead`) as DeadLetterScrobble[] ?? []); - const cachedDLength = cachedDead.length; - this.deadLetterScrobbles = cachedDead.map(x => ({...x, play: rehydratePlay(x.play), lastRetry: x.lastRetry !== undefined ? dayjs(x.lastRetry) : undefined})); + if(cachedDead.length > 0) { + this.logger.info('Migrating failed scrobbles to database...'); + let allGood = true; + for(const cDeadScrobble of cachedDead) { + const play = asPlay(cDeadScrobble.play); + const { + meta: { + lifecycle, + ...metaRest + }, + } = play; + try { + const res = await this.playRepo.createPlays([ + playToRepositoryCreatePlayOpts({ + play: { + ...play, + data: { + ...play.data, + artists: play.data?.artists === undefined ? undefined : artistNamesToCredits(play.data?.artists as unknown as string[]), + albumArtists: play.data?.albumArtists === undefined ? undefined : artistNamesToCredits(play.data?.albumArtists as unknown as string[]) + }, + meta: { + ...metaRest, + lifecycle: { + steps: [] + } + } + }, + componentId: this.dbComponent.id, + state: 'failed', + parentId: play.id + }) + ]); + this.logger.verbose(`Added Play ${res[0].uid} to database => ${buildTrackString(play)}`); + await this.queueRepo.create({ + componentId: this.dbComponent.id, + playId: res[0].id, + queueName: CLIENT_DEAD_QUEUE, + queueStatus: 'queued', + retries: cDeadScrobble.retries, + error: cDeadScrobble.error !== undefined ? {message: cDeadScrobble.error } : undefined + }); + this.logger.verbose(`Added Play ${res[0].uid} to Failed Queue`); + } catch (e) { + allGood = false; + this.logger.verbose(new Error(`Failed to migrate Play to failed queued ${buildTrackString(play)}`, {cause: e})); + } + this.logger[allGood ? 'info' : 'warn'](allGood ? 'Finished migrating all failed scrobbles.' : 'Migrated failed scrobbles with errors'); + await this.cache.cacheScrobble.delete(`${this.getMachineId()}-dead`); + this.logger.info('Deleted legacy cached failed scrobbles'); + } + } - return `Scrobbles from Cache: ${cachedQLength} Queue | ${cachedDLength} Dead Letter`; + return; } protected async postInitialize(): Promise { @@ -473,8 +692,10 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i return `${this.name}-scrobbleRange-${typeof from === 'number' ? from : from.unix()}-${typeof to === 'number' ? to :to.unix()}`; } - handleQueuedScrobbleRanges = () => { - this.scrobbleSOTRanges = groupPlaysToTimeRanges(this.queuedScrobbles.map(x => x.play).concat(this.deadLetterScrobbles.map(x => x.play)), this.scrobbleSOTRanges, {staleNowBuffer: this.config.options?.refreshStaleAfter}); + handleQueuedScrobbleRanges = async (deadRetries: number = 3) => { + const queued = await this.playRepo.getQueuedScrobbleRange(CLIENT_INGRESS_QUEUE); + const dead = await this.playRepo.getQueuedScrobbleRange(CLIENT_DEAD_QUEUE, {retries: deadRetries}); + this.scrobbleSOTRanges = groupPlaysToTimeRanges(queued.concat(dead), this.scrobbleSOTRanges, {staleNowBuffer: this.config.options?.refreshStaleAfter}); } getSOTScrobblesForPlay = async (play: PlayObject): Promise => { @@ -514,32 +735,49 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i return obj; } - addScrobbledTrack = (playObj: PlayObject, scrobbledPlay: PlayObjectLifecycleless) => { - this.scrobbledPlayObjs.add({play: playObj, scrobble: scrobbledPlay}); + addScrobbledTrack = async (playObj: PlayObject) => { + this.emitEvent('scrobble', { play: playObj }); + try { + await this.componentRepo.updateById(this.dbComponent.id, {countLive: this.dbComponent.countLive + 1}); + } catch (e) { + this.logger.warn(new Error('Unable to update scrobble count', {cause: e})); + } + //this.scrobbledPlayObjs.add({play: playObj, scrobble: scrobbledPlay}); this.scrobbledCounter.labels(this.getPrometheusLabels()).inc(); //this.lastScrobbledPlayDate = playObj.data.playDate; this.tracksScrobbled++; } - getScrobbledPlays = () => this.scrobbledPlayObjs.data.map(x => x.play) - findExistingSubmittedPlayObj = async (playObjPre: PlayObject): Promise<([undefined, undefined] | [ScrobbledPlayObject, ScrobbledPlayObject[]])> => { const playObj = await this.transformPlay(playObjPre, TRANSFORM_HOOK.candidate); - const dtInvariantMatches = (await pMap(this.scrobbledPlayObjs.data, this.staggerMappers.existing(async x => ({...x, play: await this.transformPlay(x.play, TRANSFORM_HOOK.existing)})), {concurrency: 3})) - .filter(x => playObjDataMatch(playObj, x.play)); + if(this.transformRules.compare?.existing === undefined) { + // if no existing transform then we can run cheap db match + const cheapExisting = await this.playRepo.checkExisting(playObj, {states: ['scrobbled']}); + if(cheapExisting !== undefined) { + const s: ScrobbledPlayObject = {play: cheapExisting.play, scrobble: cheapExisting.play.meta.lifecycle?.scrobble?.mergedScrobble}; + return [s, [s]]; + } + } + + const closeTemporalPlays = await this.playRepo.getTemporallyClosePlays(playObj, {states: ['scrobbled']}); + + const dtInvariantMatches = (await pMap(closeTemporalPlays.map(x => x.play), this.staggerMappers.existing(async x => (await this.transformPlay(x, TRANSFORM_HOOK.existing))), {concurrency: 3})) + .filter(x => playObjDataMatch(playObj, x)); if (dtInvariantMatches.length === 0) { return [undefined, []]; } - const matchPlayDate = dtInvariantMatches.find((x: ScrobbledPlayObject) => { - const temporalComparison = comparePlayTemporally(x.play, playObj); + const matchPlayDate = dtInvariantMatches.find((x: PlayObject) => { + const temporalComparison = comparePlayTemporally(x, playObj); return hasAcceptableTemporalAccuracy(temporalComparison.match) }); - return [matchPlayDate, dtInvariantMatches]; + const s: ScrobbledPlayObject = {play: matchPlayDate, scrobble: matchPlayDate.meta.lifecycle?.scrobble?.mergedScrobble}; + + return [s, [s]]; } public scrobble = async (playObj: PlayObject, opts?: { delay?: number | false, signal?: AbortSignal }): Promise => { @@ -703,11 +941,13 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i } while (true) { signal.throwIfAborted(); - let queueEmpty = this.queuedScrobbles.length === 0; - while (this.queuedScrobbles.length > 0) { - await this.processQueueCurrentScrobble(signal); - } - if(!queueEmpty) { + //let queueEmpty = await this.playRepo.hasQueueNext(CLIENT_INGRESS_QUEUE); // this.queuedLength; // this.queuedScrobbles.length === 0; + let nextQueued = await this.playRepo.getQueueNext(CLIENT_INGRESS_QUEUE); + if(nextQueued !== undefined) { + while (nextQueued !== undefined) { + await this.processQueueCurrentScrobble(nextQueued, signal); + nextQueued = await this.playRepo.getQueueNext(CLIENT_INGRESS_QUEUE) + } this.emitEvent('queueEmptied', {}); } await delay(signal, this.scrobbleSleep); @@ -723,23 +963,27 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i } } - protected processQueueCurrentScrobble = async (signal: AbortSignal) => { + protected processQueueCurrentScrobble = async (currQueuedPlay: PlaySelectWithQueueStates, signal: AbortSignal) => { signal.throwIfAborted(); - if (this.queuedScrobbles.length === 0) { - return; - } - - this.handleQueuedScrobbleRanges(); + //const currQueuedPlay = await this.playRepo.getQueueNext(CLIENT_INGRESS_QUEUE); + // if (currQueuedPlay === undefined) { + // this.logger.trace('Nothing queued'); + // return; + // } + await this.handleQueuedScrobbleRanges(); if (!this.upstreamRefresh.refreshEnabled) { // TODO add signal for this to scrobble match this.logger.trace('Scrobble refresh is DISABLED.'); } - let handledShiftedPlay = false; - const currQueuedPlay = this.queuedScrobbles.shift(); + //let handledShiftedPlay = false; + //const currQueuedPlay = await this.playRepo.getQueueNext(CLIENT_INGRESS_QUEUE); + //const currQueuedPlay = this.queuedScrobbles.shift(); let historicalPlays: PlayObject[] = []; let historicalError: Error | undefined; + let queueError: Error | undefined; + let successState: PlaySelect['state']; try { @@ -749,13 +993,15 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i } catch (e) { historicalError = e; if (e.message === 'Cannot get historical plays due to cached error') { - this.logger.warn(`${buildTrackString(currQueuedPlay.play)} from Source '${currQueuedPlay.source}' => Previous error while getting historical scrobbles means this scrobble cannot be compared, will queue as dead for now.`); + this.logger.warn(`${buildTrackString(currQueuedPlay.play)} from Source '${currQueuedPlay.play.meta.source}' => Previous error while getting historical scrobbles means this scrobble cannot be compared, will queue as dead for now.`); this.logger.trace(e); + queueError = e; } else { - this.logger.warn(new SimpleError(`${buildTrackString(currQueuedPlay.play)} from Source '${currQueuedPlay.source}' => cannot get historical scrobbles, will queue as dead for now.`, { cause: e, shortStack: true })); + queueError = new SimpleError(`${buildTrackString(currQueuedPlay.play)} from Source '${currQueuedPlay.play.meta.source}' => cannot get historical scrobbles, will queue as dead for now.`, { cause: e, shortStack: true }); + this.logger.warn(queueError); } - this.addDeadLetterScrobble(currQueuedPlay, e); - handledShiftedPlay = true; + await this.addDeadLetterScrobble(currQueuedPlay, e); + //handledShiftedPlay = true; } signal.throwIfAborted(); } @@ -778,16 +1024,14 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i signal.throwIfAborted(); if (transformedScrobble.meta.lifecycle === undefined) { transformedScrobble.meta.lifecycle = { - original: transformedScrobble, + //original: transformedScrobble, steps: [] }; } try { const scrobbledPlay = await this.scrobble(transformedScrobble, {signal}); - this.emitEvent('scrobble', { play: transformedScrobble }); - this.addScrobbledTrack(scrobbledPlay, scrobbledPlay.meta.lifecycle.scrobble.mergedScrobble ?? scrobbledPlay); - handledShiftedPlay = true; - signal.throwIfAborted(); + await this.addScrobbledTrack(scrobbledPlay); + //handledShiftedPlay = true; } catch (e) { currQueuedPlay.play.meta.lifecycle.scrobble = { }; @@ -802,36 +1046,64 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i currQueuedPlay.play.meta.lifecycle.scrobble.error = serializeError(e); } + queueError = e; + await this.addDeadLetterScrobble(currQueuedPlay, e); + //handledShiftedPlay = true; if (hasUpstreamError(e, false)) { - this.addDeadLetterScrobble(currQueuedPlay, e); - handledShiftedPlay = true; - this.logger.warn(new Error(`Could not scrobble ${buildTrackString(transformedScrobble)} from Source '${currQueuedPlay.source}' but error was not show stopping. Adding scrobble to Dead Letter Queue and will retry on next heartbeat.`, { cause: e })); + //handledShiftedPlay = true; + const nonShowStoppingError = new Error(`Could not scrobble ${buildTrackString(transformedScrobble)} from Source '${currQueuedPlay.play.meta.source}' but error was not show stopping. Adding scrobble to Dead Letter Queue and will retry on next heartbeat.`, { cause: e }); + this.logger.warn(nonShowStoppingError); + queueError = nonShowStoppingError; } else { - this.queuedScrobbles.unshift(currQueuedPlay); - handledShiftedPlay = true; - this.updateQueuedScrobblesCache(); - throw new Error('Error occurred while trying to scrobble', { cause: e }); + //this.queuedScrobbles.unshift(currQueuedPlay); + //handledShiftedPlay = true; + const showStoppingError = new Error('Error occurred while trying to scrobble', { cause: e }); + queueError = showStoppingError; + throw showStoppingError; } } + } else { + successState = 'duped'; } } - this.updateQueuedScrobblesCache(); - this.queuedGauge.labels(this.getPrometheusLabels()).set(this.queuedScrobbles.length); - this.emitEvent('scrobbleDequeued', { queuedScrobble: currQueuedPlay }) signal.throwIfAborted(); // reset retries if we've made this far this.scrobbleRetries = 0; } catch (e) { - if(!handledShiftedPlay) { - this.queuedScrobbles.unshift(currQueuedPlay); + if(queueError === undefined) { + queueError = e; } + // if(!handledShiftedPlay) { + // this.queuedScrobbles.unshift(currQueuedPlay); + // } throw e; + } finally { + const queueState = currQueuedPlay.queueStates.find(x => x.queueName === CLIENT_INGRESS_QUEUE); + if(queueError !== undefined) { + await this.queueRepo.updateById(queueState.id, {queueStatus: 'failed', error: queueError}); + await this.playRepo.updateById(currQueuedPlay.id, {state: 'failed', error: queueError}); + } else { + await this.queueRepo.updateById(queueState.id, {queueStatus: 'completed'}); + await this.playRepo.updateById(currQueuedPlay.id, {state: successState ?? 'scrobbled'}); + } + this.emitEvent('scrobbleDequeued', { queuedScrobble: currQueuedPlay }) + this.queuedGauge.labels(this.getPrometheusLabels()).dec(); + this.queuedLength -= 1; } } processDeadLetterQueue = async (attemptWithRetries?: number) => { - if (this.deadLetterScrobbles.length === 0) { + // if (this.deadLetterScrobbles.length === 0) { + // return; + // } + + if (!(await this.isReady())) { + this.deadLogger.warn('Cannot process dead letter scrobbles because client is not ready.'); + return; + } + if(this.deadQueueAbortController !== undefined) { + this.deadLogger.warn('Dead scrobbles are currently being processed, cannot restart right now.'); return; } @@ -842,42 +1114,93 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i } = this.config; const retries = attemptWithRetries ?? deadLetterRetries; + const removedIds = []; - const processable = this.deadLetterScrobbles.filter(x => x.retries < retries); - const queueStatus = `${processable.length} of ${this.deadLetterScrobbles.length} dead scrobbles have less than ${retries} retries, ${processable.length === 0 ? 'will skip processing.': 'processing now...'}`; - if (processable.length === 0) { - this.deadLogger.verbose(queueStatus); - return; - } - this.logger.info(queueStatus); - if(!this.upstreamRefresh.refreshEnabled) { - this.deadLogger.verbose('Scrobble refresh is DISABLED. All dead scrobbles will likely always be scrobbled (nothing to check duplicates against).'); - } - this.handleQueuedScrobbleRanges(); + this.deadQueueAbortController = new AbortController(); + this.deadQueuePromise = spawn(this.deadQueueAbortController.signal, async (signal, { defer, fork }) => { - const removedIds = []; - for (const deadScrobble of processable) { - const [scrobbled, dead] = await this.processDeadLetterScrobble(deadScrobble.id); - if (scrobbled) { - removedIds.push(deadScrobble.id); + defer(async () => { + this.deadQueueProcessing = false; + this.emitEvent('queueState', {queueName: 'dead', status: 'Idle'}); + }); + + this.emitEvent('queueState', {queueName: 'dead', status: 'Running'}); + await this.queueRepo.deadFailedToQueue(this.dbComponent.id, retries); + + const processable = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_DEAD_QUEUE]); //this.deadLetterScrobbles.filter(x => x.retries < retries); + this.deadLetterQueued = processable; + + const total = await this.queueRepo.getQueueCount(this.dbComponent.id, [CLIENT_DEAD_QUEUE], ['queued','failed']); + this.deadLetterLength = total; + const queueStatus = `${processable} of ${total} dead scrobbles have less than ${retries} retries, ${processable === 0 ? 'will skip processing.': 'processing now...'}`; + if (processable === 0) { + this.deadLogger.verbose(queueStatus); + return; } - await sleep(this.scrobbleSleep); - } - if (removedIds.length > 0) { - this.deadLogger.info(`Removed ${removedIds.length} scrobbles from dead letter queue`); - } + this.logger.info(queueStatus); + if(!this.upstreamRefresh.refreshEnabled) { + this.deadLogger.verbose('Scrobble refresh is DISABLED. All dead scrobbles will likely always be scrobbled (nothing to check duplicates against).'); + } + // await this.handleQueuedScrobbleRanges(); + + let nextQueued: PlaySelectWithQueueStates = await this.playRepo.getQueueNext(CLIENT_DEAD_QUEUE, {retries}); + if(nextQueued !== undefined) { + while(nextQueued !== undefined) { + const [scrobbled, dead] = await this.processDeadLetterScrobble(nextQueued.uid, signal); + await sleep(this.scrobbleSleep); + if(scrobbled) { + removedIds.push(dead.id); + } + nextQueued = await this.playRepo.getQueueNext(CLIENT_DEAD_QUEUE, {retries}); + } + } + + }).catch((e) => { + if (isAbortError(e)) { + const err = generateLoggableAbortReason('Dead scrrobble processing stopped', this.deadQueueAbortController.signal); + this.logger.info(err); + this.logger.trace(e) + } else { + this.logger.warn(new Error('Dead scrobble processing stopped with error', { cause: e })); + } + }).finally(() => { + if (removedIds.length > 0) { + this.deadLogger.info(`Removed ${removedIds.length} scrobbles from dead letter queue`); + } + this.deadQueueAbortController = undefined; + this.deadQueuePromise = undefined; + }); } - processDeadLetterScrobble = async (id: string): Promise<[boolean, DeadLetterScrobble?]> => { - const deadScrobbleIndex = this.deadLetterScrobbles.findIndex(x => x.id === id); - if(deadScrobbleIndex === -1) { - this.deadLogger.warn(`Could not find a dead scrobble with id ${id}`); - return [false]; + processDeadLetterScrobble = async (uid: string, signal?: AbortSignal): Promise<[boolean, PlaySelectWithQueueStates?]> => { + signal?.throwIfAborted(); + // const deadScrobbleIndex = this.deadLetterScrobbles.findIndex(x => x.id === id); + // if(deadScrobbleIndex === -1) { + // this.deadLogger.warn(`Could not find a dead scrobble with id ${id}`); + // return [false]; + // } + + + let deadQueueState: QueueStateSelect; + let deadScrobble: PlaySelectWithQueueStates = await this.playRepo.findByUid(uid, {hydrate: ['asPlay']}); + if(deadScrobble === undefined) { + throw new Error(`Play ${uid} does not exist for ${this.name}`); + } + if(deadScrobble.state === 'scrobbled') { + throw new Error(`Play ${uid} is already scrobbled.`); + } + deadQueueState = deadScrobble.queueStates.find(x => x.queueName === CLIENT_DEAD_QUEUE); + if(deadQueueState === undefined) { + throw new Error(`Play ${uid} is not currently queued in dead letter.`); } - const deadLabel = {labels: id}; - const deadScrobble = this.deadLetterScrobbles[deadScrobbleIndex]; + //const deadScrobble = await this.playRepo.getQueueNext(this.dbComponent.id, CLIENT_INGRESS_QUEUE); + const deadLabel = {labels: deadScrobble.uid}; + //const deadScrobble = this.deadLetterScrobbles[deadScrobbleIndex]; this.deadLogger.trace(deadLabel, `Processing dead scrobble => ${buildTrackString(deadScrobble.play)}`); + await this.handleQueuedScrobbleRanges(); + signal?.throwIfAborted(); + if (!(await this.isReady())) { this.deadLogger.warn(deadLabel, 'Cannot process dead letter scrobble because client is not ready.'); return [false, deadScrobble]; @@ -891,17 +1214,20 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i this.deadLogger.warn(deadLabel, `Previous error while getting historical scrobbles means this scrobble cannot be compared`); this.deadLogger.trace(e); } else { - this.deadLogger.warn(new SimpleError(`${id} - ${buildTrackString(deadScrobble.play)} from Source '${deadScrobble.source}' => cannot get historical scrobbles`, {cause: e, shortStack: true})); + this.deadLogger.warn(new SimpleError(`${deadScrobble.uid} - ${buildTrackString(deadScrobble.play)} from Source '${deadScrobble.play.meta.source}' => cannot get historical scrobbles`, {cause: e, shortStack: true})); } - deadScrobble.retries++; - deadScrobble.error = messageWithCauses(e); - deadScrobble.lastRetry = dayjs(); - this.deadLetterScrobbles[deadScrobbleIndex] = deadScrobble; - this.updateDeadLetterCache(); + + this.queueRepo.updateById(deadQueueState.id, {retries: deadQueueState.retries + 1, error: e, updatedAt: dayjs(), queueStatus: 'failed'}); + this.playRepo.updateById(deadScrobble.id, {error: e}); + // deadScrobble.retries++; + // deadScrobble.error = messageWithCauses(e); + // deadScrobble.lastRetry = dayjs(); + // this.deadLetterScrobbles[deadScrobbleIndex] = deadScrobble; this.emitEvent('updateDeadLetter', {dead: deadScrobble}); return [false, deadScrobble]; } } + signal?.throwIfAborted(); const {summary, ...matchResult} = await this.existingScrobble(deadScrobble.play, historicalPlays); const { scrobble = {}, @@ -916,11 +1242,11 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i } if(!matchResult.match) { const transformedScrobble = await this.transformPlay(deadScrobble.play, TRANSFORM_HOOK.postCompare); + signal?.throwIfAborted(); try { const scrobbledPlay = await this.scrobble(transformedScrobble); - this.emitEvent('scrobble', {play: transformedScrobble}); - this.addScrobbledTrack(transformedScrobble, scrobbledPlay); - this.removeDeadLetterScrobble(deadScrobble.id) + await this.addScrobbledTrack(scrobbledPlay); + this.removeDeadLetterScrobble(deadScrobble, 'scrobbled', true); } catch (e) { const submitError = findCauseByReference(e, ScrobbleSubmitError); @@ -933,92 +1259,191 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i deadScrobble.play.meta.lifecycle.scrobble.error = serializeError(e); } - deadScrobble.retries++; - deadScrobble.error = messageWithCauses(e); - deadScrobble.lastRetry = dayjs(); - this.deadLogger.error(new Error(`${id} - Could not scrobble ${buildTrackString(transformedScrobble)} from Source '${deadScrobble.source}' due to error`, {cause: e})); - this.deadLetterScrobbles[deadScrobbleIndex] = deadScrobble; - this.updateDeadLetterCache(); + this.queueRepo.updateById(deadQueueState.id, {retries: deadQueueState.retries + 1, error: e, updatedAt: dayjs(), queueStatus: 'failed'}); + this.playRepo.updateById(deadScrobble.id, {error: e}); + // deadScrobble.retries++; + // deadScrobble.error = messageWithCauses(e); + // deadScrobble.lastRetry = dayjs(); + this.deadLogger.error(new Error(`${deadScrobble.uid} - Could not scrobble ${buildTrackString(transformedScrobble)} from Source '${deadScrobble.play.meta.source}' due to error`, {cause: e})); + //this.deadLetterScrobbles[deadScrobbleIndex] = deadScrobble; this.emitEvent('updateDeadLetter', {dead: deadScrobble}); return [false, deadScrobble]; } } else { this.deadLogger.verbose(`Looks like ${buildTrackString(deadScrobble.play)} was already scrobbled!\n${summary}`); - this.removeDeadLetterScrobble(deadScrobble.id) + this.removeDeadLetterScrobble(deadScrobble, 'duped', true); } return [true, deadScrobble]; } - removeDeadLetterScrobble = (id: string) => { - const index = this.deadLetterScrobbles.findIndex(x => x.id === id); - if (index === -1) { - this.deadLogger.warn(`No scrobble found with ID ${id}`); - return; + removeDeadLetterScrobble = async (dead: (PlaySelect & {queueStates: QueueStateSelect[]}) | string, state: PlaySelect['state'], success: boolean) => { + + let deadScrobble: PlaySelect & {queueStates: QueueStateSelect[]}; + + if(typeof dead === 'string'){ + deadScrobble = await this.playRepo.findByUid(dead, {hydrate: ['asPlay']}); + if(deadScrobble === undefined) { + throw new Error(`Play ${dead} does not exist for ${this.name}`); + } + } else { + deadScrobble = dead; + } + // const index = this.deadLetterScrobbles.findIndex(x => x.id === id); + // if (index === -1) { + // this.deadLogger.warn(`No scrobble found with ID ${id}`); + // return; + // } + const deadQueueState = deadScrobble.queueStates.find(x => x.queueName === CLIENT_DEAD_QUEUE && x.queueStatus !== 'completed'); + const isQueued = deadQueueState.queueStatus === 'queued'; + if(deadQueueState === undefined) { + throw new Error(`Play ${deadScrobble.uid} is not currently queued in dead letter.`); } - this.deadLogger.info({labels: id}, `Removed scrobble ${buildTrackString(this.deadLetterScrobbles[index].play)} from queue`); - this.deadLetterScrobbles.splice(index, 1); - this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterScrobbles.length); - this.updateDeadLetterCache(); - this.emitEvent('removeDeadLetter', { dead: { id } }); + //this.deadLetterScrobbles.splice(index, 1); + this.deadLetterGauge.labels(this.getPrometheusLabels()).dec(); + let queueUpdate: Partial = { + updatedAt: dayjs(), + queueStatus: 'completed' + } + if(success) { + queueUpdate.error = null; + } + await this.queueRepo.updateById(deadQueueState.id, queueUpdate); + await this.playRepo.updateById(deadScrobble.id, removeUndefinedKeys({state, error: success ? null : undefined})); + this.deadLogger.info({labels: deadScrobble.uid}, `Scrobble ${buildTrackString(deadScrobble.play)} marked as completed`); + this.deadLetterLength -= 1; + if(isQueued) { + this.deadLetterQueued -= 1; + } + if(state === 'scrobbled') { + this.componentRepo.updateById(this.dbComponent.id, {countLive: this.dbComponent.countLive + 1}); + } + this.emitEvent('removeDeadLetter', { dead: { id: deadScrobble.uid } }); } - removeDeadLetterScrobbles = () => { - this.deadLetterScrobbles = []; - this.updateDeadLetterCache(); - this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterScrobbles.length); - this.logger.info('Removed all scrobbles from queue', {leaf: 'Dead Letter'}); + removeDeadLetterScrobbles = async (types: QueueStateSelect['queueStatus'][] = ['queued'], state: PlaySelect['state'], success: boolean) => { + const ids = await this.playRepo.findPlayIdentifiers({ + queues: [ + { + queueName: CLIENT_DEAD_QUEUE, + queueStatus: types + } + ] + }, 'uid'); + this.deadLogger.info(`Marking ${ids} as completed but unsuccessful...`); + await Promise.all(ids.map((x) => this.removeDeadLetterScrobble(x, state, success))); + this.deadLogger.info('Finished processing dead scrobbles.'); + await this.updateQueueStats([CLIENT_DEAD_QUEUE]); } queueScrobble = async (data: PlayObject | PlayObject[], source: string) => { - const plays = (Array.isArray(data) ? data : [data]).map(x => ({...x, meta: {...x.meta, seenAt: dayjs()}})); - for await(const play of pMapIterable(plays, this.staggerMappers.preCompare(async x => await this.transformPlay(x, TRANSFORM_HOOK.preCompare)), {concurrency: 3})) { + const playDatas = (Array.isArray(data) ? data : [data]).map(x => ({...x, meta: {...x.meta, seenAt: dayjs()}})); + + const createdQueuedPlays: PlaySelect[] = []; + + for await(const play of pMapIterable(playDatas, this.staggerMappers.preCompare(async x => await this.transformPlay(x, TRANSFORM_HOOK.preCompare)), {concurrency: 3})) { try { - const existingQueued = await this.existingScrobble(play, this.queuedScrobbles.map(x => x.play), false); - // want to be very confident of this - if(existingQueued.match && existingQueued.score > 0.99) { - this.logger.trace(`Not adding to queue because it is already in the queue\n${existingQueued.summary}`); - return; + // cheap check, looks for play data (non-meta) hash, playdate, and optionally mbid recording + const cheapExisting = await this.playRepo.checkExisting(play, {queueName: CLIENT_INGRESS_QUEUE}); + if(cheapExisting !== undefined) { + const qs = cheapExisting.queueStates.find(x => x.queueName === CLIENT_INGRESS_QUEUE); + this.logger.trace(`Not adding to queue because it is already in the queue, discovered via hash/mbid, last queued at ${todayAwareFormat(qs.createdAt)}`); + continue; + } + // then chunked queued plays + let offset = 0; + let inQueue = false; + while(true) { + const {data, meta} = await this.playRepo.getQueued(CLIENT_INGRESS_QUEUE, {offset}); + const existingQueued = await this.existingScrobble(play, data.map(x => asPlay(x.play)), false); + // want to be very confident of this + if(existingQueued.match && existingQueued.score > 0.99) { + this.logger.trace(`Not adding to queue because it is already in the queue\n${existingQueued.summary}`); + inQueue = true; + break; + } + if(data.length < meta.limit) { + break; + } + offset += meta.limit; } + + if(inQueue) { + continue; + } + + // not in queue! + const { + meta: { + // dbId, + // dbUid, + lifecycle, + ...metaRest + }, + } = play + const createPlayData = playToRepositoryCreatePlayOpts({ + play: { + ...play, + meta: { + ...metaRest, + lifecycle: { + steps: [] + } + } + }, + componentId: this.dbComponent.id, + state: 'queued', + parentId: play.id + }); + + const playRow = await this.playRepo.createPlays([createPlayData]); + await this.queueRepo.create({componentId: this.dbComponent.id, playId: playRow[0].id, queueName: CLIENT_INGRESS_QUEUE}); + createdQueuedPlays.push(playRow[0]); + this.logger.debug(`Added ${buildTrackString(play)} to the queue`); + } catch (e) { this.logger.warn(new SimpleError('Failed to check queued scrobble for existing before adding', {cause: e})); } const queuedPlay = {id: nanoid(), source, play: play} + //await this.playRepo.updateById(play.meta.dbId, {play}); this.emitEvent('scrobbleQueued', {queuedPlay: queuedPlay}); - this.queuedScrobbles.push(queuedPlay); + this.queuedLength += 1; + //this.queuedScrobbles.push(queuedPlay); this.queuedGauge.labels(this.getPrometheusLabels()).inc(); // this is wasteful but we don't want the processing loop popping out-of-order (by date) scrobbles - this.queuedScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); + //this.queuedScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); } - this.updateQueuedScrobblesCache(); + return createdQueuedPlays; } - cancelQueuedItemsBySource = (source: string): number => { - const beforeMain = this.queuedScrobbles.length; - const beforeDead = this.deadLetterScrobbles.length; - - this.queuedScrobbles = this.queuedScrobbles.filter(item => item.source !== source); - this.deadLetterScrobbles = this.deadLetterScrobbles.filter(item => item.source !== source); - - this.updateQueuedScrobblesCache(); - this.updateDeadLetterCache(); - - return (beforeMain + beforeDead) - (this.queuedScrobbles.length + this.deadLetterScrobbles.length); - } - - addDeadLetterScrobble = (data: QueuedScrobble, error: (Error | string) = 'Unspecified error') => { + addDeadLetterScrobble = async (data: PlaySelect, error: (Error | string) = 'Unspecified error') => { let eString = ''; if(typeof error === 'string') { eString = error; } else { eString = messageWithCauses(error); } - const deadData = {id: nanoid(), retries: 0, error: eString, ...data}; - this.deadLetterScrobbles.push(deadData); - this.deadLetterScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); + let e: ErrorLike; + if(isErrorLike(error)) + { + e = error; + } else if(typeof error === 'string') { + e = new Error(error); + } + this.deadLetterLength += 1; + this.deadLetterQueued += 1; + //this.playRepo.updateById(data.id, {state: 'failed', error: e}); + await this.queueRepo.create({ + componentId: this.dbComponent.id, + playId: data.id, + queueName: CLIENT_DEAD_QUEUE + }); + // TODO ? + const deadData = {id: nanoid(), retries: 0, error: eString, play: data.play}; + //this.deadLetterScrobbles.push(deadData); + //this.deadLetterScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); this.emitEvent('deadLetter', {dead: deadData}); - this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterScrobbles.length); - this.updateDeadLetterCache(); + this.deadLetterGauge.labels(this.getPrometheusLabels()).inc(); } queuePlayingNow = async (data: SourcePlayerObj, source: SourceIdentifier) => { @@ -1044,15 +1469,36 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i } processingPlayingNow = async (): Promise => { - if(this.supportsNowPlaying && this.nowPlayingEnabled) { + if(!this.supportsNowPlaying || !this.isReady()) { + return; + } + if(this.nowPlayingInit === false) { + this.initializeNowPlaying(); + } + if(this.nowPlayingEnabled) { const sourcePlayerData = this.nowPlayingFilter(this.nowPlayingQueue); if(sourcePlayerData === undefined) { return; } - if(this.shouldUpdatePlayingNow(sourcePlayerData) && (await this.shouldUpdatePlayingNowPlatformSpecific(sourcePlayerData))) { + let shouldUpdate: boolean, + clientReason: string | undefined; + const [npUpdateTop, npUpdateTopReason] = this.shouldUpdatePlayingNowResult(sourcePlayerData); + shouldUpdate = npUpdateTop; + if(!npUpdateTop) { + this.npLogger.trace(`Not updating because ${npUpdateTopReason}`); + } else { + const [clientUpdate, clientUpdateReason, level] = await this.shouldUpdatePlayingNowPlatformSpecific(sourcePlayerData); + clientReason = clientUpdateReason; + shouldUpdate = clientUpdate; + if(!clientUpdate) { + this.npLogger[level ?? 'trace'](`Not updating, ${npUpdateTopReason} --BUT-- ${clientUpdateReason}`); + } + } + if(shouldUpdate) { + this.npLogger.verbose(`Updating because ${npUpdateTopReason}${clientReason !== undefined ? ` --AND-- ${clientReason}` : ''}`); try { await this.doPlayingNow(sourcePlayerData); - this.npLogger.debug(`Now Playing updated.`); + this.npLogger.trace(`Now Playing updated.`); this.emitEvent('nowPlayingUpdated', sourcePlayerData); } catch (e) { this.npLogger.warn(new Error('Error occurred while trying to update upstream Client, will ignore', {cause: e})); @@ -1064,12 +1510,9 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i } } - shouldUpdatePlayingNow = (data: SourcePlayerObj): boolean => { + shouldUpdatePlayingNowResult = (data: SourcePlayerObj): [boolean, string?] => { if(this.nowPlayingLastPlay === undefined || this.nowPlayingLastUpdated === undefined) { - if(isDebugMode()) { - this.npLogger.debug(`Now Playing has not yet been set! Should update`); - } - return true; + return [true, 'Now Playing has not yet been set']; } const lastUpdateDiff = Math.abs(dayjs().diff(this.nowPlayingLastUpdated, 's')); @@ -1082,33 +1525,43 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i // update if play *has* changed and time since last update is greater than min interval // this prevents spamming scrobbler API with updates if user is skipping tracks and source updates frequently if(this.nowPlayingMinThreshold(data.play) < lastUpdateDiff && (playExistingDiscrepancy || playerStatusChanged || (bothPlaysExist && !playObjDataMatch(data.play, this.nowPlayingLastPlay.play)))) { - if(isDebugMode()) { - this.npLogger.debug(`New Play differs from previous Now Playing and time since update ${lastUpdateDiff}s, greater than threshold ${this.nowPlayingMinThreshold(data.play)}. Should update`); - } - return true; + return [true, `New Play differs from previous Now Playing and time since update ${lastUpdateDiff}s, greater than threshold ${this.nowPlayingMinThreshold(data.play)}`]; } // update if play *has not* changed but last update is greater than max interval // this keeps scrobbler Now Playing fresh ("active" indicator) in the event play is long if(this.nowPlayingMaxThreshold(data.play) < lastUpdateDiff && (bothPlaysExist && playObjDataMatch(data.play, this.nowPlayingLastPlay.play))) { - if(isDebugMode()) { - this.npLogger.debug(`Now Playing last updated ${lastUpdateDiff}s ago, greater than threshold ${this.nowPlayingMaxThreshold(data.play)}s. Should update`); - } - return true; + return [true, `Now Playing last updated ${lastUpdateDiff}s ago, greater than threshold ${this.nowPlayingMaxThreshold(data.play)}s`]; } - if(isDebugMode()) { - this.npLogger.debug(`Now Playing ${bothPlaysExist && playObjDataMatch(data.play, this.nowPlayingLastPlay.play) ? 'matches' : 'does not match'} and was last updated ${lastUpdateDiff}s ago (threshold ${this.nowPlayingMaxThreshold(data.play)}s), not updating`); - } - return false; + return [false, `Now Playing ${bothPlaysExist && playObjDataMatch(data.play, this.nowPlayingLastPlay.play) ? 'matches' : 'does not match'} and was last updated ${lastUpdateDiff}s ago (threshold ${this.nowPlayingMaxThreshold(data.play)}s)`]; + } + + shouldUpdatePlayingNow = (data: SourcePlayerObj): boolean => { + return this.shouldUpdatePlayingNowResult(data)[0]; } /** Implement this for specific requirements for updating playing now based on the scrobbler platform */ - protected shouldUpdatePlayingNowPlatformSpecific(data: SourcePlayerObj): Promise { + protected shouldUpdatePlayingNowPlatformSpecific(data: SourcePlayerObj): Promise<[boolean, string?, LogLevel?]> { return shouldUpdatePlayingNowPlatformWhenPlayingOnly(data); } protected doPlayingNow = (data: SourcePlayerObj): Promise => Promise.resolve(undefined) + public getQueued = (queueName: string, statuses: string[], offset?: number) => { + return this.playRepo.getQueued(queueName, {offset}); + } + + public getPlaysPaginated = (args: QueryPlaysOpts) => { + const { + limit, + offset, + with: withQuery = ['input','parent-input','queues'], + ...rest + } = args; + let parsedLimit = limit !== undefined ? Number.parseInt(limit as unknown as string) : undefined; + let parsedOffset = offset !== undefined ? Number.parseInt(offset as unknown as string) : undefined; + return this.playRepo.findPlaysPaginated({limit: parsedLimit, offset: parsedOffset, with: withQuery, ...rest}); + } public emitEvent = (eventName: string, payload: object) => { this.emitter.emit(eventName, { @@ -1118,22 +1571,6 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i from: 'client' }); } - - protected updateDeadLetterCache = () => { - this.cache.cacheScrobble.set(`${this.getMachineId()}-dead`, this.deadLetterScrobbles) - .then(() => null) - .catch((e) => this.logger.warn(new Error('Error while updating dead letter cache', {cause: e}))); - } - - protected updateQueuedScrobblesCache = () => { - this.cache.cacheScrobble.set(`${this.getMachineId()}-queue`, this.queuedScrobbles) - .then(() => { - return undefined; - }) - .catch((e) => { - this.logger.warn(new Error('Error while updating queued scrobble cache', {cause: e})) - }); - } } export const nowPlayingUpdateByPlayDuration: NowPlayingUpdateThreshold = (play?: PlayObject) => { @@ -1143,7 +1580,9 @@ export const nowPlayingUpdateByPlayDuration: NowPlayingUpdateThreshold = (play?: return (play?.data?.duration ?? 30) + 1; } -export const shouldUpdatePlayingNowPlatformWhenPlayingOnly = async (data: SourcePlayerObj): Promise => { - return (data.status.calculated === CALCULATED_PLAYER_STATUSES.playing) - || (data.nowPlayingMode && !CALCULATED_PLAYER_STATUSES.stopped); +export const shouldUpdatePlayingNowPlatformWhenPlayingOnly = async (data: SourcePlayerObj): Promise<[boolean, string]> => { + if(data.status.calculated === CALCULATED_PLAYER_STATUSES.playing || (data.nowPlayingMode && !CALCULATED_PLAYER_STATUSES.stopped)) { + return [true, `calculated player status is ${data.status.calculated}`]; + } + return [false, `calculated player status is ${data.status.calculated} but must be played/stopped`]; } \ No newline at end of file diff --git a/src/backend/scrobblers/DiscordScrobbler.ts b/src/backend/scrobblers/DiscordScrobbler.ts index 0fb06d8ad..1fc684396 100644 --- a/src/backend/scrobblers/DiscordScrobbler.ts +++ b/src/backend/scrobblers/DiscordScrobbler.ts @@ -1,4 +1,4 @@ -import { Logger } from "@foxxmd/logging"; +import { Logger, LogLevel } from "@foxxmd/logging"; import EventEmitter from "events"; import { PlayMatchResult, PlayObject, SourcePlayerObj } from "../../core/Atomic.js"; import { CALCULATED_PLAYER_STATUSES, FormatPlayObjectOptions, REPORTED_PLAYER_STATUSES, ReportedPlayerStatus, SINGLE_USER_PLATFORM_ID_STR, TimeRangeListensFetcher } from "../common/infrastructure/Atomic.js"; @@ -120,7 +120,7 @@ export default class DiscordScrobbler extends AbstractScrobbleClient { // discord does not handle scrobbles, only Now Playing // so don't bother queueing any scrobbles as we don't want to cache them // or give the user the impression they are used (in UI as a number of queued scrobbles) - return; + return []; } alreadyScrobbled = async (playObj: PlayObject, log = false): Promise<[boolean, PlayMatchResult]> => ([false, {match: false, breakdowns: [], score: 0}]) @@ -156,34 +156,33 @@ export default class DiscordScrobbler extends AbstractScrobbleClient { } } - shouldUpdatePlayingNowPlatformSpecific = async (data: SourcePlayerObj) => { + shouldUpdatePlayingNowPlatformSpecific = async (data: SourcePlayerObj): Promise<[boolean, string?, LogLevel?]> => { if ([CALCULATED_PLAYER_STATUSES.stopped, CALCULATED_PLAYER_STATUSES.paused, CALCULATED_PLAYER_STATUSES.playing].includes(data.status.calculated as ReportedPlayerStatus) || (data.nowPlayingMode && !CALCULATED_PLAYER_STATUSES.stopped) || data.status.stale) { const [sendOk, reasons, level = 'warn'] = await this.api.checkOkToSend(); if (!sendOk) { - this.npLogger[level](`Cannot update playing now because api client is ${reasons}`); - return false; + return [false, `Cannot update playing now because api client is ${reasons}`, level as LogLevel]; } if(this.api instanceof DiscordWSClient) { const [allowed, reason] = this.api.presenceIsAllowed(); if(!allowed) { this.npLogger.debug(reason); + return [false, reason]; } - - return true; } - return true; + return [true]; } else { if(!data.nowPlayingMode && ![CALCULATED_PLAYER_STATUSES.stopped, CALCULATED_PLAYER_STATUSES.paused, CALCULATED_PLAYER_STATUSES.playing].includes(data.status.calculated as ReportedPlayerStatus)) { - this.npLogger.trace(`Will not update because player is not in state: stopped | paused | playing => Found '${data.status.calculated }'`); + return [false,`player is not in state: stopped | paused | playing => Found '${data.status.calculated }'`]; } else if(data.nowPlayingMode && CALCULATED_PLAYER_STATUSES.stopped) { this.npLogger.trace(`Will not update because now playing player is stopped => Found ${data.status.calculated}`); + return [false,`playing player is stopped => Found ${data.status.calculated}` ] } else { - this.npLogger.trace('Will not update because now player is in an unexpected state for discord usage'); + return [false, 'player is in an unexpected state for discord usage'] } } } diff --git a/src/backend/scrobblers/ScrobbleClients.ts b/src/backend/scrobblers/ScrobbleClients.ts index 4ce8277f1..cc81b898a 100644 --- a/src/backend/scrobblers/ScrobbleClients.ts +++ b/src/backend/scrobblers/ScrobbleClients.ts @@ -11,8 +11,8 @@ import { ListenBrainzClientConfig, ListenBrainzData } from "../common/infrastruc import { MalojaClientConfig, MalojaData } from "../common/infrastructure/config/client/maloja.js"; import { WildcardEmitter } from "../common/WildcardEmitter.js"; import { Notifiers } from "../notifier/Notifiers.js"; -import { isDebugMode, parseBool } from "../utils.js"; -import { readJson } from '../utils/DataUtils.js'; +import { isDebugMode, nonEmptyObj, parseBool, removeUndefinedKeys } from "../utils.js"; +import { getCommonComponentEnvConfig, readJson } from '../utils/DataUtils.js'; import { validateJson } from "../utils/ValidationUtils.js"; import AbstractScrobbleClient from "./AbstractScrobbleClient.js"; import { KoitoClientConfig, KoitoData } from '../common/infrastructure/config/client/koito.js'; @@ -130,8 +130,11 @@ export default class ScrobbleClients { const { clients: mainConfigClientConfigs = [], clientDefaults: cd = {}, + database: { + retention + } = {}, } = aioConfig; - clientDefaults = cd; + clientDefaults = {retention, ...cd}; for (const [index, c] of mainConfigClientConfigs.entries()) { const {name = 'unnamed'} = c; if(c.type === undefined) { @@ -168,138 +171,149 @@ export default class ScrobbleClients { for (const clientType of clientTypes) { const defaultConfigureAs = 'client'; switch (clientType) { - case 'maloja': + case 'maloja': { // env builder for single user mode - const url = process.env.MALOJA_URL; - const apiKey = process.env.MALOJA_API_KEY; - if (url !== undefined || apiKey !== undefined) { - const malojaData: MalojaData = { - url, - apiKey - } + const data = removeUndefinedKeys({ + url: process.env.MALOJA_URL, + apiKey: process.env.MALOJA_API_KEY + }, false); + const p = getCommonComponentEnvConfig('MALOJA'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'maloja', name: 'unnamed-mlj', source: 'ENV', mode: 'single', configureAs: 'client', - data: malojaData, + data: data, + ...p, options: transformPresetEnv('MALOJA') }) } - break; - case 'lastfm': - const lfm: LastfmData = { + } break; + case 'lastfm': { + const data = removeUndefinedKeys({ apiKey: process.env.LASTFM_API_KEY, secret: process.env.LASTFM_SECRET, redirectUri: process.env.LASTFM_REDIRECT_URI, session: process.env.LASTFM_SESSION - }; - if (!Object.values(lfm).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('LASTFM'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'lastfm', name: 'unnamed-lfm', source: 'ENV', mode: 'single', configureAs: 'client', - data: lfm, + data: data, + ...p, options: transformPresetEnv('LASTFM') }) } - break; + } break; case 'librefm': { - const shouldUse = parseBool(process.env.LIBRFM_ENABLE) - const libre: LibrefmData = { + const data = removeUndefinedKeys({ apiKey: process.env.LIBREFM_API_KEY, secret: process.env.LIBREFM_SECRET, redirectUri: process.env.LIBREFM_REDIRECT_URI, session: process.env.LIBREFM_SESSION, urlBase: process.env.LIBREFM_URLBASE, - }; - if (!Object.values(libre).every(x => x === undefined) || shouldUse) { + }, false); + const p = getCommonComponentEnvConfig('LIBREFM'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'librefm', name: 'unnamed-librefm', source: 'ENV', mode: 'single', configureAs: 'client', - data: libre, + data: data, + ...p, options: transformPresetEnv('LIBREFM') }) } } break; - case 'listenbrainz': - const lz: ListenBrainzData = { + case 'listenbrainz': { + const data = removeUndefinedKeys({ url: process.env.LZ_URL, token: process.env.LZ_TOKEN, username: process.env.LZ_USER - }; - if (!Object.values(lz).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('LZ'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'listenbrainz', name: 'unnamed-lz', source: 'ENV', mode: 'single', configureAs: 'client', - data: lz, + data: data, + ...p, options: transformPresetEnv('LZ') }) } - break; - case 'koito': - const koit: KoitoData = { + } break; + case 'koito': { + const data = removeUndefinedKeys({ url: process.env.KOITO_URL, token: process.env.KOITO_TOKEN, username: process.env.KOITO_USER - }; - if (!Object.values(koit).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('KOITO'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'koito', name: 'unnamed-koito', source: 'ENV', mode: 'single', configureAs: 'client', - data: koit, + data: data, + ...p, options: transformPresetEnv('KOITO') }) } - break; - case 'tealfm': - const teal: TealData = { + } break; + case 'tealfm': { + const data: TealData = removeUndefinedKeys({ identifier: process.env.TEALFM_IDENTIFIER, appPassword: process.env.TEALFM_APP_PW, - }; - if (!Object.values(teal).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('TEALFM'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'tealfm', name: 'unnamed-tealfm', source: 'ENV', mode: 'single', configureAs: 'client', - data: teal, + data: data, + ...p, options: transformPresetEnv('TEALFM') }) } - break; - case 'rocksky': - const rocksky: RockSkyData = { + } break; + case 'rocksky': { + const data: RockSkyData = removeUndefinedKeys({ key: process.env.ROCKSKY_KEY, handle: process.env.ROCKSKY_HANDLE - }; - if (!Object.values(rocksky).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('ROCKSKY'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'rocksky', name: 'unnamed-rocksky', source: 'ENV', mode: 'single', configureAs: 'client', - data: rocksky, + data: data, + ...p, options: transformPresetEnv('ROCKSKY') }) } - break; + } break; case 'discord': { - const discord: DiscordData = { + const data: DiscordData = removeUndefinedKeys({ token: process.env.DISCORD_TOKEN, artwork: process.env.DISCORD_ARTWORK, applicationId: process.env.DISCORD_APPLICATION_ID, @@ -307,15 +321,17 @@ export default class ScrobbleClients { artworkDefaultUrl: process.env.DISCORD_ARTWORK_DEFAULT_URL, statusOverrideAllow: process.env.DISCORD_STATUS_OVERRIDE_ALLOW, listeningActivityAllow: process.env.DISCORD_LISTENING_ACTIVITY_ALLOW - } - if (!Object.values(discord).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('DISCORD'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'discord', name: 'unnamed-discord', source: 'ENV', mode: 'single', configureAs: 'client', - data: discord, + data: data, + ...p, options: transformPresetEnv('DISCORD') }) } @@ -414,7 +430,7 @@ ${sources.join('\n')}`); if (isValidConfig !== true) { throw new Error(`Config object from ${clientConfig.source || 'unknown'} with name [${clientConfig.name || 'unnamed'}] of type [${clientConfig.type || 'unknown'}] has errors: ${isValidConfig.join(' | ')}`) }*/ - const {type, name, enable = true, source, data: d = {}} = clientConfig; + const {type, name, enable = true, source, data: d = {}, options = {}} = clientConfig; if(enable === false) { this.logger.warn({labels: [`${type} - ${name}`]}, `Client from ${source} was disabled by config`); @@ -422,41 +438,41 @@ ${sources.join('\n')}`); } // add defaults - const data = {...defaults, ...d}; + const compositeOptions = {...defaults, ...options}; let newClient; this.logger.debug({labels: [`${type} - ${name}`]}, `Constructing Client from ${source}`); switch (type) { case 'maloja': const MalojaScrobbler = (await import('./MalojaScrobbler.js')).default; - newClient = new MalojaScrobbler(name, ({...clientConfig, data} as unknown as MalojaClientConfig), notifier, this.emitter, this.logger); + newClient = new MalojaScrobbler(name, ({...clientConfig, data: d, options: compositeOptions} as unknown as MalojaClientConfig), notifier, this.emitter, this.logger); break; case 'lastfm': const LastfmScrobbler = (await import('./LastfmScrobbler.js')).default; - newClient = new LastfmScrobbler(name, {...clientConfig, data } as unknown as LastfmClientConfig, this.internalConfig, notifier, this.emitter, this.logger); + newClient = new LastfmScrobbler(name, {...clientConfig, data: d, options: compositeOptions } as unknown as LastfmClientConfig, this.internalConfig, notifier, this.emitter, this.logger); break; case 'librefm': const LibrefmScrobbler = (await import('./LibrefmScrobbler.js')).default; - newClient = new LibrefmScrobbler(name, {...clientConfig, data } as unknown as LibrefmClientConfig, this.internalConfig, notifier, this.emitter, this.logger); + newClient = new LibrefmScrobbler(name, {...clientConfig, data: d, options: compositeOptions } as unknown as LibrefmClientConfig, this.internalConfig, notifier, this.emitter, this.logger); break; case 'listenbrainz': const ListenbrainzScrobbler = (await import('./ListenbrainzScrobbler.js')).default; - newClient = new ListenbrainzScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...data} } as unknown as ListenBrainzClientConfig, {}, notifier, this.emitter, this.logger); + newClient = new ListenbrainzScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...d}, options: compositeOptions } as unknown as ListenBrainzClientConfig, {}, notifier, this.emitter, this.logger); break; case 'koito': const KoitoScrobbler = (await import('./KoitoScrobbler.js')).default; - newClient = new KoitoScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...data} } as unknown as KoitoClientConfig, {}, notifier, this.emitter, this.logger); + newClient = new KoitoScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...d}, options: compositeOptions } as unknown as KoitoClientConfig, {}, notifier, this.emitter, this.logger); break; case 'tealfm': const TealScrobbler = (await import('./TealfmScrobbler.js')).default; - newClient = new TealScrobbler(name, {...clientConfig, data: {...data}} as unknown as TealClientConfig, {}, notifier, this.emitter, this.logger); + newClient = new TealScrobbler(name, {...clientConfig, data: d, options: compositeOptions} as unknown as TealClientConfig, {}, notifier, this.emitter, this.logger); break; case 'rocksky': const RockskyScrobbler = (await import('./RockskyScrobbler.js')).default; - newClient = new RockskyScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...data} } as unknown as RockSkyClientConfig, {}, notifier, this.emitter, this.logger); + newClient = new RockskyScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...d}, options: compositeOptions } as unknown as RockSkyClientConfig, {}, notifier, this.emitter, this.logger); break; case 'discord': const DiscordScrobbler = (await import('./DiscordScrobbler.js')).default; - newClient = new DiscordScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...data} } as unknown as DiscordClientConfig, {}, notifier, this.emitter, this.logger); + newClient = new DiscordScrobbler(name, {...clientConfig, data: {configDir: this.internalConfig.configDir, ...d}, options: compositeOptions } as unknown as DiscordClientConfig, {}, notifier, this.emitter, this.logger); break; default: break; diff --git a/src/backend/server/api.ts b/src/backend/server/api.ts index 94c589a4b..34843468e 100644 --- a/src/backend/server/api.ts +++ b/src/backend/server/api.ts @@ -6,6 +6,7 @@ import { FixedSizeList } from 'fixed-size-list'; import { PassThrough } from "node:stream"; import { Transform } from "stream"; import { + CLIENT_DEAD_QUEUE, ClientStatusData, DeadLetterScrobble, LeveledLogData, @@ -34,6 +35,8 @@ import ScrobbleSources from "../sources/ScrobbleSources.js"; import ScrobbleClients from "../scrobblers/ScrobbleClients.js"; import prom from 'prom-client'; import { SimpleError } from "../common/errors/MSErrors.js"; +import { QueryPlaysOpts } from "../common/database/drizzle/repositories/PlayRepository.js"; +import { playSelectToDeadScrobble } from "../common/database/drizzle/entityUtils.js"; const maxBufferSize = 300; const output: Record> = {}; @@ -236,8 +239,9 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro hasAuthInteraction: requiresAuthInteraction, authed, initialized: x.isReady(), - deadLetterScrobbles: x.deadLetterScrobbles.length, - queued: x.queuedScrobbles.length + deadLetterScrobbles: x.deadLetterQueued, // x.deadLetterScrobbles.length, + deadLetterScrobblesTotal: x.deadLetterLength, + queued: x.queuedLength // x.queuedScrobbles.length }; if (!base.initialized) { if(x.buildOK === false) { @@ -262,7 +266,9 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message scrobbleSource: source, query: { - upstream = 'false' + upstream = 'false', + next: queryNext = 'false', + ...rest } } = req; @@ -278,7 +284,11 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro return res.status(500).json({message: e.message}); } } else { - result = (source as AbstractSource).getFlatRecentlyDiscoveredPlays(); + if(queryNext === 'true') { + return res.json(await (source as AbstractSource).getRecentPlaysApi(rest)); + } + result = await (source as AbstractSource).getFlatRecentlyDiscoveredPlays(); + } } @@ -317,9 +327,21 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro const { // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message scrobbleClient: client, + query = {} } = req; - const result: DeadLetterScrobble[] = (client as AbstractScrobbleClient).deadLetterScrobbles; + const deadQuery: QueryPlaysOpts = { + ...query as Partial, + queues: [ + { + queueName: CLIENT_DEAD_QUEUE, + queueStatus: ['queued','failed'] + } + ] + } + + // @ts-ignore + const result: DeadLetterScrobble[] = (await (client as AbstractScrobbleClient).getPlaysPaginated(deadQuery)).data.map(playSelectToDeadScrobble); return res.json(result); }); @@ -350,20 +372,20 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro (client as AbstractScrobbleClient).logger.verbose(`User requested processing of dead letter scrobble ${deadId} via API call`) - const deadScrobble = (client as AbstractScrobbleClient).deadLetterScrobbles.find(x => x.id === deadId); - - if(deadScrobble === undefined) { - (client as AbstractScrobbleClient).logger.debug(`No dead letter scrobble with ID ${deadId}`) - return res.status(404).send(); - } - - const [scrobbled, dead] = await ((client as AbstractScrobbleClient).processDeadLetterScrobble(deadId)); - - if(scrobbled) { - return res.status(200).send(); + try { + const [scrobbled, dead] = await (client as AbstractScrobbleClient).processDeadLetterScrobble(deadId); + if(scrobbled) { + return res.status(200).send(); + } + return res.json(playSelectToDeadScrobble(dead)); + } catch (e) { + if(e.message.includes(`Play ${deadId} does not exist`)) { + logger.warn(e); + return res.status(404).json({error: e}); + } + logger.error(e); + return res.status(500).json({error: e}); } - - return res.json(dead); }); app.delete('/api/dead', clientMiddleFunc(true), async (req, res, next) => { @@ -374,9 +396,9 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro (client as AbstractScrobbleClient).logger.verbose('User requested deletion of all dead letter scrobbles via API'); - (client as AbstractScrobbleClient).removeDeadLetterScrobbles(); + (client as AbstractScrobbleClient).removeDeadLetterScrobbles(['queued', 'failed'], 'failed', false).then(() => null).catch((e) => logger.error(e)); - return res.json([]); + return res.sendStatus(200); }); app.delete('/api/dead/:id', clientMiddleFunc(true), async (req, res, next) => { @@ -392,26 +414,33 @@ export const setupApi = (app: Express, logger: Logger, appLoggerStream: PassThro (client as AbstractScrobbleClient).logger.verbose(`User requested removal of dead letter scrobble ${deadId} via API call`) - const deadScrobble = (client as AbstractScrobbleClient).deadLetterScrobbles.find(x => x.id === deadId); - - if(deadScrobble === undefined) { - (client as AbstractScrobbleClient).logger.verbose(`No dead letter scrobble with ID ${deadId}`) - return res.status(404).send(); + try { + await (client as AbstractScrobbleClient).removeDeadLetterScrobble(deadId,'failed', false); + return res.status(200).send(); + } catch (e) { + if(e.message.includes(`Play ${deadId} does not exist`)) { + logger.warn(e); + return res.status(404).json({error: e}); + } + logger.error(e); + return res.status(500).json({error: e}); } - - (client as AbstractScrobbleClient).removeDeadLetterScrobble(deadId); - return res.status(200).send(); }); app.get('/api/scrobbled', clientMiddleFunc(false), async (req, res, next) => { const { // @ts-expect-error scrobbleClient not part of req scrobbleClient: client, + query } = req; let result: PlayObject[] = []; if (client !== undefined) { - result = [...(client as AbstractScrobbleClient).getScrobbledPlays()].sort(sortByNewestPlayDate); + const q: Partial = { + ...query as Partial, + state: ['scrobbled'] + } + result = [...(await (client as AbstractScrobbleClient).getPlaysPaginated(q)).data.map(x => x.play)].sort(sortByNewestPlayDate); } return res.json(result); diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index 900da28ae..9a273d6e6 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -2,7 +2,7 @@ import { childLogger, LogDataPretty, LogLevel } from '@foxxmd/logging'; import dayjs, { Dayjs } from "dayjs"; import { EventEmitter } from "events"; import { FixedSizeList } from "fixed-size-list"; -import { PlayObject } from "../../core/Atomic.js"; +import { JsonPlayObject, PlayMatchResult, PlayObject } from "../../core/Atomic.js"; import { buildTrackString, capitalize, truncateStringToLength } from "../../core/StringUtils.js"; import AbstractComponent from "../common/AbstractComponent.js"; import { @@ -31,7 +31,7 @@ import { sleep, sortByOldestPlayDate, } from "../utils.js"; -import { sortByNewestPlayDate } from '../../core/PlayUtils.js'; +import { genGroupIdStr, sortByNewestPlayDate } from '../../core/PlayUtils.js'; import { formatNumber } from '../../core/DataUtils.js'; import { timeToHumanTimestamp } from "../../core/TimeUtils.js"; import { todayAwareFormat } from "../../core/TimeUtils.js"; @@ -39,13 +39,16 @@ import { getRoot } from '../ioc.js'; import { componentFileLogger } from '../common/logging.js'; import { WebhookPayload } from '../common/infrastructure/config/health/webhooks.js'; import { isAbortReasonErrorLike, messageWithCauses, messageWithCausesTruncatedDefault } from '../utils/ErrorUtils.js'; -import { genericSourcePlayMatch } from '../utils/PlayComparisonUtils.js'; +import { existingScrobble, ExistingScrobbleOpts, genericSourcePlayMatch } from '../utils/PlayComparisonUtils.js'; import { findAsync, staggerMapper, StaggerOptions } from '../utils/AsyncUtils.js'; import pMap, {pMapIterable} from 'p-map'; import prom, { Counter, Gauge } from 'prom-client'; import { normalizeStr } from '../utils/StringUtils.js'; import { spawn, catchAbortError, isAbortError, rethrowAbortError, delay, forever, AbortError, throwIfAborted } from 'abort-controller-x'; import { AbortedError, generateLoggableAbortReason } from '../common/errors/MSErrors.js'; +import { DrizzlePlayRepository, playToRepositoryCreatePlayOpts, queryArgsFromRequest, QueryPlaysOpts, RequestPlayQuery } from '../common/database/drizzle/repositories/PlayRepository.js'; +import { asPlay } from '../../core/PlayMarshalUtils.js'; +import { AsyncTask, SimpleIntervalJob, ToadScheduler } from 'toad-scheduler'; export interface RecentlyPlayedOptions { limit?: number @@ -57,7 +60,7 @@ export interface RecentlyPlayedOptions { export default abstract class AbstractSource extends AbstractComponent implements Authenticatable { name: string; - type: SourceType; + declare type: SourceType; declare config: SourceConfig; clients: string[]; @@ -88,6 +91,7 @@ export default abstract class AbstractSource extends AbstractComponent implement manualListening?: boolean + scheduler: ToadScheduler = new ToadScheduler(); emitter: EventEmitter; protected SCROBBLE_BACKLOG_COUNT: number = 30; @@ -103,8 +107,15 @@ export default abstract class AbstractSource extends AbstractComponent implement postCompare: staggerMapper({concurrency: 2}) } + declare protected componentType: 'source'; + + protected playRepo!: DrizzlePlayRepository; + + existingDiscoveredPlay: (playObjPre: PlayObject, existingScrobbles: PlayObject[], log?: boolean) => Promise + constructor(type: SourceType, name: string, config: SourceConfig, internal: InternalConfig, emitter: EventEmitter) { super(config); + this.componentType = 'source'; const {clients = [] } = config; this.type = type; this.name = name; @@ -119,6 +130,15 @@ export default abstract class AbstractSource extends AbstractComponent implement this.emitter = emitter; this.discoveredCounter = getRoot().items.sourceMetics.discovered; + + const existingScrobbleOpts: ExistingScrobbleOpts = { + logger: this.logger, + transformRules: this.transformRules, + transformPlay: this.transformPlay, + existingSubmitted: async (_) => [undefined, undefined] + } + this.existingDiscoveredPlay = (playObjPre: PlayObject, existingScrobbles: PlayObject[], log?: boolean) => existingScrobble(playObjPre, existingScrobbles, existingScrobbleOpts, log); + } async [Symbol.asyncDispose]() { @@ -127,11 +147,70 @@ export default abstract class AbstractSource extends AbstractComponent implement } } + public initTasks() { + if(this.scheduler.existsById('heartbeat') === false) { + this.logger.info('Adding Heartbeat Task and running immediately'); + this.scheduler.addSimpleIntervalJob(new SimpleIntervalJob({ + minutes: 20, + runImmediately: true + }, new AsyncTask( + 'Heartbeat', + (): Promise => { + return this.heartbeatTask().then(() => null).catch((err) => { + this.logger.error(err); + }); + }, + (err: Error) => { + this.logger.error(err); + } + ), {id: 'heartbeat'})); + } else { + this.logger.warn('Heartbeat task is already added to scheduler.'); + } + } + + protected async heartbeatTask(): Promise { + if(!this.isReady()) { + if(!this.canAuthUnattended()) { + this.logger.warn({labels: 'Heartbeat'}, 'Source is not ready but will not try to initialize because auth state is not good and cannot be corrected unattended.') + return false; + } + try { + await this.tryInitialize({force: false, notify: true, notifyTitle: 'Could not initialize automatically'}); + } catch (e) { + this.logger.error(new Error('Could not initialize automatically', {cause: e})); + return false; + } + + if('discoverDevices' in this && typeof this.discoverDevices === 'function') { + this.discoverDevices(); + } + + if (this.canPoll && !this.polling) { + if(!this.canAuthUnattended()) { + this.logger.warn({labels: 'Heartbeat'}, 'Should be polling but will not attempt to start because auth state is not good and cannot be correct unattended.'); + return false; + } else { + this.logger.info({labels: 'Heartbeat'}, 'Should be polling, attempting to start polling...'); + this.poll({force: false, notify: true}).catch(e => this.logger.error(e)); + } + return true; + } + } + return true; + } + protected async postCache(): Promise { await super.postCache(); this.generateStaggerMappers(); } + protected async postDatabase(): Promise { + this.playRepo = new DrizzlePlayRepository(this.db, {logger: this.logger}); + this.tracksDiscovered = this.dbComponent.countLive; + this.playRepo.componentId = this.dbComponent.id; + } + protected generateStaggerMappers() { const { preCompare = [], @@ -200,83 +279,91 @@ export default abstract class AbstractSource extends AbstractComponent implement // TODO make this more descriptive? or move it elsewhere recentlyPlayedTrackIsValid = (playObj: PlayObject) => true - protected addPlayToDiscovered = (play: PlayObject) => { - const platformId = this.multiPlatform ? genGroupId(play) : SINGLE_USER_PLATFORM_ID; - const list = this.recentDiscoveredPlays.get(platformId) ?? new FixedSizeList(200); - list.add(play); - this.recentDiscoveredPlays.set(platformId, list); + protected addPlayToDiscovered = async (play: PlayObject): Promise => { + const playRow = await this.playRepo.createPlays([(playToRepositoryCreatePlayOpts({play, state: 'discovered'}))]); + const recentPlays = await this.getRecentlyDiscoveredPlays(false); + // only need to update if its already in memory, + // and better to update in-memory than clear cache so we aren't refetching from db on every discover + if(recentPlays !== undefined) { + recentPlays.push(play); + recentPlays.sort(sortByOldestPlayDate); + this.cache.cacheDb.set(this.recentDiscoveredCacheKey(), recentPlays, '2m'); + } this.tracksDiscovered++; this.logger.info(`Discovered => ${buildTrackString(play)}`); this.emitEvent('discovered', {play}); this.discoveredCounter.labels(this.getPrometheusLabels()).inc(); + play.id = playRow[0].id; + play.uid = playRow[0].uid; + return play; } - getFlatRecentlyDiscoveredPlays = (): PlayObject[] => - Array.from(this.recentDiscoveredPlays.values()).map(x => x.data).flat(3).sort(sortByNewestPlayDate) - - - getRecentlyDiscoveredPlaysByPlatform = (platformId: PlayPlatformId): PlayObject[] => { - const list = this.recentDiscoveredPlays.get(platformId); - if (list !== undefined) { - const data = [...list.data]; - data.sort(sortByOldestPlayDate); - return data; - } - return []; + getFlatRecentlyDiscoveredPlays = async (): Promise => { + const list: PlayObject[] = await this.getRecentlyDiscoveredPlays(); + return list.sort(sortByNewestPlayDate); } - protected getExistingDiscoveredLists = (play: PlayObject, opts: {checkAll?: boolean} = {}): PlayObject[][] => { - const lists: PlayObject[][] = []; - if(opts.checkAll !== true) { - lists.push(this.getRecentlyDiscoveredPlaysByPlatform(this.multiPlatform ? genGroupId(play) : SINGLE_USER_PLATFORM_ID)); - } else { - // get as many as we can, optionally filtering by user - this.recentDiscoveredPlays.forEach((list, platformId) => { - if(play.meta.user !== undefined) { - if(platformId[1] === NO_USER || platformId[1] === play.meta.user) { - lists.push(this.getRecentlyDiscoveredPlaysByPlatform(platformId)); - } - } else { - lists.push(this.getRecentlyDiscoveredPlaysByPlatform(platformId)); - } - }); + getRecentPlaysApi = async (query: RequestPlayQuery) => { + const res = await this.playRepo.findPlays({ + limit: 100, + ...queryArgsFromRequest(query) + }); + return res.map((x) => { + const {id, ...rest} = x; + return rest; + }) + } + + protected recentDiscoveredCacheKey = () => { + return `recent-${this.dbComponent.id}`; + } + + getRecentlyDiscoveredPlays = async (hydrate: boolean = true): Promise => { + const cacheKey = this.recentDiscoveredCacheKey(); + let list = await this.cache.cacheDb.get(cacheKey); + if(list === undefined && hydrate) { + list = (await this.playRepo.findPlays({ + stateNot: ['queued'], + order: 'desc', + sort: 'playedAt', + limit: 200 + })).map(x => asPlay(x.play)) + list.sort(sortByOldestPlayDate); + await this.cache.cacheDb.set(cacheKey, list, '2m'); } - return lists; + return list; } - existingDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise => { - const lists: PlayObject[][] = this.getExistingDiscoveredLists(play, opts); - const candidate = await this.transformPlay(play, TRANSFORM_HOOK.candidate); - for(const list of lists) { - - const existing = await findAsync(list,async x => { - const e = await this.transformPlay(x, TRANSFORM_HOOK.existing); - return genericSourcePlayMatch(e, candidate); - }); - if(existing) { - return existing; - } + existingDiscovered = async (play: PlayObject): Promise => { + const list: PlayObject[] = await this.getRecentlyDiscoveredPlays(); + const matchResults = await this.existingDiscoveredPlay(play, list); + if(matchResults.match) { + return matchResults.closestMatchedPlay; } return undefined; } - alreadyDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise => { - const existing = await this.existingDiscovered(play, opts); - return existing !== undefined; - } - discover = async (plays: PlayObject[], options: { checkAll?: boolean, signal?: AbortSignal, [key: string]: any } = {}): Promise => { const newDiscoveredPlays: PlayObject[] = []; for await(const play of pMapIterable(plays, this.staggerMappers.preCompare(async x => await this.transformPlay(x, TRANSFORM_HOOK.preCompare)), {concurrency: 3})) { options.signal?.throwIfAborted(); - if(!(await this.alreadyDiscovered(play, options))) { + const existing = await this.existingDiscovered(play); + if(existing === undefined) { options.signal?.throwIfAborted() - this.addPlayToDiscovered(play); - newDiscoveredPlays.push(play); + const hydratedPlay = await this.addPlayToDiscovered(play); + newDiscoveredPlays.push(hydratedPlay); + } else { + this.playRepo.updateById(existing.id, {updatedAt: dayjs()}); + } + } + if(newDiscoveredPlays.length > 0) { + try { + await this.componentRepo.updateById(this.dbComponent.id, {countLive: this.dbComponent.countLive + newDiscoveredPlays.length}); + } catch (e) { + this.logger.warn(new Error('Unable to update discovered count', {cause: e})); } } - newDiscoveredPlays.sort(sortByOldestPlayDate); return newDiscoveredPlays; @@ -298,6 +385,7 @@ export default abstract class AbstractSource extends AbstractComponent implement if(newDiscoveredPlays.length > 0) { if(!this.shouldScrobble(options.discoverLocation)) { + await this.playRepo.setStateById('discarded', newDiscoveredPlays.map(x => x.id)); return; } newDiscoveredPlays.sort(sortByOldestPlayDate); @@ -500,7 +588,7 @@ export default abstract class AbstractSource extends AbstractComponent implement this.abortController.abort(reason); let elapsed = 0; let lastlog: Dayjs; - while(this.polling && elapsed < 10) { + while(this.polling && elapsed < (10 * this.stopPollingWaitInterval)) { if(lastlog === undefined || dayjs().diff(lastlog, 's') >= 2) { this.logger.verbose(`Waiting for polling stop signal to be acknowledged (waited ${formatNumber(elapsed/1000)}s)`); } @@ -548,7 +636,7 @@ export default abstract class AbstractSource extends AbstractComponent implement } - const interval = this.getInterval(); + const interval = this.getInterval(true); const maxBackoff = this.getMaxBackoff(); let sleepTime = interval; @@ -652,7 +740,7 @@ export default abstract class AbstractSource extends AbstractComponent implement return this.wakeAt; } - protected getInterval() { + protected getInterval(log?: boolean) { let interval = DEFAULT_POLLING_INTERVAL; if('interval' in this.config.data) { @@ -670,6 +758,18 @@ export default abstract class AbstractSource extends AbstractComponent implement return maxInterval - this.getInterval(); } + public getPlays = (args: QueryPlaysOpts) => { + const { + limit, + offset, + with: withQuery = ['input','parent-input','queues'], + ...rest + } = args; + let parsedLimit = limit !== undefined ? Number.parseInt(limit as unknown as string) : undefined; + let parsedOffset = offset !== undefined ? Number.parseInt(offset as unknown as string) : undefined; + return this.playRepo.findPlaysPaginated({limit: parsedLimit, offset: parsedOffset, with: withQuery, ...rest}); + } + public emitEvent = (eventName: string, payload: object = {}) => { this.emitter.emit(eventName, { type: this.type, @@ -684,7 +784,7 @@ export default abstract class AbstractSource extends AbstractComponent implement } protected async doBuildComponentLogger(): Promise { - if(this.config.options.logToFile) { + if(this.config?.options?.logToFile) { this.logger.debug('Enabling component logger...'); const root = getRoot(); const stream = root.get('loggerStream'); diff --git a/src/backend/sources/AzuracastSource.ts b/src/backend/sources/AzuracastSource.ts index 554ae4fd8..9df4c25aa 100644 --- a/src/backend/sources/AzuracastSource.ts +++ b/src/backend/sources/AzuracastSource.ts @@ -17,6 +17,7 @@ import { import { AzuracastSourceConfig, AzuraNowPlayingResponse, AzuraStationResponse } from "../common/infrastructure/config/source/azuracast.js"; import { isPortReachable, normalizeWSAddress } from "../utils/NetworkUtils.js"; import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js"; +import { artistNamesToCredits } from "../../core/StringUtils.js"; export class AzuracastSource extends MemorySource { @@ -250,7 +251,7 @@ const formatPlayObj = (obj: AzuraNowPlayingResponse, options: FormatPlayObjectOp const play: PlayObjectLifecycleless = { data: { - artists: artist !== undefined && artist !== '' ? [artist] : [], + artists: artistNamesToCredits(artist !== undefined && artist !== '' ? [artist] : []), album: album !== '' ? album : undefined, track, duration diff --git a/src/backend/sources/ChromecastSource.ts b/src/backend/sources/ChromecastSource.ts index d608af37a..d7df22a34 100644 --- a/src/backend/sources/ChromecastSource.ts +++ b/src/backend/sources/ChromecastSource.ts @@ -6,7 +6,7 @@ import dayjs from "dayjs"; import { EventEmitter } from "events"; import e from "express"; import { PlayObject, PlayObjectLifecycleless } from "../../core/Atomic.js"; -import { buildTrackString } from "../../core/StringUtils.js"; +import { artistNamesToCredits, buildTrackString } from "../../core/StringUtils.js"; import { NETWORK_ERROR_FAILURE_CODES } from "../common/errors/NodeErrors.js"; import { FormatPlayObjectOptions, @@ -722,8 +722,8 @@ export class ChromecastSource extends MemoryPositionalSource { data: { track, album, - albumArtists, - artists, + albumArtists: artistNamesToCredits(albumArtists), + artists: artistNamesToCredits(artists), duration, playDate: dayjs() }, diff --git a/src/backend/sources/DeezerInternalSource.ts b/src/backend/sources/DeezerInternalSource.ts index 71627a939..a8112c476 100644 --- a/src/backend/sources/DeezerInternalSource.ts +++ b/src/backend/sources/DeezerInternalSource.ts @@ -15,6 +15,7 @@ import { TemporalPlayComparisonOptions } from "../utils/TimeUtils.js"; import { findAsync, findIndexAsync } from "../utils/AsyncUtils.js"; import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js"; import { UpstreamError } from "../common/errors/UpstreamError.js"; +import { artistNamesToCredits } from "../../core/StringUtils.js"; interface DeezerHistoryResponse { errors: [] @@ -113,7 +114,7 @@ export default class DeezerInternalSource extends MemorySource { const {newFromSource = false} = options; const play: PlayObjectLifecycleless = { data: { - artists: [obj.ART_NAME], + artists: artistNamesToCredits([obj.ART_NAME]), album: obj.ALB_TITLE, track: obj.SNG_TITLE, duration: obj.DURATION, @@ -330,45 +331,43 @@ export default class DeezerInternalSource extends MemorySource { existingDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise => { - const lists: PlayObject[][] = this.getExistingDiscoveredLists(play, opts); + const list: PlayObject[] = await this.getRecentlyDiscoveredPlays(); const candidate = await this.transformPlay(play, TRANSFORM_HOOK.candidate); - for(const list of lists) { - const existing = await findAsync(list, async x => { + const existing = await findAsync(list, async x => { + const e = await this.transformPlay(x, TRANSFORM_HOOK.existing); + return genericSourcePlayMatch(e, candidate); + }); + if(existing) { + return existing; + } + if(this.config.options?.fuzzyDiscoveryIgnore === true || this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { + const fuzzyIndex = await findIndexAsync(list, async x => { const e = await this.transformPlay(x, TRANSFORM_HOOK.existing); - return genericSourcePlayMatch(e, candidate); - }); - if(existing) { - return existing; - } - if(this.config.options?.fuzzyDiscoveryIgnore === true || this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { - const fuzzyIndex = await findIndexAsync(list, async x => { - const e = await this.transformPlay(x, TRANSFORM_HOOK.existing); - let temporalOptions: TemporalPlayComparisonOptions = {}; - const temporalAccuracy: TemporalAccuracy[] = [TA_EXACT, TA_CLOSE, TA_FUZZY]; - if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { - temporalOptions = { - fuzzyDiffThreshold: Math.max(100, x.data.duration * 0.5), - duringReferences: ['duration', 'listenedFor', 'range'] - } - temporalAccuracy.push(TA_DURING); - } - return genericSourcePlayMatch(e, candidate, temporalAccuracy, temporalOptions); - }); - if(fuzzyIndex !== -1) { - if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { - // always return fuzzy match as existing - // likely will make MS miss scrobbles for repeated plays - return list[fuzzyIndex]; - } - if(fuzzyIndex + 1 === list.length || playObjDataMatch(list[fuzzyIndex], list[fuzzyIndex + 1])) { - // last discovered play was this one, or next played play was also this one - // so we'll assume this means the play is on repeat, don't count as existing - return undefined; + let temporalOptions: TemporalPlayComparisonOptions = {}; + const temporalAccuracy: TemporalAccuracy[] = [TA_EXACT, TA_CLOSE, TA_FUZZY]; + if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { + temporalOptions = { + fuzzyDiffThreshold: Math.max(100, x.data.duration * 0.5), + duringReferences: ['duration', 'listenedFor', 'range'] } - // next played play was *not* this one (Deezer reports play between candidate TS and fuzzy match) - // so this is likely a duplicate deezer should not have reported + temporalAccuracy.push(TA_DURING); + } + return genericSourcePlayMatch(e, candidate, temporalAccuracy, temporalOptions); + }); + if(fuzzyIndex !== -1) { + if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { + // always return fuzzy match as existing + // likely will make MS miss scrobbles for repeated plays return list[fuzzyIndex]; } + if(fuzzyIndex + 1 === list.length || playObjDataMatch(list[fuzzyIndex], list[fuzzyIndex + 1])) { + // last discovered play was this one, or next played play was also this one + // so we'll assume this means the play is on repeat, don't count as existing + return undefined; + } + // next played play was *not* this one (Deezer reports play between candidate TS and fuzzy match) + // so this is likely a duplicate deezer should not have reported + return list[fuzzyIndex]; } } return undefined; diff --git a/src/backend/sources/EndpointLastfmSource.ts b/src/backend/sources/EndpointLastfmSource.ts index 42a20e38f..126972057 100644 --- a/src/backend/sources/EndpointLastfmSource.ts +++ b/src/backend/sources/EndpointLastfmSource.ts @@ -11,13 +11,13 @@ import { REPORTED_PLAYER_STATUSES, ReportedPlayerStatus } from "../common/infrastructure/Atomic.js"; -import { parseRegexSingleOrFail } from "../utils.js"; import MemorySource from "./MemorySource.js"; import { LastFMEndpointSourceConfig } from "../common/infrastructure/config/source/endpointlfm.js"; import { LastFMScrobbleRequestPayload, scrobblePayloadToPlay } from "../common/vendor/LastfmApiClient.js"; import { Logger } from "@foxxmd/logging"; import { PlayerStateOptions } from "./PlayerState/AbstractPlayerState.js"; import { NowPlayingPlayerState } from "./PlayerState/NowPlayingPlayerState.js"; +import { parseRegexSingle } from "@foxxmd/regex-buddy-core"; const noSlugMatch = new RegExp(/(?:\/api\/lastfm\/?)$|(?:\/1\/?|\/2.0\/?)$/i); const slugMatch = new RegExp(/\/api\/lastfm\/([^\/]+)$/i); @@ -62,7 +62,7 @@ export class EndpointLastfmSource extends MemorySource { } getRecentlyPlayed = async (options = {}) => { - return this.getFlatRecentlyDiscoveredPlays(); + return await this.getFlatRecentlyDiscoveredPlays(); } isValidScrobble = (playObj: PlayObject) => { @@ -97,11 +97,11 @@ export const playStateFromRequest = (obj: LastFMScrobbleRequestPayload): PlayerS } export const parseSlugFromString = (path: string): string | false | undefined => { - const noSlug = parseRegexSingleOrFail(noSlugMatch, path); + const noSlug = parseRegexSingle(noSlugMatch, path); if (noSlug !== undefined) { return undefined; } - const slugResult = parseRegexSingleOrFail(slugMatch, path); + const slugResult = parseRegexSingle(slugMatch, path); if (slugResult !== undefined) { return slugResult.groups[0]; } diff --git a/src/backend/sources/EndpointListenbrainzSource.ts b/src/backend/sources/EndpointListenbrainzSource.ts index 77053e16d..8518e6d64 100644 --- a/src/backend/sources/EndpointListenbrainzSource.ts +++ b/src/backend/sources/EndpointListenbrainzSource.ts @@ -15,11 +15,11 @@ import { ListenbrainzEndpointSourceConfig } from "../common/infrastructure/confi import { ListenbrainzApiClient, listenPayloadToPlay } from "../common/vendor/ListenbrainzApiClient.js"; import { SubmitPayload } from '../common/vendor/listenbrainz/interfaces.js'; import { ListenPayload } from '../common/vendor/listenbrainz/interfaces.js'; -import { parseRegexSingleOrFail } from "../utils.js"; import MemorySource from "./MemorySource.js"; import { NowPlayingPlayerState } from "./PlayerState/NowPlayingPlayerState.js"; import { Logger } from "@foxxmd/logging"; import { PlayerStateOptions } from "./PlayerState/AbstractPlayerState.js"; +import { parseRegexSingle } from "@foxxmd/regex-buddy-core"; const noSlugMatch = new RegExp(/(?:\/api\/listenbrainz\/?)$|(?:\/1\/?|\/1\/submit-listens\/?\/1\/validate-token\/|)$/i); const slugMatch = new RegExp(/\/api\/listenbrainz\/([^\/]+)$/i); @@ -82,7 +82,7 @@ export class EndpointListenbrainzSource extends MemorySource { } getRecentlyPlayed = async (options = {}) => { - return this.getFlatRecentlyDiscoveredPlays(); + return await this.getFlatRecentlyDiscoveredPlays(); } isValidScrobble = (playObj: PlayObject) => { @@ -131,7 +131,7 @@ export const listenTypeAsPlayerStatus = (event: string): ReportedPlayerStatus => } export const parseTokenFromString = (str: string): string | undefined => { - const tokenMatch = parseRegexSingleOrFail(authHeaderRegex, str); + const tokenMatch = parseRegexSingle(authHeaderRegex, str); if(tokenMatch !== undefined) { return tokenMatch.groups[0]; } @@ -151,11 +151,11 @@ export const parseTokenFromRequest = (req: ExpressRequest): string | false | und } export const parseSlugFromString = (path: string): string | false | undefined => { - const noSlug = parseRegexSingleOrFail(noSlugMatch, path); + const noSlug = parseRegexSingle(noSlugMatch, path); if (noSlug !== undefined) { return undefined; } - const slugResult = parseRegexSingleOrFail(slugMatch, path); + const slugResult = parseRegexSingle(slugMatch, path); if (slugResult !== undefined) { return slugResult.groups[0]; } diff --git a/src/backend/sources/IcecastSource.ts b/src/backend/sources/IcecastSource.ts index 1ccad5c33..a748e323f 100644 --- a/src/backend/sources/IcecastSource.ts +++ b/src/backend/sources/IcecastSource.ts @@ -15,6 +15,7 @@ import IcecastMetadataStats from "icecast-metadata-stats"; import { parseArtistCredits, parseTrackCredits } from "../utils/StringUtils.js"; import { isDebugMode, sleep } from "../utils.js"; import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js"; +import { artistNamesToCredits } from "../../core/StringUtils.js"; export class IcecastSource extends MemorySource { @@ -211,7 +212,7 @@ const formatPlayObj = (obj: IcecastMetadata, options: FormatPlayObjectOptions = const play: PlayObjectLifecycleless = { data: { track, - artists + artists: artistNamesToCredits(artists) }, meta: { source: 'icecast', diff --git a/src/backend/sources/JRiverSource.ts b/src/backend/sources/JRiverSource.ts index d5a94c114..f7816e49d 100644 --- a/src/backend/sources/JRiverSource.ts +++ b/src/backend/sources/JRiverSource.ts @@ -9,6 +9,7 @@ import { Info, JRiverApiClient, PLAYER_STATE } from "../common/vendor/JRiverApiC import { RecentlyPlayedOptions } from "./AbstractSource.js"; import { MemoryPositionalSource } from "./MemoryPositionalSource.js"; import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js"; +import { artistNamesToCredits } from "../../core/StringUtils.js"; export class JRiverSource extends MemoryPositionalSource { declare config: JRiverSourceConfig; @@ -104,7 +105,7 @@ export class JRiverSource extends MemoryPositionalSource { data: { track: Name, album: album, - artists, + artists: artistNamesToCredits(artists), duration: Math.round(length), playDate: dayjs() }, diff --git a/src/backend/sources/JellyfinApiSource.ts b/src/backend/sources/JellyfinApiSource.ts index 19c7f7c1f..8521f27a9 100644 --- a/src/backend/sources/JellyfinApiSource.ts +++ b/src/backend/sources/JellyfinApiSource.ts @@ -42,8 +42,8 @@ import { from "@jellyfin/sdk/lib/index.js"; import dayjs from "dayjs"; import EventEmitter from "events"; -import { BrainzMeta, PlayObject, PlayObjectLifecycleless } from "../../core/Atomic.js"; -import { buildTrackString, combinePartsToString, truncateStringToLength } from "../../core/StringUtils.js"; +import { ArtistCredit, BrainzMeta, PlayObject, PlayObjectLifecycleless } from "../../core/Atomic.js"; +import { artistNamesToCredits, buildTrackString, combinePartsToString, truncateStringToLength } from "../../core/StringUtils.js"; import { FormatPlayObjectOptions, InternalConfig, @@ -466,9 +466,16 @@ export default class JellyfinApiSource extends MemoryPositionalSource { meta.albumArtist = [ProviderIds.MusicBrainzAlbumArtist]; } + let playArtists: ArtistCredit[] = []; + if(Artists.length === 1 && meta.artist !== undefined) { + playArtists.push({name: Artists[0], mbid: meta.artist[0]}); + } else { + playArtists = artistNamesToCredits(Artists); + } + const play: PlayObjectLifecycleless = { data: { - artists: Artists, + artists: playArtists, album: Album, track: Name, albumArtists: AlbumArtists !== undefined ? AlbumArtists.map(x => x.Name) : undefined, diff --git a/src/backend/sources/MPDSource.ts b/src/backend/sources/MPDSource.ts index b14b0c408..238792a37 100644 --- a/src/backend/sources/MPDSource.ts +++ b/src/backend/sources/MPDSource.ts @@ -20,6 +20,7 @@ import { RecentlyPlayedOptions } from "./AbstractSource.js"; import { MemoryPositionalSource } from "./MemoryPositionalSource.js"; import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js"; import { isDebugMode, sleep } from "../utils.js"; +import { artistNamesToCredits } from "../../core/StringUtils.js"; const CLIENT_PLAYER_STATE: Record = { 'play': REPORTED_PLAYER_STATUSES.playing, @@ -256,8 +257,8 @@ export class MPDSource extends MemoryPositionalSource { const play: PlayObjectLifecycleless = { data: { - artists: artists, - albumArtists, + artists: artistNamesToCredits(artists), + albumArtists: artistNamesToCredits(albumArtists), album, track: trackName, duration diff --git a/src/backend/sources/MPRISSource.ts b/src/backend/sources/MPRISSource.ts index 57473236e..7084fce30 100644 --- a/src/backend/sources/MPRISSource.ts +++ b/src/backend/sources/MPRISSource.ts @@ -21,6 +21,7 @@ import { Readable, Writable } from 'stream'; import net from 'net'; import pEvent from 'p-event'; import { baseFormatPlayObj } from '../utils/PlayTransformUtils.js'; +import { artistNamesToCredits } from '../../core/StringUtils.js'; export class MPRISSource extends MemorySource { @@ -76,8 +77,8 @@ export class MPRISSource extends MemorySource { data: { track: title, album, - artists: artist, - albumArtists: actualAlbumArtists, + artists: artistNamesToCredits(artist), + albumArtists:artistNamesToCredits(actualAlbumArtists), duration: length, playDate: dayjs() }, diff --git a/src/backend/sources/MemorySource.ts b/src/backend/sources/MemorySource.ts index 52c04d48c..3400d2a4a 100644 --- a/src/backend/sources/MemorySource.ts +++ b/src/backend/sources/MemorySource.ts @@ -368,7 +368,7 @@ export default class MemorySource extends AbstractSource { } return [false, `${stPrefix} ${EXPECTED_NON_DISCOVERED_REASON}`] } else { - const discoveredPlays = this.getRecentlyDiscoveredPlaysByPlatform(genGroupId(candidate)); + const discoveredPlays = await this.getRecentlyDiscoveredPlays(); if (discoveredPlays.length === 0 || !playObjDataMatch(discoveredPlays[0], candidate)) { // if most recent stateful play is not this track we'll add it return [true,`${stPrefix} added after ${thresholdResultSummary(thresholdResults)}. Matched other recent play but could not determine time frame due to missing duration. Allowed due to not being last played track.`]; @@ -385,7 +385,7 @@ export default class MemorySource extends AbstractSource { recentlyPlayedTrackIsValid = (playObj: any) => playObj.data.playDate.isBefore(dayjs().subtract(30, 's')) - protected getInterval(): number { + protected getInterval(log?: boolean): number { /** * If any player is progressing, reports position, and play has duration * then we can modify polling interval so that we check source data just before track is supposed to end @@ -415,7 +415,7 @@ export default class MemorySource extends AbstractSource { } } } - if(logDecrease !== undefined) { + if(logDecrease !== undefined && log) { this.logger.debug(logDecrease); } return interval; diff --git a/src/backend/sources/MopidySource.ts b/src/backend/sources/MopidySource.ts index e5641bad5..5cdd8e567 100644 --- a/src/backend/sources/MopidySource.ts +++ b/src/backend/sources/MopidySource.ts @@ -6,7 +6,7 @@ import normalizeUrl from 'normalize-url'; import pEvent from 'p-event'; import { URL } from "url"; import { PlayObject, PlayObjectLifecycleless } from "../../core/Atomic.js"; -import { buildTrackString } from "../../core/StringUtils.js"; +import { artistNamesToCredits, buildTrackString } from "../../core/StringUtils.js"; import { FormatPlayObjectOptions, InternalConfig, @@ -164,8 +164,8 @@ export class MopidySource extends MemoryPositionalSource { data: { track: name, album: albumName, - albumArtists: actualAlbumArtists.length > 0 ? actualAlbumArtists.map(x => x.name) : [], - artists: artists.length > 0 ? artists.map(x => x.name) : [], + albumArtists: artistNamesToCredits(actualAlbumArtists.length > 0 ? actualAlbumArtists.map(x => x.name) : []), + artists: artistNamesToCredits(artists.length > 0 ? artists.map(x => x.name) : []), duration: Math.round(length / 1000), playDate: dayjs() }, diff --git a/src/backend/sources/MusicCastSource.ts b/src/backend/sources/MusicCastSource.ts index 44cbb7a2a..a48d4d466 100644 --- a/src/backend/sources/MusicCastSource.ts +++ b/src/backend/sources/MusicCastSource.ts @@ -12,6 +12,7 @@ import { isPortReachable, isPortReachableConnect, joinedUrl, normalizeWebAddress import { DeviceInfoResponse, DeviceStatusResponse, MusicCastResponseCodes, MusicCastSourceConfig, playbackToReportedStatus, PlayInfoCDResponse, PlayInfoNetResponse } from "../common/infrastructure/config/source/musiccast.js"; import request, { Request, Response } from 'superagent'; import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js"; +import { artistNamesToCredits } from "../../core/StringUtils.js"; export class MusicCastSource extends MemoryPositionalSource { @@ -149,7 +150,7 @@ const formatPlayObj = (obj: PlayInfoCDResponse | PlayInfoNetResponse, options: F const play: PlayObjectLifecycleless = { data: { - artists: artist !== undefined && artist !== '' ? [artist] : [], + artists: artistNamesToCredits(artist !== undefined && artist !== '' ? [artist] : []), album: album !== '' ? album : undefined, track, // we should treat 0 time as the same as not being provided diff --git a/src/backend/sources/PlexApiSource.ts b/src/backend/sources/PlexApiSource.ts index 26ca83fc6..fdeb94d2a 100644 --- a/src/backend/sources/PlexApiSource.ts +++ b/src/backend/sources/PlexApiSource.ts @@ -1,6 +1,6 @@ import EventEmitter from "events"; import { PlayObject, PlayObjectLifecycleless, URLData } from "../../core/Atomic.js"; -import { buildTrackString, combinePartsToString, truncateStringToLength } from "../../core/StringUtils.js"; +import { artistNamesToCredits, buildTrackString, combinePartsToString, truncateStringToLength } from "../../core/StringUtils.js"; import { asPlayerStateDataMaybePlay, FormatPlayObjectOptions, @@ -376,8 +376,8 @@ export default class PlexApiSource extends MemoryPositionalSource { const play: PlayObjectLifecycleless = { data: { - artists: realArtists, - albumArtists, + artists: artistNamesToCredits(realArtists), + albumArtists: artistNamesToCredits(albumArtists), album, track, // albumArtists: AlbumArtists !== undefined ? AlbumArtists.map(x => x.Name) : undefined, diff --git a/src/backend/sources/ScrobbleSources.ts b/src/backend/sources/ScrobbleSources.ts index b11bac508..b3d4fc7d6 100644 --- a/src/backend/sources/ScrobbleSources.ts +++ b/src/backend/sources/ScrobbleSources.ts @@ -36,8 +36,8 @@ import { YTMusicData, YTMusicSourceConfig } from "../common/infrastructure/confi import { YandexMusicBridgeData, YandexMusicBridgeSourceConfig } from "../common/infrastructure/config/source/ymbridge.js"; import { SonosData, SonosSourceConfig } from "../common/infrastructure/config/source/sonos.js"; import { WildcardEmitter } from "../common/WildcardEmitter.js"; -import { parseBool } from "../utils.js"; -import { readJson } from '../utils/DataUtils.js'; +import { nonEmptyObj, parseBool, removeUndefinedKeys } from "../utils.js"; +import { getCommonComponentEnvConfig, readJson } from '../utils/DataUtils.js'; import { validateJson } from "../utils/ValidationUtils.js"; import AbstractSource from "./AbstractSource.js"; import { nonEmptyStringOrDefault } from '../../core/StringUtils.js'; @@ -54,6 +54,7 @@ import { ListenBrainzData } from '../common/infrastructure/config/client/listenb import { KoitoData } from '../common/infrastructure/config/client/koito.js'; import { TealData } from '../common/infrastructure/config/client/tealfm.js'; import { RockSkyData } from '../common/infrastructure/config/client/rocksky.js'; +import { DEFAULT_RETENTION_DELETE_AFTER } from '../common/infrastructure/config/database.js'; type groupedNamedConfigs = {[key: string]: ParsedConfig[]}; @@ -230,8 +231,11 @@ export default class ScrobbleSources { const { sources: mainConfigSourcesConfigs = [], sourceDefaults: sd = {}, + database: { + retention + } = {}, } = aioConfig; - sourceDefaults = this.buildSourceDefaults(sd); + sourceDefaults = this.buildSourceDefaults({retention, ...sd}); for (const [index, c] of mainConfigSourcesConfigs.entries()) { const {name = 'unnamed'} = c; if(c.type === undefined) { @@ -273,26 +277,28 @@ export default class ScrobbleSources { let defaultConfigureAs: ConfigureAsSource = 'source'; // env builder for single user mode switch (sourceType) { - case 'spotify': - const s: SpotifySourceData = { + case 'spotify': { + const data: SpotifySourceData = removeUndefinedKeys({ clientId: process.env.SPOTIFY_CLIENT_ID as string, clientSecret: process.env.SPOTIFY_CLIENT_SECRET as string, redirectUri: process.env.SPOTIFY_REDIRECT_URI, - }; - if (!Object.values(s).every(x => x === undefined && x !== null)) { + }, false); + const p = getCommonComponentEnvConfig('SPOTIFY'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'spotify', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: s, + data: data, + ...p, options: transformPresetEnv('SPOTIFY') }) } - break; - case 'plex': - const p: PlexApiData = { + } break; + case 'plex': { + const data: PlexApiData = removeUndefinedKeys({ url: process.env.PLEX_URL, token: process.env.PLEX_TOKEN, usersAllow: process.env.PLEX_USERS_ALLOW, @@ -301,39 +307,43 @@ export default class ScrobbleSources { devicesBlock: process.env.PLEX_DEVICES_BLOCK, librariesAllow: process.env.PLEX_LIBRARIES_ALLOW, librariesBlock: process.env.PLEX_LIBRARIES_BLOCK - }; - if (!Object.values(p).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('PLEX'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'plex', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: p, + data: data, + ...p, options: transformPresetEnv('PLEX') }) } - break; - case 'subsonic': - const sub: SubsonicData = { + } break; + case 'subsonic': { + const data: SubsonicData = removeUndefinedKeys({ user: process.env.SUBSONIC_USER, password: process.env.SUBSONIC_PASSWORD, url: process.env.SUBSONIC_URL, - }; - if (!Object.values(sub).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('SUBSONIC'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'subsonic', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: sub as SubsonicData, + data: data, + ...p, options: transformPresetEnv('SUBSONIC') }) } - break; - case 'jellyfin': - const j: JellyApiData = { + } break; + case 'jellyfin': { + const data: JellyApiData = removeUndefinedKeys({ user: process.env.JELLYFIN_USER, password: process.env.JELLYFIN_PASSWORD, apiKey: process.env.JELLYFIN_APIKEY, @@ -346,294 +356,317 @@ export default class ScrobbleSources { librariesBlock: process.env.JELLYFIN_LIBRARIES_BLOCK, frontendUrlOverride: process.env.JELLYFIN_FRONTEND_URL_OVERRIDE, allowMediaTypes: process.env.JELLYFIN_MEDIATYPES_ALLOW - }; - if (!Object.values(j).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('JELLYFIN'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'jellyfin', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: j, + data: data, + ...p, options: transformPresetEnv('JELLYFIN') }) } - break; + } break; case 'lastfm': { - const lfm: LastfmData = { + const data: LastfmData = removeUndefinedKeys({ apiKey: process.env.SOURCE_LASTFM_API_KEY, secret: process.env.SOURCE_LASTFM_SECRET, redirectUri: process.env.SOURCE_LASTFM_REDIRECT_URI, session: process.env.SOURCE_LASTFM_SESSION, - }; - if (!Object.values(lfm).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('SOURCE_LASTFM'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'lastfm', name: 'unnamed-lfm-source', source: 'ENV', mode: 'single', configureAs: 'source', - data: lfm, + data: data, + ...p, options: transformPresetEnv('SOURCE_LASTFM') }) } } break; - case 'deezer': - const d = { + case 'deezer': { + const data = removeUndefinedKeys({ clientId: process.env.DEEZER_CLIENT_ID, clientSecret: process.env.DEEZER_CLIENT_SECRET, redirectUri: process.env.DEEZER_REDIRECT_URI, accessToken: process.env.DEEZER_ACCESS_TOKEN, arl: process.env.DEEZER_ARL, accountId: process.env.DEEZER_ACCOUNT_ID - }; - if (!Object.values(d).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('DEEZER'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'deezer', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: d, + data: data, + ...p, options: transformPresetEnv('DEEZER') }); } - break; - case 'mpris': - const shouldUse = parseBool(process.env.MPRIS_ENABLE); - const mp: MPRISData = { + } break; + case 'mpris': { + const data: MPRISData = removeUndefinedKeys({ blacklist: process.env.MPRIS_BLACKLIST, whitelist: process.env.MPRIS_WHITELIST - } - if (!Object.values(mp).every(x => x === undefined) || shouldUse) { + }, false); + const p = getCommonComponentEnvConfig('MPRIS'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'mpris', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: mp as MPRISData, + data: data, + ...p, options: transformPresetEnv('MPRIS') }); } - break; + } break; case 'maloja': { - // env builder for single user mode - const url = process.env.SOURCE_MALOJA_URL; - const apiKey = process.env.SOURCE_MALOJA_API_KEY; - if (url !== undefined || apiKey !== undefined) { - const malojaData: MalojaData = { - url, - apiKey - } + const data = removeUndefinedKeys({ + url: process.env.MALOJA_URL, + apiKey: process.env.MALOJA_API_KEY + }, false); + const p = getCommonComponentEnvConfig('SOURCE_MALOJA'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'maloja', name: 'unnamed-mlj-source', source: 'ENV', mode: 'single', configureAs: 'source', - data: malojaData, + data: data, + ...p, options: transformPresetEnv('SOURCE_MALOJA') }) } } break; case 'librefm':{ - const shouldUse = parseBool(process.env.LIBRFM_ENABLE) - const libre: LibrefmData = { + const data: LibrefmData = removeUndefinedKeys({ apiKey: process.env.SOURCE_LIBREFM_API_KEY, secret: process.env.SOURCE_LIBREFM_SECRET, redirectUri: process.env.SOURCE_LIBREFM_REDIRECT_URI, session: process.env.SOURCE_LIBREFM_SESSION, urlBase: process.env.SOURCE_LIBREFM_URLBASE, - }; - if (!Object.values(libre).every(x => x === undefined) || shouldUse) { + }, false); + const p = getCommonComponentEnvConfig('SOURCE_LIBREFM'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'librefm', name: 'unnamed-librefm-source', source: 'ENV', mode: 'single', configureAs: 'source', - data: libre, + data: data, + ...p, options: transformPresetEnv('SOURCE_LIBREFM') }) } } break; case 'listenbrainz': { - const lz: ListenBrainzData = { + const data: ListenBrainzData = removeUndefinedKeys({ url: process.env.SOURCE_LZ_URL, token: process.env.SOURCE_LZ_TOKEN, username: process.env.SOURCE_LZ_USER - }; - if (!Object.values(lz).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('SOURCE_LZ'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'listenbrainz', name: 'unnamed-lz-source', source: 'ENV', mode: 'single', configureAs: 'source', - data: lz, + data: data, + ...p, options: transformPresetEnv('SOURCE_LZ') }) } } break; case 'koito': { - const koit: KoitoData = { + const data: KoitoData = removeUndefinedKeys({ url: process.env.SOURCE_KOITO_URL, token: process.env.SOURCE_KOITO_TOKEN, username: process.env.SOURCE_KOITO_USER - }; - if (!Object.values(koit).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('SOURCE_KOITO'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'koito', name: 'unnamed-koito-source', source: 'ENV', mode: 'single', configureAs: 'source', - data: koit, + data: data, + ...p, options: transformPresetEnv('SOURCE_KOITO') }) } } break; case 'tealfm': { - const teal: TealData = { + const data: TealData = removeUndefinedKeys({ identifier: process.env.SOURCE_TEALFM_IDENTIFIER, appPassword: process.env.SOURCE_TEALFM_APP_PW, - }; - if (!Object.values(teal).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('SOURCE_TEALFM'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'tealfm', name: 'unnamed-tealfm-source', source: 'ENV', mode: 'single', configureAs: 'source', - data: teal, + data: data, + ...p, options: transformPresetEnv('SOURCE_TEALFM') }) } } break; case 'rocksky': { - const rocksky: RockSkyData = { + const data: RockSkyData = removeUndefinedKeys({ key: process.env.SOURCE_ROCKSKY_KEY, handle: process.env.SOURCE_ROCKSKY_HANDLE - }; - if (!Object.values(rocksky).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('SOURCE_ROCKSKY'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'rocksky', name: 'unnamed-rocksky-source', source: 'ENV', mode: 'single', configureAs: 'source', - data: rocksky, + data: data, + ...p, options: transformPresetEnv('SOURCE_ROCKSKY') }) } } break; - case 'endpointlz': - const lzShouldUse = parseBool(process.env.LZENDPOINT_ENABLE); - const lze: ListenbrainzEndpointData = { + case 'endpointlz': { + const data: ListenbrainzEndpointData = removeUndefinedKeys({ slug: process.env.LZE_SLUG, token: process.env.LZE_TOKEN, username: process.env.LZE_USERNAME - } - if (!Object.values(lze).every(x => x === undefined) || lzShouldUse) { + }, false); + const p = getCommonComponentEnvConfig('LZE'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'endpointlz', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: lze as ListenbrainzEndpointData, + data: data, + ...p, options: transformPresetEnv('LZE') }); } - break; - case 'endpointlfm': - const lfmShouldUse = parseBool(process.env.LFMENDPOINT_ENABLE); - const lfme: LastFMEndpointData = { + } break; + case 'endpointlfm': { + const data: LastFMEndpointData = removeUndefinedKeys({ slug: process.env.LFM_SLUG, - } - if (!Object.values(lfme).every(x => x === undefined) || lfmShouldUse) { + }, false); + const p = getCommonComponentEnvConfig('LFM'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'endpointlfm', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: lfme as LastFMEndpointData, + data: data, + ...p, options: transformPresetEnv('LFM') }); } - break; + } break; case 'icecast': { const scrobbleStart = parseBool(process.env.ICECAST_SCROBBLE_START); - const icecast: IcecastData = { + const data: IcecastData = removeUndefinedKeys({ url: process.env.ICECAST_URL, - } - if (!Object.values(icecast).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('ICECAST'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'icecast', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: icecast as IcecastData, + data: data, + ...p, options: transformPresetEnv('ICECAST', { systemScrobble: scrobbleStart }) }); } } break; - case 'jriver': - const jr: JRiverData = { + case 'jriver': { + const data: JRiverData = removeUndefinedKeys({ url: process.env.JRIVER_URL, username: process.env.JRIVER_USER, password: process.env.JRIVER_PASSWORD - } - if (!Object.values(jr).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('JRIVER'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'jriver', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: jr as JRiverData, + data: data, + ...p, options: transformPresetEnv('JRIVER') }); } - break; - case 'kodi': - const ko: KodiData = { + } break; + case 'kodi': { + const data: KodiData = removeUndefinedKeys({ url: process.env.KODI_URL, username: process.env.KODI_USER, password: process.env.KODI_PASSWORD - } - if (!Object.values(ko).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('KODI'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'kodi', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: ko as KodiData, + data: data, + ...p, options: transformPresetEnv('KODI') }); } - break; - case 'webscrobbler': - const wsShouldUse = parseBool(process.env.WS_ENABLE); - const ws: WebScrobblerData = { + } break; + case 'webscrobbler': { + const data: WebScrobblerData = removeUndefinedKeys({ blacklist: process.env.WS_BLACKLIST, whitelist: process.env.WS_WHITELIST - } - if (!Object.values(ws).every(x => x === undefined) || wsShouldUse) { + }, false); + const p = getCommonComponentEnvConfig('WS'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'webscrobbler', name: 'unnamed', @@ -641,22 +674,23 @@ export default class ScrobbleSources { mode: 'single', configureAs: defaultConfigureAs, data: { - blacklist: ws.blacklist !== undefined ? (ws.blacklist as string).split(',') : [], - whitelist: ws.whitelist !== undefined ? (ws.whitelist as string).split(',') : [], + blacklist: data.blacklist !== undefined ? (data.blacklist as string).split(',') : [], + whitelist: data.whitelist !== undefined ? (data.whitelist as string).split(',') : [], }, + ...p, options: transformPresetEnv('WS') }); } - break; - case 'chromecast': - const ccShouldUse = parseBool(process.env.CC_ENABLE); - const cc: ChromecastData = { + } break; + case 'chromecast': { + const data: ChromecastData = removeUndefinedKeys({ blacklistDevices: process.env.CC_BLACKLIST_DEVICES, whitelistDevices: process.env.CC_WHITELIST_DEVICES, blacklistApps: process.env.CC_BLACKLIST_APPS, whitelistApps: process.env.CC_WHITELIST_APPS - } - if (!Object.values(cc).every(x => x === undefined) || ccShouldUse) { + }, false); + const p = getCommonComponentEnvConfig('CC'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'chromecast', name: 'unnamed', @@ -664,160 +698,177 @@ export default class ScrobbleSources { mode: 'single', configureAs: defaultConfigureAs, data: { - blacklistDevices: cc.blacklistDevices !== undefined ? (cc.blacklistDevices as string).split(',') : [], - whitelistDevices: cc.whitelistDevices !== undefined ? (cc.whitelistDevices as string).split(',') : [], - blacklistApps: cc.blacklistApps !== undefined ? (cc.blacklistApps as string).split(',') : [], - whitelistApps: cc.whitelistApps !== undefined ? (cc.whitelistApps as string).split(',') : [], + blacklistDevices: data.blacklistDevices !== undefined ? (data.blacklistDevices as string).split(',') : [], + whitelistDevices: data.whitelistDevices !== undefined ? (data.whitelistDevices as string).split(',') : [], + blacklistApps: data.blacklistApps !== undefined ? (data.blacklistApps as string).split(',') : [], + whitelistApps: data.whitelistApps !== undefined ? (data.whitelistApps as string).split(',') : [], }, + ...p, options: transformPresetEnv('CC') }); } - break; - case 'musiccast': - const musecase: MusicCastData = { + } break; + case 'musiccast': { + const data: MusicCastData = removeUndefinedKeys({ url: process.env.MCAST_URL, - } - if (!Object.values(musecase).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('MCAST'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'musiccast', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: musecase as MusicCastData, + data: data, + ...p, options: transformPresetEnv('MCAST') }); } - break; - case 'musikcube': - const mc: MusikcubeData = { + } break; + case 'musikcube': { + const data: MusikcubeData = removeUndefinedKeys({ url: process.env.MC_URL, password: process.env.MC_PASSWORD - } - if (!Object.values(mc).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('MC'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'musikcube', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: mc as MusikcubeData, + data: data as MusikcubeData, + ...p, options: transformPresetEnv('MC') }); } - break; - case 'mpd': - const mpd: MPDData = { + } break; + case 'mpd': { + const data: MPDData = removeUndefinedKeys({ url: process.env.MPD_URL, password: process.env.MPD_PASSWORD - } - if (!Object.values(mpd).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('MPD'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'mpd', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: mpd as MPDData, + data: data, + ...p, options: transformPresetEnv('MPD') }); } - break; - case 'vlc': - const vlc: VLCData = { + } break; + case 'vlc': { + const data: VLCData = removeUndefinedKeys({ url: process.env.VLC_URL, password: process.env.VLC_PASSWORD - } - if (!Object.values(vlc).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('VLC'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'vlc', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: vlc as VLCData, + data: data, + ...p, options: transformPresetEnv('VLC') }); } - break; - case 'ytmusic': - const ytm: YTMusicData = { + } break; + case 'ytmusic': { + const data: YTMusicData = removeUndefinedKeys({ redirectUri: process.env.YTM_REDIRECT_URI, clientId: process.env.YTM_CLIENT_ID, clientSecret: process.env.YTM_CLIENT_SECRET, cookie: process.env.YTM_COOKIE - } - if (!Object.values(ytm).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('YTM'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'ytmusic', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: ytm as YTMusicData, + data: data, + ...p, options: transformPresetEnv('YTM') }); } - break; + } break; case 'azuracast': { - const azura: AzuracastData = { + const data: AzuracastData = removeUndefinedKeys({ station: process.env.AZURA_STATION, url: process.env.AZURA_URL, apiKey: process.env.AZURA_KEY - } + }, false); const listenerNum = process.env.AZURA_LISTENERS_NUM ?? ''; if(listenerNum.trim() !== '') { - azura.monitorWhenListeners = !isNaN(Number.parseInt(listenerNum)) ? Number.parseInt(listenerNum) : parseBool(listenerNum); + data.monitorWhenListeners = !isNaN(Number.parseInt(listenerNum)) ? Number.parseInt(listenerNum) : parseBool(listenerNum); } const live = process.env.AZURA_LIVE ?? ''; if(live.trim() !== '') { - azura.monitorWhenLive = parseBool(live); + data.monitorWhenLive = parseBool(live); } - if (!Object.values(azura).every(x => x === undefined)) { + const p = getCommonComponentEnvConfig('AZURA'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'azuracast', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: azura as unknown as AzuracastData, + data: data, + ...p, options: transformPresetEnv('AZURA') }); } - } break; + } break; case 'sonos': { - const sonos: SonosData = { + const data: SonosData = removeUndefinedKeys({ host: process.env.SONOS_HOST, devicesAllow: process.env.SONOS_DEVICES_ALLOW, devicesBlock: process.env.SONOS_DEVICES_BLOCK, groupsAllow: process.env.SONOS_GROUPS_ALLOW, groupsBlock: process.env.SONOS_GROUPS_BLOCK - } - if (!Object.values(sonos).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('SONOS'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'sonos', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: sonos as unknown as SonosData, + data: data, + ...p, options: transformPresetEnv('SONOS') }); } } break; case 'ymbridge': { - const yandex: YandexMusicBridgeData = { + const data: YandexMusicBridgeData = removeUndefinedKeys({ url: process.env.YMBRIDGE_URL, apiKey: process.env.YMBRIDGE_API_KEY, - } - if (!Object.values(yandex).every(x => x === undefined)) { + }, false); + const p = getCommonComponentEnvConfig('YMBRIDGE'); + if (nonEmptyObj(data) || nonEmptyObj(p)) { configs.push({ type: 'ymbridge', name: 'unnamed', source: 'ENV', mode: 'single', configureAs: defaultConfigureAs, - data: yandex, + data: data, + ...p, options: transformPresetEnv('YMBRIDGE') }); } diff --git a/src/backend/sources/SonosSource.ts b/src/backend/sources/SonosSource.ts index 1af0b5e41..a53bbe510 100644 --- a/src/backend/sources/SonosSource.ts +++ b/src/backend/sources/SonosSource.ts @@ -22,6 +22,7 @@ import { buildStatePlayerPlayIdententifyingInfo, hashObject, parseArrayFromMaybe import { isDebugMode, playObjDataMatch, sleep } from "../utils.js"; import dayjs, { Dayjs } from "dayjs"; import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js"; +import { artistNamesToCredits } from "../../core/StringUtils.js"; export interface DeviceState { device: SonosDevice @@ -377,7 +378,7 @@ export const formatPlayObj = (obj: SonosState, options: FormatPlayObjectOptions data: { track: titleStr, album: Album, - artists: Artist === undefined ? undefined : [Artist], + artists: Artist === undefined ? undefined : artistNamesToCredits([Artist]), duration: dur, }, meta: { diff --git a/src/backend/sources/SpotifySource.ts b/src/backend/sources/SpotifySource.ts index 909a0cabc..f014164fd 100644 --- a/src/backend/sources/SpotifySource.ts +++ b/src/backend/sources/SpotifySource.ts @@ -3,7 +3,7 @@ import EventEmitter from "events"; import SpotifyWebApi from "spotify-web-api-node"; import request from 'superagent'; import { BrainzMeta, PlayObject, PlayObjectLifecycleless, SCROBBLE_TS_SOC_END, SCROBBLE_TS_SOC_START, ScrobbleTsSOC, SpotifyMeta } from "../../core/Atomic.js"; -import { combinePartsToString, truncateStringToLength } from "../../core/StringUtils.js"; +import { artistNamesToCredits, artistNameToCredit, combinePartsToString, truncateStringToLength } from "../../core/StringUtils.js"; import { isNodeNetworkException } from "../common/errors/NodeErrors.js"; import { hasUpstreamError, UpstreamError } from "../common/errors/UpstreamError.js"; import { @@ -214,8 +214,8 @@ export default class SpotifySource extends MemoryPositionalSource implements Pag const play: PlayObjectLifecycleless = { data: { - artists: artists.map(x => x.name), - albumArtists: actualAlbumArtists.map(x => x.name), + artists: artists.map(x => artistNameToCredit(x.name)), + albumArtists: actualAlbumArtists.map(x => artistNameToCredit(x.name)), album: albumName, track: name, duration: duration_ms / 1000, @@ -232,7 +232,7 @@ export default class SpotifySource extends MemoryPositionalSource implements Pag } }, meta: { - deviceId: deviceId ?? `${NO_DEVICE}-${NO_USER}`, + deviceId: deviceId ?? `${NO_DEVICE}`, source: 'Spotify', musicService: 'Spotify', trackId: id, diff --git a/src/backend/sources/WebScrobblerSource.ts b/src/backend/sources/WebScrobblerSource.ts index 8100fb445..f6857961a 100644 --- a/src/backend/sources/WebScrobblerSource.ts +++ b/src/backend/sources/WebScrobblerSource.ts @@ -23,6 +23,7 @@ import { Logger } from "@foxxmd/logging"; import { PlayerStateOptions } from "./PlayerState/AbstractPlayerState.js"; import { NowPlayingPlayerState } from "./PlayerState/NowPlayingPlayerState.js"; import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js"; +import { artistCreditToName, artistNameToCredit } from "../../core/StringUtils.js"; export class WebScrobblerSource extends MemorySource { @@ -127,9 +128,9 @@ export class WebScrobblerSource extends MemorySource { const play: PlayObjectLifecycleless = { data: { track, - artists: [artist], + artists: [artistNameToCredit(artist)], album: album === null ? undefined : album, - albumArtists: albumArtist === null ? undefined : [albumArtist], + albumArtists: albumArtist === null ? undefined : [artistNameToCredit(albumArtist)], playDate: dayjs.unix(startTimestamp), duration: duration === null ? undefined : duration, meta: { @@ -155,7 +156,7 @@ export class WebScrobblerSource extends MemorySource { return baseFormatPlayObj(obj, play); } - getRecentlyPlayed = async (options = {}) => this.getFlatRecentlyDiscoveredPlays() + getRecentlyPlayed = async (options = {}) => await this.getFlatRecentlyDiscoveredPlays() isValidScrobble = (playObj: PlayObject) => { if (playObj.meta?.scrobbleAllowed === false) { diff --git a/src/backend/sources/YTMusicSource.ts b/src/backend/sources/YTMusicSource.ts index 1fed9eda3..c19abc16c 100644 --- a/src/backend/sources/YTMusicSource.ts +++ b/src/backend/sources/YTMusicSource.ts @@ -18,11 +18,12 @@ import { playsAreSortConsistent } from "../utils/PlayComparisonUtils.js"; import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource.js"; -import { buildTrackString, truncateStringToLength } from "../../core/StringUtils.js"; +import { artistNamesToCredits, buildTrackString, truncateStringToLength } from "../../core/StringUtils.js"; import { joinedUrl } from "../utils/NetworkUtils.js"; import { todayAwareFormat } from "../../core/TimeUtils.js"; import { parseArrayFromMaybeString, parseArtistCredits, parseCredits } from "../utils/StringUtils.js"; import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js"; +import { FixedSizeList } from "fixed-size-list"; export interface HistoryIngressResult { plays: PlayObject[], @@ -118,6 +119,7 @@ export default class YTMusicSource extends AbstractSource { declare config: YTMusicSourceConfig recentlyPlayed: PlayObject[] = []; + transientDiscovered: FixedSizeList = new FixedSizeList(200); yti: Innertube; userCode?: string; @@ -149,6 +151,10 @@ export default class YTMusicSource extends AbstractSource { this.config.options = {...rest, logDiff: diffVal}; } } + + this.emitter.on('discovered', (play) => { + this.transientDiscovered.add(play); + }) } public additionalApiData(): Record { @@ -371,7 +377,7 @@ Redirect URI : ${this.redirectUri}`); duration: dur, // string timestamp } = obj; - let artists = [], + let artists: string[] = [], album = undefined, duration = undefined; if(artistsData !== undefined) { @@ -415,8 +421,8 @@ Redirect URI : ${this.redirectUri}`); } const play: PlayObjectLifecycleless = { data: { - artists, - albumArtists, + artists: artistNamesToCredits(artists), + albumArtists: artistNamesToCredits(albumArtists), album, track: title, duration, @@ -588,7 +594,7 @@ Redirect URI : ${this.redirectUri}`); if(consistent && newPlays.length > 1) { const interimPlays = newPlays.slice(0, newPlays.length - 1); // check enough time has passed since last discovery - const discovered = this.getFlatRecentlyDiscoveredPlays(); + const discovered = this.transientDiscovered.data; if(discovered.length > 0) { const lastDiscovered = discovered[0].data.playDate; // the assumption in behavior is that user skips 1 or more tracks which then get recorded to YTM history @@ -681,10 +687,11 @@ ${humanDiff}`; const reversedPlays = [...referencePlays]; // actual order they were discovered in (oldest to newest) reversedPlays.reverse(); - if(this.getFlatRecentlyDiscoveredPlays().length === 0) { + if(this.transientDiscovered.data.length === 0) { // and add to discovered since its empty for(const refPlay of reversedPlays) { - this.addPlayToDiscovered(refPlay); + //this.transientDiscovered.add(refPlay); + await this.addPlayToDiscovered(refPlay); } } } diff --git a/src/backend/sources/YandexMusicBridgeSource.ts b/src/backend/sources/YandexMusicBridgeSource.ts index e1b7ce4a3..587cc920d 100644 --- a/src/backend/sources/YandexMusicBridgeSource.ts +++ b/src/backend/sources/YandexMusicBridgeSource.ts @@ -15,6 +15,7 @@ import { YandexMusicBridgeSourceConfig } from "../common/infrastructure/config/s import { isPortReachableConnect, joinedUrl, normalizeWebAddress } from "../utils/NetworkUtils.js"; import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js"; import { UpstreamError } from "../common/errors/UpstreamError.js"; +import { artistNamesToCredits } from "../../core/StringUtils.js"; interface BridgeTrackData { title?: string @@ -437,7 +438,7 @@ const formatPlayObj = (obj: BridgeTrackData, playerId: string): PlayObject => { const play: PlayObjectLifecycleless = { data: { - artists, + artists: artistNamesToCredits(artists), album: obj.album ?? undefined, track: obj.title ?? undefined, duration: obj.duration_ms !== undefined && obj.duration_ms !== null diff --git a/src/backend/tasks/heartbeatClients.ts b/src/backend/tasks/heartbeatClients.ts deleted file mode 100644 index b8e02cf15..000000000 --- a/src/backend/tasks/heartbeatClients.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { childLogger, Logger } from '@foxxmd/logging'; -import { PromisePool } from "@supercharge/promise-pool"; -import { AsyncTask } from "toad-scheduler"; -import ScrobbleClients from "../scrobblers/ScrobbleClients.js"; - -export const createHeartbeatClientsTask = (clients: ScrobbleClients, parentLogger: Logger) => { - const logger = childLogger(parentLogger, ['Heartbeat', 'Clients']); - - return new AsyncTask( - 'Heartbeat', - (): Promise => { - logger.verbose('Starting check...'); - return PromisePool - .withConcurrency(1) - .for(clients.clients) - .process(async (client) => { - if(!client.isReady()) { - if(!client.canAuthUnattended()) { - client.logger.warn({labels: 'Heartbeat'}, 'Client is not ready but will not try to initialize because auth state is not good and cannot be correct unattended.') - return 0; - } - try { - await client.tryInitialize({force: false, notify: true, notifyTitle: 'Could not initialize automatically'}); - } catch (e) { - client.logger.error(new Error('Could not initialize automatically', {cause: e})); - return 1; - } - } - - if(!client.canAuthUnattended()) { - client.logger.warn({label: 'Heartbeat'}, 'Should be monitoring scrobbles but will not attempt to start because auth state is not good and cannot be correct unattended.'); - return 0; - } - - await client.processDeadLetterQueue(); - if(!client.scrobbling) { - client.logger.info({labels: 'Heartbeat'}, 'Should be processing scrobbles! Attempting to restart scrobbling...'); - client.initScrobbleMonitoring(); - return 1; - } - }).then(({results, errors}) => { - logger.verbose(`Checked Dead letter queue for ${clients.clients.length} clients.`); - const restarted = results.reduce((acc, curr) => acc += curr, 0); - if (restarted > 0) { - logger.info(`Attempted to start ${restarted} clients that were not processing scrobbles.`); - } - if (errors.length > 0) { - logger.error(`Encountered errors!`); - for (const err of errors) { - logger.error(err); - } - } - }); - }, - (err: Error) => { - logger.error(err); - } - ); -} diff --git a/src/backend/tasks/heartbeatSources.ts b/src/backend/tasks/heartbeatSources.ts deleted file mode 100644 index 6317e4b31..000000000 --- a/src/backend/tasks/heartbeatSources.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { childLogger, Logger } from '@foxxmd/logging'; -import { PromisePool } from "@supercharge/promise-pool"; -import { AsyncTask } from "toad-scheduler"; -import { ChromecastSource } from "../sources/ChromecastSource.js"; -import ScrobbleSources from "../sources/ScrobbleSources.js"; - -export const createHeartbeatSourcesTask = (sources: ScrobbleSources, parentLogger: Logger) => { - const logger = childLogger(parentLogger, ['Heartbeat', 'Sources']); - - return new AsyncTask( - 'Heartbeat', - (): Promise => { - logger.verbose('Starting check...'); - return PromisePool - .withConcurrency(1) - .for(sources.sources) - .process(async (source) => { - if(!source.isReady()) { - if(!source.canAuthUnattended()) { - source.logger.warn({label: 'Heartbeat'}, 'Source is not ready but will not try to initialize because auth state is not good and cannot be correct unattended.'); - return 0; - } - try { - await source.tryInitialize({force: false, notify: true, notifyTitle: 'Could not initialize automatically'}); - } catch (e) { - source.logger.error(new Error('Could not initialize source automatically', {cause: e})); - return 1; - } - } - - if(source.type === 'chromecast') { - (source as ChromecastSource).discoverDevices(); - } - - if (source.canPoll && !source.polling) { - if(!source.canAuthUnattended()) { - source.logger.warn({label: 'Heartbeat'}, 'Should be polling but will not attempt to start because auth state is not good and cannot be correct unattended.'); - return 0; - } else { - source.logger.info({label: 'Heartbeat'}, 'Should be polling, attempting to start polling...'); - source.poll({force: false, notify: true}).catch(e => source.logger.error(e)); - } - return 1; - } - - return 0; - }).then(({results, errors}) => { - logger.verbose(`Checked ${sources.sources.length} sources for start signals.`); - const restarted = results.reduce((acc, curr) => acc += curr, 0); - if (restarted > 0) { - logger.info(`Attempted to start ${restarted} sources.`); - } - if (errors.length > 0) { - logger.error(`Encountered errors!`); - for (const err of errors) { - logger.error(err); - } - } - }); - }, - (err: Error) => { - logger.error(err); - } - ); -} diff --git a/src/backend/tasks/retentionCleanup.ts b/src/backend/tasks/retentionCleanup.ts new file mode 100644 index 000000000..cb623b21d --- /dev/null +++ b/src/backend/tasks/retentionCleanup.ts @@ -0,0 +1,44 @@ +import { childLogger, Logger } from '@foxxmd/logging'; +import { AsyncTask } from "toad-scheduler"; +import ScrobbleSources from "../sources/ScrobbleSources.js"; +import ScrobbleClients from '../scrobblers/ScrobbleClients.js'; + +export const createRetentionCleanupTask = (sources: ScrobbleSources, clients: ScrobbleClients, parentLogger: Logger) => { + const logger = childLogger(parentLogger, ['Schedule', 'Retention Cleanup']); + + return new AsyncTask( + 'Retention', + (): Promise => { + return retentionTask(sources, clients, logger).then(() => null).catch((err) => { + logger.error(err); + }); + }, + (err: Error) => { + logger.error(err); + } + ); +} + +const retentionTask = async (sources: ScrobbleSources, clients: ScrobbleClients, logger: Logger): Promise => { + // todo may want to implement abort controllers for these in case they don't finish + + logger.verbose('Starting client cleanup...'); + const validClients = clients.clients.filter(x => x.databaseOK); + const invalidClients = clients.clients.filter(x => x.databaseOK !== true); + if(invalidClients.length > 0) { + logger.debug(`Not running cleanup for ${invalidClients.length} clients because their database state is not OK: ${invalidClients.map(x => x.getSafeExternalName()).join(',')}`); + } + const clientCleanupPromises = validClients.map(x => x.retentionCleanup().catch((err) => x.logger.warn(new Error('Failed to catch retention cleanup error!', {cause: err})))); + await Promise.all(clientCleanupPromises); + logger.verbose('Client cleanup done!'); + + logger.verbose('Starting source cleanup...'); + const validSources = sources.sources.filter(x => x.databaseOK); + const invalidSources= sources.sources.filter(x => x.databaseOK !== true); + if(invalidClients.length > 0) { + logger.debug(`Not running cleanup for ${invalidSources.length} sources because their database state is not OK: ${invalidSources.map(x => x.getSafeExternalName()).join(',')}`); + } + const sourceCleanupPromises = validSources.map(x => x.retentionCleanup().catch((err) => x.logger.warn(new Error('Failed to catch retention cleanup error!', {cause: err})))); + await Promise.all(sourceCleanupPromises); + logger.verbose('Source cleanup done!'); +} \ No newline at end of file diff --git a/src/backend/tests/cache/cache.test.ts b/src/backend/tests/cache/cache.test.ts index d88b5a1c7..c21f2f689 100644 --- a/src/backend/tests/cache/cache.test.ts +++ b/src/backend/tests/cache/cache.test.ts @@ -4,15 +4,10 @@ import asPromised from 'chai-as-promised'; import { after, before, describe, it } from 'mocha'; import dayjs from "dayjs"; import withLocalTmpDir from 'with-local-tmp-dir'; -import { initFileCache, initMemoryCache, initValkeyCache, MSCache } from "../../common/Cache.js"; -import { generatePlays } from "../../../core/PlayTestUtils.js"; +import { initFileCache, initMemoryCache, initValkeyCache } from "../../common/Cache.js"; import { ListenProgressPositional, ListenProgressTS } from "../../sources/PlayerState/ListenProgress.js"; import { isPortReachableConnect } from "../../utils/NetworkUtils.js"; -import { getRoot } from "../../ioc.js"; -import { transientCache } from "../utils/CacheTestUtils.js"; -import { TestScrobbler } from "../scrobbler/TestScrobbler.js"; import { sleep } from "../../utils.js"; -import {promises} from 'node:fs'; chai.use(asPromised); @@ -56,16 +51,24 @@ describe('#Caching', function () { it('File cache serializes and deserializes dayjs', async function () { - withLocalTmpDir(async () => { + await withLocalTmpDir(async () => { + // for some reason this *recreates* an empty cache after the test has finished (when running the full suite) + // i think its because of long persist interval compared to test time? + // + // so: + // make intervals very small + // call destroy on both cache instances (shouldn't be necessary) + // sleep for longer than persist interal so tmp dir callback can (hopefully) properly delete any files - const [keyv, flat] = await initFileCache({ cacheDir: process.cwd() }); + const [keyv, flat] = await initFileCache({ cacheDir: process.cwd(), persistInterval: 5, expirationInterval: 4 }); const now = dayjs(); await keyv.set('foo', now); - flat.save(); + flat.save(true); + keyv.disconnect(); - const [cleanKeyv, cleanFlat] = await initFileCache({ cacheDir: process.cwd() }); + const [cleanKeyv, cleanFlat] = await initFileCache({ cacheDir: process.cwd(), persistInterval: 5, expirationInterval: 4 }); const time = await cleanKeyv.get('foo'); @@ -73,31 +76,9 @@ describe('#Caching', function () { expect(time instanceof dayjs).is.true; expect(now.toJSON()).eq((time as any).toJSON()); flat.destroy(); - - }, { unsafeCleanup: true }); - }); - - it('File cache serializes and deserializes ListenProgress', async function () { - - withLocalTmpDir(async () => { - - const [keyv, flat] = await initFileCache({ cacheDir: process.cwd() }); - - const prog = new ListenProgressPositional({ timestamp: dayjs(), position: 35, positionPercent: 50 }); - - await keyv.set('foo', prog); - await flat.save(); - - const [cleanKeyv, cleanFlat] = await initFileCache({ cacheDir: process.cwd() }); - - const cachedProg = await cleanKeyv.get('foo'); - - expect(cachedProg).to.not.be.undefined; - expect(cachedProg instanceof ListenProgressTS).is.true; - expect(cachedProg.timestamp.toJSON()).eq(prog.timestamp.toJSON()); - flat.destroy(); - - }, { unsafeCleanup: true }); + cleanFlat.destroy(); + await sleep(10); + }, { unsafeCleanup: true, postfix: 'fileCacheDajys' }); }); }); @@ -127,62 +108,5 @@ describe('#Caching', function () { expect(now.toJSON()).eq((time as any).toJSON()); }); - - it('Valkey cache serializes and deserializes ListenProgress', async function () { - - const keyv = await initValkeyCache('test', 'redis://valkey:6379'); - await keyv.clear(); - - const prog = new ListenProgressPositional({ timestamp: dayjs(), position: 35, positionPercent: 50 }); - - await keyv.set('foo', prog); - - const cachedProg = await keyv.get('foo'); - - expect(cachedProg).to.not.be.undefined; - expect(cachedProg instanceof ListenProgressTS).is.true; - expect(cachedProg.timestamp.toJSON()).eq(prog.timestamp.toJSON()); - - }); - }); - - describe('#ScrobbleCache', function () { - - afterEach(function () { - const root = getRoot(); - root.upsert({ cache: () => transientCache }); - root.items.cache().init(); - }); - - it('Preserves scrobbles', async function () { - - this.timeout(100000); - - // why does this take so long? - await withLocalTmpDir(async () => { - - const root = getRoot(); - root.upsert({ cache: () => () => new MSCache(loggerTest, { scrobble: { provider: 'file', connection: process.cwd(), persistInterval: 100 }, auth: {provider: 'memory'}, metadata: {provider: 'memory'} }) }); - - await using test = new TestScrobbler(); - await test.initialize(); - const plays = generatePlays(100, {}, {}, {listenRanges: true}); - await test.queueScrobble(plays, 'testSource'); - const queued = test.queuedScrobbles.map(x => x.play); - await sleep(101); - const dirContents = await promises.readdir('.'); - const hasCache = dirContents.some(x => x === 'ms-scrobble.cache'); - expect(hasCache).is.true; - - await using newTest = new TestScrobbler(); - await newTest.initialize(); - expect(newTest.queuedScrobbles.length).to.eq(plays.length); - expect(newTest.queuedScrobbles[0].play.data.track).to.eq(queued[0].data.track); - - }, { unsafeCleanup: true }); - - }); - }); - }); diff --git a/src/backend/tests/component/transformers.test.ts b/src/backend/tests/component/transformers.test.ts index b7766a42a..b73218296 100644 --- a/src/backend/tests/component/transformers.test.ts +++ b/src/backend/tests/component/transformers.test.ts @@ -15,9 +15,10 @@ import { initMemoryCache } from "../../common/Cache.js"; import { Cacheable } from "cacheable"; import { TransformerCommonConfig } from "../../../core/Atomic.js"; import TransformerManager from "../../common/transforms/TransformerManager.js"; -import { transientCache } from "../utils/CacheTestUtils.js"; +import { transientCache } from "../utils/TransientTestUtils.js"; import dayjs from "dayjs"; import clone from "clone"; +import { artistCreditsToNames, artistNamesToCredits } from "../../../core/StringUtils.js"; chai.use(asPromised); @@ -350,10 +351,10 @@ describe('Play Transforms', function () { } component.buildTransformRules(); - const play = generatePlay({ artists: ['My Artist One / My Artist Two / Another Guy'] }); + const play = generatePlay({ artists: artistNamesToCredits(['My Artist One / My Artist Two / Another Guy']) }); const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.artists).length(1) - expect(transformed.data.artists[0]).equal('My Artist One'); + expect(transformed.data.artists[0].name).equal('My Artist One'); }); it('Removes title when transform replaces with empty string', async function () { @@ -402,10 +403,10 @@ describe('Play Transforms', function () { } component.buildTransformRules(); - const play = generatePlay({ artists: ['something', 'big'] }); + const play = generatePlay({ artists: artistNamesToCredits(['something', 'big']) }); const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.artists!.length).is.eq(1) - expect(transformed.data.artists![0]).is.eq('big') + expect(transformed.data.artists![0].name).is.eq('big') }); }); @@ -421,10 +422,10 @@ describe('Play Transforms', function () { await t.tryInitialize(); const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}}); - const play = generatePlay({artists: [str]}); + const play = generatePlay({artists: artistNamesToCredits([str])}); const transformedPlay = await t.handle(t.parseConfig({type: 'native'}), play); - expect(transformedPlay.data.artists).eql(primaries.concat(secondaries)); + expect(artistCreditsToNames(transformedPlay.data.artists)).eql(primaries.concat(secondaries)); }); it('Ignores artists', async function() { @@ -435,10 +436,10 @@ describe('Play Transforms', function () { await t.tryInitialize(); - const play = generatePlay({artists: [str], track: 'My Test'}); + const play = generatePlay({artists: artistNamesToCredits([str]), track: 'My Test'}); const transformedPlay = await t.handle(t.parseConfig({type: 'native'}), play); - expect(transformedPlay.data.artists).eql([str]); + expect(artistCreditsToNames(transformedPlay.data.artists)).eql([str]); }); it('Uses custom delimiters artists', async function() { @@ -456,10 +457,10 @@ describe('Play Transforms', function () { await t.tryInitialize(); - const play = generatePlay({artists: [str], track: 'My Test'}); + const play = generatePlay({artists: artistNamesToCredits([str]), track: 'My Test'}); const transformedPlay = await t.handle(t.parseConfig({type: 'native'}), play); - expect(transformedPlay.data.artists).eql(primaries.concat(secondaries)); + expect(artistCreditsToNames(transformedPlay.data.artists)).eql(primaries.concat(secondaries)); }); }); @@ -484,10 +485,10 @@ describe('Play Transforms', function () { } component.buildTransformRules(); - const play = generatePlay({ artists: ['something', 'big'], album: 'It Has No Match' }); + const play = generatePlay({ artists: artistNamesToCredits(['something', 'big']), album: 'It Has No Match' }); const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.artists!.length).is.eq(2) - expect(transformed.data.artists![0]).is.eq('something') + expect(transformed.data.artists![0].name).is.eq('something') }); it('Does run hook if when conditions matches', async function () { @@ -507,10 +508,10 @@ describe('Play Transforms', function () { } component.buildTransformRules(); - const play = generatePlay({ artists: ['something', 'big'], album: 'It Has This Match' }); + const play = generatePlay({ artists: artistNamesToCredits(['something', 'big']), album: 'It Has This Match' }); const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.artists!.length).is.eq(1) - expect(transformed.data.artists![0]).is.eq('big') + expect(transformed.data.artists![0].name).is.eq('big') }); }); @@ -537,10 +538,10 @@ describe('Play Transforms', function () { } component.buildTransformRules(); - const play = generatePlay({ artists: ['something', 'big'], album: 'It Has No Match' }); + const play = generatePlay({ artists: artistNamesToCredits(['something', 'big']), album: 'It Has No Match' }); const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.artists!.length).is.eq(2) - expect(transformed.data.artists![0]).is.eq('something') + expect(transformed.data.artists![0].name).is.eq('something') }); it('Does run hook if when conditions matches', async function () { @@ -565,10 +566,10 @@ describe('Play Transforms', function () { } component.buildTransformRules(); - const play = generatePlay({ artists: ['something', 'big'], album: 'It Has This Match' }); + const play = generatePlay({ artists: artistNamesToCredits(['something', 'big']), album: 'It Has This Match' }); const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.artists!.length).is.eq(1) - expect(transformed.data.artists![0]).is.eq('big') + expect(transformed.data.artists![0].name).is.eq('big') }); }); @@ -632,10 +633,10 @@ describe('Play Transforms', function () { const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}}); component.buildTransformRules(); - const play = generatePlay({ track: 'My cool something track', artists: [str] }); + const play = generatePlay({ track: 'My cool something track', artists: artistNamesToCredits([str]) }); const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.track).equal('My cool bar track'); - expect(transformed.data.artists).eql(primaries.concat(secondaries)); + expect(artistCreditsToNames(transformed.data.artists)).eql(primaries.concat(secondaries)); }); }); @@ -667,17 +668,17 @@ describe('Play Transforms', function () { const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}}); component.buildTransformRules(); - const play = generatePlay({ track: 'My cool something track', artists: [str], playDate: dayjs().subtract(10, 'm') }); + const play = generatePlay({ track: 'My cool something track', artists: artistNamesToCredits([str]), playDate: dayjs().subtract(10, 'm') }); const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare, 'all'); expect(transformed.data.track).equal('My cool bar track'); - expect(transformed.data.artists).eql(primaries.concat(secondaries)); + expect(artistCreditsToNames(transformed.data.artists)).eql(primaries.concat(secondaries)); const cachablePlay = clone(play); const laterDate = dayjs().subtract(5, 'm'); cachablePlay.data.playDate = laterDate; const cacheTransformed = await component.transformPlay(cachablePlay, TRANSFORM_HOOK.preCompare, 'all'); expect(cacheTransformed.data.track).equal('My cool bar track'); - expect(cacheTransformed.data.artists).eql(primaries.concat(secondaries)); + expect(artistCreditsToNames(cacheTransformed.data.artists)).eql(primaries.concat(secondaries)); expect(cacheTransformed.data.playDate.isSame(cachablePlay.data.playDate)); }); diff --git a/src/backend/tests/config/config.test.ts b/src/backend/tests/config/config.test.ts index 10717d911..7482074a2 100644 --- a/src/backend/tests/config/config.test.ts +++ b/src/backend/tests/config/config.test.ts @@ -44,7 +44,7 @@ describe('Sample Configs', function () { let reset: any; beforeEach(async function() { - reset = await withLocalTmpDir({unsafeCleanup: true}); + reset = await withLocalTmpDir({unsafeCleanup: true, postfix: 'sourceConfigParse'}); }); afterEach(async function() { @@ -78,7 +78,7 @@ describe('Sample Configs', function () { let reset: any; beforeEach(async function() { - reset = await withLocalTmpDir({unsafeCleanup: true}); + reset = await withLocalTmpDir({unsafeCleanup: true, postfix: 'clientConfigParse'}); }); afterEach(async function() { diff --git a/src/backend/tests/database/drizzle.test.ts b/src/backend/tests/database/drizzle.test.ts new file mode 100644 index 000000000..ca148f453 --- /dev/null +++ b/src/backend/tests/database/drizzle.test.ts @@ -0,0 +1,689 @@ +import chai, { assert, expect } from 'chai'; +import asPromised from 'chai-as-promised'; +import { getDb, getMigratedDb, migrateDb, shouldBackupDb } from '../../common/database/drizzle/drizzleUtils.js'; +import withLocalTmpDir from 'with-local-tmp-dir'; +import { components, playInputs, plays, queueStates } from '../../common/database/drizzle/schema/schema.js'; +import dayjs from 'dayjs'; +import { generatePlay } from '../../../core/PlayTestUtils.js'; +import { getDbPath } from '../../common/database/Database.js'; +import { x } from 'tinyexec'; +import * as path from 'path'; +import * as fs from 'fs/promises'; +import { projectDir } from '../../common/index.js'; +import { DatabaseSync } from 'node:sqlite'; +import { fixtureCreateComponent, fixtureCreateInput, fixtureCreatePlay, getPrepopulatedFSPGlite, getPrepopulatedMemoryPGlite } from '../utils/databaseFixtures.js'; +import { DrizzlePlayRepository, RepositoryCreatePlayOpts } from '../../common/database/drizzle/repositories/PlayRepository.js'; +import { generatePlayWithLifecycle, generateRandomObj } from '../../../core/tests/utils/fixtures.js'; +import { formatNumber, generateArray } from '../../../core/DataUtils.js'; +import { objectsEqual } from '../../utils/DataUtils.js'; +import { eq, sql } from 'drizzle-orm'; +import { PlaySelect } from '../../common/database/drizzle/drizzleTypes.js'; +import { loggerDebug } from '@foxxmd/logging'; +import { transientDb } from '../utils/TransientTestUtils.js'; +import { dataDir } from '@electric-sql/pglite-prepopulatedfs' + +// would be great to push migrations directly from schema but doesn't seem supported in newest beta +// https://github.com/drizzle-team/drizzle-orm/discussions/4373 + +describe('Migrations', function () { + + it('Detects non-existent db', async function () { + + this.timeout(5000); + + await withLocalTmpDir(async () => { + const [db, isNew] = await getMigratedDb(getDbPath('notreal', process.cwd()), { loadDataDir: dataDir() }); + expect(isNew).is.true; + db.$client.close(); + }, {postfix: 'noDb'}); + + }); + + it('Detects abnormal db', async function () { + this.timeout(5000); + + // database exists but there is no __drizzle_migrations table + const db = await getDb(':memory:', { loadDataDir: dataDir() }); + const [shouldBackup, pending] = await shouldBackupDb(db); + expect(shouldBackup).is.true; + db.$client.close(); + expect(pending).length(0); + + }); + + it('Detects pending migrations', async function () { + + this.timeout(5000); + + const allFiles = await fs.readdir(path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations')); + const migrationFiles = allFiles + .sort(); + + await withLocalTmpDir(async () => { + + // copy first migration + await fs.mkdir('migrations'); + try { + await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true }); + const mf = path.resolve('./migrations'); + const db = await getDb(':memory:', {loadDataDir: dataDir()}); + await migrateDb(db, { migrationsFolder: mf }); + const res = await x('drizzle-kit', [ + 'generate', + '--name', + 'newMigration', + '--out', + `${mf}`, + '--custom', + '--schema', + path.resolve(projectDir, 'src/backend/common/database/drizzle/schema'), + '--dialect', + 'postgresql' + ], {throwOnError: true}); + const [shouldBackup, pending] = await shouldBackupDb(db, { migrationsFolder: mf }); + expect(shouldBackup).is.true; + expect(pending).length(1); + expect(pending[0]).includes('newMigration'); + db.$client.close(); + } catch (e) { + throw e; + } + }, { unsafeCleanup: true, postfix: 'pendingMigrations' }); + }); + + it('Detects no pending migrations correctly', async function () { + + this.timeout(5000); + + const allFiles = await fs.readdir(path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations')); + const migrationFiles = allFiles + .sort(); + + await withLocalTmpDir(async () => { + + // copy first migration + await fs.mkdir('migrations'); + try { + await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true }); + const mf = path.resolve('./migrations'); + const db = await getDb(':memory:', {loadDataDir: dataDir()}); + await migrateDb(db, { migrationsFolder: mf }); + const [shouldBackup, pending] = await shouldBackupDb(db, { migrationsFolder: mf }); + expect(shouldBackup).is.false; + expect(pending).length(0); + db.$client.close(); + } catch (e) { + throw e; + } + }, { unsafeCleanup: true, postfix: 'noMigrations' }); + }); + + it('Backs up database when migrations are pending', async function () { + + // this can be slow due to all the io + this.timeout(5000); + + const allFiles = await fs.readdir(path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations')); + const migrationFiles = allFiles + .sort(); + + await withLocalTmpDir(async () => { + + const dbPath = getDbPath('msDb', process.cwd()); + // copy first migration + await fs.mkdir('migrations'); + try { + await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true }); + const mf = path.resolve('./migrations'); + const [db, _] = await getMigratedDb(dbPath, {migrationsFolder: mf, loadDataDir: dataDir()}); + await db.$client.close(); + const res = await x('drizzle-kit', [ + 'generate', + '--name', + 'newMigration', + '--out', + `${mf}`, + '--custom', + '--schema', + path.resolve(projectDir, 'src/backend/common/database/drizzle/schema'), + '--dialect', + 'postgresql' + ], {throwOnError: true}); + + // add dummy data to migration so migrate() doesn't fail + const newMigrationFolder = (await fs.readdir(path.resolve('./migrations/'))).find(x => x.includes('newMigration')); + await fs.appendFile(path.resolve('./migrations/',newMigrationFolder, 'migration.sql'),`\nselect count(*) from plays;`); + + await getMigratedDb(dbPath, {migrationsFolder: mf}); + const contents = await fs.readdir(path.resolve('./')); + const backupPattern = new RegExp(/msDb-\d+\.bak/) + expect(contents.some(x => backupPattern.test(x))).is.true; + } catch (e) { + throw e; + } + }, { unsafeCleanup: true, postfix: 'dbBackup' }); + }); + +}); + +describe('Basic DB Operations', function () { + + it('Should create a play', async function () { + + const db = await transientDb(); + + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + + const playRow = await db.insert(plays).values({ + componentId: component[0].id, + state: 'queued', + playedAt: dayjs(), + seenAt: dayjs(), + play: generatePlay() + }).returning(); + + expect(playRow.length).eq(1); + db.$client.close(); + }); + + it('Should create a play with relations', async function () { + + const db = await transientDb(); + + try { + + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + + const playRow = await db.insert(plays).values(fixtureCreatePlay({ componentId: component[0].id })).returning(); + + const input = await db.insert(playInputs).values(fixtureCreateInput({ + playId: playRow[0].id, + play: playRow[0].play + })).returning(); + + const twoQueues = await db.insert(queueStates).values([ + { + playId: playRow[0].id, + componentId: component[0].id, + queueName: 'foo' + }, + { + playId: playRow[0].id, + componentId: component[0].id, + queueName: 'bar', + queueStatus: 'completed' + } + ]); + + const fullPlay = await db.query.plays.findFirst({ + with: { + input: true, + queueStates: true, + }, + }); + + + expect(fullPlay.queueStates).to.not.be.undefined; + expect(fullPlay.queueStates).length(2); + + expect(fullPlay.input).to.not.be.undefined; + + } catch (e) { + throw e; + } + db.$client.close(); + }); + + it('deletes all dependent relations when a Play is deleted', async function () { + + const db = await transientDb(); + + try { + + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + + const playRow = await db.insert(plays).values(fixtureCreatePlay({ componentId: component[0].id })).returning(); + + const input = await db.insert(playInputs).values(fixtureCreateInput({ + playId: playRow[0].id, + play: playRow[0].play + })).returning(); + + const twoQueues = await db.insert(queueStates).values([ + { + playId: playRow[0].id, + componentId: component[0].id, + queueName: 'foo' + }, + { + playId: playRow[0].id, + componentId: component[0].id, + queueName: 'bar', + queueStatus: 'completed' + } + ]).returning(); + + const fullPlay = await db.query.plays.findFirst({ + with: { + input: true, + queueStates: true, + }, + }); + + + expect(fullPlay.queueStates).to.not.be.undefined; + expect(fullPlay.queueStates).length(2); + expect(fullPlay.input).to.not.be.undefined; + + await db.delete(plays).where(eq(plays.id, fullPlay.id)); + const deletedPlay = await db.query.plays.findFirst({ + where: { + id: fullPlay.id + } + }); + expect(deletedPlay).to.be.undefined; + + const deletedInput = await db.query.playInputs.findFirst({ + where: { + id: input[0].id + } + }); + expect(deletedInput).to.be.undefined; + + const deletedQueues = await db.query.queueStates.findMany({ + where: { + id: { + in: [twoQueues[0].id, twoQueues[1].id] + } + } + }); + expect(deletedQueues).length(0); + } catch (e) { + throw e; + } + db.$client.close(); + }); + +}); + +describe('Repository Operations', function () { + + it('creates Plays and inputs', async function () { + + const db = await transientDb(); + + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + + const repo = new DrizzlePlayRepository(db, {componentId: component[0].id}); + + const numPlays = 3; + + const playData = generateArray(numPlays, () => ({ ...fixtureCreatePlay(), state: 'queued', input: { data: generateRandomObj(undefined, { allowUndefined: false }) } })) + + const rows = await repo.createPlays(playData); + expect(rows).length(numPlays); + const fullPlays = await db.query.plays.findMany({ + with: { + input: true + } + }); + fullPlays.forEach((play, index) => { + const ref = playData[index]; + + expect(play.play.data.track).eq(ref.play.data.track); + expect(play.input).to.not.undefined; + expect(objectsEqual(play.input.data, ref.input.data)).is.true; + }) + + }); + + it('finds Plays by state', async function () { + + const db = await transientDb(); + await migrateDb(db); + + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + + const repo = new DrizzlePlayRepository(db, {componentId: component[0].id}); + + const numPlays = 3; + + const playData = generateArray(numPlays, () => ({ + ...fixtureCreatePlay(), + state: 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + })); + const discovered = { + ...fixtureCreatePlay(), + state: 'discovered' as 'discovered', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }; + playData.push(discovered) + + await repo.createPlays(playData); + + const plays = await repo.findPlays({ state: ['discovered'] }); + expect(plays).length(1); + expect(plays[0].play.data.track).eq(discovered.play.data.track); + }); + + it('finds Plays by date range', async function () { + + const db = await transientDb(); + await migrateDb(db); + + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + + const repo = new DrizzlePlayRepository(db, {componentId: component[0].id}); + + const playData: RepositoryCreatePlayOpts[] = [ + { + ...fixtureCreatePlay({ play: generatePlay({ playDate: dayjs().subtract(2, 'm') }) }), + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay({ play: generatePlay({ playDate: dayjs().subtract(6, 'm') }) }), + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay({ play: generatePlay({ playDate: dayjs().subtract(8, 'm') }) }), + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay({ play: generatePlay({ playDate: dayjs().subtract(10, 'm') }) }), + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + ] + + await repo.createPlays(playData); + + const newerPlays = await repo.findPlays({ playedAt: { type: 'gt', date: dayjs().subtract(3, 'm') } }); + expect(newerPlays).length(1); + expect(newerPlays[0].play.data.track).eq(playData[0].play.data.track); + + const olderPlays = await repo.findPlays({ playedAt: { type: 'lt', date: dayjs().subtract(6, 'm').subtract(5, 's') } }); + expect(olderPlays).length(2); + expect(olderPlays[0].play.data.track).eq(playData[2].play.data.track); + expect(olderPlays[1].play.data.track).eq(playData[3].play.data.track); + + const bwPlays = await repo.findPlays({ playedAt: { type: 'between', range: [dayjs().subtract(9, 'm'), dayjs().subtract(3, 'm')] } }); + expect(bwPlays).length(2); + expect(bwPlays[0].play.data.track).eq(playData[1].play.data.track); + expect(bwPlays[1].play.data.track).eq(playData[2].play.data.track); + }); + + it('finds Plays by component', async function () { + + const db = await transientDb(); + await migrateDb(db); + + const component1 = await db.insert(components).values(fixtureCreateComponent()).returning(); + const component2 = await db.insert(components).values(fixtureCreateComponent({ uid: 'test2', name: 'jelly2' })).returning(); + const component3 = await db.insert(components).values(fixtureCreateComponent({ uid: 'test3', name: 'jelly3' })).returning(); + + const repo = new DrizzlePlayRepository(db); + + const playData: RepositoryCreatePlayOpts[] = [ + { + ...fixtureCreatePlay(), + componentId: component1[0].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay(), + componentId: component3[0].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay(), + componentId: component3[0].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + } + ] + + await repo.createPlays(playData); + + const plays = await repo.findPlays({ componentId: component3[0].id }); + expect(plays).length(2); + expect(plays[0].play.data.track).eq(playData[1].play.data.track); + expect(plays[1].play.data.track).eq(playData[2].play.data.track); + + const plays1 = await repo.findPlays({ componentId: component1[0].id }); + expect(plays1).length(1); + expect(plays1[0].play.data.track).eq(playData[0].play.data.track); + + const noPlays = await repo.findPlays({ componentId: component2[0].id }); + expect(noPlays).length(0); + }); + + it('finds purgable Plays', async function () { + + const db = await transientDb(); + await migrateDb(db); + + const component1 = await db.insert(components).values(fixtureCreateComponent()).returning(); + const component2 = await db.insert(components).values(fixtureCreateComponent({ uid: 'test2', name: 'jelly2' })).returning(); + + const repo = new DrizzlePlayRepository(db); + + const playData: RepositoryCreatePlayOpts[] = [ + { + ...fixtureCreatePlay({play: generatePlay({},{seenAt: dayjs().subtract(25, 'h')})}), + componentId: component1[0].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay({play: generatePlay({},{seenAt: dayjs().subtract(26, 'h')})}), + componentId: component1[0].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay({play: generatePlay({},{seenAt: dayjs().subtract(26, 'h')})}), + componentId: component2[0].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + { + ...fixtureCreatePlay({play: generatePlay({},{seenAt: dayjs().subtract(25, 'h').subtract(1, 'm')})}), + componentId: component1[0].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + ] + + const initialPlays = await repo.createPlays(playData); + + const childPlays = await repo.createPlays([ + { + ...fixtureCreatePlay({play: generatePlay({},{seenAt: dayjs().subtract(25, 'h')})}), + componentId: component2[0].id, + parentId: initialPlays[1].id, + state: 'queued' as 'queued', + input: { data: generateRandomObj(undefined, { allowUndefined: false }) } + }, + ]) + + // does not return newer plays + expect((await repo.findPurgablePlayIds(dayjs().subtract(27, 'h'), {componentId: 1}))).length(0); + + const pPlays = await repo.findPurgablePlayIds(dayjs().subtract(24, 'h'), {componentId: 1}); + // only returns plays that do not have children + expect(pPlays).length(2); + expect(pPlays[0]).to.eq(initialPlays[0].id); + expect(pPlays[1]).to.eq(initialPlays[3].id); + + // only finds plays by component + // and allows purging if they have parent id + const p2Plays = await repo.findPurgablePlayIds(dayjs().subtract(23, 'h'), {componentId: 2}); + expect(p2Plays).length(2); + expect(p2Plays[0]).to.eq(initialPlays[2].id); + expect(p2Plays[1]).to.eq(childPlays[0].id); + }); + + // it('Get json property from play', async function () { + + // const db = getDb(':memory:', { workingDirectory: process.cwd() }); + // await migrateDb(db); + + // try { + + // const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + + // const playRows = await db.insert(plays).values([ + // fixtureCreatePlay({ componentId: component[0].id, play: generatePlay({}, {source: 'test1'}) }), + // fixtureCreatePlay({ componentId: component[0].id, play: generatePlay({}, {source: 'test2'}) }) + // ]).returning(); + + // let result: PlaySelect[]; + // // https://github.com/drizzle-team/drizzle-orm/discussions/938#discussioncomment-6542336 + // result = await db.select().from(plays).where( + // sql`json_extract(${plays.play}, '$.meta.source') = 'test1'` + // ); + + // expect(result).length(1); + // expect(result[0].play.meta.source).eq('test1'); + + // } catch (e) { + // throw e; + // } + // db.$client.close(); + // }); + +}); + +describe('DB Size Stats', function () { + + + before(function () { + if (process.env.DB_SIZE_TEST !== 'true') { + this.skip(); + } + }); + + it('get empty db size stats', async function () { + + await withLocalTmpDir(async () => { + try { + let db = await getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd()))); + await migrateDb(db); + const play100Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]); + loggerDebug.debug(`100 Plays => ${formatNumber((play100Component / 1024) / 1024, {toFixed: 2})}mb`); + } catch (e) { + throw e; + } + }, { unsafeCleanup: true, postfix: 'dbStatEmpty' }); + }); + + it('get db plays size stats', async function () { + + this.timeout(10000); + + await withLocalTmpDir(async () => { + try { + let db = await getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd()))); + await migrateDb(db); + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + + const playRepo = new DrizzlePlayRepository(db); + const playData = generateArray(100, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlay() }), state: 'queued', input: { data: undefined } })); + await playRepo.createPlays(playData); + + const play100Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]); + loggerDebug.debug(`100 Plays => ${formatNumber((play100Component / 1024) / 1024, {toFixed: 2})}mb`); + + const morePlayData = generateArray(900, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlay() }), state: 'queued', input: { data: undefined }})); + await playRepo.createPlays(morePlayData); + const play1000Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]); + loggerDebug.debug(`1000 Plays => ${formatNumber((play1000Component / 1024) / 1024, {toFixed: 2})}mb`); + + for(let i = 0; i < 9; i++) { + const evenMorePlayData = generateArray(1000, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlay() }), state: 'queued', input: { data: undefined }})); + await playRepo.createPlays(evenMorePlayData); + } + + const play10000Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]); + loggerDebug.debug(`10000 Plays => ${formatNumber((play10000Component / 1024) / 1024, {toFixed: 2})}mb`); + } catch (e) { + throw e; + } + }, { unsafeCleanup: true, postfix: 'dbStatPlain' }); + }); + + it('get db plays size stats with input', async function () { + + this.timeout(10000); + + await withLocalTmpDir(async () => { + try { + let db = await getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd()))); + await migrateDb(db); + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + + const playRepo = new DrizzlePlayRepository(db); + const playData = generateArray(100, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlay() }), state: 'queued', input: { data: generateRandomObj(undefined, { allowUndefined: false }) } })); + await playRepo.createPlays(playData); + + const play100Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]); + loggerDebug.debug(`100 Plays => ${formatNumber((play100Component / 1024) / 1024, {toFixed: 2})}mb`); + + const morePlayData = generateArray(900, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlay() }), state: 'queued', input: { data: generateRandomObj(undefined, { allowUndefined: false }) } })); + await playRepo.createPlays(morePlayData); + const play1000Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]); + loggerDebug.debug(`1000 Plays => ${formatNumber((play1000Component / 1024) / 1024, {toFixed: 2})}mb`); + + for(let i = 0; i < 9; i++) { + const evenMorePlayData = generateArray(1000, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlay() }), state: 'queued', input: { data: generateRandomObj(undefined, { allowUndefined: false }) } })); + await playRepo.createPlays(evenMorePlayData); + } + const play10000Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]); + loggerDebug.debug(`10000 Plays => ${formatNumber((play10000Component / 1024) / 1024, {toFixed: 2})}mb`); + } catch (e) { + throw e; + } + }, { unsafeCleanup: true, postfix: 'dbStatInput' }); + }); + + it('get db plays size stats with input and lifecycle', async function () { + + this.timeout(10000); + + await withLocalTmpDir(async () => { + try { + let db = await getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd()))); + await migrateDb(db); + const component = await db.insert(components).values(fixtureCreateComponent()).returning(); + + const playRepo = new DrizzlePlayRepository(db); + const playData = generateArray(100, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlayWithLifecycle({lifecycleSteps: {preCompare: 1}}) }), state: 'queued', input: { data: generateRandomObj(undefined, { allowUndefined: false }) } })); + await playRepo.createPlays(playData); + + const play100Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]); + loggerDebug.debug(`100 Plays => ${formatNumber((play100Component / 1024) / 1024, {toFixed: 2})}mb`); + + const morePlayData = generateArray(900, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlayWithLifecycle({lifecycleSteps: {preCompare: 1}}) }), state: 'queued', input: { data: generateRandomObj(undefined, { allowUndefined: false }) } })); + await playRepo.createPlays(morePlayData); + const play1000Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]); + loggerDebug.debug(`1000 Plays => ${formatNumber((play1000Component / 1024) / 1024, {toFixed: 2})}mb`); + + for(let i = 0; i < 9; i++) { + const evenMorePlayData = generateArray(1000, () => ({ ...fixtureCreatePlay({ componentId: component[0].id, play: generatePlayWithLifecycle({lifecycleSteps: {preCompare: 1}}) }), state: 'queued', input: { data: generateRandomObj(undefined, { allowUndefined: false }) } })); + await playRepo.createPlays(evenMorePlayData); + } + + const play10000Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]); + loggerDebug.debug(`10000 Plays => ${formatNumber((play10000Component / 1024) / 1024, {toFixed: 2})}mb`); + } catch (e) { + throw e; + } + }, { unsafeCleanup: true, postfix: 'dbStatAll' }); + }); +}) \ No newline at end of file diff --git a/src/backend/tests/lastfm/lastfm.test.ts b/src/backend/tests/lastfm/lastfm.test.ts index 1623f4548..a9400e51d 100644 --- a/src/backend/tests/lastfm/lastfm.test.ts +++ b/src/backend/tests/lastfm/lastfm.test.ts @@ -9,16 +9,17 @@ import { http, HttpResponse, delay } from "msw"; import { loggerDebug } from '@foxxmd/logging'; import { configDir, projectDir } from '../../common/index.js'; import { LastFMGeo } from 'lastfm-ts-api'; +import { artistNamesToCredits } from '../../../core/StringUtils.js'; chai.use(asPromised); describe('#LFM Scrobble Payload Behavior', function () { it('Should remove VA from album artist', function() { - const play = generatePlay({albumArtists: ['VA']}); + const play = generatePlay({albumArtists: artistNamesToCredits(['VA'])}); expect(playToClientPayload(play).albumArtist).to.be.undefined; - const okPlay = generatePlay({albumArtists: ['My Dude']}); + const okPlay = generatePlay({albumArtists: artistNamesToCredits(['My Dude'])}); expect(playToClientPayload(okPlay).albumArtist).eq('My Dude'); }); }); @@ -55,17 +56,17 @@ describe('#LFM Track to Play', function() { const toArtText = generateLastfmTrackObject(); delete toArtText.artist.name; expect(toArtText.artist['#text']).to.not.be.undefined; - expect(formatPlayObj(toArtText).data.artists[0]).to.eq(toArtText.artist['#text']); + expect(formatPlayObj(toArtText).data.artists[0].name).to.eq(toArtText.artist['#text']); const toArtTextEmptyNAme = generateLastfmTrackObject(); toArtTextEmptyNAme.artist.name = ''; expect(toArtTextEmptyNAme.artist['#text']).to.not.be.undefined; - expect(formatPlayObj(toArtTextEmptyNAme).data.artists[0]).to.eq(toArtTextEmptyNAme.artist['#text']); + expect(formatPlayObj(toArtTextEmptyNAme).data.artists[0].name).to.eq(toArtTextEmptyNAme.artist['#text']); const toArtName = generateLastfmTrackObject(); delete toArtName.artist['#text']; expect(toArtName.artist.name).to.not.be.undefined; - expect(formatPlayObj(toArtName).data.artists[0]).to.eq(toArtName.artist.name); + expect(formatPlayObj(toArtName).data.artists[0].name).to.eq(toArtName.artist.name); }); }); diff --git a/src/backend/tests/listenbrainz/listenbrainz.test.ts b/src/backend/tests/listenbrainz/listenbrainz.test.ts index 88155d7ca..67b6ce1f1 100644 --- a/src/backend/tests/listenbrainz/listenbrainz.test.ts +++ b/src/backend/tests/listenbrainz/listenbrainz.test.ts @@ -25,6 +25,7 @@ import incorrectMultiArtistsTrackName from './incorrectlyMapped/multiArtistsInTr import veryWrong from './incorrectlyMapped/veryWrong.json' with { type: "json" }; import { generatePlay } from "../../../core/PlayTestUtils.js"; import { defaultLifecycle } from "../../utils/PlayTransformUtils.js"; +import { artistCreditsToNames, artistNamesToCredits } from "../../../core/StringUtils.js"; interface LZTestFixture { data: ListenResponse @@ -37,7 +38,7 @@ describe('#PlayParse Listenbrainz Listen Parsing', function () { for(const test of noArtistMapping as unknown as LZTestFixture[]) { const play = listenResponseToPlay(test.data); assert.equal(play.data.track, test.expected.track); - assert.sameDeepMembers(play.data.artists, test.expected.artists); + assert.sameDeepMembers(artistCreditsToNames(play.data.artists), test.expected.artists); } }); @@ -45,7 +46,7 @@ describe('#PlayParse Listenbrainz Listen Parsing', function () { for(const test of veryWrong as unknown as LZTestFixture[]) { const play = listenResponseToPlay(test.data); assert.equal(play.data.track, test.expected.track); - assert.sameDeepMembers( play.data.artists, test.expected.artists); + assert.sameDeepMembers( artistCreditsToNames(play.data.artists), test.expected.artists); } }); @@ -53,7 +54,7 @@ describe('#PlayParse Listenbrainz Listen Parsing', function () { for(const test of incorrectMultiArtistsTrackName as unknown as LZTestFixture[]) { const play = listenResponseToPlay(test.data); assert.equal(play.data.track, test.expected.track); - assert.sameDeepMembers(play.data.artists, test.expected.artists); + assert.sameDeepMembers(artistCreditsToNames(play.data.artists), test.expected.artists); } }); }) @@ -65,7 +66,7 @@ describe('#PlayParse Listenbrainz Listen Parsing', function () { for(const test of slightlyDifferentNames as unknown as LZTestFixture[]) { const play = listenResponseToPlay(test.data); assert.equal(play.data.track, test.expected.track); - assert.sameDeepMembers(play.data.artists, test.expected.artists); + assert.sameDeepMembers(artistCreditsToNames(play.data.artists), test.expected.artists); } }); @@ -73,7 +74,7 @@ describe('#PlayParse Listenbrainz Listen Parsing', function () { for(const test of multiMappedArtistsWithSingleUserArtist as unknown as LZTestFixture[]) { const play = listenResponseToPlay(test.data); assert.equal(play.data.track, test.expected.track); - assert.sameDeepMembers(play.data.artists, test.expected.artists); + assert.sameDeepMembers(artistCreditsToNames(play.data.artists), test.expected.artists); } }); @@ -81,7 +82,7 @@ describe('#PlayParse Listenbrainz Listen Parsing', function () { for(const test of artistWithProperJoiner as unknown as LZTestFixture[]) { const play = listenResponseToPlay(test.data); assert.equal(play.data.track, test.expected.track); - assert.sameDeepMembers( play.data.artists, test.expected.artists); + assert.sameDeepMembers( artistCreditsToNames(play.data.artists), test.expected.artists); } }); @@ -89,7 +90,7 @@ describe('#PlayParse Listenbrainz Listen Parsing', function () { for(const test of multiArtistInArtistName as unknown as LZTestFixture[]) { const play = listenResponseToPlay(test.data); assert.equal(play.data.track, test.expected.track); - assert.sameDeepMembers(play.data.artists, test.expected.artists); + assert.sameDeepMembers(artistCreditsToNames(play.data.artists), test.expected.artists); } }); @@ -97,7 +98,7 @@ describe('#PlayParse Listenbrainz Listen Parsing', function () { for(const test of multiArtistsInTrackName as unknown as LZTestFixture[]) { const play = listenResponseToPlay(test.data); assert.equal(play.data.track, test.expected.track); - assert.sameDeepMembers( play.data.artists, test.expected.artists); + assert.sameDeepMembers(artistCreditsToNames(play.data.artists), test.expected.artists); } }); @@ -105,7 +106,7 @@ describe('#PlayParse Listenbrainz Listen Parsing', function () { for(const test of normalizedValues as unknown as LZTestFixture[]) { const play = listenResponseToPlay(test.data); assert.equal(play.data.track, test.expected.track); - assert.sameDeepMembers( play.data.artists, test.expected.artists); + assert.sameDeepMembers(artistCreditsToNames(play.data.artists), test.expected.artists); } }); }); @@ -128,7 +129,7 @@ describe('Listenbrainz Response Behavior', function() { async function() { const play: PlayObject = { data: { - artists: ['Celldweller'], + artists: artistNamesToCredits(['Celldweller']), album: 'The Complete Cellout, Volume 01', track: 'Frozen', duration: 299, @@ -172,7 +173,7 @@ describe('Listenbrainz Response Behavior', function() { async function() { const play: PlayObject = { data: { - artists: ['Celldweller'], + artists: artistNamesToCredits(['Celldweller']), album: 'The Complete Cellout, Volume 01', track: 'Frozen', duration: 299, @@ -202,7 +203,7 @@ describe('Listenbrainz Endpoint Behavior', function() { it('Should combine artist and artist_names', function() { - const play = generatePlay({artists: ['Artist A'], albumArtists: []}); + const play = generatePlay({artists: artistNamesToCredits(['Artist A']), albumArtists: []}); const submitPayload = playToListenPayload(play); const additionalArtists = [...submitPayload.track_metadata.additional_info.artist_names, 'Artist B']; @@ -211,13 +212,13 @@ describe('Listenbrainz Endpoint Behavior', function() { const playFromPayload = listenPayloadToPlay(submitPayload); - expect(playFromPayload.data.artists).to.be.eql(additionalArtists) + expect(artistCreditsToNames(playFromPayload.data.artists)).to.be.eql(additionalArtists) }); it('Should combine artist and artist_names into a unique array', function() { - const play = generatePlay({artists: ['Artist A'], albumArtists: []}); + const play = generatePlay({artists: artistNamesToCredits(['Artist A']), albumArtists: []}); const submitPayload = playToListenPayload(play); const additionalArtists = ['Artist A', 'Artist B']; @@ -226,13 +227,13 @@ describe('Listenbrainz Endpoint Behavior', function() { const playFromPayload = listenPayloadToPlay(submitPayload); - expect(playFromPayload.data.artists).to.be.eql(['Artist A', 'Artist B']) + expect(artistCreditsToNames(playFromPayload.data.artists)).to.be.eql(['Artist A', 'Artist B']) }); it('Should set music_service_name from source', function() { - const play = generatePlay({artists: ['Artist A'], albumArtists: []}, {source: 'Plex'}); + const play = generatePlay({artists: artistNamesToCredits(['Artist A']), albumArtists: []}, {source: 'Plex'}); const submitPayload = playToListenPayload(play); expect(submitPayload.track_metadata.additional_info.music_service_name).to.be.eql('Plex') @@ -243,7 +244,7 @@ describe('Listenbrainz Endpoint Behavior', function() { const playFromPayload = listenPayloadToPlay(submit); - expect(playFromPayload.data.artists).to.be.eql(submit.track_metadata.additional_info.artist_names); + expect(artistCreditsToNames(playFromPayload.data.artists)).to.be.eql(submit.track_metadata.additional_info.artist_names); }); diff --git a/src/backend/tests/listenbrainz/submitListensRequestExample.json b/src/backend/tests/listenbrainz/submitListensRequestExample.json new file mode 100644 index 000000000..88a740180 --- /dev/null +++ b/src/backend/tests/listenbrainz/submitListensRequestExample.json @@ -0,0 +1,70 @@ +{ + "listen_type": "single", + "payload": [ + { + "listened_at": 0, + "track_metadata": { + "release_name": "Adore (2014 Remaster)", + "additional_info": { + "date": "date", + "artist_names": [ + "artist_names", + "artist_names" + ], + "recording_mbid": "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "release_artist_name": "release_artist_name", + "artist_mbids": [ + "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "046b6c7f-0b8a-43b9-b35d-6489e6daee91" + ], + "work_mbids": [ + "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "046b6c7f-0b8a-43b9-b35d-6489e6daee91" + ], + "duration": 5, + "discnumber": 9, + "spotify_album_artist_ids": [ + "spotify_album_artist_ids", + "spotify_album_artist_ids" + ], + "genre": "genre", + "media_player_version": "media_player_version", + "origin_url": "origin_url", + "submission_client": "submission_client", + "trackNumber": "trackNumber", + "spotify_id": "spotify_id", + "spotify_album_id": "spotify_album_id", + "submission_client_version": "submission_client_version", + "isrc": "isrc", + "release_artist_names": [ + "release_artist_names", + "release_artist_names" + ], + "youtube_id": "youtube_id", + "tags": [ + "tags", + "tags" + ], + "duration_ms": 2, + "spotify_artist_ids": [ + "spotify_artist_ids", + "spotify_artist_ids" + ], + "music_service_name": "music_service_name", + "recording_msid": "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "release_group_mbid": "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "comment": "comment", + "music_service": "music_service", + "track_mbid": "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "listening_from": "listening_from", + "media_player": "media_player", + "release_mbid": "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "tracknumber": 7, + "albumartist": "albumartist" + }, + "artist_name": "The Smashing Pumpkins", + "track_name": "17" + } + } + ] +} \ No newline at end of file diff --git a/src/backend/tests/listenbrainz/submitListensRequestMinimalExample.json b/src/backend/tests/listenbrainz/submitListensRequestMinimalExample.json new file mode 100644 index 000000000..ce8c056b1 --- /dev/null +++ b/src/backend/tests/listenbrainz/submitListensRequestMinimalExample.json @@ -0,0 +1,16 @@ +{ + "listen_type": "single", + "payload": [ + { + "listened_at": 1778211134, + "track_metadata": { + "release_name": "Adore (2014 Remaster)", + "additional_info": { + "duration": 20 + }, + "artist_name": "The Smashing Pumpkins", + "track_name": "17" + } + } + ] +} \ No newline at end of file diff --git a/src/backend/tests/musicbrainz/musicbrainz.test.ts b/src/backend/tests/musicbrainz/musicbrainz.test.ts index 16150b2b1..a07f1ac69 100644 --- a/src/backend/tests/musicbrainz/musicbrainz.test.ts +++ b/src/backend/tests/musicbrainz/musicbrainz.test.ts @@ -16,6 +16,7 @@ import { generatePlay, withBrainz } from '../../../core/PlayTestUtils.js'; import { intersect, missingMbidTypes } from '../../utils.js'; import { defaultLifecycle } from '../../utils/PlayTransformUtils.js'; import { CoverArtApiClient, CoverArtApiConfig } from '../../common/vendor/musicbrainz/CoverArtApiClient.js'; +import { artistCreditToName, artistNamesToCredits, artistNameToCredit } from '../../../core/StringUtils.js'; const envPath = path.join(projectDir, '.env'); dotenv.config({ path: envPath }); @@ -62,7 +63,7 @@ describe('Musicbrainz API', function () { const play: PlayObject = { data: { track: "Little Joe and Mary ii", - artists: ["Khruangbin"], + artists: artistNamesToCredits(["Khruangbin"]), album: "The Universe Smiles Upon You ii" }, meta: { @@ -89,7 +90,7 @@ describe('Musicbrainz API', function () { track: 'Cyber Space (CrossWorlds Remix): Final Lap (No Chants)', album: "Sonic Racing: CrossWorlds Original Soundtrack - Echoes of Dimensions", artists: [ - "Kanon Oguni" + artistNameToCredit("Kanon Oguni") ] }, meta: { @@ -118,7 +119,7 @@ describe('Musicbrainz API', function () { const play: PlayObject = { data: { track: "Fake", - artists: ["Fake"], + artists: artistNamesToCredits(["Fake"]), album: "Fake", meta: { brainz: { @@ -151,7 +152,7 @@ describe('Musicbrainz API', function () { const play: PlayObject = { data: { track: "Fake", - artists: ["Fake"], + artists: artistNamesToCredits(["Fake"]), album: "Fake", isrc: 'GBAHT1600302' }, @@ -177,8 +178,8 @@ describe('Musicbrainz API', function () { const play: PlayObject = { data: { track: "Berghain", - artists: ["ROSALÍA", "Björk", "Yves Tumor"], - albumArtists: ["ROSALÍA"], + artists: artistNamesToCredits(["ROSALÍA", "Björk", "Yves Tumor"]), + albumArtists: artistNamesToCredits(["ROSALÍA"]), album: "LUX", meta: { brainz: { @@ -211,8 +212,8 @@ describe('Musicbrainz API', function () { const play: PlayObject = { data: { track: "Berghain", - artists: ["ROSALÍA", "Björk", "Yves Tumor"], - albumArtists: ["ROSALÍA"], + artists: artistNamesToCredits(["ROSALÍA", "Björk", "Yves Tumor"]), + albumArtists: artistNamesToCredits(["ROSALÍA"]), album: "LUX", meta: { brainz: { @@ -246,7 +247,7 @@ describe('Musicbrainz API', function () { const play: PlayObject = { data: { track: "Roulette Road (CrossWorlds Remix)", - artists: ["Takahiro Kai, SEGA GAME MUSIC & SEGA SOUND TEAM"], + artists: artistNamesToCredits(["Takahiro Kai, SEGA GAME MUSIC & SEGA SOUND TEAM"]), album: "Sonic Racing: CrossWorlds Original Soundtrack - Echoes of Dimensions" }, meta: { @@ -271,7 +272,7 @@ describe('Musicbrainz API', function () { const play: PlayObject = { data: { track: "Undefeatable (feat. Kellin Quinn)", - artists: ["SEGA Sound Team / Tomoya Ohtani"], + artists: artistNamesToCredits(["SEGA Sound Team / Tomoya Ohtani"]), }, meta: { lifecycle: defaultLifecycle() @@ -296,7 +297,7 @@ describe('Musicbrainz API', function () { const play: PlayObject = { data: { track: "Bad Apple!! feat.SEKAI", - artists: ["、ナイトコードで。"], + artists: artistNamesToCredits(["、ナイトコードで。"]), album: "25時、ナイトコードで。 SEKAI ALBUM Vol.3" }, meta: { @@ -321,7 +322,7 @@ describe('Musicbrainz API', function () { const play: PlayObject = { data: { track: "HIBANA - Reloaded - (feat. 星乃一歌 & Hatsune Miku)", - artists: ["Leo/need"], + artists: artistNamesToCredits(["Leo/need"]), album: "Leo / need SEKAI ALBUM Vol.1" }, meta: { @@ -350,7 +351,7 @@ describe('Musicbrainz API', function () { const play: PlayObject = { data: { track: "Price", - artists: ["ATLUS Sound Team"], + artists: artistNamesToCredits(["ATLUS Sound Team"]), album: "PERSONA5 ORIGINAL SOUNDTRACK", isrc: 'JPK651601515' }, @@ -395,7 +396,7 @@ describe('Musicbrainz API', function () { const play: PlayObject = { data: { track: "Little Joe and Mary ii", - artists: ["Khruangbin"], + artists: artistNamesToCredits(["Khruangbin"]), album: "The Universe Smiles Upon You ii" }, meta: { @@ -429,7 +430,7 @@ describe('Musicbrainz API', function () { const play: PlayObject = { data: { track: "Little Joe and Mary ii", - artists: ["Khruangbin"], + artists: artistNamesToCredits(["Khruangbin"]), album: "The Universe Smiles Upon You ii" }, meta: { diff --git a/src/backend/tests/plays/mixedDuration.json b/src/backend/tests/plays/mixedDuration.json index a8ee964eb..b3c856fb3 100644 --- a/src/backend/tests/plays/mixedDuration.json +++ b/src/backend/tests/plays/mixedDuration.json @@ -2,7 +2,7 @@ { "data": { "artists": [ - "Kaidi Tatham" + {"name": "Kaidi Tatham"} ], "track": "Fricassee", "duration": 305, @@ -12,7 +12,7 @@ { "data": { "artists": [ - "Kuna Maze" + {"name": "Kuna Maze"} ], "track": "Jimbó", "playDate": "2023-09-20T14:46:00.000Z" @@ -21,8 +21,8 @@ { "data": { "artists": [ - "Norman Person", - "Shamek Farrah" + {"name": "Norman Person"}, + {"name": "Shamek Farrah"} ], "track": "Aisha", "duration": 559, @@ -32,8 +32,8 @@ { "data": { "artists": [ - "Frédéric Chopin", - "Krystian Zimerman" + {"name": "Frédéric Chopin"}, + {"name": "Krystian Zimerman"} ], "track": "Ballade No. 4 in F Minor, Op. 52", "duration": 575, @@ -43,8 +43,8 @@ { "data": { "artists": [ - "Nidia Gongora", - "The Bongo Hop" + {"name": "Nidia Gongora"}, + {"name": "The Bongo Hop"} ], "track": "Sonora", "duration": 327, @@ -55,7 +55,7 @@ { "data": { "artists": [ - "Cheo Feliciano" + {"name": "Cheo Feliciano"} ], "track": "Aprieta (Oye Cómo Va)", "playDate": "2023-09-20T15:38:12.000Z" @@ -64,7 +64,7 @@ { "data": { "artists": [ - "Pink Floyd" + {"name": "Pink Floyd"} ], "track": "Another Brick in the Wall, Pt. 1", "playDate": "2023-09-20T15:39:12.000Z" @@ -73,8 +73,8 @@ { "data": { "artists": [ - "Frédéric Chopin", - "Krystian Zimerman" + {"name": "Frédéric Chopin"}, + {"name": "Krystian Zimerman"} ], "track": "Ballade No. 1 in G Minor, Op. 23", "duration": 575, @@ -85,9 +85,9 @@ { "data": { "artists": [ - "A Collection of Boogie", - "Jazz Funk", - "Disco with Mehdi El" + {"name": "A Collection of Boogie"}, + {"name": "Jazz Funk"}, + {"name": "Disco with Mehdi El"} ], "track": "Aquil", "listenedFor": 300, @@ -97,7 +97,7 @@ { "data": { "artists": [ - "Outkast" + {"name": "Outkast"} ], "track": "Da Art of Storytellin' (Pt. 1)", "duration": 422, @@ -107,13 +107,13 @@ { "data": { "artists": [ - "9th Wonder", - "Cordae", - "Dinner Party", - "Kamasi Washington", - "Phoelix", - "Robert Glasper", - "Terrace Martin" + {"name": "9th Wonder"}, + {"name": "Cordae"}, + {"name": "Dinner Party"}, + {"name": "Kamasi Washington"}, + {"name": "Phoelix"}, + {"name": "Robert Glasper"}, + {"name": "Terrace Martin"} ], "track": "Freeze Tag", "album": "Dinner Party: Dessert", diff --git a/src/backend/tests/plays/withDuration.json b/src/backend/tests/plays/withDuration.json index 38949e7d6..f20e32e38 100644 --- a/src/backend/tests/plays/withDuration.json +++ b/src/backend/tests/plays/withDuration.json @@ -2,7 +2,7 @@ { "data": { "artists": [ - "Kaidi Tatham" + {"name": "Kaidi Tatham"} ], "track": "Fricassee", "duration": 305, @@ -16,7 +16,7 @@ { "data": { "artists": [ - "Kuna Maze" + {"name": "Kuna Maze"} ], "track": "Jimbó", "duration": 255, @@ -30,8 +30,8 @@ { "data": { "artists": [ - "Norman Person", - "Shamek Farrah" + {"name": "Norman Person"}, + {"name": "Shamek Farrah"} ], "track": "Aisha", "duration": 559, @@ -45,8 +45,8 @@ { "data": { "artists": [ - "Frédéric Chopin", - "Krystian Zimerman" + {"name": "Frédéric Chopin"}, + {"name": "Krystian Zimerman"} ], "track": "Ballade No. 4 in F Minor, Op. 52", "duration": 575, @@ -60,8 +60,8 @@ { "data": { "artists": [ - "Nidia Gongora", - "The Bongo Hop" + {"name": "Nidia Gongora"}, + {"name": "The Bongo Hop"} ], "track": "Sonora", "duration": 327, @@ -75,7 +75,7 @@ { "data": { "artists": [ - "Cheo Feliciano" + {"name": "Cheo Feliciano"} ], "track": "Aprieta (Oye Cómo Va)", "duration": 195, @@ -89,7 +89,7 @@ { "data": { "artists": [ - "Pink Floyd" + {"name": "Pink Floyd"} ], "track": "Another Brick in the Wall, Pt. 1", "duration": 192, @@ -103,8 +103,8 @@ { "data": { "artists": [ - "Frédéric Chopin", - "Krystian Zimerman" + {"name": "Frédéric Chopin"}, + {"name": "Krystian Zimerman"} ], "track": "Ballade No. 1 in G Minor, Op. 23", "duration": 575, @@ -118,9 +118,9 @@ { "data": { "artists": [ - "A Collection of Boogie", - "Jazz Funk", - "Disco with Mehdi El" + {"name": "A Collection of Boogie"}, + {"name": "Jazz Funk"}, + {"name": "Disco with Mehdi El"} ], "track": "Aquil", "duration": 230, @@ -134,7 +134,7 @@ { "data": { "artists": [ - "Outkast" + {"name": "Outkast"} ], "track": "Da Art of Storytellin' (Pt. 1)", "duration": 422, diff --git a/src/backend/tests/scrobbler/TestScrobbler.ts b/src/backend/tests/scrobbler/TestScrobbler.ts index bd17997eb..60eeaea7c 100644 --- a/src/backend/tests/scrobbler/TestScrobbler.ts +++ b/src/backend/tests/scrobbler/TestScrobbler.ts @@ -7,18 +7,27 @@ import { CommonClientConfig, CommonClientOptions, NowPlayingOptions } from "../. import clone from "clone"; import { TimeRangeListensFetcher } from "../../common/infrastructure/Atomic.js"; import { loggerNoop } from "../../common/MaybeLogger.js"; +import { DrizzlePlayRepository, RepositoryCreatePlayOpts } from "../../common/database/drizzle/repositories/PlayRepository.js"; +import { DrizzleQueueRepository } from "../../common/database/drizzle/repositories/QueueRepository.js"; +import { PlaySelect } from "../../common/database/drizzle/drizzleTypes.js"; +import { loggerDebug } from "@foxxmd/logging"; export class TestScrobbler extends AbstractScrobbleClient { testRecentScrobbles: PlayObject[] = []; getScrobblesForTimeRange: TimeRangeListensFetcher; + public playRepoTest: DrizzlePlayRepository; + public queueRepoTest: DrizzleQueueRepository; + constructor(config: CommonClientConfig = {name: 'test'}) { const logger = loggerNoop; const notifier = new Notifiers(new EventEmitter(), new EventEmitter(), new EventEmitter(), logger); super('test', 'Test', {name: 'test', ...config}, notifier, new EventEmitter(), logger); this.supportsNowPlaying = false; - this.getScrobblesForTimeRange = async (_) => this.testRecentScrobbles; + this.getScrobblesForTimeRange = async (_) => { + return this.testRecentScrobbles; + } this.scrobbleDelay = 10; this.scrobbleSleep = 20; this.scrobbleWaitStopInterval = 20; @@ -33,12 +42,21 @@ export class TestScrobbler extends AbstractScrobbleClient { return super.doParseCache(); } - + protected async postDatabase(): Promise { + super.postDatabase(); + this.playRepoTest = this.playRepo; + this.queueRepoTest = this.queueRepo; + } playToClientPayload(playObject: PlayObject): object { return playObject; } + addScrobbled = async (plays: PlayObject[]): Promise => { + const newPlayData: RepositoryCreatePlayOpts[] = plays.map(x => ({play: x, state: 'scrobbled', input: {}})); + return await this.playRepoTest.createPlays(newPlayData); + } + } export class TestAuthScrobbler extends TestScrobbler { diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts index a49d17a09..9dca0e895 100644 --- a/src/backend/tests/scrobbler/scrobblers.test.ts +++ b/src/backend/tests/scrobbler/scrobblers.test.ts @@ -6,7 +6,7 @@ import dayjs from "dayjs"; import { after, before, describe, it } from 'mocha'; import { http, HttpResponse } from 'msw'; import pEvent from 'p-event'; -import { PlayObject, SOURCE_SOT } from "../../../core/Atomic.js"; +import { CLIENT_INGRESS_QUEUE, PlayObject, SOURCE_SOT } from "../../../core/Atomic.js"; import { sleep, sortByOldestPlayDate } from "../../utils.js"; import { genGroupIdStr } from '../../../core/PlayUtils.js'; import mixedDuration from '../plays/mixedDuration.json' with { type: 'json' }; @@ -20,10 +20,15 @@ import { PaginatedTimeRangeOptions, PlayPlatformId, REFRESH_STALE_DEFAULT } from import { defaultLifecycle } from '../../utils/PlayTransformUtils.js'; import { shuffleArray } from '../../utils/DataUtils.js'; import { DEFAULT_CONSOLIDATE_DURATION, DEFAULT_GROUP_DURATION, groupPlaysToTimeRanges } from '../../utils/ListenFetchUtils.js'; -import { asPlay } from '../../../core/tests/utils/fixtures.js'; +import { asPlay } from '../../../core/PlayMarshalUtils.js'; import { nanoid } from 'nanoid'; import { getRoot } from '../../ioc.js'; -import { transientCache } from '../utils/CacheTestUtils.js'; +import { transientCache } from '../utils/TransientTestUtils.js'; +import { generateArray } from '../../../core/DataUtils.js'; +import { RepositoryCreatePlayOpts } from '../../common/database/drizzle/repositories/PlayRepository.js'; +import { fixtureCreatePlay } from '../utils/databaseFixtures.js'; +import { isAbortError } from 'abort-controller-x'; +import { artistNamesToCredits } from '../../../core/StringUtils.js'; chai.use(asPromised); @@ -38,7 +43,7 @@ const normalizedWithMixedDur = normalizePlays(mixedDurPlays, {initialDate: first const normalizedWithMixedDurOlder = normalizePlays(mixedDurPlays, {initialDate: olderFirstPlayDate}); -const generateTestScrobbler = () => { +const generateTestScrobbler = async () => { const testScrobbler = new TestScrobbler(); testScrobbler.verboseOptions = { match: { @@ -47,6 +52,7 @@ const generateTestScrobbler = () => { confidenceBreakdown: true } }; + await testScrobbler.tryInitialize(); return testScrobbler; } @@ -118,7 +124,7 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu it('It is not detected as duplicate when play date is newer than most recent', async function () { - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = normalizedWithMixedDur; const newScrobble = generatePlay({ @@ -131,7 +137,7 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu it('It is not detected as duplicate when play date is close to an existing scrobble', async function () { - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = normalizedWithMixedDur; const newScrobble = generatePlay({ @@ -143,13 +149,11 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu it('It handles unique detection when no existing scrobble matches above a score of 0', async function () { - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = normalizedWithMixedDur; const uniquePlay = generatePlay({ - artists: [ - "２８１４" - ], + artists: artistNamesToCredits(["２８１４"]), track: "新宿ゴールデン街", duration: 130, playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(6, 'minutes') @@ -164,7 +168,7 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu it('Is not detected as duplicate when artist is same, time is similar, but track is different', async function () { - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = normalizedWithMixedDur; const diffPlay = clone(normalizedWithMixedDur[1]); @@ -176,12 +180,12 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu it('Is not detected as duplicate when track is same, time is similar, but artist is different', async function () { - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = normalizedWithMixedDur; const diffPlay = clone(normalizedWithMixedDur[1]); diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's'); - diffPlay.data.artists = ['A Different Artist']; + diffPlay.data.artists = artistNamesToCredits(['A Different Artist']); assert.isFalse((await testScrobbler.alreadyScrobbled(diffPlay))[0]); }); @@ -189,7 +193,7 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu it('Is not detected as duplicate when play date is different by more than 10 seconds (high granularity source)', async function () { - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = normalizedWithMixedDur; const timeOffPos = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]); @@ -208,7 +212,7 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu initialDate: firstPlayDate, defaultMeta: {source: 'subsonic'} }); - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = recent; const timeOffPos = clone(recent[recent.length - 1]); @@ -225,7 +229,7 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu it('A track with continuity to the previous track is not detected as a duplicate', async function () { - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = normalizedWithDur; const brickPt1 = normalizedWithDur.find(x => x.data.track.includes('Another Brick')); @@ -255,7 +259,7 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu initialDate: firstPlayDate, defaultMeta: {source: 'jellyfin'} }); - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = recent; const repeatPlay = clone(recent[recent.length - 1]); @@ -270,13 +274,13 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu it('Is not detected as duplicate when play date matches fuzzy and play source SOT is history', async function () { const play = generatePlay({ - artists: ['Nejad'], + artists: artistNamesToCredits(['Nejad']), track: 'CODE', album: undefined, playDate: dayjs().subtract(179, 's'), duration: 179 }); - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = [play]; const newPlay = clone(play); @@ -293,13 +297,13 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu describe('When scrobble is a duplicate (title/artists/album)', function () { it('Is detected as duplicate when an exact match', async function () { - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = normalizedWithMixedDur; assert.isTrue((await testScrobbler.alreadyScrobbled(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]))[0]); }); it('Is detected as duplicate when artist/title differences are whitespace or case', async function () { - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = normalizedWithMixedDur; const ref = normalizedWithMixedDur[3]; @@ -316,15 +320,15 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu diffPlay.data.track = ref.data.track.replaceAll(' ', ' '); assert.isTrue((await testScrobbler.alreadyScrobbled(diffPlay))[0]); - diffPlay.data.artists = ref.data.artists.map(x => x.toUpperCase()); + diffPlay.data.artists = ref.data.artists.map(x => ({...x, name: x.name.toUpperCase()})); assert.isTrue((await testScrobbler.alreadyScrobbled(diffPlay))[0]); - diffPlay.data.artists = ref.data.artists.map(x => x.replaceAll(' ', ' ')); + diffPlay.data.artists = ref.data.artists.map(x => ({...x, name: x.name.replaceAll(' ', ' ')})); assert.isTrue((await testScrobbler.alreadyScrobbled(diffPlay))[0]); }); it('Is detected as duplicate when artist/title differences are from unicode normalization', async function () { - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = normalizedWithMixedDur; const ref = normalizedWithMixedDur.find(x => x.data.track === 'Jimbó'); @@ -336,7 +340,7 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu it('Is detected as duplicate when play date is off by 10 seconds or less (high granularity source)', async function () { - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = normalizedWithMixedDur; const timeOffPos = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]); @@ -366,7 +370,7 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu initialDate: firstPlayDate, defaultMeta: {source: 'subsonic'} }); - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = recent; const timeOffPos = clone(recent[recent.length - 1]); @@ -380,7 +384,7 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu }); it('Is detected as duplicate when title is exact, artist is similar, and time is similar', async function () { - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = normalizedWithMixedDur; const ref = normalizedWithMixedDur[3]; @@ -406,7 +410,7 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu const spotifyPlay: PlayObject = { data: { - artists: [ + artists: artistNamesToCredits([ "Terrace Martin", "Robert Glasper", "9th Wonder", @@ -414,7 +418,7 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu "Dinner Party", "Cordae", "Phoelix" - ], + ]), album: "Dinner Party: Dessert", track: "Freeze Tag (feat. Cordae & Phoelix)", "duration": 191.375, @@ -425,7 +429,7 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu lifecycle: defaultLifecycle() } } - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = normalizedWithMixedDurOlder.concat(ref); assert.isTrue((await testScrobbler.alreadyScrobbled(spotifyPlay))[0]); @@ -435,7 +439,7 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu it('Is detected as duplicate when play date is close to the end of an existing scrobble', async function () { - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = normalizedWithDur; const timeEnd = clone(normalizedWithDur[normalizedWithMixedDur.length - 2]); @@ -465,7 +469,7 @@ describe('Detects duplicate and unique scrobbles using actively tracked scrobble playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds') }); - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); testScrobbler.testRecentScrobbles = normalizedWithMixedDur; const [matchedPlay, matchedData] = await testScrobbler.findExistingSubmittedPlayObj(newScrobble); @@ -477,9 +481,10 @@ describe('Detects duplicate and unique scrobbles using actively tracked scrobble const newScrobble = generatePlay({ playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds') }); - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); + await testScrobbler.playRepoTest.createPlays([{play: newScrobble, state: 'scrobbled', input: {}}]) testScrobbler.testRecentScrobbles = normalizedWithMixedDur; - testScrobbler.addScrobbledTrack(newScrobble, newScrobble); + //testScrobbler.addScrobbledTrack(newScrobble, newScrobble); const [matchedPlay, matchedData] = await testScrobbler.findExistingSubmittedPlayObj(newScrobble); @@ -491,9 +496,10 @@ describe('Detects duplicate and unique scrobbles using actively tracked scrobble const newScrobble = generatePlay({ playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds') }); - await using testScrobbler = generateTestScrobbler(); - testScrobbler.testRecentScrobbles = normalizedWithMixedDur; - testScrobbler.addScrobbledTrack(newScrobble, newScrobble); + await using testScrobbler = await generateTestScrobbler(); + //testScrobbler.testRecentScrobbles = normalizedWithMixedDur; + await testScrobbler.playRepoTest.createPlays([{play: newScrobble, state: 'scrobbled', input: {}}]) + //testScrobbler.addScrobbledTrack(newScrobble, newScrobble); const dupScrobble = clone(newScrobble); dupScrobble.data.playDate = newScrobble.data.playDate.add(2, 'seconds'); @@ -515,7 +521,7 @@ describe('Upstream Scrobbles', function() { it('Calls timerange func to get SOT scrobbles when none exists', async function() { const existingPlays = normalizePlays(generatePlays(3), {initialDate: dayjs().subtract(1, 'hour')}); - await using scrobbler = generateTestScrobbler(); + await using scrobbler = await generateTestScrobbler(); scrobbler.testRecentScrobbles = []; await scrobbler.tryInitialize(); scrobbler.testRecentScrobbles = existingPlays; @@ -534,7 +540,7 @@ describe('Upstream Scrobbles', function() { it('Uses cached timerange for closely grouped scrobbles', async function() { const existingPlays = normalizePlays(generatePlays(3), {initialDate: dayjs().subtract(1, 'hour')}); - await using scrobbler = generateTestScrobbler(); + await using scrobbler = await generateTestScrobbler(); scrobbler.testRecentScrobbles = []; await scrobbler.tryInitialize(); scrobbler.testRecentScrobbles = existingPlays; @@ -554,7 +560,7 @@ describe('Upstream Scrobbles', function() { it('Uses separate timerange calls when scrobbles are not closely grouped', async function() { const existingPlays = normalizePlays(generatePlays(3), {initialDate: dayjs().subtract(1, 'hour')}); - await using scrobbler = generateTestScrobbler(); + await using scrobbler = await generateTestScrobbler(); scrobbler.testRecentScrobbles = []; await scrobbler.tryInitialize(); scrobbler.testRecentScrobbles = existingPlays; @@ -575,7 +581,7 @@ describe('Upstream Scrobbles', function() { it('Gets fresh timerange if TTL of staleAfter has passed', async function() { const existingPlays = normalizePlays(generatePlays(3), {initialDate: dayjs().subtract(1, 'hour')}); - await using scrobbler = generateTestScrobbler(); + await using scrobbler = await generateTestScrobbler(); scrobbler.testRecentScrobbles = []; await scrobbler.tryInitialize(); scrobbler.testRecentScrobbles = existingPlays; @@ -604,23 +610,29 @@ describe('Upstream Scrobbles', function() { describe('Dead Scrobbles', function() { it('Processes all dead scrobbles', async function () { - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); await testScrobbler.initialize(); testScrobbler.testRecentScrobbles = []; - const deadPlays = generatePlays(3); - for(const dead of deadPlays) { - testScrobbler.addDeadLetterScrobble({source: 'test', play: dead, id: nanoid()}); + const queuedPlayed = await testScrobbler.playRepoTest.createPlays(generateArray(3, () => ({ ...fixtureCreatePlay(), state: 'queued', input: {} }))) + + for(const dead of queuedPlayed) { + await testScrobbler.addDeadLetterScrobble(dead); } + await testScrobbler.processDeadLetterQueue(); - await testScrobbler.tryStopScrobbling() - expect(testScrobbler.deadLetterScrobbles.length).eq(0); + await Promise.race([ + sleep(15000), + pEvent(testScrobbler.emitter, 'queueState') + ]) + + expect(testScrobbler.deadLetterQueued).eq(0); }); }); const normalizedScrobbler = async () => { - await using testScrobbler = generateTestScrobbler(); + await using testScrobbler = await generateTestScrobbler(); await testScrobbler.initialize(); testScrobbler.testRecentScrobbles = normalizedWithMixedDur; testScrobbler.scrobbleSleep = 500; @@ -669,7 +681,8 @@ describe('Scrobble client uses transform plays correctly', function() { track: 'my cool track' }); await testScrobbler.queueScrobble(newScrobble, 'test'); - expect(testScrobbler.queuedScrobbles[0].play.data.track).is.eq('my cool track'); + const queuedPlayedData = await testScrobbler.playRepoTest.getQueued(CLIENT_INGRESS_QUEUE); + expect(queuedPlayedData.data[0].play.data.track).is.eq('my cool track'); testScrobbler.scrobbleSleep = 100; testScrobbler.initScrobbleMonitoring().catch(console.error); @@ -731,12 +744,16 @@ describe('Scrobble client uses transform plays correctly', function() { }); const normalizedMonitoringScrobbler = async () => { - const testScrobbler = generateTestScrobbler(); + const testScrobbler = await generateTestScrobbler(); await testScrobbler.initialize(); testScrobbler.testRecentScrobbles = normalizedWithMixedDur; testScrobbler.scrobbleSleep = 100; testScrobbler.scrobbleDelay = 0; - testScrobbler.initScrobbleMonitoring().catch(console.error); + testScrobbler.initScrobbleMonitoring().catch((e) => { + if(!isAbortError(e)) { + console.error(e); + } + }); return testScrobbler; } @@ -1025,7 +1042,7 @@ describe('Now Playing', function() { await using npScrobbler = new NowPlayingScrobbler(); npScrobbler.nowPlayingTaskInterval = 10; await npScrobbler.initialize(); - npScrobbler.scheduler.startById('pn_task'); + npScrobbler.initTasks(); await npScrobbler.queuePlayingNow(generateSourcePlayerObj({play:generatePlay({}, {deviceId: genGroupIdStr(generatePlayPlatformId())})}), {type: 'jellyfin', name: 'test'}); @@ -1039,13 +1056,13 @@ describe('Now Playing', function() { await using npScrobbler = new NowPlayingScrobbler(); npScrobbler.nowPlayingTaskInterval = 10; await npScrobbler.initialize(); - npScrobbler.scheduler.startById('pn_task'); + npScrobbler.initTasks(); const now = dayjs(); await npScrobbler.queuePlayingNow(generateSourcePlayerObj({play:generatePlay({}, {deviceId: genGroupIdStr(generatePlayPlatformId())})}), {type: 'jellyfin', name: 'test'}); - const res = await Promise.race([pEvent(npScrobbler.emitter, 'nowPlayingUpdated'), sleep(12)]); + const res = await Promise.race([pEvent(npScrobbler.emitter, 'nowPlayingUpdated'), sleep(20)]); expect(res).is.not.undefined; @@ -1053,7 +1070,7 @@ describe('Now Playing', function() { await npScrobbler.queuePlayingNow(generateSourcePlayerObj({play:generatePlay({}, {deviceId: genGroupIdStr(generatePlayPlatformId())})}), {type: 'jellyfin', name: 'test'}); - const resUpdate = await Promise.race([pEvent(npScrobbler.emitter, 'nowPlayingUpdated'), sleep(12)]); + const resUpdate = await Promise.race([pEvent(npScrobbler.emitter, 'nowPlayingUpdated'), sleep(20)]); expect(resUpdate).is.not.undefined; }); diff --git a/src/backend/tests/setup.ts b/src/backend/tests/setup.ts index ecc3656a1..5e8d0d5a0 100644 --- a/src/backend/tests/setup.ts +++ b/src/backend/tests/setup.ts @@ -1,6 +1,19 @@ import { loggerTest } from '@foxxmd/logging'; import { getRoot } from "../ioc.js"; -import { transientCache } from './utils/CacheTestUtils.js'; +import { transientCache, transientDb } from './utils/TransientTestUtils.js'; -const root = getRoot({cache: transientCache, logger: loggerTest}); +// let transientD: DbConcrete; +// const transientDbFactory = () => { +// return getDb(transientD.$client.clone()) +// } + +// export async function mochaGlobalSetup() { +// transientD = getDb(':memory:'); +// await migrateDb(transientD); + +// const root = getRoot({cache: transientCache, logger: loggerTest, db: transientDb}); +// root.items.cache().init(); +// } + +const root = getRoot({cache: transientCache, logger: loggerTest, db: transientDb}); root.items.cache().init(); \ No newline at end of file diff --git a/src/backend/tests/source/source.test.ts b/src/backend/tests/source/source.test.ts index 8710c0cb3..48a2f4f32 100644 --- a/src/backend/tests/source/source.test.ts +++ b/src/backend/tests/source/source.test.ts @@ -19,24 +19,29 @@ import { RT_TICK_DEFAULT, setRtTick } from "../../sources/PlayerState/RealtimePl import { sleep } from "../../utils.js"; import DeezerInternalSource from "../../sources/DeezerInternalSource.js"; import { DeezerInternalSourceOptions } from "../../common/infrastructure/config/source/deezer.js"; +import { artistCreditsToNames } from "../../../core/StringUtils.js"; chai.use(asPromised); const emitter = new EventEmitter(); -const generateSource = () => { - return new TestSource('spotify', 'test', {}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter); +const generateSource = async () => { + const source = new TestSource('spotify', 'test-basic', {}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter); + await source.tryInitialize(); + return source; } -const generateMemorySource = (config: SourceConfig = {}) => { - const s = new TestMemorySource('spotify', 'test', config, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter); - s.buildTransformRules(); +const generateMemorySource = async (config: SourceConfig = {}) => { + const s = new TestMemorySource('spotify', 'test-memory', config, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter); + await s.tryInitialize(); + // s.buildTransformRules(); s.scheduler.stop(); return s; } -const generateMemoryPositionalSource = (config: SourceConfig = {}) => { - const s = new TestMemoryPositionalSource('spotify', 'test', config, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter); - s.buildTransformRules(); +const generateMemoryPositionalSource = async (config: SourceConfig = {}) => { + const s = new TestMemoryPositionalSource('spotify', 'test-positional', config, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter); + await s.tryInitialize(); + //s.buildTransformRules(); s.scheduler.stop(); return s; } @@ -44,7 +49,7 @@ const generateMemoryPositionalSource = (config: SourceConfig = {}) => { describe('Sources use transform plays correctly', function () { it('Transforms play on preCompare', async function() { - await using source = generateSource(); + await using source = await generateSource(); source.config.options = { playTransform: { preCompare: { @@ -67,7 +72,7 @@ describe('Sources use transform plays correctly', function () { }); it('Transforms play on postCompare', async function() { - await using source = generateSource(); + await using source = await generateSource(); source.config.options = { playTransform: { postCompare: { @@ -96,7 +101,7 @@ describe('Sources use transform plays correctly', function () { }); it('Transforms play existing comparison', async function() { - await using source = generateSource(); + await using source = await generateSource(); source.config.options = { playTransform: { compare: { @@ -123,7 +128,7 @@ describe('Sources use transform plays correctly', function () { }); it('Transforms play candidate comparison', async function() { - await using source = generateSource(); + await using source = await generateSource(); source.config.options = { playTransform: { compare: { @@ -159,7 +164,7 @@ describe('Sources correctly parse incoming payloads', function () { const play = SpotifySource.formatPlayObj(noAAPayload as SpotifyApi.CurrentPlaybackResponse); expect(play.data.track).eq('The Sandpits Of Zonhoven'); expect(play.data.album).eq('Bloodbags And Downtube Shifters'); - expect(play.data.artists).eql(['Dubmood', 'MASTER BOOT RECORD']); + expect(artistCreditsToNames(play.data.artists)).eql(['Dubmood', 'MASTER BOOT RECORD']); expect(play.data.albumArtists).to.be.empty; }); @@ -167,8 +172,8 @@ describe('Sources correctly parse incoming payloads', function () { const play = SpotifySource.formatPlayObj(spotifyPayload as SpotifyApi.CurrentPlaybackResponse); expect(play.data.track).eq('The Sandpits Of Zonhoven'); expect(play.data.album).eq('Bloodbags And Downtube Shifters'); - expect(play.data.artists).eql(['Dubmood', 'MASTER BOOT RECORD']); - expect(play.data.albumArtists).eql(['Dubmood']); + expect(artistCreditsToNames(play.data.artists)).eql(['Dubmood', 'MASTER BOOT RECORD']); + expect(artistCreditsToNames(play.data.albumArtists)).eql(['Dubmood']); }); it('Spotify parses payload with identical album artists correctly', function() { @@ -177,8 +182,8 @@ describe('Sources correctly parse incoming payloads', function () { const identicalArtistsPlay = SpotifySource.formatPlayObj(identicalArtistsPayload as SpotifyApi.CurrentPlaybackResponse); expect(identicalArtistsPlay.data.track).eq('The Sandpits Of Zonhoven'); expect(identicalArtistsPlay.data.album).eq('Bloodbags And Downtube Shifters'); - expect(identicalArtistsPlay.data.artists).eql(['Dubmood', 'MASTER BOOT RECORD']); - expect(identicalArtistsPlay.data.albumArtists).to.be.empty; + expect(artistCreditsToNames(identicalArtistsPlay.data.artists)).eql(['Dubmood', 'MASTER BOOT RECORD']); + expect(artistCreditsToNames(identicalArtistsPlay.data.albumArtists)).to.be.empty; }); }); @@ -192,8 +197,8 @@ describe('Player Cleanup', function () { setRtTick(1); }); - const cleanedUpDuration = async (generateSource: (config: SourceConfig) => MemorySource) => { - await using source = generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}}); + const cleanedUpDuration = async (generateSource: (config: SourceConfig) => Promise) => { + await using source = await generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}}); const initialDate = dayjs(); const initialState = generatePlayerStateData({position: 0, playData: {duration: 50}, stateUpdatedAt: initialDate, status: REPORTED_PLAYER_STATUSES.playing}); expect((await source.processRecentPlays([initialState])).length).to.be.eq(0); @@ -235,9 +240,9 @@ describe('Player Cleanup', function () { await cleanedUpDuration(generateMemoryPositionalSource); }); - const noScrobbleRediscoveryOnActive = async (generateSource: (config: SourceConfig) => MemorySource) => { + const noScrobbleRediscoveryOnActive = async (generateSource: (config: SourceConfig) => Promise) => { - await using source = generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}}); + await using source = await generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}}); const initialDate = dayjs(); const initialState = generatePlayerStateData({position: 0, playData: {duration: 50}, stateUpdatedAt: initialDate, status: REPORTED_PLAYER_STATUSES.playing}); expect((await source.processRecentPlays([initialState])).length).to.be.eq(0); @@ -303,9 +308,9 @@ describe('Player Cleanup', function () { await noScrobbleRediscoveryOnActive(generateMemoryPositionalSource); }); - const noScrobbleStale = async (generateSource: (config: SourceConfig) => MemorySource) => { + const noScrobbleStale = async (generateSource: (config: SourceConfig) => Promise) => { - await using source = generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}}); + await using source = await generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}}); const initialDate = dayjs(); // if player incorrectly counted stale time then 30s of actual play + 20s of stale time > scrobble threshold of 50% of 90s @@ -348,9 +353,9 @@ describe('Player Cleanup', function () { await noScrobbleStale(generateMemoryPositionalSource); }); - const scrobbleRediscoveryOnActive = async (generateSource: (config: SourceConfig) => MemorySource) => { + const scrobbleRediscoveryOnActive = async (generateSource: (config: SourceConfig) => Promise) => { - await using source = generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}}); + await using source = await generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}}); const initialDate = dayjs(); // if player incorrectly counted stale time then 30s of actual play + 20s of stale time > scrobble threshold of 50% of 90s @@ -421,8 +426,19 @@ describe('Player Cleanup', function () { }); }); -const generateDeezerSource = (options: DeezerInternalSourceOptions = {}) => { - return new DeezerInternalSource('test', {data: {arl: 'test'}, options}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter); +class DeezerTestSource extends DeezerInternalSource { + protected async doCheckConnection(): Promise { + return; + } + doAuthentication = async () => { + return true; + } +} + +const generateDeezerSource = async (options: DeezerInternalSourceOptions = {}) => { + const source = new DeezerTestSource('test', {data: {arl: 'test'}, options}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter); + await source.tryInitialize(); + return source; } const firstPlayDate = dayjs().subtract(1, 'hour'); const normalizedPlays = normalizePlays(generatePlays(6), {initialDate: firstPlayDate}); @@ -438,7 +454,7 @@ describe('Deezer Internal Source', function() { const fuzzyPlay = clone(targetPlay); fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); - const source = generateDeezerSource(); + const source = await generateDeezerSource(); source.discover([...normalizedPlays, interimPlay]); const discovered = await source.discover([fuzzyPlay]); @@ -455,7 +471,7 @@ describe('Deezer Internal Source', function() { const fuzzyPlay = clone(targetPlay); fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); - await using source = generateDeezerSource({fuzzyDiscoveryIgnore: true}); + await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: true}); await source.discover([...normalizedPlays, interimPlay]); const discovered = await source.discover([fuzzyPlay]); @@ -468,7 +484,7 @@ describe('Deezer Internal Source', function() { const fuzzyPlay = clone(targetPlay); fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); - await using source = generateDeezerSource({fuzzyDiscoveryIgnore: true}); + await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: true}); await source.discover(normalizedPlays); const discovered = await source.discover([fuzzyPlay]); @@ -482,7 +498,7 @@ describe('Deezer Internal Source', function() { fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); const morePlays = normalizePlays([...normalizedPlays, fuzzyPlay, ...generatePlays(2)], {initialDate: firstPlayDate}); - await using source = generateDeezerSource({fuzzyDiscoveryIgnore: true}); + await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: true}); const discovered = await source.discover(morePlays); expect(discovered.length).to.eq(morePlays.length); @@ -497,7 +513,7 @@ describe('Deezer Internal Source', function() { const fuzzyPlay = clone(targetPlay); fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); - await using source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); + await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); await source.discover([...normalizedPlays, interimPlay]); const discovered = await source.discover([fuzzyPlay]); @@ -511,7 +527,7 @@ describe('Deezer Internal Source', function() { const duringPlay = clone(targetPlay); duringPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration * 0.5, 's'); - await using source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); + await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); await source.discover([...normalizedPlays, interimPlay]); const discovered = await source.discover([duringPlay]); @@ -525,7 +541,7 @@ describe('Deezer Internal Source', function() { const fuzzyPlay = clone(targetPlay); fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration + 39, 's'); - await using source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); + await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); await source.discover([...normalizedPlays, interimPlay]); const discovered = await source.discover([fuzzyPlay]); @@ -538,7 +554,7 @@ describe('Deezer Internal Source', function() { const fuzzyPlay = clone(targetPlay); fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); - await using source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); + await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); await source.discover(normalizedPlays); const discovered = await source.discover([fuzzyPlay]); @@ -552,7 +568,7 @@ describe('Deezer Internal Source', function() { fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); const morePlays = normalizePlays([...normalizedPlays, fuzzyPlay, ...generatePlays(2)], {initialDate: firstPlayDate}); - await using source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); + await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); const discovered = await source.discover(morePlays); expect(discovered.length).to.eq(morePlays.length - 1); diff --git a/src/backend/tests/tealfm/tealfm.test.ts b/src/backend/tests/tealfm/tealfm.test.ts index f7ba19d33..0aa935f11 100644 --- a/src/backend/tests/tealfm/tealfm.test.ts +++ b/src/backend/tests/tealfm/tealfm.test.ts @@ -4,6 +4,7 @@ import { after, before, describe, it } from 'mocha'; import { generateLastfmTrackObject, generateMbid, generatePlay, generateTealPlayRecord } from "../../../core/PlayTestUtils.js"; import { AbstractBlueSkyApiClient, listRecordToPlay } from '../../common/vendor/bluesky/AbstractBlueSkyApiClient.js'; import dayjs from 'dayjs'; +import { artistCreditsToNames } from '../../../core/StringUtils.js'; chai.use(asPromised); @@ -19,7 +20,7 @@ describe('#tealfm Record to Play', function() { expect(play.data.album).eq(rec.value.releaseName); expect(play.data.playDate.unix()).eq(dayjs(rec.value.playedTime).unix()); expect(play.data.duration).eq(rec.value.duration); - expect(play.data.artists).eql(rec.value.artists.map(x => x.artistName)); + expect(artistCreditsToNames(play.data.artists)).eql(rec.value.artists.map(x => x.artistName)); expect(play.meta.user).eq(`did:plc:${did}`); expect(play.meta.playId).eq(tid); }); diff --git a/src/backend/tests/utils/CacheTestUtils.ts b/src/backend/tests/utils/CacheTestUtils.ts deleted file mode 100644 index 3b537f41c..000000000 --- a/src/backend/tests/utils/CacheTestUtils.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { loggerTest } from "@foxxmd/logging"; -import { MSCache } from "../../common/Cache.js"; - -export const transientCache = () => new MSCache(loggerTest, {scrobble: {provider: 'memory'}, auth: {provider: 'memory'}, metadata: {provider: 'memory'}}); \ No newline at end of file diff --git a/src/backend/tests/utils/TransientTestUtils.ts b/src/backend/tests/utils/TransientTestUtils.ts new file mode 100644 index 000000000..b327de1c1 --- /dev/null +++ b/src/backend/tests/utils/TransientTestUtils.ts @@ -0,0 +1,18 @@ +import { loggerTest } from "@foxxmd/logging"; +import { MSCache } from "../../common/Cache.js"; +import { getDb, migrateDbSync, migrateDb, DbConcrete } from "../../common/database/drizzle/drizzleUtils.js"; +import { getPrepopulatedMemoryPGlite } from "./databaseFixtures.js"; +import { PGlite } from "@electric-sql/pglite"; + +export const transientCache = () => new MSCache(loggerTest); + +let baseDb: PGlite; + +export const transientDb = async () => { + if(baseDb === undefined) { + baseDb = await getPrepopulatedMemoryPGlite(); + await migrateDb(await getDb(baseDb)); + } + const db = getDb((await baseDb.clone()) as Awaited); + return db; +} \ No newline at end of file diff --git a/src/backend/tests/utils/databaseFixtures.ts b/src/backend/tests/utils/databaseFixtures.ts new file mode 100644 index 000000000..135a5e6ed --- /dev/null +++ b/src/backend/tests/utils/databaseFixtures.ts @@ -0,0 +1,53 @@ +import { generatePlay } from "../../../core/PlayTestUtils.js"; +import { generateRandomObj } from "../../../core/tests/utils/fixtures.js"; +import { generateComponentEntity, generateInputEntity, generatePlayEntity } from "../../common/database/drizzle/entityUtils.js"; +import { PlayNew } from "../../common/database/drizzle/drizzleTypes.js"; +import { PlayInputNew } from "../../common/database/drizzle/drizzleTypes.js"; +import { ComponentNew } from "../../common/database/drizzle/drizzleTypes.js"; +import { ObjectPlayData } from "../../../core/Atomic.js"; +import { PGlite } from '@electric-sql/pglite' +import { dataDir } from '@electric-sql/pglite-prepopulatedfs' + +export const fixtureCreateComponent = (data: Partial = {}): ComponentNew => { + return generateComponentEntity( + { + uid: 'test', + mode: 'source', + type: 'jellyfin', + name: 'myJelly', + ...data + }); +} + +export const fixtureCreatePlay = (data: Partial = {}): PlayNew => { + const { + play = generatePlay(), + ...rest + } = data; + return generatePlayEntity(play, {seenAt: play.meta.seenAt ?? play.data.playDate, updatedAt: play.meta.seenAt ?? play.data.playDate, ...rest}); +} + +export const fixtureCreateInput = (data: PlayInputNew & { data?: object | false }): PlayInputNew => { + const { + data: inputData = generateRandomObj(), + ...rest + } = data; + let realData: undefined; + if(inputData !== false) { + realData = inputData; + } + return generateInputEntity({...rest, data: realData}); +} + +export const getPrepopulatedFSPGlite = async (dir: string) => { + return PGlite.create({ + dataDir: dir, + loadDataDir: await dataDir() + }); +} + +export const getPrepopulatedMemoryPGlite = async () => { + return PGlite.create({ + loadDataDir: await dataDir() + }); +} \ No newline at end of file diff --git a/src/backend/tests/ytm/ytm.test.ts b/src/backend/tests/ytm/ytm.test.ts index 838d8fd79..809d6a28c 100644 --- a/src/backend/tests/ytm/ytm.test.ts +++ b/src/backend/tests/ytm/ytm.test.ts @@ -14,7 +14,7 @@ import { ApiResponse } from 'youtubei.js'; chai.use(asPromised); -const createYtSource = (opts?: { +const createYtSource = async (opts?: { config?: YTMusicSourceConfig emitter?: EventEmitter }) => { @@ -27,7 +27,7 @@ const createYtSource = (opts?: { emitter = new EventEmitter } = opts || {}; const source = new YTMusicSource('test', config, { localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test' }, emitter); - source.buildTransformRules(); + await source.buildDatabase(); return source; } @@ -48,7 +48,7 @@ describe('Handles temporal inconsistency in history', function () { it(`Adds new, prepended track`, async function () { - const source = createYtSource(); + const source = await createYtSource(); const plays = [...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Yesterday' })]; @@ -72,7 +72,7 @@ describe('Handles temporal inconsistency in history', function () { it(`Adds bumped, prepended track`, async function () { - const source = createYtSource(); + const source = await createYtSource(); const plays = [...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Yesterday' })]; @@ -98,7 +98,7 @@ describe('Handles temporal inconsistency in history', function () { it(`Does not add appended track`, async function () { - const source = createYtSource(); + const source = await createYtSource(); const plays = [...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Yesterday' })]; @@ -122,7 +122,7 @@ describe('Handles temporal inconsistency in history', function () { this.timeout(3700); - const source = createYtSource(); + const source = await createYtSource(); const plays = [...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Yesterday' })]; @@ -139,7 +139,7 @@ describe('Handles temporal inconsistency in history', function () { const prependedPlays = [newPlay, ...plays]; expect(source.parseRecentAgainstResponse(prependedPlays).plays).length(1); - await sleep(1000); + await sleep(50); // YT returns outdated history // should be detected as append since "removed" track in last position from previous history is seen again @@ -147,12 +147,12 @@ describe('Handles temporal inconsistency in history', function () { expect(badAppend).to.deep.include({consistent: false, diffType: 'added', plays: []}); expect(badAppend.diffResults[2]).eq('append'); - await sleep(500); + await sleep(10); // contiuned outdated history expect(source.parseRecentAgainstResponse(plays)).to.deep.include({consistent: true, plays: []}); - await sleep(500); + await sleep(10); // correct, current history is finally returned correctly const recentHistoryResult = source.parseRecentAgainstResponse(prependedPlays); @@ -166,7 +166,7 @@ describe('Handles interim tracks', function () { it(`Does not add skipped plays`, async function () { - const source = createYtSource(); + const source = await createYtSource(); const plays = [...generatePlays(10, {playDate: dayjs().subtract(20, 'seconds')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(20, 'seconds')}, { comment: 'Yesterday' })]; @@ -193,7 +193,7 @@ describe('Handles interim tracks', function () { it(`Adds interim plays when discover time is plausible`, async function () { - const source = createYtSource(); + const source = await createYtSource(); const plays = [...generatePlays(10, {playDate: dayjs().subtract(2, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(2, 'minutes')}, { comment: 'Yesterday' })]; @@ -211,7 +211,7 @@ describe('Handles interim tracks', function () { const interimPlays = [generatePlay({duration: 40}, { comment: 'Today' }), generatePlay({duration: 200}, { comment: 'Today' })] const prependedPlays = [firstPlay, ...interimPlays, ...plays]; const prependResult = source.parseRecentAgainstResponse(prependedPlays); - expect(prependResult.plays).length(2); + expect(prependResult.plays).length(1); expect(prependResult.plays[prependResult.plays.length - 1].data.track).eq(firstPlay.data.track) }); }); \ No newline at end of file diff --git a/src/backend/utils.ts b/src/backend/utils.ts index e1085fed7..567114945 100644 --- a/src/backend/utils.ts +++ b/src/backend/utils.ts @@ -200,7 +200,7 @@ export const spreadDelay = (retries: any, multiplier: any) => { return s; } -export const removeUndefinedKeys = >(obj: T): T | undefined => { +export const removeUndefinedKeys = >(obj: T, returnUndefined: boolean = true): T | undefined => { const newObj: any = {}; Object.keys(obj).forEach((key) => { if(Array.isArray(obj[key])) { @@ -214,7 +214,10 @@ export const removeUndefinedKeys = >(obj: T): T | } }); if(Object.keys(newObj).length === 0) { - return undefined; + if(returnUndefined) { + return undefined; + } + return newObj; } Object.keys(newObj).forEach(key => { if(newObj[key] === undefined || (null !== newObj[key] && typeof newObj[key] === 'object' && Object.keys(newObj[key]).length === 0)) { @@ -225,6 +228,20 @@ export const removeUndefinedKeys = >(obj: T): T | return newObj; } +export const removeEmptyArrays = >(obj: T): T => { + const newObj: any = {}; + Object.keys(obj).forEach((key) => { + if(Array.isArray(obj[key])) { + if(obj[key].length !== 0) { + newObj[key] = obj[key]; + } + } else { + newObj[key] = obj[key]; + } + }); + return newObj; +} + export const remoteHostIdentifiers = (req: Request): RemoteIdentityParts => { const remote = req.connection.remoteAddress; const proxyRemote = Array.isArray(req.headers["x-forwarded-for"]) ? req.headers["x-forwarded-for"][0] : req.headers["x-forwarded-for"]; @@ -293,7 +310,10 @@ export function parseBool(value: any, prev: any = false): boolean { throw new Error(`'${value.toString()}' is not a boolean value.`); } -export function parseBoolStrict(value: string): boolean { +export function parseBoolStrict(value: string | boolean): boolean { + if(typeof value === 'boolean') { + return value; + } const strTrue = ['1', 'true', 'yes'].includes(value.toLocaleLowerCase().trim()); if (strTrue) { return strTrue; @@ -338,46 +358,6 @@ export const pollingBackoff = (attempt: number, scaleFactor: number = 1): number return Math.round(backoffStrat(attempt + 1) / 1000); } -export const parseRegex = (reg: RegExp, val: string): RegExResult[] | undefined => { - - if (reg.global) { - const g = Array.from(val.matchAll(reg)); - if (g.length === 0) { - return undefined; - } - return g.map(x => { - return { - match: x[0], - index: x.index, - groups: x.slice(1), - named: x.groups || {}, - } as RegExResult; - }); - } - - const m = val.match(reg) - if (m === null) { - return undefined; - } - return [{ - match: m[0], - index: m.index as number, - groups: m.slice(1), - named: m.groups || {} - }]; -} - -export const parseRegexSingleOrFail = (reg: RegExp, val: string): RegExResult | undefined => { - const results = parseRegex(reg, val); - if (results !== undefined) { - if (results.length > 1) { - throw new Error(`Expected Regex to match once but got ${results.length} results. Either Regex must NOT be global (using 'g' flag) or parsed value must only match regex once. Given: ${val} || Regex: ${reg.toString()}`); - } - return results[0]; - } - return undefined; -} - export const intersect = (a: Array, b: Array) => { const setA = new Set(a); const setB = new Set(b); @@ -549,6 +529,10 @@ export const isEmptyArrayOrUndefined = (arr: unknown[] | undefined, return true; } +export const nonEmptyObj = (obj: object): boolean => { + return Object.keys(obj).length > 0; +} + /** * Runs the function `fn` * and retries automatically if it fails. diff --git a/src/backend/utils/DataUtils.ts b/src/backend/utils/DataUtils.ts index 176f1adb2..60480bac0 100644 --- a/src/backend/utils/DataUtils.ts +++ b/src/backend/utils/DataUtils.ts @@ -2,6 +2,9 @@ import JSON5 from "json5"; import { constants, promises } from "fs"; import { MaybeLogger } from '../common/MaybeLogger.js'; import { deepEqual } from 'fast-equals'; +import { CommonConfigPrimitives } from "../common/infrastructure/config/common.js"; +import { parseBoolStrict, removeUndefinedKeys } from "../utils.js"; +import { nonEmptyStringOrDefault } from "../../core/StringUtils.js"; export const asArray = (data: T | T[]): T[] => { if (Array.isArray(data)) { @@ -175,4 +178,13 @@ export const objectsEqual = (a: object, b: object) => { } catch (e) { throw new Error('Could not compare objects', {cause: e}); } +} + +export const getCommonComponentEnvConfig = (prefix: string): Partial => { + const e = nonEmptyStringOrDefault(process.env[`${prefix}_ENABLE`], undefined); + return removeUndefinedKeys({ + id: nonEmptyStringOrDefault(process.env[`${prefix}_ID`], undefined), + name: nonEmptyStringOrDefault(process.env[`${prefix}_NAME`], undefined), + enable: e !== undefined ? parseBoolStrict(e) : undefined + }, false); } \ No newline at end of file diff --git a/src/backend/utils/ErrorUtils.ts b/src/backend/utils/ErrorUtils.ts index 2e7bee38b..f4fbfa146 100644 --- a/src/backend/utils/ErrorUtils.ts +++ b/src/backend/utils/ErrorUtils.ts @@ -86,7 +86,7 @@ export const MessageTransformerDefault = (val: string) => val; /** * Adapted from https://github.com/voxpelli/pony-cause * */ -const _messageWithCauses = (err: Error, seen = new Set(), msgTransform: MessageTransformer = MessageTransformerDefault) => { +const _messageWithCauses = (err: Error, seen = new Set(), msgTransform: MessageTransformer = MessageTransformerDefault): string => { if (!(err instanceof Error)) return ''; const message = err.message; diff --git a/src/backend/utils/FSUtils.ts b/src/backend/utils/FSUtils.ts index dca0d5174..8ce631e46 100644 --- a/src/backend/utils/FSUtils.ts +++ b/src/backend/utils/FSUtils.ts @@ -33,6 +33,7 @@ export async function readText(path: any) { // }); // }); } + export const fileOrDirectoryIsWriteable = (location: string) => { const pathInfo = pathUtil.parse(location); const isDir = pathInfo.ext === ''; @@ -63,3 +64,20 @@ export const fileOrDirectoryIsWriteable = (location: string) => { } }; +export const fileExists = (location: string) => { + const pathInfo = pathUtil.parse(location); + const isDir = pathInfo.ext === ''; + try { + accessSync(location, constants.R_OK); + return true; + } catch (err: any) { + const { code } = err; + if (code === 'ENOENT') { + return false; + } else if (code === 'EACCES') { + throw new Error(`${isDir ? 'Directory' : 'File'} exists at ${location} but application does not have permission to write to it.`); + } else { + throw new Error(`${isDir ? 'Directory' : 'File'} exists at ${location} but application is unable to access it due to a system error`, { cause: err }); + } + } +}; \ No newline at end of file diff --git a/src/backend/utils/NetworkUtils.ts b/src/backend/utils/NetworkUtils.ts index 3e50bceb0..410ea225a 100644 --- a/src/backend/utils/NetworkUtils.ts +++ b/src/backend/utils/NetworkUtils.ts @@ -4,7 +4,7 @@ import address from "address"; import net from 'node:net'; import normalizeUrl from "normalize-url"; import { join as joinPath } from "path"; -import { getFirstNonEmptyVal, isDebugMode, parseRegexSingleOrFail } from "../utils.js"; +import { getFirstNonEmptyVal, isDebugMode} from "../utils.js"; import { URLData } from "../../core/Atomic.js"; import { CloseEvent, ErrorEvent, RetryEvent } from 'iso-websocket' import { WEBSOCKET_CLOSE_CODE_REASONS } from "../common/infrastructure/Atomic.js"; @@ -225,7 +225,7 @@ export const getAddress = (host = '0.0.0.0', logger?: Logger): { v4?: string, v6 } const IPV4_REGEX = new RegExp(/^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}$/); export const isIPv4 = (address: string): boolean => { - return parseRegexSingleOrFail(IPV4_REGEX, address) !== undefined; + return parseRegexSingle(IPV4_REGEX, address) !== undefined; } export const formatWebsocketClose = (e: CloseEvent): string => { diff --git a/src/backend/utils/PlayComparisonUtils.ts b/src/backend/utils/PlayComparisonUtils.ts index 0a0c3f415..4672cf3b1 100644 --- a/src/backend/utils/PlayComparisonUtils.ts +++ b/src/backend/utils/PlayComparisonUtils.ts @@ -61,6 +61,53 @@ export const playContentInvariantTransform = (play: PlayObject): PlayObjectLifec } } +export const playContentBasicInvariantTransform = (play: PlayObject): PlayObjectLifecycleless => { + const { + data: { + playDate, + repeat, + playDateCompleted, + listenRanges, + listenedFor, + meta, + ...rest + } + } = play; + return { + data: { + ...rest + }, + meta: {} + } +} + +export const playMbidIdentifier = (play: PlayObject): string | undefined => { + const { + data: { + meta: { + brainz: { + recording, + album, + track + } = {} + } = {} + } = {} + } = play; + + // track mbid is a unique combo of recording on release + // so we only need it to identifier what should be track + album + artist + if(track !== undefined) { + return track; + } + // recording is independent of release + // so to make sure the corresponding track + album is the same + // we need both recording and release + if(recording !== undefined && album !== undefined) { + return `${recording}-${album}` + } + return undefined; +} + export type PlayTransformer = (play: PlayObject) => PlayObjectLifecycleless; export type ListTransformers = PlayTransformer[]; @@ -265,8 +312,8 @@ export const comparePlayArtistsNormalized = (existing: PlayObject, candidate: Pl artists: candidateArtists = [], } = {} } = candidate; - const normExisting = existingArtists.map(x => normalizeStr(x, {keepSingleWhitespace: true})); - const candidateExisting = candidateArtists.map(x => normalizeStr(x, {keepSingleWhitespace: true})); + const normExisting = existingArtists.map(x => normalizeStr(x.name, {keepSingleWhitespace: true})); + const candidateExisting = candidateArtists.map(x => normalizeStr(x.name, {keepSingleWhitespace: true})); const wholeMatches = setIntersection(new Set(normExisting), new Set(candidateExisting)).size; return [Math.min(compareScrobbleArtists(existing, candidate)/100, 1), wholeMatches] diff --git a/src/backend/utils/PlayTransformUtils.ts b/src/backend/utils/PlayTransformUtils.ts index fb999739a..fd8a243a9 100644 --- a/src/backend/utils/PlayTransformUtils.ts +++ b/src/backend/utils/PlayTransformUtils.ts @@ -148,8 +148,8 @@ export const testWhen = (parts: WhenParts, play: PlayObject, options?: S } if(parts.artists !== undefined) { // allows user to test if artists are empty - const artists = parts.artists.length === 0 ? [''] : play.data.artists; - if(artists.every(x => !testMaybeRegex(parts.artists, x)[0])) { + const artists = parts.artists.length === 0 ? [{name: ''}] : play.data.artists; + if(artists.every(x => !testMaybeRegex(parts.artists, x.name)[0])) { return false; } } diff --git a/src/backend/utils/StringUtils.ts b/src/backend/utils/StringUtils.ts index 7fe7e3ddd..fb7530dc2 100644 --- a/src/backend/utils/StringUtils.ts +++ b/src/backend/utils/StringUtils.ts @@ -2,9 +2,10 @@ import { strategies, stringSameness, StringSamenessResult } from "@foxxmd/string import { hasher } from 'node-object-hash'; import { PlayObject } from "../../core/Atomic.js"; import { asPlayerStateData, DELIMITERS, DELIMITERS_NO_AMP, PlayerStateDataMaybePlay } from "../common/infrastructure/Atomic.js"; -import { getPlatformIdFromData, intersect, parseBool, parseBoolStrict, parseRegexSingleOrFail } from "../utils.js"; +import { getPlatformIdFromData, intersect, parseBool, parseBoolStrict } from "../utils.js"; import { genGroupIdStr } from '../../core/PlayUtils.js'; import { buildTrackString } from "../../core/StringUtils.js"; +import { parseRegexSingle } from "@foxxmd/regex-buddy-core"; const {levenStrategy, diceStrategy} = strategies; @@ -104,7 +105,7 @@ export const parseCredits = (str: string, delimiters?: boolean | string[]): Play let primary: string | undefined; let secondary: string[] = []; let suffix: string | undefined; - const results = parseRegexSingleOrFail(PRIMARY_SECONDARY_SECTIONS_REGEX, str); + const results = parseRegexSingle(PRIMARY_SECONDARY_SECTIONS_REGEX, str); if(results !== undefined) { let delims: string[] | undefined; @@ -116,7 +117,7 @@ export const parseCredits = (str: string, delimiters?: boolean | string[]): Play primary = results.named.primary.trim(); for(const strat of SECONDARY_REGEX_STRATS) { - const secCredits = parseRegexSingleOrFail(strat, results.named.secondary); + const secCredits = parseRegexSingle(strat, results.named.secondary); if(secCredits !== undefined) { secondary = parseContextAwareStringList(secCredits.named.credits as string, delims) suffix = secCredits.named.creditsSuffix; @@ -281,7 +282,7 @@ export const compareScrobbleArtists = (existing: PlayObject, candidate: PlayObje } } = candidate; - return compareNormalizedStrings(existingArtists.reduce((acc, curr) => `${acc} ${curr}`, ''), candidateArtists.reduce((acc, curr) => `${acc} ${curr}`, '')).highScore; + return compareNormalizedStrings(existingArtists.reduce((acc, curr) => `${acc} ${curr.name}`, ''), candidateArtists.reduce((acc, curr) => `${acc} ${curr.name}`, '')).highScore; } /** @@ -437,7 +438,7 @@ export const buildStatePlayerPlayIdententifyingInfo = (data: PlayObject | Player export const LZ_VERSION_PATH: RegExp = new RegExp(/\/?1\/?$/); export const normalizeListenbrainzUrl = (urlVal: string): string | undefined => { - if (parseRegexSingleOrFail(LZ_VERSION_PATH, urlVal)) { + if (parseRegexSingle(LZ_VERSION_PATH, urlVal)) { return urlVal.replace(LZ_VERSION_PATH, ''); } return undefined; diff --git a/src/backend/utils/TimeUtils.ts b/src/backend/utils/TimeUtils.ts index 15a692005..36e2aef48 100644 --- a/src/backend/utils/TimeUtils.ts +++ b/src/backend/utils/TimeUtils.ts @@ -17,7 +17,7 @@ import { TemporalPlayComparison, UnixTimestamp, } from "../../core/Atomic.js"; -import { capitalize } from "../../core/StringUtils.js"; +import { capitalize, stringIsOnlyNumbers } from "../../core/StringUtils.js"; import { DEFAULT_CLOSE_POSITION_ABSOLUTE, DEFAULT_CLOSE_POSITION_PERCENT, @@ -25,11 +25,16 @@ import { DEFAULT_DURATION_REPEAT_PERCENT, DEFAULT_SCROBBLE_DURATION_THRESHOLD, DEFAULT_SCROBBLE_PERCENT_THRESHOLD, + DurationValue, lowGranularitySources, ScrobbleThresholdResult, } from "../common/infrastructure/Atomic.js"; import { ScrobbleThresholds } from "../common/infrastructure/config/source/index.js"; import { formatNumber } from '../../core/DataUtils.js'; +import { InvalidRegexError, SimpleError } from "../common/errors/MSErrors.js"; +import { NamedGroup, parseRegex } from "@foxxmd/regex-buddy-core"; +import { Duration } from "dayjs/plugin/duration.js"; +import { SourceType } from "../common/infrastructure/config/source/sources.js"; //dayjs.extend(isToday); @@ -105,7 +110,7 @@ export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: P const [candidateTsSOCDate, candidateTsSOC] = getScrobbleTsSOCDateWithContext(candidatePlay); const { - diffThreshold = lowGranularitySources.some(x => x.toLocaleLowerCase() === source) ? 60 : 10, + diffThreshold = getTemporalAccuracyCloseVal(source as SourceType), fuzzyDuration = false, fuzzyDiffThreshold = 10, duringReferences = ['range'] @@ -255,6 +260,10 @@ export const temporalAccuracyToString = (acc: TemporalAccuracy): string => { } } +export const getTemporalAccuracyCloseVal = (source: SourceType): number => { + return lowGranularitySources.includes(source) ? 60 : 10; +} + export const getScrobbleTsSOCDateWithContext = (data: PlayObject): [Dayjs, ScrobbleTsSOC] => { const { meta: { @@ -399,4 +408,59 @@ export const repeatDurationPlayed = (play: PlayObject, duration: number, thresho /** Convert unix timestamp in microseconds to unix timestamp in seconds */ export const usecToUnix = (usec: number): UnixTimestamp => { return Math.floor(usec / 1000); +} + +// string must only contain ISO8601 optionally wrapped by whitespace +const ISO8601_REGEX: RegExp = /^\s*((-?)P(?=\d|T\d)(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)([DW]))?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?)\s*$/; +// finds ISO8601 in any part of a string +const ISO8601_SUBSTRING_REGEX: RegExp = /((-?)P(?=\d|T\d)(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)([DW]))?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?)/g; +// string must only duration optionally wrapped by whitespace +const DURATION_REGEX: RegExp = /^\s*(?