From c6b0f5acaff5176f5404877fc358e91674ff7bff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Arnauts?= Date: Tue, 11 Aug 2026 10:14:45 +0200 Subject: [PATCH 1/2] update telenet --- sites/telenet.tv/telenet.tv.config.js | 196 ++++++++++++++++++-------- sites/telenet.tv/telenet.tv.test.js | 36 ++++- 2 files changed, 170 insertions(+), 62 deletions(-) diff --git a/sites/telenet.tv/telenet.tv.config.js b/sites/telenet.tv/telenet.tv.config.js index b832da8610..b685b8e45c 100644 --- a/sites/telenet.tv/telenet.tv.config.js +++ b/sites/telenet.tv/telenet.tv.config.js @@ -1,10 +1,15 @@ const axios = require('axios') const dayjs = require('dayjs') +const doFetch = require('@ntlab/sfetch') const API_STATIC_ENDPOINT = 'https://staticqbr-prod-be.gnp.cloud.telenet.tv/eng/web/epg-service-lite/be' const API_PROD_ENDPOINT = 'https://spark-prod-be.gnp.cloud.telenet.tv/eng/web/linear-service/v2' const API_IMAGE_ENDPOINT = 'https://staticqbr-prod-be.gnp.cloud.telenet.tv/image-service' +// a segment holds 6 hours of guide for every channel at once, so it is kept around +// and shared between all the channels of the same day +const segments = {} + module.exports = { site: 'telenet.tv', days: 2, @@ -13,63 +18,41 @@ module.exports = { ttl: 60 * 60 * 1000 // 1 hour } }, - url: function ({ date, channel }) { - return `${API_STATIC_ENDPOINT}/${channel.lang}/events/segments/${date.format('YYYYMMDD')}000000` + url: function ({ date, channel, segment = 0 }) { + return `${API_STATIC_ENDPOINT}/${channel.lang}/events/segments/${date.format( + 'YYYYMMDD' + )}${segment.toString().padStart(2, '0')}0000` }, async parser({ content, channel, date }) { - let programs = [] - let items = parseItems(content, channel) - if (!items.length) return programs - const promises = [ - axios.get( - `${API_STATIC_ENDPOINT}/${channel.lang}/events/segments/${date.format('YYYYMMDD')}060000`, - { - responseType: 'arraybuffer' - } - ), - axios.get( - `${API_STATIC_ENDPOINT}/${channel.lang}/events/segments/${date.format('YYYYMMDD')}120000`, - { - responseType: 'arraybuffer' - } - ), - axios.get( - `${API_STATIC_ENDPOINT}/${channel.lang}/events/segments/${date.format('YYYYMMDD')}180000`, - { - responseType: 'arraybuffer' - } - ) - ] - - await Promise.allSettled(promises) - .then(results => { - results.forEach(r => { - if (r.status === 'fulfilled') { - const parsed = parseItems(r.value.data, channel) - - items = items.concat(parsed) - } - }) - }) - .catch(console.error) - - for (let item of items) { - const detail = await loadProgramDetails(item, channel) - programs.push({ + const items = await loadItems({ content, channel, date }) + if (!items.length) return [] + + const details = await loadProgramDetails(items, channel) + + return items.map((item, index) => { + const detail = details[index] || {} + + return { title: item.title, - subTitle: detail.episodeName, + subTitle: detail.episodeName || item.seriesName, icon: parseIcon(item), - description: detail.longDescription, + description: detail.longDescription || detail.shortDescription, category: detail.genres, actors: detail.actors, + directors: detail.directors, + producers: detail.producers, season: parseSeason(detail), episode: parseEpisode(detail), + date: parseYear(detail), + country: detail.countryOfOrigin, + rating: parseRating(detail, item), + language: parseLanguages(detail, item), + subtitles: parseSubtitles(detail, item), + new: Boolean(detail.premiere || item.premiere), start: parseStart(item), stop: parseStop(item) - }) - } - - return programs + } + }) }, async channels() { const data = await axios @@ -87,15 +70,63 @@ module.exports = { } } -async function loadProgramDetails(item, channel) { - if (!item.id) return {} - const url = `${API_PROD_ENDPOINT}/replayEvent/${item.id}?returnLinearContent=true&language=${channel.lang}` - const data = await axios - .get(url) - .then(r => r.data) - .catch(console.log) +async function loadItems({ content, channel, date }) { + const urls = [0, 6, 12, 18].map(segment => module.exports.url({ date, channel, segment })) + + // the first segment has already been downloaded by the grabber itself + cacheSegment(urls[0], content) + + const missing = urls.filter(url => segments[url] === undefined) + if (missing.length) { + await doFetch(missing, (url, res) => cacheSegment(url, res)) + } + + const items = urls.flatMap(url => findEvents(segments[url], channel)) + + // an event that spans a segment boundary is listed in both segments + return uniqueItems(items) +} + +// `res` is left out when a request fails, but only when sfetch is checking results, which is a +// setting shared with the other sites using it, so every program is built from its event first +async function loadProgramDetails(items, channel) { + const details = [] + const queues = items + .map((item, index) => ({ item, index })) + .filter(({ item }) => item.id) + .map(({ item, index }) => ({ + url: `${API_PROD_ENDPOINT}/replayEvent/${item.id}?returnLinearContent=true&language=${channel.lang}`, + index + })) + + if (queues.length) { + await doFetch(queues, (queue, res) => { + if (res) details[queue.index] = res + }) + } + + return details +} - return data || {} +function cacheSegment(url, content) { + if (segments[url] !== undefined) return + const entries = parseEntries(content) + // a failed download must not be cached, the next channel gets to retry it + if (entries.length) segments[url] = entries +} + +// the grabber hands over the raw response, while axios has already parsed the ones fetched here +function parseEntries(content) { + if (!content) return [] + try { + const data = Array.isArray(content.entries) ? content : JSON.parse(content) + + return Array.isArray(data.entries) ? data.entries : [] + } catch (err) { + console.error(`Unable to parse guide: ${err.message}!`) + + return [] + } } function parseStart(item) { @@ -106,16 +137,25 @@ function parseStop(item) { return dayjs.unix(item.endTime) } -function parseItems(content, channel) { - if (!content) return [] - const data = JSON.parse(content) - if (!data || !Array.isArray(data.entries)) return [] - const channelData = data.entries.find(e => e.channelId === channel.site_id) +function findEvents(entries, channel) { + if (!Array.isArray(entries)) return [] + const channelData = entries.find(e => e.channelId === channel.site_id) if (!channelData) return [] return Array.isArray(channelData.events) ? channelData.events : [] } +function uniqueItems(items) { + const seen = new Set() + + return items.filter(item => { + if (seen.has(item.id)) return false + seen.add(item.id) + + return true + }) +} + function parseSeason(detail) { if (!detail.seasonNumber) return null if (String(detail.seasonNumber).length > 2) return null @@ -128,6 +168,42 @@ function parseEpisode(detail) { return detail.episodeNumber } +function parseYear(detail) { + if (!detail.productionDate) return null + if (!/^\d{4}$/.test(String(detail.productionDate))) return null + return detail.productionDate +} + +// the minimum age follows the Kijkwijzer (NICAM) classification +function parseRating(detail, item) { + const minimumAge = detail.minimumAge != null ? detail.minimumAge : item.minimumAge + if (minimumAge == null || minimumAge === '') { + if (detail.isAdult || item.isAdult) return { system: 'Kijkwijzer', value: '18' } + return null + } + return { system: 'Kijkwijzer', value: String(minimumAge) } +} + +function parseLanguages(detail, item) { + return parseLangCodes(detail.audioLanguages || item.audioLanguages) +} + +function parseSubtitles(detail, item) { + const captions = parseLangCodes(detail.captionLanguages || item.captionLanguages) + const signed = parseLangCodes(detail.signLanguages || item.signLanguages) + + return [ + ...captions.map(language => ({ language })), + ...signed.map(language => ({ type: 'deaf-signed', language })) + ] +} + +// the same language can be listed more than once, once per purpose (e.g. audio description) +function parseLangCodes(languages) { + if (!Array.isArray(languages)) return [] + return [...new Set(languages.map(language => language.lang).filter(Boolean))] +} + function parseIcon(item) { return `${API_IMAGE_ENDPOINT}/intent/${item.id}/posterTile` } diff --git a/sites/telenet.tv/telenet.tv.test.js b/sites/telenet.tv/telenet.tv.test.js index 08fbe7f2af..df73cfa63c 100644 --- a/sites/telenet.tv/telenet.tv.test.js +++ b/sites/telenet.tv/telenet.tv.test.js @@ -78,12 +78,44 @@ it('can parse response', async () => { 'Sharon Gless' ], season: 5, - episode: 8 + episode: 8, + date: '2005', + country: 'US', + rating: { system: 'Kijkwijzer', value: '16' }, + language: ['nl'], + subtitles: [], + new: false + }) +}) + +it('can parse response without program details', async () => { + const content = fs.readFileSync(path.resolve(__dirname, '__data__/content_0000.json')) + + axios.get.mockImplementation(() => Promise.resolve({ data: '' })) + + let results = await parser({ content, channel, date }) + results = results.map(p => { + p.start = p.start.toJSON() + p.stop = p.stop.toJSON() + return p + }) + + expect(results[0]).toMatchObject({ + start: '2022-10-29T23:56:00.000Z', + stop: '2022-10-30T01:44:00.000Z', + title: 'Queer as Folk USA', + rating: { system: 'Kijkwijzer', value: '16' }, + language: ['nl'] }) }) it('can handle empty guide', async () => { - let results = await parser({ content: '', channel, date }) + // a date of its own, the segments of `date` are still held by the cache of the parser + const emptyDate = dayjs.utc('2022-11-30', 'YYYY-MM-DD').startOf('d') + + axios.get.mockImplementation(() => Promise.resolve({ data: '' })) + + let results = await parser({ content: '', channel, date: emptyDate }) expect(results).toMatchObject([]) }) From cbe4e3f3cecb8c51c4224c10539c48a665b74686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Arnauts?= Date: Tue, 11 Aug 2026 15:03:47 +0200 Subject: [PATCH 2/2] deduplicate --- sites/telenet.tv/telenet.tv.config.js | 20 +++++++++++++++--- sites/telenet.tv/telenet.tv.test.js | 30 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/sites/telenet.tv/telenet.tv.config.js b/sites/telenet.tv/telenet.tv.config.js index b685b8e45c..89f816065b 100644 --- a/sites/telenet.tv/telenet.tv.config.js +++ b/sites/telenet.tv/telenet.tv.config.js @@ -10,6 +10,10 @@ const API_IMAGE_ENDPOINT = 'https://staticqbr-prod-be.gnp.cloud.telenet.tv/image // and shared between all the channels of the same day const segments = {} +// which day emitted an event, per channel, so a broadcast running past midnight is not repeated +// by the day it ends in +const claimed = {} + module.exports = { site: 'telenet.tv', days: 2, @@ -83,8 +87,8 @@ async function loadItems({ content, channel, date }) { const items = urls.flatMap(url => findEvents(segments[url], channel)) - // an event that spans a segment boundary is listed in both segments - return uniqueItems(items) + // an event that spans a boundary is listed twice: by both segments, or by both days + return uniqueItems(items, channel, date) } // `res` is left out when a request fails, but only when sfetch is checking results, which is a @@ -145,12 +149,22 @@ function findEvents(entries, channel) { return Array.isArray(channelData.events) ? channelData.events : [] } -function uniqueItems(items) { +// Both guards are needed: the local set drops an event listed by two segments of this day, while +// the claim drops one the previous day already emitted. Re-parsing a day it claimed itself stays +// idempotent, so parsing the same day twice keeps returning the same programs. +function uniqueItems(items, channel, date) { + const day = date.format('YYYYMMDD') + const claimedByChannel = claimed[channel.site_id] || (claimed[channel.site_id] = new Map()) const seen = new Set() return items.filter(item => { if (seen.has(item.id)) return false + + const owner = claimedByChannel.get(item.id) + if (owner !== undefined && owner !== day) return false + seen.add(item.id) + claimedByChannel.set(item.id, day) return true }) diff --git a/sites/telenet.tv/telenet.tv.test.js b/sites/telenet.tv/telenet.tv.test.js index df73cfa63c..add8baf981 100644 --- a/sites/telenet.tv/telenet.tv.test.js +++ b/sites/telenet.tv/telenet.tv.test.js @@ -119,3 +119,33 @@ it('can handle empty guide', async () => { expect(results).toMatchObject([]) }) + +it('only lists a broadcast running past midnight on the day it starts', async () => { + // dates of their own again, so this does not run into the segments cached by the tests above + const first = dayjs.utc('2022-12-01', 'YYYY-MM-DD').startOf('d') + const second = first.add(1, 'd') + const content = JSON.stringify({ + entries: [ + { + channelId: channel.site_id, + events: [ + { + id: 'overnight', + title: 'Overnight', + startTime: dayjs.utc('2022-12-01T23:30:00Z').unix(), + endTime: dayjs.utc('2022-12-02T04:00:00Z').unix() + } + ] + } + ] + }) + + axios.get.mockImplementation(() => Promise.resolve({ data: '' })) + + // both days list the event, because it is on air when either of them begins or ends + expect(await parser({ content, channel, date: first })).toHaveLength(1) + expect(await parser({ content, channel, date: second })).toHaveLength(0) + + // and the day that claimed it keeps returning it + expect(await parser({ content, channel, date: first })).toHaveLength(1) +})