Fixes for telenet.be: remove duplicate programmes, cache segments, add metadata

sites/telenet.tv/telenet.tv.config.js

@@ -1,10 +1,19 @@
 const axios = require('axios')
 const dayjs = require('dayjs')
+const doFetch = require('@ntlab/sfetch')
 
 const API_STATIC_ENDPOINT = 'https://staticqbr-prod-be.gnp.cloud.telenet.tv/eng/web/epg-service-lite/be'
 const API_PROD_ENDPOINT = 'https://spark-prod-be.gnp.cloud.telenet.tv/eng/web/linear-service/v2'
 const API_IMAGE_ENDPOINT = 'https://staticqbr-prod-be.gnp.cloud.telenet.tv/image-service'
 
+// a segment holds 6 hours of guide for every channel at once, so it is kept around
+// and shared between all the channels of the same day
+const segments = {}
+
+// which day emitted an event, per channel, so a broadcast running past midnight is not repeated
+// by the day it ends in
+const claimed = {}
+
 module.exports = {
   site: 'telenet.tv',
   days: 2,
@@ -13,63 +22,41 @@ module.exports = {
       ttl: 60 * 60 * 1000 // 1 hour
     }
   },
-  url: function ({ date, channel }) {
-    return `${API_STATIC_ENDPOINT}/${channel.lang}/events/segments/${date.format('YYYYMMDD')}000000`
+  url: function ({ date, channel, segment = 0 }) {
+    return `${API_STATIC_ENDPOINT}/${channel.lang}/events/segments/${date.format(
+      'YYYYMMDD'
+    )}${segment.toString().padStart(2, '0')}0000`
   },
   async parser({ content, channel, date }) {
-    let programs = []
-    let items = parseItems(content, channel)
-    if (!items.length) return programs
-    const promises = [
-      axios.get(
-        `${API_STATIC_ENDPOINT}/${channel.lang}/events/segments/${date.format('YYYYMMDD')}060000`,
-        {
-          responseType: 'arraybuffer'
-        }
-      ),
-      axios.get(
-        `${API_STATIC_ENDPOINT}/${channel.lang}/events/segments/${date.format('YYYYMMDD')}120000`,
-        {
-          responseType: 'arraybuffer'
-        }
-      ),
-      axios.get(
-        `${API_STATIC_ENDPOINT}/${channel.lang}/events/segments/${date.format('YYYYMMDD')}180000`,
-        {
-          responseType: 'arraybuffer'
-        }
-      )
-    ]
-
-    await Promise.allSettled(promises)
-      .then(results => {
-        results.forEach(r => {
-          if (r.status === 'fulfilled') {
-            const parsed = parseItems(r.value.data, channel)
-
-            items = items.concat(parsed)
-          }
-        })
-      })
-      .catch(console.error)
-
-    for (let item of items) {
-      const detail = await loadProgramDetails(item, channel)
-      programs.push({
+    const items = await loadItems({ content, channel, date })
+    if (!items.length) return []
+
+    const details = await loadProgramDetails(items, channel)
+
+    return items.map((item, index) => {
+      const detail = details[index] || {}
+
+      return {
         title: item.title,
-        subTitle: detail.episodeName,
+        subTitle: detail.episodeName || item.seriesName,
         icon: parseIcon(item),
-        description: detail.longDescription,
+        description: detail.longDescription || detail.shortDescription,
         category: detail.genres,
         actors: detail.actors,
+        directors: detail.directors,
+        producers: detail.producers,
         season: parseSeason(detail),
         episode: parseEpisode(detail),
+        date: parseYear(detail),
+        country: detail.countryOfOrigin,
+        rating: parseRating(detail, item),
+        language: parseLanguages(detail, item),
+        subtitles: parseSubtitles(detail, item),
+        new: Boolean(detail.premiere || item.premiere),
         start: parseStart(item),
         stop: parseStop(item)
-      })
-    }
-
-    return programs
+      }
+    })
   },
   async channels() {
     const data = await axios
@@ -87,15 +74,63 @@ module.exports = {
   }
 }
 
-async function loadProgramDetails(item, channel) {
-  if (!item.id) return {}
-  const url = `${API_PROD_ENDPOINT}/replayEvent/${item.id}?returnLinearContent=true&language=${channel.lang}`
-  const data = await axios
-    .get(url)
-    .then(r => r.data)
-    .catch(console.log)
+async function loadItems({ content, channel, date }) {
+  const urls = [0, 6, 12, 18].map(segment => module.exports.url({ date, channel, segment }))
+
+  // the first segment has already been downloaded by the grabber itself
+  cacheSegment(urls[0], content)
+
+  const missing = urls.filter(url => segments[url] === undefined)
+  if (missing.length) {
+    await doFetch(missing, (url, res) => cacheSegment(url, res))
+  }
+
+  const items = urls.flatMap(url => findEvents(segments[url], channel))
+
+  // an event that spans a boundary is listed twice: by both segments, or by both days
+  return uniqueItems(items, channel, date)
+}
+
+// `res` is left out when a request fails, but only when sfetch is checking results, which is a
+// setting shared with the other sites using it, so every program is built from its event first
+async function loadProgramDetails(items, channel) {
+  const details = []
+  const queues = items
+    .map((item, index) => ({ item, index }))
+    .filter(({ item }) => item.id)
+    .map(({ item, index }) => ({
+      url: `${API_PROD_ENDPOINT}/replayEvent/${item.id}?returnLinearContent=true&language=${channel.lang}`,
+      index
+    }))
+
+  if (queues.length) {
+    await doFetch(queues, (queue, res) => {
+      if (res) details[queue.index] = res
+    })
+  }
+
+  return details
+}
+
+function cacheSegment(url, content) {
+  if (segments[url] !== undefined) return
+  const entries = parseEntries(content)
+  // a failed download must not be cached, the next channel gets to retry it
+  if (entries.length) segments[url] = entries
+}
+
+// the grabber hands over the raw response, while axios has already parsed the ones fetched here
+function parseEntries(content) {
+  if (!content) return []
+  try {
+    const data = Array.isArray(content.entries) ? content : JSON.parse(content)
+
+    return Array.isArray(data.entries) ? data.entries : []
+  } catch (err) {
+    console.error(`Unable to parse guide: ${err.message}!`)
 
-  return data || {}
+    return []
+  }
 }
 
 function parseStart(item) {
@@ -106,16 +141,35 @@ function parseStop(item) {
   return dayjs.unix(item.endTime)
 }
 
-function parseItems(content, channel) {
-  if (!content) return []
-  const data = JSON.parse(content)
-  if (!data || !Array.isArray(data.entries)) return []
-  const channelData = data.entries.find(e => e.channelId === channel.site_id)
+function findEvents(entries, channel) {
+  if (!Array.isArray(entries)) return []
+  const channelData = entries.find(e => e.channelId === channel.site_id)
   if (!channelData) return []
 
   return Array.isArray(channelData.events) ? channelData.events : []
 }
 
+// Both guards are needed: the local set drops an event listed by two segments of this day, while
+// the claim drops one the previous day already emitted. Re-parsing a day it claimed itself stays
+// idempotent, so parsing the same day twice keeps returning the same programs.
+function uniqueItems(items, channel, date) {
+  const day = date.format('YYYYMMDD')
+  const claimedByChannel = claimed[channel.site_id] || (claimed[channel.site_id] = new Map())
+  const seen = new Set()
+
+  return items.filter(item => {
+    if (seen.has(item.id)) return false
+
+    const owner = claimedByChannel.get(item.id)
+    if (owner !== undefined && owner !== day) return false
+
+    seen.add(item.id)
+    claimedByChannel.set(item.id, day)
+
+    return true
+  })
+}
+
 function parseSeason(detail) {
   if (!detail.seasonNumber) return null
   if (String(detail.seasonNumber).length > 2) return null
@@ -128,6 +182,42 @@ function parseEpisode(detail) {
   return detail.episodeNumber
 }
 
+function parseYear(detail) {
+  if (!detail.productionDate) return null
+  if (!/^\d{4}$/.test(String(detail.productionDate))) return null
+  return detail.productionDate
+}
+
+// the minimum age follows the Kijkwijzer (NICAM) classification
+function parseRating(detail, item) {
+  const minimumAge = detail.minimumAge != null ? detail.minimumAge : item.minimumAge
+  if (minimumAge == null || minimumAge === '') {
+    if (detail.isAdult || item.isAdult) return { system: 'Kijkwijzer', value: '18' }
+    return null
+  }
+  return { system: 'Kijkwijzer', value: String(minimumAge) }
+}
+
+function parseLanguages(detail, item) {
+  return parseLangCodes(detail.audioLanguages || item.audioLanguages)
+}
+
+function parseSubtitles(detail, item) {
+  const captions = parseLangCodes(detail.captionLanguages || item.captionLanguages)
+  const signed = parseLangCodes(detail.signLanguages || item.signLanguages)
+
+  return [
+    ...captions.map(language => ({ language })),
+    ...signed.map(language => ({ type: 'deaf-signed', language }))
+  ]
+}
+
+// the same language can be listed more than once, once per purpose (e.g. audio description)
+function parseLangCodes(languages) {
+  if (!Array.isArray(languages)) return []
+  return [...new Set(languages.map(language => language.lang).filter(Boolean))]
+}
+
 function parseIcon(item) {
   return `${API_IMAGE_ENDPOINT}/intent/${item.id}/posterTile`
 }

sites/telenet.tv/telenet.tv.test.js

@@ -78,12 +78,74 @@ it('can parse response', async () => {
       'Sharon Gless'
     ],
     season: 5,
-    episode: 8
+    episode: 8,
+    date: '2005',
+    country: 'US',
+    rating: { system: 'Kijkwijzer', value: '16' },
+    language: ['nl'],
+    subtitles: [],
+    new: false
+  })
+})
+
+it('can parse response without program details', async () => {
+  const content = fs.readFileSync(path.resolve(__dirname, '__data__/content_0000.json'))
+
+  axios.get.mockImplementation(() => Promise.resolve({ data: '' }))
+
+  let results = await parser({ content, channel, date })
+  results = results.map(p => {
+    p.start = p.start.toJSON()
+    p.stop = p.stop.toJSON()
+    return p
+  })
+
+  expect(results[0]).toMatchObject({
+    start: '2022-10-29T23:56:00.000Z',
+    stop: '2022-10-30T01:44:00.000Z',
+    title: 'Queer as Folk USA',
+    rating: { system: 'Kijkwijzer', value: '16' },
+    language: ['nl']
   })
 })
 
 it('can handle empty guide', async () => {
-  let results = await parser({ content: '', channel, date })
+  // a date of its own, the segments of `date` are still held by the cache of the parser
+  const emptyDate = dayjs.utc('2022-11-30', 'YYYY-MM-DD').startOf('d')
+
+  axios.get.mockImplementation(() => Promise.resolve({ data: '' }))
+
+  let results = await parser({ content: '', channel, date: emptyDate })
 
   expect(results).toMatchObject([])
 })
+
+it('only lists a broadcast running past midnight on the day it starts', async () => {
+  // dates of their own again, so this does not run into the segments cached by the tests above
+  const first = dayjs.utc('2022-12-01', 'YYYY-MM-DD').startOf('d')
+  const second = first.add(1, 'd')
+  const content = JSON.stringify({
+    entries: [
+      {
+        channelId: channel.site_id,
+        events: [
+          {
+            id: 'overnight',
+            title: 'Overnight',
+            startTime: dayjs.utc('2022-12-01T23:30:00Z').unix(),
+            endTime: dayjs.utc('2022-12-02T04:00:00Z').unix()
+          }
+        ]
+      }
+    ]
+  })
+
+  axios.get.mockImplementation(() => Promise.resolve({ data: '' }))
+
+  // both days list the event, because it is on air when either of them begins or ends
+  expect(await parser({ content, channel, date: first })).toHaveLength(1)
+  expect(await parser({ content, channel, date: second })).toHaveLength(0)
+
+  // and the day that claimed it keeps returning it
+  expect(await parser({ content, channel, date: first })).toHaveLength(1)
+})