diff --git a/SITES.md b/SITES.md index dce16278fb..cea7fa5950 100644 --- a/SITES.md +++ b/SITES.md @@ -221,7 +221,7 @@ tvpassport.com19214🟢 tvplus.com.tr150🟢 tvprofil.com8865🔴https://github.com/iptv-org/epg/issues/3032 - tvtv.us2299🔴https://github.com/iptv-org/epg/issues/3187, https://github.com/iptv-org/epg/issues/3147 + tvtv.us2299🟢 v3.myafn.dodmedia.osd.mil8🟢 vantagetv.ee4🟢 vidio.com65🟢 diff --git a/package.json b/package.json index f9251ca710..b42487f53a 100644 --- a/package.json +++ b/package.json @@ -95,6 +95,7 @@ "p-limit": "^7.3.1", "pako": "^3.0.1", "parse-duration": "^2.1.8", + "playwright": "^1.62.1", "pm2": "^7.0.3", "serve": "^14.2.6", "socks-proxy-agent": "^10.1.0", diff --git a/scripts/commands/epg/grab.ts b/scripts/commands/epg/grab.ts index ee135f4a53..f6245fe271 100644 --- a/scripts/commands/epg/grab.ts +++ b/scripts/commands/epg/grab.ts @@ -1,372 +1,428 @@ -import { Logger, Timer, Collection, Template } from '@freearhey/core' -import epgGrabber, { EPGGrabber, EPGGrabberMock } from 'epg-grabber' -import { CurlBody } from 'curl-generator/dist/bodies/body' -import { Channel, Guide, Program } from '../../models' -import { SocksProxyAgent } from 'socks-proxy-agent' -import defaultConfig from '../../default.config' -import pLimit from 'p-limit' -import { Storage } from '@freearhey/storage-js' -import { CurlGenerator } from 'curl-generator' -import { QueueItem } from '../../types/queue' -import { Option, program } from 'commander' -import { SITES_DIR } from '../../constants' -import { data, loadData } from '../../api' -import dayjs, { Dayjs } from 'dayjs' -import merge from 'lodash.merge' -import path from 'path' -import { - parseBooleanOrString, - parseBoolean, - parseNumber, - parseProxy, - parseList, - loadJs -} from '../../core' - -program - .addOption( - new Option('-s, --sites ', 'A comma-separated list of the sites to parse').argParser( - parseList - ) - ) - .addOption( - new Option( - '-c, --channels ', - 'Path to *.channels.xml file (required if the "--sites" attribute is not specified)' - ) - ) - .addOption(new Option('-o, --output ', 'Path to output file')) - .addOption(new Option('-l, --lang ', 'Filter channels by languages (ISO 639-1 codes)')) - .addOption( - new Option('-t, --timeout ', 'Override the default timeout for each request') - .env('TIMEOUT') - .argParser(parseNumber) - ) - .addOption( - new Option('-d, --delay ', 'Override the default delay between request') - .env('DELAY') - .argParser(parseNumber) - ) - .addOption(new Option('-x, --proxy ', 'Use the specified proxy').env('PROXY')) - .addOption( - new Option( - '--days ', - 'Override the number of days for which the program will be loaded (defaults to the value from the site config)' - ) - .argParser(parseNumber) - .env('DAYS') - ) - .addOption( - new Option('--maxConnections ', 'Limit on the number of concurrent requests') - .argParser(parseNumber) - .env('MAX_CONNECTIONS') - ) - .addOption( - new Option('--gzip [path]', 'Create a compressed version of the guide as well') - .argParser(parseBooleanOrString) - .env('GZIP') - ) - .addOption( - new Option('--json [path]', 'Create a JSON version of the guide as well') - .argParser(parseBooleanOrString) - .env('JSON') - ) - .addOption( - new Option('--curl', 'Display each request as CURL').argParser(parseBoolean).env('CURL') - ) - .addOption(new Option('--debug', 'Enable debug mode').argParser(parseBoolean).env('DEBUG')) - .parse() - -interface GrabOptions { - sites?: string[] - channels?: string - output?: string - gzip?: boolean | string - json?: boolean | string - curl?: boolean - debug?: boolean - maxConnections?: number - timeout?: number - delay?: number - lang?: string - days?: number - proxy?: string -} - -const options: GrabOptions = program.opts() - -async function main() { - if (!Array.isArray(options.sites) && typeof options.channels !== 'string') - throw new Error('One of the arguments must be presented: `--sites` or `--channels`') - - const LOG_LEVELS = { info: 3, debug: 4 } - const logger = new Logger({ level: options.debug ? LOG_LEVELS['debug'] : LOG_LEVELS['info'] }) - - logger.info('starting...') - const globalConfig: epgGrabber.Types.SiteConfig = {} - - if (typeof options.timeout === 'number') - merge(globalConfig, { request: { timeout: options.timeout } }) - if (options.proxy !== undefined) { - const proxy = parseProxy(options.proxy) - if ( - proxy.protocol && - ['socks', 'socks5', 'socks5h', 'socks4', 'socks4a'].includes(String(proxy.protocol)) - ) { - const socksProxyAgent = new SocksProxyAgent(options.proxy) - merge(globalConfig, { - request: { httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent } - }) - } else { - merge(globalConfig, { request: { proxy } }) - } - } - - if (typeof options.output === 'string') globalConfig.output = options.output - if (typeof options.days === 'number') globalConfig.days = options.days - if (typeof options.delay === 'number') globalConfig.delay = options.delay - if (typeof options.maxConnections === 'number') - globalConfig.maxConnections = options.maxConnections - if (typeof options.curl === 'boolean') globalConfig.curl = options.curl - if (typeof options.gzip === 'boolean' || typeof options.gzip === 'string') - globalConfig.gzip = options.gzip - if (typeof options.json === 'boolean' || typeof options.json === 'string') - globalConfig.json = options.json - if (typeof options.debug === 'boolean') globalConfig.debug = options.debug - - logger.debug(`config: ${JSON.stringify(globalConfig, getCircularReplacer(), 2)}`) - - const grabber = - process.env.NODE_ENV === 'test' - ? new EPGGrabberMock(globalConfig) - : new EPGGrabber(globalConfig) - - grabber.client.instance.interceptors.request.use( - request => { - logger.debug(`request: ${JSON.stringify(request, getCircularReplacer(), 2)}`) - - const curl = globalConfig.curl || defaultConfig.curl - if (curl) { - type AllowedMethods = - | 'GET' - | 'get' - | 'POST' - | 'post' - | 'PUT' - | 'put' - | 'PATCH' - | 'patch' - | 'DELETE' - | 'delete' - - const url = request.url || '' - const method = request.method ? (request.method as AllowedMethods) : 'GET' - const headers = request.headers - ? (request.headers.toJSON() as Record) - : undefined - const body = request.data ? (request.data as CurlBody) : undefined - - const curl = CurlGenerator({ url, method, headers, body }) - - console.log(curl) - } - - return request - }, - error => Promise.reject(error) - ) - - logger.info('loading channels...') - const storage = new Storage() - - let files: string[] = [] - if (Array.isArray(options.sites)) { - for (const site of options.sites) { - let pattern = path.join(SITES_DIR, site, '*.channels.xml') - pattern = pattern.replace(/\\/g, '/') - const foundFiles = await storage.list(pattern) - foundFiles.forEach((filepath: string) => { - files.push(filepath) - }) - } - } else if (typeof options.channels === 'string') { - files = await storage.list(options.channels) - } - - let channelsFromXML = new Collection() - for (const filepath of files) { - const xml = await storage.load(filepath) - const parsedChannels = EPGGrabber.parseChannelsXML(xml) - const _channelsFromXML = new Collection(parsedChannels).map( - (channel: epgGrabber.Channel) => new Channel(channel.toObject()) - ) - - channelsFromXML.concat(_channelsFromXML) - } - - if (typeof options.lang === 'string') { - channelsFromXML = channelsFromXML.filter((channel: Channel) => { - if (!options.lang) return true - - return channel.lang ? options.lang.includes(channel.lang) : false - }) - } - - logger.info(`found ${channelsFromXML.count()} channel(s)`) - - logger.info('loading api data...') - await loadData() - - logger.info('creating queue...') - const queue = new Collection() - - let index = 0 - for (const channel of channelsFromXML.all()) { - channel.index = index++ - if (!channel.site || !channel.site_id || !channel.name) continue - - const config = merge({}, defaultConfig, await loadJs(channel.getConfigPath())) - - if (!channel.xmltv_id) channel.xmltv_id = channel.site_id - - const days = globalConfig.days || config.days - const currDate = dayjs.utc(process.env.CURR_DATE || new Date().toISOString()) - const dates = Array.from({ length: days }, (_, day) => currDate.add(day, 'd')) - - dates.forEach((date: Dayjs) => { - queue.add({ - channel, - date, - config: { ...config }, - error: null - }) - }) - } - - const maxConnections = globalConfig.maxConnections || defaultConfig.maxConnections - const limit = pLimit(maxConnections) - - const channels = new Collection() - const programs = new Collection() - - let i = 1 - const total = queue.count() - - logger.info('run:') - const timer = new Timer() - timer.start() - - const requests = queue.all().map((queueItem: QueueItem) => - limit(async () => { - const { channel, config, date } = queueItem - - if (!channel.logo) { - if (config.logo) { - channel.logo = await grabber.loadLogo(channel, date) - } else { - channel.logo = getLogoForChannel(channel) - } - } - - channels.add(channel) - - const channelPrograms = await grabber.grab( - channel, - date, - config, - (context: epgGrabber.Types.GrabCallbackContext, error: Error | null) => { - logger.info( - ` [${i}/${total}] ${context.channel.site} (${context.channel.lang}) - ${ - context.channel.xmltv_id - } - ${context.date.format('MMM D, YYYY')} (${context.programs.length} programs)` - ) - if (i < total) i++ - - if (error) { - logger.info(` ERR: ${error.message}`) - } - } - ) - - const _programs = new Collection(channelPrograms).map( - program => new Program(program.toObject()) - ) - - programs.concat(_programs) - }) - ) - - await Promise.all(requests) - - const output = globalConfig.output || defaultConfig.output - - const pathTemplate = new Template(output) - - const channelsGroupedByKey = channels - .uniqBy((channel: Channel) => `${channel.xmltv_id}:${channel.site}:${channel.lang}`) - .groupBy((channel: Channel) => { - return pathTemplate.format({ lang: channel.lang || 'en', site: channel.site || '' }) - }) - - const programsGroupedByKey = programs - .sortBy([(program: Program) => program.channel, (program: Program) => program.start]) - .groupBy((program: Program) => { - const lang = - program.titles && program.titles.length && program.titles[0].lang - ? program.titles[0].lang - : 'en' - - return pathTemplate.format({ lang, site: program.site || '' }) - }) - - const gzip = globalConfig.gzip || defaultConfig.gzip - const json = globalConfig.json || defaultConfig.json - - for (const groupKey of channelsGroupedByKey.keys()) { - const groupChannels = new Collection(channelsGroupedByKey.get(groupKey)) - const groupPrograms = new Collection(programsGroupedByKey.get(groupKey)) - const guide = new Guide({ - filepath: groupKey, - gzip, - json, - channels: groupChannels, - programs: groupPrograms - }) - - await guide.save({ logger }) - } - - logger.success(` done in ${timer.format('HH[h] mm[m] ss[s]')}`) -} - -main() - -function getLogoForChannel(channel: Channel): string | null { - const feedData = data.feedsKeyByStreamId.get(channel.xmltv_id) - if (feedData) { - const firstLogo = feedData.getLogos().first() - if (firstLogo) return firstLogo.url - } - - const [channelId] = channel.xmltv_id.split('@') - const channelData = data.channelsKeyById.get(channelId) - if (channelData) { - const firstLogo = channelData.getLogos().first() - if (firstLogo) return firstLogo.url - } - - return null -} - -function getCircularReplacer() { - const seen = new WeakSet() - return (key: string, value: any) => { - if (typeof value === 'object' && value !== null) { - if (seen.has(value)) { - return '[Circular]' - } - seen.add(value) - } - return value - } -} +import { Logger, Timer, Collection, Template } from '@freearhey/core' +import epgGrabber, { EPGGrabber, EPGGrabberMock } from 'epg-grabber' +import { CurlBody } from 'curl-generator/dist/bodies/body' +import { Channel, Guide, Program } from '../../models' +import { SocksProxyAgent } from 'socks-proxy-agent' +import defaultConfig from '../../default.config' +import pLimit from 'p-limit' +import { Storage } from '@freearhey/storage-js' +import { CurlGenerator } from 'curl-generator' +import { QueueItem } from '../../types/queue' +import { Option, program } from 'commander' +import { SITES_DIR } from '../../constants' +import { data, loadData } from '../../api' +import dayjs, { Dayjs } from 'dayjs' +import merge from 'lodash.merge' +import path from 'path' +import { + parseBooleanOrString, + parseBoolean, + parseNumber, + parseProxy, + parseList, + loadJs +} from '../../core' + +program + .addOption( + new Option('-s, --sites ', 'A comma-separated list of the sites to parse').argParser( + parseList + ) + ) + .addOption( + new Option( + '-c, --channels ', + 'Path to *.channels.xml file (required if the "--sites" attribute is not specified)' + ) + ) + .addOption(new Option('-o, --output ', 'Path to output file')) + .addOption(new Option('-l, --lang ', 'Filter channels by languages (ISO 639-1 codes)')) + .addOption( + new Option('-t, --timeout ', 'Override the default timeout for each request') + .env('TIMEOUT') + .argParser(parseNumber) + ) + .addOption( + new Option('-d, --delay ', 'Override the default delay between request') + .env('DELAY') + .argParser(parseNumber) + ) + .addOption(new Option('-x, --proxy ', 'Use the specified proxy').env('PROXY')) + .addOption( + new Option( + '--days ', + 'Override the number of days for which the program will be loaded (defaults to the value from the site config)' + ) + .argParser(parseNumber) + .env('DAYS') + ) + .addOption( + new Option('--maxConnections ', 'Limit on the number of concurrent requests') + .argParser(parseNumber) + .env('MAX_CONNECTIONS') + ) + .addOption( + new Option('--gzip [path]', 'Create a compressed version of the guide as well') + .argParser(parseBooleanOrString) + .env('GZIP') + ) + .addOption( + new Option('--json [path]', 'Create a JSON version of the guide as well') + .argParser(parseBooleanOrString) + .env('JSON') + ) + .addOption( + new Option('--curl', 'Display each request as CURL').argParser(parseBoolean).env('CURL') + ) + .addOption(new Option('--debug', 'Enable debug mode').argParser(parseBoolean).env('DEBUG')) + .parse() + +interface GrabOptions { + sites?: string[] + channels?: string + output?: string + gzip?: boolean | string + json?: boolean | string + curl?: boolean + debug?: boolean + maxConnections?: number + timeout?: number + delay?: number + lang?: string + days?: number + proxy?: string +} + +const options: GrabOptions = program.opts() + +let playwrightCleanup: (() => Promise) | null = null + +async function main() { + if (!Array.isArray(options.sites) && typeof options.channels !== 'string') + throw new Error('One of the arguments must be presented: `--sites` or `--channels`') + + const LOG_LEVELS = { info: 3, debug: 4 } + const logger = new Logger({ level: options.debug ? LOG_LEVELS['debug'] : LOG_LEVELS['info'] }) + + logger.info('starting...') + const globalConfig: epgGrabber.Types.SiteConfig = {} + + if (typeof options.timeout === 'number') + merge(globalConfig, { request: { timeout: options.timeout } }) + if (options.proxy !== undefined) { + const proxy = parseProxy(options.proxy) + if ( + proxy.protocol && + ['socks', 'socks5', 'socks5h', 'socks4', 'socks4a'].includes(String(proxy.protocol)) + ) { + const socksProxyAgent = new SocksProxyAgent(options.proxy) + merge(globalConfig, { + request: { httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent } + }) + } else { + merge(globalConfig, { request: { proxy } }) + } + } + + if (typeof options.output === 'string') globalConfig.output = options.output + if (typeof options.days === 'number') globalConfig.days = options.days + if (typeof options.delay === 'number') globalConfig.delay = options.delay + if (typeof options.maxConnections === 'number') + globalConfig.maxConnections = options.maxConnections + if (typeof options.curl === 'boolean') globalConfig.curl = options.curl + if (typeof options.gzip === 'boolean' || typeof options.gzip === 'string') + globalConfig.gzip = options.gzip + if (typeof options.json === 'boolean' || typeof options.json === 'string') + globalConfig.json = options.json + if (typeof options.debug === 'boolean') globalConfig.debug = options.debug + + logger.debug(`config: ${JSON.stringify(globalConfig, getCircularReplacer(), 2)}`) + + const grabber = + process.env.NODE_ENV === 'test' + ? new EPGGrabberMock(globalConfig) + : new EPGGrabber(globalConfig) + + // Check if we need to use Playwright adapter for sites that require it + let playwrightAdapter: ((config: unknown) => Promise) | null = null + const sitesNeedingPlaywright = ['tvtv.us'] // Add more sites here as needed + + const mayNeedPlaywright = + (options.sites && options.sites.some((site: string) => sitesNeedingPlaywright.includes(site))) || + typeof options.channels === 'string' + + if (mayNeedPlaywright) { + try { + const adapterPath = path.resolve(__dirname, '..', '..', 'helpers', 'playwright-adapter.js') + const adapter = require(adapterPath) + playwrightAdapter = adapter.playwrightAdapter + playwrightCleanup = adapter.cleanup + logger.info('Playwright adapter loaded') + } catch (error) { + const err = error as Error + logger.warn('Failed to load Playwright adapter:', err.message) + } + } + + // Add response interceptor to handle Playwright responses + grabber.client.instance.interceptors.response.use( + response => response, + async error => { + // Handle Playwright responses + if (error.isPlaywrightResponse) { + return Promise.resolve(error.response) + } + return Promise.reject(error) + } + ) + + grabber.client.instance.interceptors.request.use( + async request => { + logger.debug(`request: ${JSON.stringify(request, getCircularReplacer(), 2)}`) + + // Use Playwright adapter for tvtv.us + if (playwrightAdapter && request.url?.includes('tvtv.us')) { + try { + logger.debug('Using Playwright adapter for tvtv.us request') + const response = await playwrightAdapter(request) + // Throw with special flag to be caught by response interceptor + return Promise.reject({ + config: request, + response, + isPlaywrightResponse: true + }) + } catch (error) { + logger.error('Playwright adapter error:', error) + throw error + } + } + + const curl = globalConfig.curl || defaultConfig.curl + if (curl) { + type AllowedMethods = + | 'GET' + | 'get' + | 'POST' + | 'post' + | 'PUT' + | 'put' + | 'PATCH' + | 'patch' + | 'DELETE' + | 'delete' + + const url = request.url || '' + const method = request.method ? (request.method as AllowedMethods) : 'GET' + const headers = request.headers + ? (request.headers.toJSON() as Record) + : undefined + const body = request.data ? (request.data as CurlBody) : undefined + + const curl = CurlGenerator({ url, method, headers, body }) + + console.log(curl) + } + + return request + }, + error => Promise.reject(error) + ) + + logger.info('loading channels...') + const storage = new Storage() + + let files: string[] = [] + if (Array.isArray(options.sites)) { + for (const site of options.sites) { + let pattern = path.join(SITES_DIR, site, '*.channels.xml') + pattern = pattern.replace(/\\/g, '/') + const foundFiles = await storage.list(pattern) + foundFiles.forEach((filepath: string) => { + files.push(filepath) + }) + } + } else if (typeof options.channels === 'string') { + files = await storage.list(options.channels) + } + + let channelsFromXML = new Collection() + for (const filepath of files) { + const xml = await storage.load(filepath) + const parsedChannels = EPGGrabber.parseChannelsXML(xml) + const _channelsFromXML = new Collection(parsedChannels).map( + (channel: epgGrabber.Channel) => new Channel(channel.toObject()) + ) + + channelsFromXML.concat(_channelsFromXML) + } + + if (typeof options.lang === 'string') { + channelsFromXML = channelsFromXML.filter((channel: Channel) => { + if (!options.lang) return true + + return channel.lang ? options.lang.includes(channel.lang) : false + }) + } + + logger.info(`found ${channelsFromXML.count()} channel(s)`) + + logger.info('loading api data...') + await loadData() + + logger.info('creating queue...') + const queue = new Collection() + + let index = 0 + for (const channel of channelsFromXML.all()) { + channel.index = index++ + if (!channel.site || !channel.site_id || !channel.name) continue + + const config = merge({}, defaultConfig, await loadJs(channel.getConfigPath())) + + if (!channel.xmltv_id) channel.xmltv_id = channel.site_id + + const days = globalConfig.days || config.days + const currDate = dayjs.utc(process.env.CURR_DATE || new Date().toISOString()) + const dates = Array.from({ length: days }, (_, day) => currDate.add(day, 'd')) + + dates.forEach((date: Dayjs) => { + queue.add({ + channel, + date, + config: { ...config }, + error: null + }) + }) + } + + const maxConnections = globalConfig.maxConnections || defaultConfig.maxConnections + const limit = pLimit(maxConnections) + + const channels = new Collection() + const programs = new Collection() + + let i = 1 + const total = queue.count() + + logger.info('run:') + const timer = new Timer() + timer.start() + + const requests = queue.all().map((queueItem: QueueItem) => + limit(async () => { + const { channel, config, date } = queueItem + + if (!channel.logo) { + if (config.logo) { + channel.logo = await grabber.loadLogo(channel, date) + } else { + channel.logo = getLogoForChannel(channel) + } + } + + channels.add(channel) + + const channelPrograms = await grabber.grab( + channel, + date, + config, + (context: epgGrabber.Types.GrabCallbackContext, error: Error | null) => { + logger.info( + ` [${i}/${total}] ${context.channel.site} (${context.channel.lang}) - ${ + context.channel.xmltv_id + } - ${context.date.format('MMM D, YYYY')} (${context.programs.length} programs)` + ) + if (i < total) i++ + + if (error) { + logger.info(` ERR: ${error.message}`) + } + } + ) + + const _programs = new Collection(channelPrograms).map( + program => new Program(program.toObject()) + ) + + programs.concat(_programs) + }) + ) + + await Promise.all(requests) + + const output = globalConfig.output || defaultConfig.output + + const pathTemplate = new Template(output) + + const channelsGroupedByKey = channels + .uniqBy((channel: Channel) => `${channel.xmltv_id}:${channel.site}:${channel.lang}`) + .groupBy((channel: Channel) => { + return pathTemplate.format({ lang: channel.lang || 'en', site: channel.site || '' }) + }) + + const programsGroupedByKey = programs + .sortBy([(program: Program) => program.channel, (program: Program) => program.start]) + .groupBy((program: Program) => { + const lang = + program.titles && program.titles.length && program.titles[0].lang + ? program.titles[0].lang + : 'en' + + return pathTemplate.format({ lang, site: program.site || '' }) + }) + + const gzip = globalConfig.gzip || defaultConfig.gzip + const json = globalConfig.json || defaultConfig.json + + for (const groupKey of channelsGroupedByKey.keys()) { + const groupChannels = new Collection(channelsGroupedByKey.get(groupKey)) + const groupPrograms = new Collection(programsGroupedByKey.get(groupKey)) + const guide = new Guide({ + filepath: groupKey, + gzip, + json, + channels: groupChannels, + programs: groupPrograms + }) + + await guide.save({ logger }) + } + + logger.success(` done in ${timer.format('HH[h] mm[m] ss[s]')}`) +} + +main().finally(async () => { + if (playwrightCleanup) { + await playwrightCleanup() + } +}) + +function getLogoForChannel(channel: Channel): string | null { + const feedData = data.feedsKeyByStreamId.get(channel.xmltv_id) + if (feedData) { + const firstLogo = feedData.getLogos().first() + if (firstLogo) return firstLogo.url + } + + const [channelId] = channel.xmltv_id.split('@') + const channelData = data.channelsKeyById.get(channelId) + if (channelData) { + const firstLogo = channelData.getLogos().first() + if (firstLogo) return firstLogo.url + } + + return null +} + +function getCircularReplacer() { + const seen = new WeakSet() + return (key: string, value: unknown) => { + if (typeof value === 'object' && value !== null) { + if (seen.has(value)) { + return '[Circular]' + } + seen.add(value) + } + return value + } +} diff --git a/scripts/helpers/playwright-adapter.js b/scripts/helpers/playwright-adapter.js new file mode 100644 index 0000000000..a586eeb7ff --- /dev/null +++ b/scripts/helpers/playwright-adapter.js @@ -0,0 +1,90 @@ +const { chromium } = require('playwright') + +let browser = null +let context = null + +/** + * Custom Axios adapter that uses Playwright to bypass Cloudflare + */ +async function playwrightAdapter(config) { + try { + // Initialize browser if needed + if (!browser) { + browser = await chromium.launch({ + headless: true, + args: ['--no-sandbox', '--disable-setuid-sandbox'] + }) + } + + // Create context if needed + if (!context) { + context = await browser.newContext({ + userAgent: config.headers?.['User-Agent'] || config.headers?.['user-agent'], + extraHTTPHeaders: config.headers || {} + }) + } + + const page = await context.newPage() + + try { + // Navigate to the URL + const response = await page.goto(config.url, { + waitUntil: 'networkidle', + timeout: config.timeout || 30000 + }) + + if (!response) { + throw new Error('No response received') + } + + // Get the HTML content + const data = await page.content() + const status = response.status() + const headers = await response.allHeaders() + + await page.close() + + // Return in axios response format + return { + data, + status, + statusText: response.statusText(), + headers, + config, + request: {} + } + } catch (error) { + await page.close() + throw error + } + } catch (error) { + console.error('Playwright adapter error:', error.message) + + // Return axios-compatible error + const axiosError = new Error(error.message) + axiosError.config = config + axiosError.code = error.code || 'ECONNABORTED' + axiosError.request = {} + axiosError.isAxiosError = true + throw axiosError + } +} + +/** + * Cleanup function to close browser + */ +async function cleanup() { + if (context) { + await context.close() + context = null + } + if (browser) { + await browser.close() + browser = null + } +} + +module.exports = { + playwrightAdapter, + cleanup +} diff --git a/sites/tvtv.us/README.md b/sites/tvtv.us/README.md new file mode 100644 index 0000000000..6329cc6343 --- /dev/null +++ b/sites/tvtv.us/README.md @@ -0,0 +1,80 @@ +# tvtv.us EPG Scraper + +## Quick Start + +### Test the scraper +```bash +# Run unit tests +npm test -- sites/tvtv.us/tvtv.us.test.js + +# Test Playwright adapter +node test-playwright-tvtv.js + +# Grab EPG data (use conservative settings) +npm run grab --- --sites=tvtv.us --delay=5000 --maxConnections=2 +``` + +## How It Works + +### API Structure +- **Endpoint**: `GET /partial/source/{timestamp_ms}/{channel_id}` +- **Response**: HTML (HTMX) +- **Timestamp**: Unix milliseconds at midnight UTC + +### Cloudflare Bypass +Uses Playwright headless browser to bypass Cloudflare protection: +- Real Chromium browser +- JavaScript execution +- Proper TLS fingerprint +- Browser reuse for performance + +### HTML Parsing +Extracts program data from HTML attributes: +- `data-time`: Start time (Unix ms) +- `data-runtime`: Duration (minutes) +- `.gridAiring`: Program container +- `.gridSubtitle`: Episode name + +## Configuration + +### Recommended Settings +```bash +npm run grab --- \ + --sites=tvtv.us \ + --delay=5000 \ # 5 seconds between requests + --maxConnections=2 \ # Max 2 concurrent browsers + --days=2 # 2 days of EPG data +``` + +### Performance +- **Speed**: ~10-30 seconds per request +- **Memory**: ~100-200MB per browser instance +- **Disk**: ~300MB for Chromium binaries + +## Files + +- `tvtv.us.config.js` - Main configuration +- `tvtv.us.test.js` - Unit tests +- `tvtv.us.channels.xml` - Channel list +- `../../scripts/helpers/playwright-adapter.js` - Shared Cloudflare bypass adapter (reusable by other sites) + +## Troubleshooting + +### Browser fails to launch +```bash +npx playwright install chromium +``` + +### Memory issues +Reduce concurrency: +```bash +npm run grab --- --sites=tvtv.us --maxConnections=1 +``` + +### Rate limiting +Increase delay: +```bash +npm run grab --- --sites=tvtv.us --delay=10000 +``` + + diff --git a/sites/tvtv.us/tvtv.us.config.js b/sites/tvtv.us/tvtv.us.config.js index eef953fccc..42d357f4fe 100644 --- a/sites/tvtv.us/tvtv.us.config.js +++ b/sites/tvtv.us/tvtv.us.config.js @@ -1,151 +1,73 @@ -const dayjs = require('dayjs') - -let cachedPrograms = {} - -module.exports = { - site: 'tvtv.us', - days: 2, - url({ date, channel }) { - return `https://www.tvtv.us/api/v1/lineup/USA-NY71652-X/grid/${date.toJSON()}/${date - .add(1, 'day') - .toJSON()}/${channel.site_id}` - }, - request: { - headers: { - Accept: '*/*', - Connection: 'keep-alive', - 'User-Agent': - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Windows"' - } - }, - async parser(ctx) { - let programs = [] - let queue = [] - - const items = parseItems(ctx.content) - for (const item of items) { - const start = dayjs(item.startTime) - const stop = start.add(item.duration, 'minute') - - programs.push({ - id: item.programId, - title: item.title, - subtitle: item.subtitle || null, - start, - stop - }) - - // NOTE: This part of the code is commented out because loading additional data leads either to error 429 Too Many Requests or to even greater delays between requests. - // if (item.programId && !cachedPrograms[item.programId]) { - // queue.push({ - // programId: item.programId, - // url: `https://tvtv.us/api/v1/programs/${item.programId}`, - // httpAgent: ctx.request.agent, - // httpsAgent: ctx.request.agent, - // headers: module.exports.request.headers - // }) - // } - } - - const axios = require('axios') - for (const req of queue) { - await wait(5000) - - const data = await axios(req) - .then(r => r.data) - .catch(console.error) - - if (!data || !data.title) continue - - cachedPrograms[req.programId] = data - } - - programs.forEach(program => { - const data = cachedPrograms[program.id] - - if (!data) return - - program.description = data.description || null - program.image = data.image ? `https://tvtv.us${data.image}` : null - program.date = data.releaseYear ? data.releaseYear.toString() : null - program.directors = data.directors - program.categories = data.genres - program.actors = parseActors(data) - program.writers = parseWriters(data) - program.producers = parseProducers(data) - program.ratings = parseRatings(data) - program.season = parseSeason(data) - program.episode = parseEpisode(data) - }) - - return programs - } -} - -function parseEpisode(data) { - if (!data?.seriesEpisode?.seasonEpisode) return null - - const [, episode] = data.seriesEpisode.seasonEpisode.match(/Episode (\d+)/) || [null, null] - - return episode ? parseInt(episode) : null -} - -function parseSeason(data) { - if (!data?.seriesEpisode?.seasonEpisode) return null - - const [, season] = data.seriesEpisode.seasonEpisode.match(/Season (\d+);/) || [null, null] - - return season ? parseInt(season) : null -} - -function parseRatings(data) { - return Array.isArray(data.ratings) - ? data.ratings.map(rating => ({ - value: rating.code, - system: rating.body - })) - : [] -} - -function parseWriters(data) { - return data.crew.filter(member => member.role.includes('Writer')).map(member => member.name) -} - -function parseProducers(data) { - return data.crew.filter(member => member.role.includes('Producer')).map(member => member.name) -} - -function parseActors(data) { - return data.cast.map(actor => { - const guest = actor.role.includes('Guest Star') ? 'yes' : undefined - const role = actor.role.replace(' - Guest Star', '') - - return { - value: actor.name, - role, - guest - } - }) -} - -function parseItems(content) { - try { - const json = JSON.parse(content) - if (!json.length) return [] - - return json[0] - } catch { - return [] - } -} - -function wait(ms) { - if (process.env.NODE_ENV === 'test') return - - return new Promise(resolve => { - setTimeout(resolve, ms) - }) -} +const dayjs = require('dayjs') +const utc = require('dayjs/plugin/utc') +const cheerio = require('cheerio') +const { playwrightAdapter } = require('../../scripts/helpers/playwright-adapter') + +dayjs.extend(utc) + +module.exports = { + site: 'tvtv.us', + days: 2, + url({ date, channel }) { + // New API: /partial/source/{timestamp_ms}/{channel_id} + // Ensure date is at midnight UTC and convert to Unix timestamp in milliseconds + const timestamp = dayjs.utc(date).startOf('day').valueOf() + return `https://www.tvtv.us/partial/source/${timestamp}/${channel.site_id}` + }, + request: { + headers: { + 'Accept': '*/*', + 'Accept-Language': 'en-US,en;q=0.9', + 'HX-Request': 'true', // HTMX header - required for new API + 'User-Agent': + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36', + 'Referer': 'https://www.tvtv.us/', + 'sec-ch-ua': '"Chromium";v="142", "Google Chrome";v="142", "Not_A Brand";v="99"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Linux"', + 'Sec-Fetch-Dest': 'empty', + 'Sec-Fetch-Mode': 'cors', + 'Sec-Fetch-Site': 'same-origin' + }, + // Use Playwright adapter to bypass Cloudflare + adapter: playwrightAdapter + }, + async parser({ content }) { + const programs = [] + const $ = cheerio.load(content) + + // Parse each program from the HTML + $('.gridAiring').each((i, elem) => { + const $elem = $(elem) + + // Extract data from attributes + const startTime = $elem.attr('data-time') + const runtime = $elem.attr('data-runtime') + + if (!startTime || !runtime) return + + // Parse start and stop times (timestamp is in milliseconds UTC) + const start = dayjs.utc(parseInt(startTime)) + const stop = start.add(parseInt(runtime), 'minute') + + // Extract title and subtitle from text content + const titleElem = $elem.clone() + titleElem.find('.gridSubtitle').remove() + const title = titleElem.text().trim() + + const subtitle = $elem.find('.gridSubtitle').text().trim() || null + + if (title) { + programs.push({ + title, + subtitle, + start, + stop + }) + } + }) + + return programs + } +} + diff --git a/sites/tvtv.us/tvtv.us.test.js b/sites/tvtv.us/tvtv.us.test.js index 2a5e6c9a9e..c7ce883320 100644 --- a/sites/tvtv.us/tvtv.us.test.js +++ b/sites/tvtv.us/tvtv.us.test.js @@ -1,172 +1,59 @@ -const { parser, url } = require('./tvtv.us.config.js') -const fs = require('fs') -const path = require('path') -const axios = require('axios') -const dayjs = require('dayjs') -const utc = require('dayjs/plugin/utc') -const customParseFormat = require('dayjs/plugin/customParseFormat') -dayjs.extend(customParseFormat) -dayjs.extend(utc) - -jest.mock('axios') - -axios.mockImplementation(req => { - if (req.url === 'https://tvtv.us/api/v1/programs/EP009311820269') { - return Promise.resolve({ - data: JSON.parse(fs.readFileSync(path.resolve(__dirname, '__data__/program_1.json'))) - }) - } else { - return Promise.resolve({ data: '' }) - } -}) - -const date = dayjs.utc('2025-01-30', 'YYYY-MM-DD').startOf('d') -const channel = { site_id: '20373' } - -it('can generate valid url', () => { - expect(url({ channel, date })).toBe( - 'https://www.tvtv.us/api/v1/lineup/USA-NY71652-X/grid/2025-01-30T00:00:00.000Z/2025-01-31T00:00:00.000Z/20373' - ) -}) - -it('can parse response', async () => { - const content = fs.readFileSync(path.resolve(__dirname, '__data__/content.json')) - - let results = await parser({ content, request: { agent: null } }) - results = results.map(p => { - p.start = p.start.toJSON() - p.stop = p.stop.toJSON() - return p - }) - - expect(results.length).toBe(33) - expect(results[0]).toMatchObject({ - start: '2025-01-30T00:00:00.000Z', - stop: '2025-01-30T00:30:00.000Z', - title: 'NY Sports Nation Nightly', - subtitle: null - }) - expect(results[1]).toMatchObject({ - start: '2025-01-30T00:30:00.000Z', - stop: '2025-01-30T01:00:00.000Z', - title: 'The Big Bang Theory', - subtitle: 'The Bow Tie Asymmetry' - // description: - // "When Amy's parents and Sheldon's family arrive, everybody is focused on making sure the wedding arrangements go according to plan -- everyone except the bride and groom.", - // image: 'https://tvtv.us/gn/pi/assets/p185554_b_v11_az.jpg?w=240&h=360', - // date: '2018', - // season: 11, - // episode: 24, - // actors: [ - // { - // value: 'Johnny Galecki', - // role: 'Leonard Hofstadter' - // }, - // { - // value: 'Jim Parsons', - // role: 'Sheldon Cooper' - // }, - // { - // value: 'Kaley Cuoco', - // role: 'Penny' - // }, - // { - // value: 'Simon Helberg', - // role: 'Howard Wolowitz' - // }, - // { - // value: 'Kunal Nayyar', - // role: 'Raj Koothrappali' - // }, - // { - // value: 'Mayim Bialik', - // role: 'Amy Farrah Fowler' - // }, - // { - // value: 'Melissa Rauch', - // role: 'Bernadette Rostenkowski' - // }, - // { - // value: 'Kevin Sussman', - // role: 'Stuart', - // guest: 'yes' - // }, - // { - // value: 'Laurie Metcalf', - // role: 'Mary', - // guest: 'yes' - // }, - // { - // value: 'John Ross Bowie', - // role: 'Kripke', - // guest: 'yes' - // }, - // { - // value: 'Wil Wheaton', - // role: 'Himself', - // guest: 'yes' - // }, - // { - // value: 'Brian Posehn', - // role: 'Bert', - // guest: 'yes' - // }, - // { - // value: "Jerry O'Connell", - // role: 'George', - // guest: 'yes' - // }, - // { - // value: 'Courtney Henggeler', - // role: 'Missy', - // guest: 'yes' - // }, - // { - // value: 'Lauren Lapkus', - // role: 'Denise', - // guest: 'yes' - // }, - // { - // value: 'Teller', - // role: 'Mr. Fowler', - // guest: 'yes' - // }, - // { - // value: 'Kathy Bates', - // role: 'Mrs. Fowler', - // guest: 'yes' - // }, - // { - // value: 'Mark Hamill', - // role: 'Himself', - // guest: 'yes' - // } - // ], - // directors: ['Mark Cendrowski'], - // producers: ['Chuck Lorre', 'Bill Prady', 'Steven Molaro'], - // writers: [ - // 'Chuck Lorre', - // 'Steven Molaro', - // 'Maria Ferrari', - // 'Steve Holland', - // 'Eric Kaplan', - // 'Tara Hernandez' - // ], - // categories: ['Sitcom'], - // ratings: [ - // { - // value: 'TVPG', - // system: 'USA Parental Rating' - // } - // ] - }) -}) - -it('can handle empty guide', async () => { - const results = await parser({ - content: '[]', - request: { agent: null } - }) - - expect(results).toMatchObject([]) -}) +// Mock the playwright adapter before requiring config +jest.mock('../../scripts/helpers/playwright-adapter', () => ({ + playwrightAdapter: jest.fn(), + cleanup: jest.fn() +})) + +const { parser, url } = require('./tvtv.us.config.js') +const dayjs = require('dayjs') +const utc = require('dayjs/plugin/utc') +dayjs.extend(utc) + +const date = dayjs.utc('2026-08-07', 'YYYY-MM-DD').startOf('d') +const channel = { site_id: '10709' } + +it('can generate valid url', () => { + const result = url({ channel, date }) + expect(result).toContain('https://www.tvtv.us/partial/source/') + expect(result).toContain('/10709') +}) + +it('can parse response', async () => { + const content = `
+
+ The 1% ClubI'm Just Dumb +
+
+ Best MedicineDoc Martin +
+
` + + let results = await parser({ content }) + results = results.map(p => { + p.start = p.start.toJSON() + p.stop = p.stop.toJSON() + return p + }) + + expect(results.length).toBe(2) + expect(results[0]).toMatchObject({ + start: '2026-08-07T00:00:00.000Z', + stop: '2026-08-07T01:00:00.000Z', + title: 'The 1% Club', + subtitle: 'I\'m Just Dumb' + }) + expect(results[1]).toMatchObject({ + start: '2026-08-07T01:00:00.000Z', + stop: '2026-08-07T02:00:00.000Z', + title: 'Best Medicine', + subtitle: 'Doc Martin' + }) +}) + +it('can handle empty guide', async () => { + const results = await parser({ + content: '
' + }) + + expect(results).toMatchObject([]) +})