From 1bf82e1dc38bdbe4695a7c37bc2df60e328b1cce Mon Sep 17 00:00:00 2001 From: Alexey <34516115+siniakinaa@users.noreply.github.com> Date: Mon, 27 Oct 2025 17:47:44 +0300 Subject: [PATCH 01/64] piktochart.com: update test URL (#598) --- plugins/domains/piktochart.com/piktochart.com.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/domains/piktochart.com/piktochart.com.js b/plugins/domains/piktochart.com/piktochart.com.js index 610ca2814..a4901f154 100644 --- a/plugins/domains/piktochart.com/piktochart.com.js +++ b/plugins/domains/piktochart.com/piktochart.com.js @@ -19,7 +19,7 @@ export default { tests: [ "https://magic.piktochart.com/output/16505417-projet-immigration-copy", - "https://magic.piktochart.com/output/14330224-descripciones-fisicas", + "https://magic.piktochart.com/output/7850098-pm-syllabus", "https://create.piktochart.com/output/986319273bed-trump-tariffs", "https://create.piktochart.com/output/4407f4418cb3-setmana-del-govern-obert-2024" From 0cf3cc7111e5ac9efb2801c61173e5019c1b44b8 Mon Sep 17 00:00:00 2001 From: Aliaksei <34516115+siniakinaa@users.noreply.github.com> Date: Thu, 30 Oct 2025 12:35:59 +0300 Subject: [PATCH 02/64] livestream.com: livestream is no longer available (#599) --- plugins/domains/livestream.com.js | 49 ------------------------------- 1 file changed, 49 deletions(-) delete mode 100644 plugins/domains/livestream.com.js diff --git a/plugins/domains/livestream.com.js b/plugins/domains/livestream.com.js deleted file mode 100644 index 8dc5386c6..000000000 --- a/plugins/domains/livestream.com.js +++ /dev/null @@ -1,49 +0,0 @@ -export default { - - re: [ - /^https?:\/\/livestream\.com\/(\w+)\/events\/(\d+)(\/videos?\/\d+)?/i, - /^https?:\/\/livestream\.com\/accounts\/(\d+)\/events\/(\d+)(\/videos?\/\d+)?/i, - /^https?:\/\/livestream\.com\/accounts\/(\d+)\/(\w+)(\/videos?\/\d+)?/i, - /^https?:\/\/livestream\.com\/(\w+)\/events\/(\d+)(\/videos?\/\d+)?/i, - /^https?:\/\/livestream\.com\/(\w+)\/(\w+)(\/videos?\/\d+)/i, - /^https?:\/\/livestream\.com\/(\w+)\/(\w+)(?:\/?\?t=\d+)?$/i - ], - - mixins: [ - "*" - ], - - getLink: function (meta, urlMatch) { - - var account_id = /^\d+$/.test(urlMatch[1]) ? urlMatch[1] : null; - var event_id = /^\d+$/.test(urlMatch[2]) ? urlMatch[2] : null; - var video_id = urlMatch[3] ? urlMatch[3] : ''; - - if (!account_id && meta['apple-itunes-app'] && /https?:\/\/livestream\.com\/accounts\/\d+\/events\/\d+\/?/i.test(meta['apple-itunes-app'])) { - account_id = meta['apple-itunes-app'].match(/https?:\/\/livestream\.com\/accounts\/(\d+)\/events\/\d+/i)[1]; - } - - if (!event_id && meta['apple-itunes-app'] && /https?:\/\/livestream\.com\/accounts\/\d+\/events\/\d+\/?/i.test(meta['apple-itunes-app'])) { - event_id = meta['apple-itunes-app'].match(/https?:\/\/livestream\.com\/accounts\/\d+\/events\/(\d+)/i)[1]; - } - - if (event_id && account_id) { - return { - href: "https://livestream.com/accounts/" + account_id + "/events/" + event_id + video_id + "/player?autoPlay=false", - accept: CONFIG.T.text_html, - rel: CONFIG.R.player, - "aspect-ratio": 16/9, - autoplay: "autoPlay=true" - } - } - - }, - - tests: [ - "https://livestream.com/accounts/11707815/events/4299357", - "https://livestream.com/ironman/events/7777204/videos/163920520", - "https://livestream.com/accounts/18968940/events/6670218/videos/145352362", - "https://livestream.com/accounts/26021522/events/8730585/videos/214364915", - "https://livestream.com/accounts/771055/live/videos/253665646" - ] -}; \ No newline at end of file From 271bb8c6aa56735da6e06d439d456c482ee0ab25 Mon Sep 17 00:00:00 2001 From: Ivan Paramonau Date: Thu, 30 Oct 2025 15:08:21 -0400 Subject: [PATCH 03/64] Domains: fix Google Maps consent redirect for servers in the EU --- config.local.js.SAMPLE | 4 +++- plugins/domains/google.com/google.maps.js | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config.local.js.SAMPLE b/config.local.js.SAMPLE index 6eea66258..f8c8d6e77 100644 --- a/config.local.js.SAMPLE +++ b/config.local.js.SAMPLE @@ -237,7 +237,9 @@ export default { }, google: { // https://developers.google.com/maps/documentation/embed/guide#api_key - maps_key: "INSERT YOUR VALUE" + maps_key: "INSERT YOUR VALUE", + // If your servers are in EU, google maps will redirect to /consent/ page and the plugins won't work. The option below ignores the redirect. + // follow_http_redirect: true }, /* diff --git a/plugins/domains/google.com/google.maps.js b/plugins/domains/google.com/google.maps.js index c133600ee..30a2ab6c5 100644 --- a/plugins/domains/google.com/google.maps.js +++ b/plugins/domains/google.com/google.maps.js @@ -149,6 +149,8 @@ export default { // responseStatusCode: 415, // No error, fall back to generic parsers instead message: "Google requires your own key for Maps Embeds API. Get one and add it to the provider options." }); + } else if (options.getProviderOptions('google.follow_http_redirect', false)) { + options.followHTTPRedirect = true; } var gmap = {}; From fe0e4938805013f32a48d47e12dfd8413f6f2e79 Mon Sep 17 00:00:00 2001 From: Ivan Paramonau Date: Tue, 4 Nov 2025 16:35:49 -0500 Subject: [PATCH 04/64] [Domains] Review oEmbed Meta deprecation for Facebook and Instagram --- plugins/domains/facebook.com/facebook.meta.js | 32 +++--------------- .../domains/instagram.com/instagram.com.js | 33 ++----------------- 2 files changed, 8 insertions(+), 57 deletions(-) diff --git a/plugins/domains/facebook.com/facebook.meta.js b/plugins/domains/facebook.com/facebook.meta.js index 456cb2b74..2fc743b73 100644 --- a/plugins/domains/facebook.com/facebook.meta.js +++ b/plugins/domains/facebook.com/facebook.meta.js @@ -2,13 +2,6 @@ import * as entities from 'entities'; export default { - /** - * HEADS-UP: New endpoints as of Oct 24, 2020: - * https://developers.facebook.com/docs/plugins/oembed/ - * Please configure your `access_token` in your local config file - * as desribed on https://github.com/itteco/iframely/issues/284. - */ - re: [ 'facebook.post', 'facebook.video' @@ -16,29 +9,14 @@ export default { mixins: [ "domain-icon", - "oembed-author", - "oembed-canonical", "oembed-site" ], - getMeta: function(oembed) { - - if (oembed.html) { - - var description = oembed.html.match(/

([^<>]+)<\/p>/i); - description = description ? description[1]: ''; - - var author = oembed.html.match(/Posted by ]+)>([^<>]+)<\/a>/); - author = author ? author[1]: oembed.author_name; - - var title = oembed.html.match(/>([^<>]+)<\/a>

/i); - title = title ? title[1] : author; - - return { - title: title ? entities.decodeHTML(title) : oembed.title, - description: description ? entities.decodeHTML(description) : oembed.description, - author: author - }; + getMeta: function(__allowFBThumbnail, meta) { + return { + title: meta.twitter?.title || meta.og?.title, + description: meta.twitter?.description || meta.og?.description, + canonical: meta.og?.url } }, diff --git a/plugins/domains/instagram.com/instagram.com.js b/plugins/domains/instagram.com/instagram.com.js index 80936355b..b1811db86 100644 --- a/plugins/domains/instagram.com/instagram.com.js +++ b/plugins/domains/instagram.com/instagram.com.js @@ -29,37 +29,10 @@ export default { provides: ['ipOG', '__allowInstagramMeta'], getMeta: function (oembed, urlMatch, ipOG) { - - var title = ipOG.title; - var description = ipOG.description || oembed.title; - - if (!description || !title || /login/i.test(title)) { - var $container = cheerio('

'); - try { - $container.html(decodeHTML5(oembed.html)); - } catch (ex) {} - - if (!title || /login/i.test(title)) { - var $a = $container.find(`p a[href*="${oembed.author_name}"], p a[href*="${urlMatch[1]}"]`); - - if ($a.length == 1) { - title = $a.text(); - title += /@/.test(title) ? '' : (oembed.author_name ? ` (@${oembed.author_name})` : ''); - } else if (oembed.author_name) { - title = `Instagram (@${oembed.author_name})`; - } - } - - if (!description) { - var $a = $container.find(`p a[href*="${urlMatch[1]}"]`); - description = $a.text(); - } - } - return { - title: title, - description: description, - canonical: `https://www.instagram.com/p/${urlMatch[1]}` + title: ipOG.title, + description: ipOG.description, + canonical: ipOG.url || `https://www.instagram.com/p/${urlMatch[1]}` } }, From 4e545426cd9ccca57eb9bafcf727c96c2032f07b Mon Sep 17 00:00:00 2001 From: Ivan Paramonau Date: Tue, 4 Nov 2025 16:41:13 -0500 Subject: [PATCH 05/64] remove obsolete imports --- plugins/domains/instagram.com/instagram.com.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/plugins/domains/instagram.com/instagram.com.js b/plugins/domains/instagram.com/instagram.com.js index b1811db86..06a05d35f 100644 --- a/plugins/domains/instagram.com/instagram.com.js +++ b/plugins/domains/instagram.com/instagram.com.js @@ -1,7 +1,3 @@ -import cheerio from 'cheerio'; - -import { decodeHTML5 } from 'entities'; - export default { /** @@ -63,7 +59,7 @@ export default { // Remove below error when and if it's fixed. Validators will remove the link og_image.error = 'Unfortunatelly Instagram\'s OG image is cropped'; } else if (!oembed.thumbnail_url) { - og_image.message = "Unfortunatelly, Instagram removed full images on October 1, 2025"; + og_image.message = "Unfortunatelly, Instagram removed full images on November 3, 2025"; } links.push(og_image); } From cde8c2c793bf6d4a86e5f871b8afaea55b6bf624 Mon Sep 17 00:00:00 2001 From: Nazar Leush <39333+nleush@users.noreply.github.com> Date: Mon, 10 Nov 2025 16:53:22 +0200 Subject: [PATCH 06/64] [Chores] Update package dependencies 2025-10 (#602) * update some libs * update `cheerio` import * oembed: fix getting iframe * html-utils: update cheerio usage to generate html code * fix oembed-description cheerio usage * review cheerio usage, add comments * fix tumblr.text * fix youtube.video * simpler oembed.js * update `cheerio.load` usage * remove todos * fix `tumblr.api` cheerio usage * fix `tumblr` cheerio usage * oembed: getIframe - reorganize ifs * update `supertest` 6.3.3 to 7.1.4 * remove `mocha` obsolete dependency * update `chai` from 4.3.2 to 6.2.0 * add new verion of `mocha` * fix node supported to 20 only * use pnpm for tests * update `chokidar` * tests: fix install pnpm * force fix deep dependencies * replace `_.find` to native array method * Delete pnpm-workspace.yaml * fix lock file * Revert "fix lock file" This reverts commit 78e0dd48b91a2a963320c5502149aa39ebdb4f5d. * Revert "Delete pnpm-workspace.yaml" This reverts commit 3ddd6ac28ce33aa39287016da67f6c382d5b3153. * try fix pnpm workspace * tests: fix pnpm version * Revert "try fix pnpm workspace" This reverts commit fb5340078259e73489212a612631b62342af17a4. * `.keys` -> `Object.keys` * `_.extend` -> `Object.assign` * `_.values` -> `Object.values` * `_.all -> array.every` * `_.compact` -> `array.filter(Boolean)` * `_.flatten` -> `array.flat` * `_.isString` -> typeof * remove `_.isArray` and `_.object` * move `mocha` to `devDependencies` * fix parsing multiple cookies * remove debug log * bugfix unefined * bugfix in log * remove underscore --- .github/workflows/tests.yml | 17 +- lib/core.js | 18 +- lib/fetch.js | 37 +- lib/html-utils.js | 39 +- lib/oembed.js | 10 +- lib/plugins/system/cheerio.js | 4 +- lib/plugins/system/htmlparser/htmlparser.js | 2 +- lib/plugins/system/oembed/oembed.js | 25 +- lib/utils.js | 4 +- lib/whitelist.js | 8 +- modules/api/views.js | 17 +- modules/tests-ui/tester.js | 18 +- modules/tests-ui/utils.js | 20 +- modules/tests-ui/views.js | 16 +- package.json | 21 +- plugins/domains/google.com/docs.google.com.js | 4 +- .../domains/instagram.com/instagram.com.js | 2 +- plugins/domains/tumblr.com/tumblr.api.js | 8 +- plugins/domains/tumblr.com/tumblr.photo.js | 6 +- plugins/domains/tumblr.com/tumblr.text.js | 7 +- plugins/domains/tumblr.com/tumblr.video.js | 8 +- plugins/domains/youtube.com/youtube.video.js | 20 +- plugins/links/embedURL/embedURL.js | 6 +- plugins/links/embedURL/ld-video.js | 12 +- plugins/links/oembed-video.js | 4 +- plugins/meta/ld-article.js | 8 +- plugins/meta/ld-product.js | 6 +- plugins/meta/oembed-description.js | 8 +- pnpm-lock.yaml | 1444 +++++++++-------- pnpm-workspace.yaml | 5 + static/js/debug.js | 8 +- test/custom_plugins.js | 8 +- test/e2e.js | 32 +- utils.js | 27 +- 34 files changed, 999 insertions(+), 880 deletions(-) create mode 100644 pnpm-workspace.yaml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 03a5b9e23..c404cd17b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,16 +9,25 @@ jobs: strategy: matrix: - node-version: ['18.x', '20.x'] + node-version: ['20.x'] steps: - uses: actions/checkout@v4 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10 + run_install: false + - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} - cache: 'npm' + cache: 'pnpm' + - name: Install dependencies - run: npm i + run: pnpm install + - name: Running tests - run: npm test \ No newline at end of file + run: pnpm test diff --git a/lib/core.js b/lib/core.js index ecc5e7db7..8c11cd9e1 100644 --- a/lib/core.js +++ b/lib/core.js @@ -1,5 +1,4 @@ - import * as _ from 'underscore'; import * as urlLib from 'url'; import * as pluginUtils from './loader/utils.js'; import * as utils from './utils.js'; @@ -12,6 +11,7 @@ import * as htmlUtils from './html-utils.js'; import * as metaUtils from './plugins/system/meta/utils.js'; import mediaPlugin from './plugins/validators/media.js'; + import { difference, intersection } from '../utils.js'; const plugins = pluginLoader._plugins, pluginsModules = pluginLoader._pluginsModules, @@ -92,7 +92,7 @@ * mandatoryParams - list of new params not used by plugins. Core will find what can use them. * 'mandatoryParams' enables mandatory mode: function will use _only_ methods which has this input 'mandatoryParams'. * This is used for "go down by tree" algorithm. - * var mandatoryParams = _.difference(loadedParams, Object.keys(usedParams)); + * var mandatoryParams = difference(loadedParams, Object.keys(usedParams)); * mandatoryParams = [ * paramName * ] @@ -140,13 +140,13 @@ // If mandatory params mode. if (mandatoryParams && mandatoryParams.length > 0) { - if (_.intersection(params, mandatoryParams).length === 0) { + if (intersection(params, mandatoryParams).length === 0) { // Skip method if its not using mandatory params. continue; } } - var absentParams = _.difference(params, loadedParams); + var absentParams = difference(params, loadedParams); // If "__" (as in "__statusCode") or "...Error" (as in "oembedError") // super mandatory params are absent - skip the plugin. @@ -362,7 +362,7 @@ var loadedParams = Object.keys(context); - // var mandatoryParams = _.difference(loadedParams, Object.keys(usedParams)); + // var mandatoryParams = difference(loadedParams, Object.keys(usedParams)); // Reset scanned plugins for each iteration. var scannedPluginsIds = {}; @@ -523,7 +523,7 @@ for(var k = 0; k < paramPlugins.length; k++) { var foundPluginId = paramPlugins[k]; - var exists = _.find(initialPlugins, function(plugin) { + var exists = initialPlugins.find(function(plugin) { return plugin.id === foundPluginId; }); if (!exists) { @@ -830,7 +830,7 @@ usedPluginMethods[method.name] = usedPluginMethods[method.name] - 1; if (r.error && options.debug) { - console.error(" -- Plugin error", method.pluginId, method.name, result.error); + console.error(" -- Plugin error", method.pluginId, method.name, r.error); } // Collect total result. @@ -1013,7 +1013,7 @@ links = [links]; } - links = _.compact(links); + links = links.filter(Boolean); for(var j = 0; j < links.length; j++) { var link = links[j]; @@ -1296,7 +1296,7 @@ // Sort links in order of REL according to CONFIG.REL_GROUPS. function getRelIndex(rel) { - var rels = _.intersection(rel, CONFIG.REL_GROUPS); + var rels = intersection(rel, CONFIG.REL_GROUPS); var gr = CONFIG.REL_GROUPS.length + 1; if (rels.length > 0) { for(var i = 0; i < rels.length; i++) { diff --git a/lib/fetch.js b/lib/fetch.js index 242a520bf..dabae460a 100644 --- a/lib/fetch.js +++ b/lib/fetch.js @@ -61,6 +61,11 @@ function doFetch(fetch_func, h1_fetch_func, options) { a_fetch_func(uri, fetch_options) .then(response => { var headers = response.headers.plain(); + var cookies = response.headers.raw()['set-cookie']; + if (cookies) { + // Keep cookies as array of strings. + headers['set-cookie'] = cookies; + } var stream = response.body; stream.on('end', () => { clearTimeout(timeoutTimerId); @@ -264,20 +269,32 @@ const cookiesOptions = [ 'SameSite', ]; -export function extendCookiesJar(jar, headers) { - if (headers && headers['set-cookie']) { - var cookies = parseCookie(headers['set-cookie']); - // Filter cookies options. - cookies = Object.fromEntries(Object.entries(cookies).filter(([k,v]) => !cookiesOptions.includes(k))); - jar = jar || {}; - jar = {...jar, ...cookies}; +export function extendCookiesJar(uri, jar, headers) { + var cookiesValue = headers && headers['set-cookie']; + if (cookiesValue) { + var cookiesArray = Array.isArray(cookiesValue) ? cookiesValue : [cookiesValue]; + try { + var cookies = cookiesArray.reduce((allCookies, cookieStr) => { + return { ...allCookies, ...parseCookie(cookieStr) }; + }, {}); + // Filter cookies options. + cookies = Object.fromEntries(Object.entries(cookies).filter(([k,v]) => !cookiesOptions.includes(k))); + jar = jar || {}; + jar = {...jar, ...cookies}; + } catch(ex) { + log('Error parse cookie', uri, ex.message); + } } return jar; } -export function setCookieFromJar(headers, jar) { +export function setCookieFromJar(uri, headers, jar) { if (jar) { - var cookies = Object.entries(jar).map(([k,v]) => serializeCookie(k, v)).join('; '); - headers['Cookie'] = cookies; + try{ + var cookies = Object.entries(jar).map(([k,v]) => serializeCookie(k, v)).join('; '); + headers['Cookie'] = cookies; + } catch(ex) { + log('Error serialize cookie', uri, ex.message); + } } } diff --git a/lib/html-utils.js b/lib/html-utils.js index 3a97f5ded..3913a954f 100644 --- a/lib/html-utils.js +++ b/lib/html-utils.js @@ -1,10 +1,12 @@ - import cheerio from 'cheerio'; - - import * as _ from 'underscore'; + import * as cheerio from 'cheerio'; import CONFIG from '../config.loader.js'; var defaultPaddingBottom = 100 / CONFIG.DEFAULT_ASPECT_RATIO; + function createCheerioElement(name) { + return cheerio.load(`<${name}>`)(name); + } + function wrapContainer($element, data, options) { var aspectWrapperClass = options && options.aspectWrapperClass; @@ -30,7 +32,7 @@ } } - var $container = cheerio('
') + var $container = createCheerioElement('div') .append($element); if (aspectWrapperClass) { @@ -45,7 +47,7 @@ var hasMaxWidth = media && (media["max-width"] || media["min-width"] || media["width"] || verticalAspect); if (hasMaxWidth || forceWidthLimitContainer) { - $widthLimitContainer = cheerio('
') + $widthLimitContainer = createCheerioElement('div') .append($container); } @@ -166,7 +168,7 @@ && data.href; }, generate: function(data) { - var $img = cheerio('') + var $img = createCheerioElement('img') .attr('src', data.href); if (data.title) { $img @@ -190,7 +192,7 @@ var givf = data.rel.indexOf('gifv') > -1; var autoplay = data.rel.indexOf('autoplay') > -1 || givf; - var $video = cheerio('Your browser does not support HTML5 video.'); + var $video = cheerio.load('Your browser does not support HTML5 video.')('video'); if (iframelyData && iframelyData.links) { @@ -279,8 +281,7 @@ return data.type === "text/html" && data.href; }, generate: function(data, options) { - - var $iframe = cheerio('