|
${getPlatformLabel(section.platform)}
${cards}`; + }) + .join(""); + + const chatUrl = `${CHAT_APP_URL}/?q=${encodeURIComponent("tell me about my artist's latest posts")}`; + const html = `
+
|
${s.platform}
- ${s.postUrls
- .map(u => `
- ${u} `) - .join("")}
New posts found for ${who}:
${sectionHtml}`, + subject, + html, }); } diff --git a/lib/apify/digest/truncateText.ts b/lib/apify/digest/truncateText.ts new file mode 100644 index 00000000..13c36986 --- /dev/null +++ b/lib/apify/digest/truncateText.ts @@ -0,0 +1,4 @@ +/** Truncates to `max` characters with a trailing ellipsis. */ +export function truncateText(text: string, max: number): string { + return text.length <= max ? text : `${text.slice(0, max - 1).trimEnd()}…`; +} diff --git a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts index 2a985014..d8310dbd 100644 --- a/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts +++ b/lib/apify/instagram/__tests__/handleInstagramProfileScraperResults.test.ts @@ -97,7 +97,7 @@ describe("handleInstagramProfileScraperResults", () => { expect(upsertPosts).toHaveBeenCalledOnce(); expect(upsertSocialPosts).toHaveBeenCalledOnce(); - expect(sendApifyWebhookEmail).toHaveBeenCalledWith(expect.any(Object), ["x@y.com"]); + expect(sendApifyWebhookEmail).toHaveBeenCalledWith(expect.any(Object), ["x@y.com"], ["u1"]); expect(handleInstagramProfileFollowUpRuns).toHaveBeenCalledOnce(); expect(result.social).toEqual({ id: "s1" }); expect(result.posts).toEqual(posts); diff --git a/lib/apify/instagram/handleInstagramProfileScraperResults.ts b/lib/apify/instagram/handleInstagramProfileScraperResults.ts index 3244e8eb..de38d4c4 100644 --- a/lib/apify/instagram/handleInstagramProfileScraperResults.ts +++ b/lib/apify/instagram/handleInstagramProfileScraperResults.ts @@ -112,6 +112,7 @@ export async function handleInstagramProfileScraperResults(parsed: ApifyWebhookP sentEmails = await sendApifyWebhookEmail( firstResult, accountEmails.map(e => e.email).filter(Boolean), + newPostUrls, ); } } catch (error) { diff --git a/lib/apify/sendApifyWebhookEmail.ts b/lib/apify/sendApifyWebhookEmail.ts index 3510dccd..1f07ce8d 100644 --- a/lib/apify/sendApifyWebhookEmail.ts +++ b/lib/apify/sendApifyWebhookEmail.ts @@ -1,48 +1,37 @@ -import generateText from "@/lib/ai/generateText"; import { sendEmailWithResend } from "@/lib/emails/sendEmail"; import { RECOUP_FROM_EMAIL } from "@/lib/const"; -import { getFrontendBaseUrl } from "@/lib/composio/getFrontendBaseUrl"; +import { renderScrapeDigestHtml } from "@/lib/apify/digest/renderScrapeDigestHtml"; +import { extractPostsFromDatasetItems } from "@/lib/apify/digest/extractPostsFromDatasetItems"; import type { ApifyInstagramProfileResult } from "@/lib/apify/types"; /** - * Sends an Apify-webhook summary email to the given recipients using - * Resend. Generates the email body via an LLM based on the first - * profile result. + * Sends the legacy single-platform Instagram alert (non-batch scrape runs). + * Batch runs get the consolidated digest instead. Body comes from the shared + * deterministic renderer — the per-send LLM body is gone (chat#1855): stable + * branding, direct links to the new posts, no vendor jargon. * * @param profile - First dataset result from the profile scraper. - * @param emails - Recipient email addresses. - * @returns The Resend response, or `null` when there are no recipients. + * @param emails - Recipient email addresses (cross-tenant — BCC only). + * @param newPostUrls - Post URLs genuinely new to the platform. + * @returns The Resend response, or `null` when there is nothing to send. */ export async function sendApifyWebhookEmail( profile: ApifyInstagramProfileResult, emails: string[], + newPostUrls: string[] = [], ) { - if (!emails?.length) return null; + if (!emails?.length || !newPostUrls.length) return null; - const prompt = `You have a new Apify dataset update. Here is the data: - -Key Data -Full Name: ${profile.fullName} -Username: ${profile.username} -Profile URL: ${profile.url} -Profile Picture: ${profile.profilePicUrl} -Biography: ${profile.biography} -Followers: ${profile.followersCount} -Following: ${profile.followsCount} -Latest Posts: ${(Array.isArray(profile.latestPosts) ? profile.latestPosts : []).map(p => JSON.stringify(p)).join(", ")} -`; - - const { text } = await generateText({ - system: `you are a record label services manager for Recoup. - write beautiful html email. - subject: New Apify Dataset Notification. you're notifying music managers about new posts being available for one of their roster musician's Instagram profile. - include a link to view the instagram profile. - call to action is to open a chat link to learn more about the latest posts using Recoup Chat (AI Agents): ${getFrontendBaseUrl()}/?q=tell%20me%20about%20my%20latest%20Ig%20posts - You'll be passed a dataset summary for a musician profile and their latest posts on instagram. - your goal is to get the recipient to click a cta link to open a chat link to learn more about the latest posts using Recoup Chat (AI Agents). - only include the email body html. - no headers or subject`, - prompt, + // The profile IS dataset item 0, so the shared extractor enriches the new + // URLs with caption/media/timestamp from latestPosts already in hand. + const { subject, html } = renderScrapeDigestHtml({ + sections: [ + { + platform: "instagram", + posts: extractPostsFromDatasetItems("instagram", [profile], newPostUrls), + }, + ], + artistName: profile.fullName ?? profile.username ?? null, }); // Recipients span multiple customer accounts (the social's watchers are @@ -52,7 +41,7 @@ Latest Posts: ${(Array.isArray(profile.latestPosts) ? profile.latestPosts : []). from: RECOUP_FROM_EMAIL, to: [RECOUP_FROM_EMAIL], bcc: emails, - subject: `${profile.fullName ?? profile.username ?? "Your artist"} has new posts on Instagram`, - html: text, + subject, + html, }); } diff --git a/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts b/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts index 0497e058..3d53e946 100644 --- a/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts +++ b/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts @@ -76,4 +76,22 @@ describe("handleTwitterProfileScraperResults", () => { expect(upsertSocials).not.toHaveBeenCalled(); expect(persistPostsForSocial).not.toHaveBeenCalled(); }); + + it("excludes retweets and replies from persisted posts and the newness diff", async () => { + listItems.mockResolvedValue({ + items: [ + { ...REAL_ITEM, url: "https://x.com/a/status/1" }, + { ...REAL_ITEM, url: "https://x.com/other/status/2", isRetweet: true }, + { ...REAL_ITEM, url: "https://x.com/a/status/3", isReply: true }, + ], + }); + await handleTwitterProfileScraperResults(payload as never); + expect(persistPostsForSocial).toHaveBeenCalledWith( + expect.objectContaining({ + postRows: [ + { post_url: "https://x.com/a/status/1", updated_at: "2026-07-02T17:21:21.000Z" }, + ], + }), + ); + }); }); diff --git a/lib/apify/twitter/__tests__/isOriginalTweet.test.ts b/lib/apify/twitter/__tests__/isOriginalTweet.test.ts new file mode 100644 index 00000000..6ac47cec --- /dev/null +++ b/lib/apify/twitter/__tests__/isOriginalTweet.test.ts @@ -0,0 +1,20 @@ +import { describe, it, expect } from "vitest"; +import { isOriginalTweet } from "@/lib/apify/twitter/isOriginalTweet"; + +describe("isOriginalTweet", () => { + it("keeps original tweets", () => { + expect(isOriginalTweet({ isRetweet: false, isQuote: false, isReply: false })).toBe(true); + }); + it("keeps quote tweets — the artist adds their own words and earns the stats", () => { + expect(isOriginalTweet({ isRetweet: false, isQuote: true, isReply: false })).toBe(true); + }); + it("drops retweets — the content and metrics belong to someone else", () => { + expect(isOriginalTweet({ isRetweet: true, isQuote: false, isReply: false })).toBe(false); + }); + it("drops replies — conversational noise, not roster posts", () => { + expect(isOriginalTweet({ isRetweet: false, isQuote: false, isReply: true })).toBe(false); + }); + it("keeps items without flags (defensive default)", () => { + expect(isOriginalTweet({})).toBe(true); + }); +}); diff --git a/lib/apify/twitter/__tests__/startTwitterProfileScraping.test.ts b/lib/apify/twitter/__tests__/startTwitterProfileScraping.test.ts index 325e1f48..43fc1605 100644 --- a/lib/apify/twitter/__tests__/startTwitterProfileScraping.test.ts +++ b/lib/apify/twitter/__tests__/startTwitterProfileScraping.test.ts @@ -11,11 +11,11 @@ beforeEach(() => { }); describe("startTwitterProfileScraping", () => { - it("defaults to the legacy single-item snapshot (maxItems: 1) when posts is omitted", async () => { + it("defaults to a 10-item fetch when posts is omitted — retweets/replies are filtered AFTER the fetch, so depth 1 would rarely surface an authored post", async () => { const r = await startTwitterProfileScraping("sweetman_eth"); expect(apifyClient.actor).toHaveBeenCalledWith("apidojo/twitter-scraper-lite"); expect(start).toHaveBeenCalledWith( - { twitterHandles: ["sweetman_eth"], sort: "Latest", maxItems: 1 }, + { twitterHandles: ["sweetman_eth"], sort: "Latest", maxItems: 10 }, { webhooks: expect.any(Array) }, ); expect(r).toEqual({ runId: "run-1", datasetId: "ds-1" }); diff --git a/lib/apify/twitter/handleTwitterProfileScraperResults.ts b/lib/apify/twitter/handleTwitterProfileScraperResults.ts index 4634fae0..766aa6e1 100644 --- a/lib/apify/twitter/handleTwitterProfileScraperResults.ts +++ b/lib/apify/twitter/handleTwitterProfileScraperResults.ts @@ -4,12 +4,16 @@ import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; import { persistPostsForSocial } from "@/lib/apify/persistPostsForSocial"; import { filterNewPostUrls } from "@/lib/socials/filterNewPostUrls"; import { toIsoDate } from "@/lib/apify/toIsoDate"; +import { isOriginalTweet } from "@/lib/apify/twitter/isOriginalTweet"; import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; import type { TablesInsert } from "@/types/database.types"; /** Tweet item from apidojo~twitter-scraper-lite (real shape, run ALVMZYXkh3WHgeGfT; * tweet fields verified on run bx3asRqfbNnkKgogG). */ type TweetItem = { + isRetweet?: boolean; + isQuote?: boolean; + isReply?: boolean; url?: string; createdAt?: string; author?: { @@ -50,8 +54,12 @@ export async function handleTwitterProfileScraperResults(parsed: ApifyWebhookPay // Tweet URLs keep the author's display casing — the path segment is the // case-sensitive status id's context, and unlike profile keys they are // stored as-is (posts upsert keys on exact post_url). + // Only the artist's own posts count — retweets are someone else's content + // and stats, replies are conversation (chat#1855). const postRows: TablesInsert<"posts">[] = (items as TweetItem[]).flatMap(item => - item.url ? [{ post_url: item.url, updated_at: toIsoDate(item.createdAt) }] : [], + item.url && isOriginalTweet(item) + ? [{ post_url: item.url, updated_at: toIsoDate(item.createdAt) }] + : [], ); // Diff before persisting so the digest can report genuinely new posts (chat#1855). const newPostUrls = await filterNewPostUrls(postRows.map(p => p.post_url)); diff --git a/lib/apify/twitter/isOriginalTweet.ts b/lib/apify/twitter/isOriginalTweet.ts new file mode 100644 index 00000000..22d435db --- /dev/null +++ b/lib/apify/twitter/isOriginalTweet.ts @@ -0,0 +1,14 @@ +type TweetFlags = { isRetweet?: boolean; isQuote?: boolean; isReply?: boolean }; + +/** + * True for tweets the artist actually authored — originals and quote tweets + * (their own words on top). Retweets are someone else's content and carry the + * ORIGINAL author's engagement stats, so persisting or reporting them + * misattributes both (chat#1855 feedback: an all-retweet digest section + * showed "hundreds of likes" the artist never earned). Replies are excluded + * as conversational noise rather than roster posts. Items without flags pass + * (defensive default — never silently drop a scrape result on schema drift). + */ +export function isOriginalTweet(item: TweetFlags): boolean { + return !item.isRetweet && !item.isReply; +} diff --git a/lib/apify/twitter/startTwitterProfileScraping.ts b/lib/apify/twitter/startTwitterProfileScraping.ts index 16821355..5de2e1bf 100644 --- a/lib/apify/twitter/startTwitterProfileScraping.ts +++ b/lib/apify/twitter/startTwitterProfileScraping.ts @@ -15,7 +15,10 @@ const startTwitterProfileScraping = async ( const input = { twitterHandles: [cleanHandle], sort: "Latest", - maxItems: posts ?? 1, + // Timeline items include retweets/replies, which are dropped AFTER the + // fetch (isOriginalTweet) — depth 1 would rarely surface an authored + // post for active retweeters (chat#1855). + maxItems: posts ?? 10, }; const run = await apifyClient diff --git a/lib/const.ts b/lib/const.ts index 5b178dd0..4a05312c 100644 --- a/lib/const.ts +++ b/lib/const.ts @@ -15,6 +15,16 @@ export const PRIVY_PROJECT_SECRET = process.env.PRIVY_PROJECT_SECRET; /** Base URL for the public API documentation site */ export const DOCS_BASE_URL = "https://docs.recoupable.dev"; +/** Public marketing site. */ +export const WEBSITE_URL = "https://recoupable.com"; + +/** Chat app — customer-facing emails must always link here, never a derived + * deployment URL (previews would point at the API deployment itself). */ +export const CHAT_APP_URL = "https://chat.recoupable.dev"; + +/** Brand icon hosted on prod chat (email clients need an absolute PNG). */ +export const RECOUP_LOGO_URL = "https://chat.recoupable.dev/icon-with-background.png"; + /** Domain for receiving inbound emails (e.g., support@recoupable.dev) */ export const INBOUND_EMAIL_DOMAIN = "@recoupable.dev"; diff --git a/lib/emails/escapeHtml.ts b/lib/emails/escapeHtml.ts new file mode 100644 index 00000000..c0d8bf6d --- /dev/null +++ b/lib/emails/escapeHtml.ts @@ -0,0 +1,11 @@ +/** + * Escapes text for safe interpolation into email/notification HTML, so + * user- or scraper-supplied content can never inject markup. + */ +export function escapeHtml(text: string): string { + return text + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +}