From b7e4dc2ae40fdf9f3e5d4d3827fc10b8c53817c6 Mon Sep 17 00:00:00 2001 From: Fabrice LAMANT Date: Wed, 4 Feb 2026 09:28:16 +0100 Subject: [PATCH] official website update --- scraper/cache.ts | 2 +- scraper/ics-generator.ts | 91 ++++++++--- scraper/index.ts | 13 +- scraper/package-lock.json | 42 +++++ scraper/package.json | 1 + scraper/scraper.ts | 258 ++++++++++++++--------------- scraper/tsconfig.json | 3 +- scraper/types.d.ts | 73 ++------- ui/app/flag.tsx | 3 +- ui/app/page.tsx | 330 ++++++++++++++++++++++---------------- ui/lib/text.ts | 12 ++ 11 files changed, 469 insertions(+), 359 deletions(-) diff --git a/scraper/cache.ts b/scraper/cache.ts index 8d4e1dc2b..3f6c26390 100644 --- a/scraper/cache.ts +++ b/scraper/cache.ts @@ -5,7 +5,7 @@ export class Cache { private debug = Debug("olympics-calendar:cache"); private cachePath = (key: string): string => { - return `./cache/${key}.cached`; + return `./cache/${key}.json`; }; public get(key: string): string | null { diff --git a/scraper/ics-generator.ts b/scraper/ics-generator.ts index 02339137c..abd388dd2 100644 --- a/scraper/ics-generator.ts +++ b/scraper/ics-generator.ts @@ -13,6 +13,23 @@ export class ICSGenerator { this.calendar = calendar; } + private cleanLine(line: string): string { + if (line.length <= 75) { + return line; + } + const chunks: string[] = []; + let index = 0; + while (index < line.length) { + let chunk = line.slice(index, index + 75); + if (index > 0) { + chunk = " " + chunk.trim(); + } + chunks.push(chunk); + index += 75; + } + return chunks.join("\n"); + } + private generateICSFile( sportKey: string | null, nocKey: string | null, @@ -22,27 +39,28 @@ export class ICSGenerator { sportKey || "all-sports", nocKey || "all-nocs", ); - this.calendar.languages.forEach((lang) => { + + for (const lang of this.calendar.languages) { const pathSportKey = sportKey ? sportKey : "all-sports"; const pathNocKey = nocKey ? nocKey : "calendar"; const filepath = `./output/${lang.code.toLowerCase()}/${pathSportKey.toLowerCase()}/${pathNocKey.toLowerCase()}.ics`; mkdirSync(filepath.split("/").slice(0, -1).join("/"), { recursive: true }); - const titleComponents = []; + const titleComponents: string[] = []; if (nocKey) { titleComponents.push( `${this.calendar.nocs.find((n) => n.key === nocKey)!.name[lang.code]}`, ); } if (sportKey) { - titleComponents.push(this.calendar.sports.find((s) => s.key === sportKey)!.name[lang.code]); + titleComponents.push(this.calendar.sports.find((s) => s.key === sportKey)!.name[lang.code] || ""); } titleComponents.push("Milano Cortina 2026"); const title = titleComponents.join(" - "); - const lines = []; + const lines: string[] = []; lines.push("BEGIN:VCALENDAR"); lines.push("VERSION:2.0"); @@ -54,17 +72,11 @@ export class ICSGenerator { this.calendar.events .filter((event) => { - if (sportKey && event.sport !== sportKey) return false; - if (nocKey) { - if (event.match) { - const team1Key = event.match.team1.key; - const team2Key = event.match.team2.key; - if (team1Key !== nocKey && team2Key !== nocKey) { - return false; - } - } else { - return false; - } + if ((sportKey && event.sport !== sportKey) || event.sport === "CER") { + return false; + } + if (nocKey && !event.nocs.includes(nocKey)) { + return false; } return true; }) @@ -79,27 +91,54 @@ export class ICSGenerator { const sport = this.calendar.sports.find( (s) => s.key === event.sport, )!; - lines.push(`DESCRIPTION:${sport.name[lang.code]} - ${event.name[lang.code] || ""}`); - const summary = `SUMMARY:${event.name[lang.code] || ""}`; + let description = `DESCRIPTION:${sport.name[lang.code]} - ${event.name[lang.code] || ""}`; + let summary = `SUMMARY:${event.name[lang.code] || ""}`; - if (event.match) { - const team1Name = event.match.team1.name[lang.code] || event.match.team1.key; - const team1Flag = getFlag(event.match.team1.key); - const team2Name = event.match.team2.name[lang.code] || event.match.team2.key; - const team2Flag = getFlag(event.match.team2.key); - if (team1Name && team2Name) { - lines.push(`SUMMARY:${team1Flag} ${team1Name} - ${team2Name} ${team2Flag}`); + if (event.competitors?.length === 2) { + const competitor1 = this.getCompetitor(event.competitors[0]!, lang.code); + const competitor2 = this.getCompetitor(event.competitors[1]!, lang.code); + + if (competitor1 && competitor2) { + summary = `SUMMARY:${competitor1?.flag} ${competitor1.name} - ${competitor2?.name} ${competitor2.flag}`; } + } else if (event.competitors?.length > 0) { + const competitors = event.competitors + .map((competitorId) => this.getCompetitor(competitorId, lang.code)) + .map((competitor) => `\\n${competitor.flag} ${competitor.name}`).join(""); + description += `${competitors}`; + } lines.push(summary); + lines.push(this.cleanLine(description)); lines.push("END:VEVENT"); }); lines.push("END:VCALENDAR"); - writeFileSync(filepath, lines.join("\n")); - }); + if (lines.length <= 10) { + this.debug("Skipping empty ICS file:", filepath); + } else { + writeFileSync(filepath, lines.join("\n")); + } + } + } + + private getCompetitor(competitorId: string, lang: string) { + if (competitorId.startsWith("team:")) { + const team = this.calendar.nocs.find(noc => noc.key === competitorId.replace("team:", "")); + return { + noc: team?.key, + name: team?.name[lang] || team?.key || competitorId, + flag: getFlag(team?.key || ""), + }; + } + const competitor = this.calendar.competitors.find(comp => comp.code === competitorId)!; + return { + noc: competitor.noc, + name: competitor.name, + flag: getFlag(competitor.noc), + }; } public generate(): void { diff --git a/scraper/index.ts b/scraper/index.ts index 963a1ef2c..762ee3bbf 100644 --- a/scraper/index.ts +++ b/scraper/index.ts @@ -1,11 +1,18 @@ +import { removeSync } from "fs-extra/esm"; import nodeCron from "node-cron"; import { Scraper } from "./scraper"; -const main = async () => { - nodeCron.schedule("* * * * *", async () => { +const main = () => { + nodeCron.schedule("*/1 * * * *", () => { + removeSync("./cache/schedules"); const scraper = new Scraper(); - await scraper.scrape(); + scraper.scrape(); + }); + + nodeCron.schedule("0 0 * * *", () => { + removeSync("./cache/disciplinesevents"); + removeSync("./cache/nocs"); }); }; diff --git a/scraper/package-lock.json b/scraper/package-lock.json index aee2b7652..dae2ba288 100644 --- a/scraper/package-lock.json +++ b/scraper/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "axios": "^1.13.4", "debug": "^4.4.3", + "fs-extra": "^11.3.3", "node-cron": "^4.2.1", "nodemon": "^3.1.11", "ts-node": "^10.9.2" @@ -1268,6 +1269,20 @@ "node": ">= 6" } }, + "node_modules/fs-extra": { + "version": "11.3.3", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.3.tgz", + "integrity": "sha512-VWSRii4t0AFm6ixFFmLLx1t7wS1gh+ckoa84aOeapGum0h+EZd1EhEumSB+ZdDLnEPuucsVB9oB7cxJHap6Afg==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -1366,6 +1381,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -1541,6 +1562,18 @@ "dev": true, "license": "MIT" }, + "node_modules/jsonfile": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", + "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -2135,6 +2168,15 @@ "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", "license": "MIT" }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", diff --git a/scraper/package.json b/scraper/package.json index 6e741a5f4..9628f6243 100644 --- a/scraper/package.json +++ b/scraper/package.json @@ -13,6 +13,7 @@ "dependencies": { "axios": "^1.13.4", "debug": "^4.4.3", + "fs-extra": "^11.3.3", "node-cron": "^4.2.1", "nodemon": "^3.1.11", "ts-node": "^10.9.2" diff --git a/scraper/scraper.ts b/scraper/scraper.ts index 4c19a33c2..4519baf62 100644 --- a/scraper/scraper.ts +++ b/scraper/scraper.ts @@ -1,129 +1,123 @@ -import { get } from "axios"; import Debug from "debug"; import { writeFileSync } from "fs"; import { Cache } from "./cache"; import { ICSGenerator } from "./ics-generator"; -import { Calendar, Event, Language, PageData, Sport, Team } from "./types"; +import { Calendar, Event, Language, Sport, NOC, Competitor } from "./types"; -const BASE_URL = "https://www.olympics.com"; -const BASE_SCHEDULE_PATH = "milano-cortina-2026/schedule/overview"; export class Scraper { - private cache = new Cache(); - private debug = Debug("olympics-calendar:scraper"); + private readonly cache = new Cache(); - private events: Event[] = []; - private languages: Language[] = []; - private nocs: Team[] = []; - private sports: Sport[] = []; + private readonly competitors: Competitor[] = []; - private async getPageData(path: string): Promise { - this.debug(`getPageData: path=${path}`); - if (!this.cache.has(path)) { - const url = `${BASE_URL}${path}`; - this.debug(url); - const response = await get(url, { - headers: { + private readonly debug = Debug("olympics-calendar:scraper"); + + private readonly events: Event[] = []; + private readonly languages: Language[] = [ + { code: "en", name: "English", code3: "ENG" }, + { code: "it", name: "Italiano", code3: "ITA" }, + { code: "fr", name: "Français", code3: "FRA" }, + { code: "de", name: "Deutsch", code3: "DEU" }, + { code: "pt", name: "Português", code3: "POR" }, + { code: "es", name: "Español", code3: "SPA" }, + { code: "ja", name: "日本語", code3: "JPN" }, + { code: "zh", name: "中文", code3: "CHI" }, + { code: "hi", name: "हिन्दी", code3: "HIN" }, + { code: "ko", name: "한국어", code3: "KOR" }, + { code: "ru", name: "Русский", code3: "RUS" }, + ]; + + private readonly nocs: NOC[] = []; + private readonly sports: Sport[] = []; + + private async getJSONData(url: string, cacheKey: string): Promise { + this.debug(`getJSONData: url=${url}`); + + if (!this.cache.has(cacheKey)) { + const response = await fetch(url, { + "headers": { "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", - "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36" + "accept-language": "en-US,en;q=0.9,fr-FR;q=0.8,fr;q=0.7", + "cache-control": "max-age=0", + "priority": "u=0, i", + "sec-ch-ua": "\"Not(A:Brand\";v=\"8\", \"Chromium\";v=\"144\", \"Google Chrome\";v=\"144\"", + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": "\"macOS\"", + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "none", + "sec-fetch-user": "?1", + "upgrade-insecure-requests": "1", }, + "body": null, + "method": "GET" }); - const page = await response.data; - const dataMatch = page.match( - /