From fef4690b0b02dc5ea7f7687167245c36bb89db1d Mon Sep 17 00:00:00 2001 From: Fabrice Lamant Date: Sat, 6 Dec 2025 14:55:13 +0100 Subject: [PATCH] add docker --- Dockerfile | 0 cache/.gitkeep | 0 docker-compose.yaml | 21 + scraper/.gitignore | 2 + scraper/Dockerfile | 16 + scraper/cache.ts | 46 ++- scraper/eslint.config.mjs | 80 +++- scraper/ics-generator.ts | 120 ++++++ scraper/ics.ts | 150 ------- scraper/index.ts | 141 +------ scraper/nocs.ts | 2 +- scraper/nodemon.json | 4 +- scraper/package-lock.json | 594 ++++++++++++++++++++++++++- scraper/package.json | 20 +- scraper/scraper.ts | 188 +++++++++ scraper/tsconfig.json | 131 ++---- scraper/types.d.ts | 59 ++- ui/.gitignore | 2 +- ui/Dockerfile | 21 + ui/app/api/data/[[...slug]]/route.ts | 24 ++ ui/app/flag.tsx | 2 +- ui/app/page.tsx | 5 +- ui/lib/data.ts | 2 +- ui/next.config.ts | 2 +- 24 files changed, 1186 insertions(+), 446 deletions(-) delete mode 100644 Dockerfile delete mode 100644 cache/.gitkeep create mode 100644 docker-compose.yaml create mode 100644 scraper/.gitignore create mode 100644 scraper/Dockerfile create mode 100644 scraper/ics-generator.ts delete mode 100644 scraper/ics.ts create mode 100644 scraper/scraper.ts create mode 100644 ui/Dockerfile create mode 100644 ui/app/api/data/[[...slug]]/route.ts diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index e69de29bb..000000000 diff --git a/cache/.gitkeep b/cache/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 000000000..ef24021b7 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,21 @@ +services: + scraper: + container_name: scraper + build: ./scraper + restart: unless-stopped + volumes: + - cache:/app/cache + - data:/app/output + + ui: + container_name: ui + build: ./ui + ports: + - "3000:3000" + restart: unless-stopped + volumes: + - data:/app/data + +volumes: + cache: + data: diff --git a/scraper/.gitignore b/scraper/.gitignore new file mode 100644 index 000000000..ab3bb1740 --- /dev/null +++ b/scraper/.gitignore @@ -0,0 +1,2 @@ +cache/ +output/ diff --git a/scraper/Dockerfile b/scraper/Dockerfile new file mode 100644 index 000000000..84130befe --- /dev/null +++ b/scraper/Dockerfile @@ -0,0 +1,16 @@ +FROM node:lts-alpine AS base +WORKDIR /app +COPY package*.json ./ +RUN npm install + +FROM base AS build +WORKDIR /app +COPY . . +RUN npm run build + +FROM node:lts-alpine AS runner +WORKDIR /app +COPY --from=base /app/node_modules ./node_modules +COPY --from=build /app/dist . +ENV DEBUG=olympics-calendar:* +CMD ["node", "index.js"] diff --git a/scraper/cache.ts b/scraper/cache.ts index aa61c4eb9..8d4e1dc2b 100644 --- a/scraper/cache.ts +++ b/scraper/cache.ts @@ -1,30 +1,32 @@ import Debug from "debug"; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs" +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; -const debug = Debug(`olympics-calendar:cache`); +export class Cache { + private debug = Debug("olympics-calendar:cache"); -const cachePath = (key: string): string => { - return `../cache/${key}.cached`; -} + private cachePath = (key: string): string => { + return `./cache/${key}.cached`; + }; -export const get = (key: string): string | null => { - debug(`get: key=${key}`); - const path = cachePath(key); - if (existsSync(path)) { - return readFileSync(path, "utf-8"); + public get(key: string): string | null { + this.debug("get", key); + const path = this.cachePath(key); + if (existsSync(path)) { + return readFileSync(path, "utf-8"); + } + return null; } - return null; -} -export const has = (key: string): boolean => { - debug(`has: key=${key}`); - const path = cachePath(key); - return existsSync(path); -} + public has(key: string): boolean { + this.debug(`has: key=${key}`); + const path = this.cachePath(key); + return existsSync(path); + } -export const set = (key: string, data: string): void => { - debug(`set: key=${key}`); - const path = cachePath(key); - mkdirSync(path.split("/").slice(0, -1).join("/"), { recursive: true }); - writeFileSync(path, data); + public set(key: string, data: string): void { + this.debug(`set: key=${key}`); + const path = this.cachePath(key); + mkdirSync(path.split("/").slice(0, -1).join("/"), { recursive: true }); + writeFileSync(path, data); + } } diff --git a/scraper/eslint.config.mjs b/scraper/eslint.config.mjs index ddec96ac2..fa3486c2d 100644 --- a/scraper/eslint.config.mjs +++ b/scraper/eslint.config.mjs @@ -1,26 +1,76 @@ -import globals from "globals"; -import pluginJs from "@eslint/js"; +import eslint from "@eslint/js"; +import perfectionist from "eslint-plugin-perfectionist"; +import { defineConfig, globalIgnores } from "eslint/config"; +import tseslint from "typescript-eslint"; - -export default [ +export default defineConfig( + globalIgnores(["./dist/**", "./node_modules/**"]), + eslint.configs.recommended, + tseslint.configs.recommended, { - files: ["**/*.js"], - languageOptions: { - sourceType: "commonjs", + plugins: { + perfectionist, }, }, - { - languageOptions: { - globals: globals.node, - }, - }, - pluginJs.configs.recommended, { rules: { - "comma-dangle": ["error", "always-multiline"], + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/no-unsafe-function-type": "off", + "@typescript-eslint/no-unused-vars": ["error", { caughtErrors: "none" }], + "no-case-declarations": "off", + "comma-dangle": ["error", "only-multiline"], complexity: ["error", 15], quotes: ["error", "double"], semi: ["error", "always"], + "perfectionist/sort-imports": [ + "error", + { order: "asc", type: "natural" }, + ], + "perfectionist/sort-classes": [ + "error", + { + groups: [ + "index-signature", + + ["private-static-property", "private-static-accessor-property"], + ["protected-static-property", "protected-static-accessor-property"], + ["static-property", "static-accessor-property"], + + ["private-property", "private-accessor-property"], + ["protected-property", "protected-accessor-property"], + ["property", "accessor-property"], + + "constructor", + + ["private-static-get-method", "private-static-set-method"], + ["protected-static-get-method", "protected-static-set-method"], + ["static-get-method", "static-set-method"], + + ["private-get-method", "private-set-method"], + ["protected-get-method", "protected-set-method"], + ["get-method", "set-method"], + + ["private-static-method", "private-static-function-property"], + ["protected-static-method", "protected-static-function-property"], + ["static-method", "static-function-property"], + + ["private-method", "private-function-property"], + ["protected-method", "protected-function-property"], + ["method", "function-property"], + + "static-block", + "unknown", + ], + ignoreCase: true, + order: "asc", + type: "alphabetical", + }, + ], + "perfectionist/sort-named-exports": [ + "error", + { order: "asc", type: "natural" }, + ], + "no-console": "off", }, }, -]; +); diff --git a/scraper/ics-generator.ts b/scraper/ics-generator.ts new file mode 100644 index 000000000..02339137c --- /dev/null +++ b/scraper/ics-generator.ts @@ -0,0 +1,120 @@ +import Debug from "debug"; +import { mkdirSync, writeFileSync } from "fs"; + +import { getFlag } from "./nocs"; +import { Calendar } from "./types"; + +export class ICSGenerator { + private calendar: Calendar; + + private debug = Debug("olympics-calendar:ics-generator"); + + constructor(calendar: Calendar) { + this.calendar = calendar; + } + + private generateICSFile( + sportKey: string | null, + nocKey: string | null, + ): void { + this.debug( + "generateICSFile", + sportKey || "all-sports", + nocKey || "all-nocs", + ); + this.calendar.languages.forEach((lang) => { + const pathSportKey = sportKey ? sportKey : "all-sports"; + const pathNocKey = nocKey ? nocKey : "calendar"; + + const filepath = `./output/${lang.code.toLowerCase()}/${pathSportKey.toLowerCase()}/${pathNocKey.toLowerCase()}.ics`; + mkdirSync(filepath.split("/").slice(0, -1).join("/"), { recursive: true }); + + const titleComponents = []; + if (nocKey) { + titleComponents.push( + `${this.calendar.nocs.find((n) => n.key === nocKey)!.name[lang.code]}`, + ); + } + if (sportKey) { + titleComponents.push(this.calendar.sports.find((s) => s.key === sportKey)!.name[lang.code]); + } + titleComponents.push("Milano Cortina 2026"); + + const title = titleComponents.join(" - "); + + const lines = []; + + lines.push("BEGIN:VCALENDAR"); + lines.push("VERSION:2.0"); + lines.push( + `PRODID:-//fabrice404//olympics-calendar//${lang.code}/${pathSportKey}/${pathNocKey}`, + ); + lines.push(`X-WR-CALNAME:${title}`); + lines.push(`NAME:${title}`); + + this.calendar.events + .filter((event) => { + if (sportKey && event.sport !== sportKey) return false; + if (nocKey) { + if (event.match) { + const team1Key = event.match.team1.key; + const team2Key = event.match.team2.key; + if (team1Key !== nocKey && team2Key !== nocKey) { + return false; + } + } else { + return false; + } + } + return true; + }) + .forEach((event) => { + lines.push("BEGIN:VEVENT"); + lines.push(`UID:${event.key.replace(/--/g, "-")}`); + lines.push(`DTSTAMP:${event.start.replace(/[-:]/g, "").replace(/\.\d+Z$/, "Z")}`); + lines.push(`DTSTART:${event.start.replace(/[-:]/g, "").replace(/\.\d+Z$/, "Z")}`); + lines.push(`DTEND:${event.end.replace(/[-:]/g, "").replace(/\.\d+Z$/, "Z")}`); + lines.push(`LOCATION:${event.location[lang.code] || ""}`); + + const sport = this.calendar.sports.find( + (s) => s.key === event.sport, + )!; + lines.push(`DESCRIPTION:${sport.name[lang.code]} - ${event.name[lang.code] || ""}`); + const summary = `SUMMARY:${event.name[lang.code] || ""}`; + + if (event.match) { + const team1Name = event.match.team1.name[lang.code] || event.match.team1.key; + const team1Flag = getFlag(event.match.team1.key); + const team2Name = event.match.team2.name[lang.code] || event.match.team2.key; + const team2Flag = getFlag(event.match.team2.key); + if (team1Name && team2Name) { + lines.push(`SUMMARY:${team1Flag} ${team1Name} - ${team2Name} ${team2Flag}`); + } + } + + lines.push(summary); + lines.push("END:VEVENT"); + }); + + lines.push("END:VCALENDAR"); + + writeFileSync(filepath, lines.join("\n")); + }); + } + + public generate(): void { + this.debug("generate"); + this.generateICSFile(null, null); + + this.calendar.sports.forEach((sport) => { + this.generateICSFile(sport.key, null); + this.calendar.nocs.forEach((noc) => { + this.generateICSFile(sport.key, noc.key); + }); + }); + + this.calendar.nocs.forEach((noc) => { + this.generateICSFile(null, noc.key); + }); + } +} diff --git a/scraper/ics.ts b/scraper/ics.ts deleted file mode 100644 index f00fd8a30..000000000 --- a/scraper/ics.ts +++ /dev/null @@ -1,150 +0,0 @@ -// BEGIN:VCALENDAR -// VERSION:2.0 -// PRODID:-//fabrice404//olympics-calendar//archery/AUS//EN -// X-WR-CALNAME:šŸ‡¦šŸ‡ŗ Australia Archery | Paris 2024 -// NAME:šŸ‡¦šŸ‡ŗ Australia Archery | Paris 2024 -// BEGIN:VEVENT -// UID:20240725T073000Z-archery-WOMENS-INDIVIDUAL-RANKING-ROUND -// DTSTAMP:20240725T073000Z -// DTSTART:20240725T073000Z -// DTEND:20240725T103000Z -// DESCRIPTION:Archery - Women's Individual Ranking Round\nšŸ‡ØšŸ‡³ AN -// Qixuan\nšŸ‡²šŸ‡½ Alejandra VALENCIA\nšŸ‡²šŸ‡© Alexandra MIRCA\nšŸ‡µšŸ‡· Alondra -// RIVERA\nšŸ‡«šŸ‡· Amelie CORDEAU\nšŸ‡§šŸ‡· Ana Luiza SLIACHTICAS CAETANO\nšŸ‡ØšŸ‡“ Ana -// RENDON MARTINEZ\nšŸ‡²šŸ‡½ Ana VAZQUEZ\nšŸ‡²šŸ‡½ Angela RUIZ\nšŸ‡®šŸ‡³ Ankita -// BHAKAT\nšŸ‡²šŸ‡¾ Ariana Nur Dania MOHAMAD ZAIRI\nšŸ‡®šŸ‡³ Bhajan KAUR\nšŸ‡¬šŸ‡§ -// Bryony PITMAN\nšŸ‡¹šŸ‡¼ CHIU Yi-Ching\nšŸ‡«šŸ‡· Caroline LOPEZ\nšŸ‡ŗšŸ‡ø Casey -// KAUFHOLD\nšŸ‡ŗšŸ‡ø Catalina GNORIEGA\nšŸ‡©šŸ‡Ŗ Charline SCHWARZ\nšŸ‡®šŸ‡¹ Chiara -// REBAGLIATI\nšŸ‡®šŸ‡³ Deepika KUMARI\nšŸ‡øšŸ‡° Denisa BARANKOVA\nšŸ‡®šŸ‡© Diananda -// CHOIRUNISA\nšŸ‡ŖšŸ‡ø Elia CANALES\nšŸ‡¹šŸ‡· Elif Berra GOKKIR\nšŸ‡¦šŸ‡¹ Elisabeth -// STRAKA\nšŸ‡¬šŸ‡³ Fatoumata SYLLA\nšŸ‡³šŸ‡± Gaby SCHLOESSER\nšŸ‡øšŸ‡² Giorgia -// CESARINI\nšŸ‡°šŸ‡· JEON Hunyoung\nšŸ‡ŖšŸ‡¬ Jana ALI\nšŸ‡ŗšŸ‡ø Jennifer MUCINO\nšŸ‡©šŸ‡Ŗ -// Katharina BAUER\nšŸ‡©šŸ‡° Kirstine DANSTRUP ANDERSEN\nšŸ‡¹šŸ‡¼ LEI -// Chien-Ying\nšŸ‡ØšŸ‡³ LI Jiaman\nšŸ‡¹šŸ‡¼ LI Tsai-Chi\nšŸ‡°šŸ‡· LIM Sihyeon\nšŸ‡¦šŸ‡ŗ -// Laura PAEGLIS\nšŸ‡³šŸ‡± Laura van der WINKEL\nšŸ‡«šŸ‡· Lisa BARBELIN\nšŸ‡·šŸ‡“ -// Madalina AMAISTROAIE\nšŸ‡ØšŸ‡æ Marie HORACKOVA\nšŸ‡¬šŸ‡§ Megs HAVERS\nšŸ‡©šŸ‡Ŗ -// Michelle KROPPEN\nšŸ‡®šŸ‡± Mikaella MOSHE\nšŸ‡®šŸ‡· Mobina FALLAH\nšŸ‡°šŸ‡· NAM -// Suhyeon\nšŸ‡ÆšŸ‡µ NODA Satsuki\nšŸ‡²šŸ‡¾ Nurul Azreena MOHAMAD FAZIL\nšŸ‡¬šŸ‡§ Penny -// HEALEY\nšŸ‡³šŸ‡± Quinty ROEFFEN\nšŸ‡ŖšŸ‡Ŗ Reena PARNAT\nšŸ‡®šŸ‡© Rezza OCTAVIA\nšŸ‡¹šŸ‡³ -// Rihab ELWALID\nšŸ‡²šŸ‡¾ Syaqiera MASHAYIKH\nšŸ‡®šŸ‡© Syifa Nurafifah KAMAL\nšŸ‡»šŸ‡³ -// Thi Anh Nguyet DO\nšŸ‡ŗšŸ‡¦ Veronika MARCHENKO\nšŸ‡ØšŸ‡¦ Virginie CHENIER\nšŸ‡µšŸ‡± -// Wioleta MYSZOR\nšŸ‡ØšŸ‡³ YANG Xiaolei\nšŸ‡¦šŸ‡æ Yaylagul RAMAZANOVA\nšŸ‡øšŸ‡® Zana -// PINTARIC\nšŸ‡ŗšŸ‡æ Ziyodakhon ABDUSATTOROVA -// SUMMARY:šŸ¹ Women's Individual Ranking Round -// LOCATION:Invalides -// END:VEVENT - -import { mkdirSync, writeFileSync } from "fs"; -import { Calendar } from "./types"; -import { getFlag } from "./nocs"; - - -// BEGIN:VCALENDAR -// VERSION:2.0 -// PRODID:-//fabrice404//olympics-calendar//3x3-basketball/AUS//EN -// X-WR-CALNAME:šŸ‡¦šŸ‡ŗ Australia 3x3 Basketball | Paris 2024 -// NAME:šŸ‡¦šŸ‡ŗ Australia 3x3 Basketball | Paris 2024 -// BEGIN:VEVENT -// UID:20240730T160000Z-3x3-basketball-WOMENS-POOL-ROUND-AUS-CAN -// DTSTAMP:20240730T160000Z -// DTSTART:20240730T160000Z -// DTEND:20240730T162500Z -// DESCRIPTION:3x3 Basketball - Women's Pool Round -// SUMMARY:šŸ€ AUS šŸ‡¦šŸ‡ŗ - šŸ‡ØšŸ‡¦ CAN -// LOCATION:La Concorde 1 -// END:VEVENT - -// END:VCALENDAR - -export const generateICSFiles = (calendar: Calendar): void => { - - generateICSFile(calendar, null, null); - - calendar.sports.forEach((sport) => { - generateICSFile(calendar, sport.key, null); - calendar.nocs.forEach((noc) => { - generateICSFile(calendar, sport.key, noc.key); - }); - }); - - calendar.nocs.forEach((noc) => { - generateICSFile(calendar, null, noc.key); - }); -}; - - -export const generateICSFile = (calendar: Calendar, sportKey: string | null, nocKey: string | null): void => { - calendar.languages.forEach((lang) => { - const pathSportKey = sportKey ? sportKey : "all-sports"; - const pathNocKey = nocKey ? nocKey : "calendar" - - const filepath = `../ui/public/data/${lang.code.toLowerCase()}/${pathSportKey.toLowerCase()}/${pathNocKey.toLowerCase()}.ics`; - mkdirSync(filepath.split('/').slice(0, -1).join('/'), { recursive: true }); - - const titleComponents = []; - if (nocKey) { - titleComponents.push(`${calendar.nocs.find(n => n.key === nocKey)!.name[lang.code]}`); - } - if (sportKey) { - titleComponents.push(calendar.sports.find(s => s.key === sportKey)!.name[lang.code]); - } - titleComponents.push("Milano Cortina 2026"); - - const title = titleComponents.join(' - '); - - const lines = []; - - lines.push("BEGIN:VCALENDAR"); - lines.push("VERSION:2.0"); - lines.push(`PRODID:-//fabrice404//olympics-calendar//${lang.code}/${pathSportKey}/${pathNocKey}`); - lines.push(`X-WR-CALNAME:${title}`); - lines.push(`NAME:${title}`); - - calendar.events - .filter((event) => { - if (sportKey && event.sport !== sportKey) return false; - if (nocKey) { - if (event.match) { - const team1Key = event.match.team1.key; - const team2Key = event.match.team2.key; - if (team1Key !== nocKey && team2Key !== nocKey) { - return false; - } - } else { - return false; - } - } - return true; - }) - .forEach((event) => { - lines.push("BEGIN:VEVENT"); - lines.push(`UID:${event.key.replace(/--/g, '-')}`); - lines.push(`DTSTAMP:${event.start.replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z')}`); - lines.push(`DTSTART:${event.start.replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z')}`); - lines.push(`DTEND:${event.end.replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z')}`); - lines.push(`LOCATION:${event.location[lang.code] || ''}`); - - const sport = calendar.sports.find(s => s.key === event.sport)!; - lines.push(`DESCRIPTION:${sport.name[lang.code]} - ${event.name[lang.code] || ''}`); - let summary = `SUMMARY:${event.name[lang.code] || ''}` - - if (event.match) { - const team1Name = event.match.team1.name[lang.code] || event.match.team1.key; - const team1Flag = getFlag(event.match.team1.key); - const team2Name = event.match.team2.name[lang.code] || event.match.team2.key; - const team2Flag = getFlag(event.match.team2.key); - if (team1Name && team2Name) { - lines.push(`SUMMARY:${team1Flag} ${team1Name} - ${team2Name} ${team2Flag}`); - } - } - - lines.push(summary); - lines.push(`END:VEVENT`); - - }) - - lines.push("END:VCALENDAR"); - - writeFileSync(filepath, lines.join('\n')); - }); -}; diff --git a/scraper/index.ts b/scraper/index.ts index a2b55044e..963a1ef2c 100644 --- a/scraper/index.ts +++ b/scraper/index.ts @@ -1,141 +1,12 @@ -import Debug from "debug"; +import nodeCron from "node-cron"; -import * as cache from "./cache"; -import { mkdirSync, writeFileSync } from "fs"; -import { Calendar, Event, Sport, Team } from "./types"; -import { generateICSFiles } from "./ics"; - -const baseUrl = "https://www.olympics.com"; -const basePath = "/milano-cortina-2026/schedule/overview"; - -const debug = Debug(`olympics-calendar:index`); - - -const getScheduleOverview = async (language: string) => { - debug(`getScheduleOverview: language=${language}`); - - const scheduleOverviewKey = `${language}/schedule-overview`; - - if (!cache.has(scheduleOverviewKey)) { - debug(`Fetching ${baseUrl}/${language}${basePath}`); - const response = await fetch(`${baseUrl}/${language}/${basePath}`); - const page = await response.text(); - const dataMatch = page.match(/