import { PlaywrightCrawler } from "crawlee"; import * as dateFns from "date-fns"; import type { Request } from "express"; import type { FoundEventDto } from "../../dtos/foundEvents.js"; import { inject } from "../../infrastructure/di/index.js"; import { Injectable } from "../../infrastructure/di/injectable-decorator.js"; import { DatabaseService } from "../db/prisma.js"; import { EnvironmentService } from "../environmentService.js"; import { parseTUS, parseTUSDetail } from "./parseTUS.js"; import { FoundEventSource } from "./sources.js"; @Injectable() export class FoundEventsService { private readonly environmnent = inject(EnvironmentService); private readonly database = inject(DatabaseService); private readonly crawler: PlaywrightCrawler; public constructor() { const top = this; this.crawler = new PlaywrightCrawler({ maxRequestsPerMinute: 100, maxConcurrency: 2, requestHandler: async ({ request, page, log }) => { // if (request.retryCount > 0) return; const label = request.label; switch (label) { case "TUS": const tusDetails = await parseTUS(page, top.database); await top.crawler.addRequests( tusDetails.map(url => ({ url: url, label: "TUS_DETAIL", })), ); break; case "TUS_DETAIL": await parseTUSDetail( request.loadedUrl, page, top.database, ); break; // case "DTU": // events = parseDTU(page); // break; // case "BTU": // events = parseBTU(page); // break; // case "MAGOSCH": // events = parseMAGOSCH(page); // break; default: log.warning(`Unknown source ${label}`); } }, headless: !this.environmnent.isDev(), }); } public async crawl() { const sources = Object.entries(FoundEventSource).map(([key, url]) => ({ url, label: key, })); return await this.crawler.run(sources); } public async getFutureEvents(request: Request): Promise { const allEvents = await this.database.doRequest( async prisma => prisma.foundEvents.findMany({ include: { attachedFiles: true, }, }), request, ); const today = new Date(Date.now()); return allEvents.filter(event => { const parts = event.eventDate?.split("-"); let date: Date | null = null; if (!parts) { return true; } if (parts.length == 1) { // like '10.05.2012' date = dateFns.parse(parts[0]!, "dd.MM.yyyy", new Date()); } if (parts.length == 2) { // like '12.-25.04.2026' date = dateFns.parse(parts[1]!, "dd.MM.yyyy", new Date()); } if (!date) { console.warn("Unknown Date Format", event.eventDate); return true; } return isNaN(date.valueOf()) || date >= today; }); } }