toni
106 lines
3.7 KiB
TypeScript
106 lines
3.7 KiB
TypeScript
import { PlaywrightCrawler } from "crawlee";
|
|
import * as dateFns from "date-fns";
|
|
import type { Request } from "express";
|
|
import type { FoundEventDto } from "../../dtos/foundEvents.js";
|
|
import { inject } from "../../infrastructure/di/index.js";
|
|
import { Injectable } from "../../infrastructure/di/injectable-decorator.js";
|
|
import { DatabaseService } from "../db/prisma.js";
|
|
import { EnvironmentService } from "../environmentService.js";
|
|
import { parseTUS, parseTUSDetail } from "./parseTUS.js";
|
|
import { FoundEventSource } from "./sources.js";
|
|
|
|
@Injectable()
|
|
export class FoundEventsService {
|
|
private readonly environmnent = inject(EnvironmentService);
|
|
private readonly database = inject(DatabaseService);
|
|
private readonly crawler: PlaywrightCrawler;
|
|
|
|
public constructor() {
|
|
const top = this;
|
|
|
|
this.crawler = new PlaywrightCrawler({
|
|
maxRequestsPerMinute: 100,
|
|
maxConcurrency: 2,
|
|
requestHandler: async ({ request, page, log }) => {
|
|
// if (request.retryCount > 0) return;
|
|
|
|
const label = request.label;
|
|
|
|
switch (label) {
|
|
case "TUS":
|
|
const tusDetails = await parseTUS(page, top.database);
|
|
await top.crawler.addRequests(
|
|
tusDetails.map(url => ({
|
|
url: url,
|
|
label: "TUS_DETAIL",
|
|
})),
|
|
);
|
|
break;
|
|
case "TUS_DETAIL":
|
|
await parseTUSDetail(
|
|
request.loadedUrl,
|
|
page,
|
|
top.database,
|
|
);
|
|
break;
|
|
// case "DTU":
|
|
// events = parseDTU(page);
|
|
// break;
|
|
// case "BTU":
|
|
// events = parseBTU(page);
|
|
// break;
|
|
// case "MAGOSCH":
|
|
// events = parseMAGOSCH(page);
|
|
// break;
|
|
default:
|
|
log.warning(`Unknown source ${label}`);
|
|
}
|
|
},
|
|
headless: !this.environmnent.isDev(),
|
|
});
|
|
}
|
|
|
|
public async crawl() {
|
|
const sources = Object.entries(FoundEventSource).map(([key, url]) => ({
|
|
url,
|
|
label: key,
|
|
}));
|
|
return await this.crawler.run(sources);
|
|
}
|
|
|
|
public async getFutureEvents(request: Request): Promise<FoundEventDto[]> {
|
|
const allEvents = await this.database.doRequest(
|
|
async prisma =>
|
|
prisma.foundEvents.findMany({
|
|
include: {
|
|
attachedFiles: true,
|
|
},
|
|
}),
|
|
request,
|
|
);
|
|
|
|
const today = new Date(Date.now());
|
|
return allEvents.filter(event => {
|
|
const parts = event.eventDate?.split("-");
|
|
let date: Date | null = null;
|
|
if (!parts) {
|
|
return true;
|
|
}
|
|
if (parts.length == 1) {
|
|
// like '10.05.2012'
|
|
date = dateFns.parse(parts[0]!, "dd.MM.yyyy", new Date());
|
|
}
|
|
if (parts.length == 2) {
|
|
// like '12.-25.04.2026'
|
|
date = dateFns.parse(parts[1]!, "dd.MM.yyyy", new Date());
|
|
}
|
|
|
|
if (!date) {
|
|
console.warn("Unknown Date Format", event.eventDate);
|
|
return true;
|
|
}
|
|
return isNaN(date.valueOf()) || date >= today;
|
|
});
|
|
}
|
|
}
|