scraper

2026-03-09 20:40:53 +01:00
parent 10f4750755
commit 71ca4d81e1
17 changed files with 2157 additions and 7 deletions
@@ -0,0 +1,65 @@
+import { PlaywrightCrawler } from "crawlee";
+import { inject } from "../../infrastructure/di/index.js";
+import { Injectable } from "../../infrastructure/di/injectable-decorator.js";
+import { DatabaseService } from "../db/prisma.js";
+import { EnvironmentService } from "../environmentService.js";
+import { parseTUS, parseTUSDetail } from "./parseTUS.js";
+import { FoundEventSource } from "./sources.js";
+
+@Injectable()
+export class FoundEventsService {
+    private readonly environmnent = inject(EnvironmentService);
+    private readonly database = inject(DatabaseService);
+    private readonly crawler: PlaywrightCrawler;
+
+    public constructor() {
+        const top = this;
+
+        this.crawler = new PlaywrightCrawler({
+            async requestHandler({ request, page, log }) {
+                if (request.retryCount > 0) return;
+
+                const label = request.label;
+
+                switch (label) {
+                    case "TUS":
+                        const tusDetails = await parseTUS(page, top.database);
+                        await top.crawler.addRequests(
+                            tusDetails.map(url => ({
+                                url: url,
+                                label: "TUS_DETAIL",
+                            })),
+                        );
+                        break;
+                    case "TUS_DETAIL":
+                        await parseTUSDetail(
+                            request.loadedUrl,
+                            page,
+                            top.database,
+                        );
+                        break;
+                    // case "DTU":
+                    //     events = parseDTU(page);
+                    //     break;
+                    // case "BTU":
+                    //     events = parseBTU(page);
+                    //     break;
+                    // case "MAGOSCH":
+                    //     events = parseMAGOSCH(page);
+                    //     break;
+                    default:
+                        log.warning(`Unknown source ${label}`);
+                }
+            },
+            headless: !this.environmnent.isDev(),
+        });
+    }
+
+    public async crawl() {
+        const sources = Object.entries(FoundEventSource).map(([key, url]) => ({
+            url,
+            label: key,
+        }));
+        return await this.crawler.run(sources);
+    }
+}