scraper

packages/api/src/services/db/prisma.ts

@@ -56,10 +56,11 @@ export class DatabaseService {
 
     public async doRequest<T>(
         command: (prisma: SafeClient) => Promise<T>,
-        request: Request,
+        request: Request | null,
     ): Promise<T> {
         const transaction =
-            (request as any).transaction ?? (this.prismaClient as SafeClient);
+            (request as any | null)?.transaction ??
+            (this.prismaClient as SafeClient);
         return await command(transaction);
     }
 }

packages/api/src/services/foundEvents/foundEventsService.ts

@@ -0,0 +1,65 @@
+import { PlaywrightCrawler } from "crawlee";
+import { inject } from "../../infrastructure/di/index.js";
+import { Injectable } from "../../infrastructure/di/injectable-decorator.js";
+import { DatabaseService } from "../db/prisma.js";
+import { EnvironmentService } from "../environmentService.js";
+import { parseTUS, parseTUSDetail } from "./parseTUS.js";
+import { FoundEventSource } from "./sources.js";
+
+@Injectable()
+export class FoundEventsService {
+    private readonly environmnent = inject(EnvironmentService);
+    private readonly database = inject(DatabaseService);
+    private readonly crawler: PlaywrightCrawler;
+
+    public constructor() {
+        const top = this;
+
+        this.crawler = new PlaywrightCrawler({
+            async requestHandler({ request, page, log }) {
+                if (request.retryCount > 0) return;
+
+                const label = request.label;
+
+                switch (label) {
+                    case "TUS":
+                        const tusDetails = await parseTUS(page, top.database);
+                        await top.crawler.addRequests(
+                            tusDetails.map(url => ({
+                                url: url,
+                                label: "TUS_DETAIL",
+                            })),
+                        );
+                        break;
+                    case "TUS_DETAIL":
+                        await parseTUSDetail(
+                            request.loadedUrl,
+                            page,
+                            top.database,
+                        );
+                        break;
+                    // case "DTU":
+                    //     events = parseDTU(page);
+                    //     break;
+                    // case "BTU":
+                    //     events = parseBTU(page);
+                    //     break;
+                    // case "MAGOSCH":
+                    //     events = parseMAGOSCH(page);
+                    //     break;
+                    default:
+                        log.warning(`Unknown source ${label}`);
+                }
+            },
+            headless: !this.environmnent.isDev(),
+        });
+    }
+
+    public async crawl() {
+        const sources = Object.entries(FoundEventSource).map(([key, url]) => ({
+            url,
+            label: key,
+        }));
+        return await this.crawler.run(sources);
+    }
+}

packages/api/src/services/foundEvents/parseTUS.ts

@@ -0,0 +1,88 @@
+import type { Page } from "playwright";
+import { expect } from "playwright/test";
+import type { DatabaseService } from "../db/prisma.js";
+import { FoundEventSource } from "./sources.js";
+
+export async function parseTUS(
+    page: Page,
+    database: DatabaseService,
+): Promise<string[]> {
+    let rowLocators = await page.locator("table").locator("tr").all();
+    rowLocators = rowLocators.filter(
+        // skip --- and header rows
+        (_row, index) => index > 0 && index % 2 === 0,
+    );
+    const links = await Promise.all(
+        rowLocators.map(async row => {
+            return {
+                url: await row.locator("a").getAttribute("href"),
+                title: await row.locator("td").nth(1).textContent(),
+                date: await row.locator("td").nth(0).textContent(),
+            };
+        }),
+    );
+
+    const urls = links.map(link => link.url).filter(url => url != null);
+    const existingEvents = await database.doRequest(
+        async prisma =>
+            prisma.foundEvents.findMany({
+                where: {
+                    url: { in: urls },
+                },
+            }),
+        null,
+    );
+
+    const newLinks = links.filter(
+        newEvent =>
+            !existingEvents.some(
+                oldEvent =>
+                    oldEvent.url === newEvent.url ||
+                    (oldEvent.title === newEvent.title &&
+                        oldEvent.eventDate === newEvent.date),
+            ),
+    );
+
+    const datePages = newLinks.map(link => link.url).filter(url => url != null);
+    return datePages;
+}
+
+export async function parseTUSDetail(
+    url: string,
+    page: Page,
+    database: DatabaseService,
+): Promise<void> {
+    const content = page.locator("div#content");
+    const title = await content.locator("h1").textContent();
+    expect(content.locator("td").nth(0)).toHaveText("Datum:");
+    const dateString = await content.locator("td").nth(1).textContent();
+    const description = (await content.locator("span").allTextContents()).join(
+        " ",
+    );
+    const attachedFileUrls = (
+        await Promise.all(
+            (await content.locator("a").all()).map(async locator =>
+                locator.getAttribute("href"),
+            ),
+        )
+    ).filter(url => url != null);
+
+    await database.doRequest(async prisma => {
+        const newEvent = await prisma.foundEvents.create({
+            data: {
+                title: title?.trim() ?? null,
+                eventDate: dateString,
+                source: FoundEventSource.TUS,
+                url: url,
+                description: description?.trim() ?? null,
+            },
+        });
+
+        await prisma.foundEventAttachments.createMany({
+            data: attachedFileUrls.map(url => ({
+                url: url,
+                eventId: newEvent.id,
+            })),
+        });
+    }, null);
+}

packages/api/src/services/foundEvents/sources.ts

@@ -0,0 +1,6 @@
+export enum FoundEventSource {
+    TUS = "https://www.taekwondo-union-sachsen.de/index.php?page=Termine-TUS",
+    // DTU = "https://www.dtu.de/termine",
+    // BTU = "https://btu-online.de/termine/",
+    // MAGOSCH = "https://www.taekwondo-magosch.de/termine",
+}