packages/api/.gitignore

@@ -8,3 +8,5 @@ keycloak/realm-export
 
 src/cert.pem
 src/key.pem
+
+storage

packages/api/dev/postgres/initdb/webapi.sql

@@ -0,0 +1,16 @@
+CREATE ROLE webapi WITH
+  LOGIN
+  NOSUPERUSER
+  INHERIT
+  NOCREATEDB
+  NOCREATEROLE
+  NOREPLICATION
+  NOBYPASSRLS
+  PASSWORD 'devpassword';
+
+ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA public
+GRANT DELETE, INSERT, SELECT, UPDATE ON TABLES TO webapi;
+
+ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA public
+GRANT USAGE ON TYPES TO webapi;
+ 

packages/api/package.json

@@ -4,10 +4,11 @@
     "scripts": {
         "start": "node dist/index.js",
         "build": "tsc -p .",
-        "dev": "pnpm -w run dev"
+        "dev": "pnpm -w run dev",
+        "postinstall": "playwright install"
     },
     "type": "module",
-    "packageManager": "pnpm@10.24.0+sha512.01ff8ae71b4419903b65c60fb2dc9d34cf8bb6e06d03bde112ef38f7a34d6904c424ba66bea5cdcf12890230bf39f9580473140ed9c946fef328b6e5238a345a",
+    "packageManager": "pnpm@10.31.0+sha512.e3927388bfaa8078ceb79b748ffc1e8274e84d75163e67bc22e06c0d3aed43dd153151cbf11d7f8301ff4acb98c68bdc5cadf6989532801ffafe3b3e4a63c268",
     "private": true,
     "devDependencies": {
         "@types/connect-pg-simple": "^7.0.3",
@@ -27,12 +28,14 @@
         "@tsoa/runtime": "^6.6.0",
         "connect-pg-simple": "^10.0.0",
         "cors": "^2.8.6",
+        "crawlee": "^3.16.0",
         "dotenv": "^17.3.1",
         "express": "^5.2.1",
         "express-session": "^1.19.0",
         "helmet": "^8.1.0",
         "keycloak-connect": "^26.1.1",
         "pg": "^8.20.0",
+        "playwright": "^1.58.2",
         "rxjs": "~7.8.2",
         "tsoa": "^6.6.0"
     }

packages/api/prisma/migrations/20260308214122_scraper/migration.sql

@@ -0,0 +1,13 @@
+-- CreateTable
+CREATE TABLE "FoundEvents" (
+    "id" UUID NOT NULL DEFAULT uuidv7(),
+    "source" VARCHAR NOT NULL,
+    "url" VARCHAR NOT NULL,
+    "foundDate" DATE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "eventDate" DATE NOT NULL,
+    "title" VARCHAR NOT NULL,
+    "description" VARCHAR NOT NULL,
+    "attachedFile" BYTEA NOT NULL,
+
+    CONSTRAINT "FoundEvents_pkey" PRIMARY KEY ("id")
+);

packages/api/prisma/migrations/20260308234748_found_events2/migration.sql

@@ -0,0 +1,6 @@
+-- AlterTable
+ALTER TABLE "FoundEvents" ALTER COLUMN "eventDate" DROP NOT NULL,
+ALTER COLUMN "title" DROP NOT NULL,
+ALTER COLUMN "description" DROP NOT NULL,
+ALTER COLUMN "attachedFile" DROP NOT NULL,
+ALTER COLUMN "attachedFile" SET DATA TYPE VARCHAR;

packages/api/prisma/migrations/20260308235352_event_attachments/migration.sql

@@ -0,0 +1,20 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `attachedFile` on the `FoundEvents` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "FoundEvents" DROP COLUMN "attachedFile";
+
+-- CreateTable
+CREATE TABLE "FoundEventAttachments" (
+    "id" UUID NOT NULL DEFAULT uuidv7(),
+    "url" VARCHAR NOT NULL,
+    "eventId" UUID NOT NULL,
+
+    CONSTRAINT "FoundEventAttachments_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "FoundEventAttachments" ADD CONSTRAINT "FoundEventAttachments_eventId_fkey" FOREIGN KEY ("eventId") REFERENCES "FoundEvents"("id") ON DELETE RESTRICT ON UPDATE CASCADE;

packages/api/prisma/migrations/20260309001330_eventdaterange/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "FoundEvents" ALTER COLUMN "eventDate" SET DATA TYPE VARCHAR;

packages/api/prisma/schema.prisma

@@ -35,3 +35,21 @@ model session {
 enum Topic {
   TEST
 }
+
+model FoundEvents {
+  id            String    @id @default(dbgenerated("uuidv7()")) @db.Uuid
+  source        String    @db.VarChar
+  url           String    @db.VarChar
+  foundDate     DateTime  @db.Date @default(now())
+  eventDate     String?   @db.VarChar
+  title         String?   @db.VarChar
+  description   String?   @db.VarChar
+  attachedFiles FoundEventAttachments[]
+}
+
+model FoundEventAttachments {
+  id      String      @id @default(dbgenerated("uuidv7()")) @db.Uuid
+  url     String      @db.VarChar
+  event   FoundEvents @relation(fields: [eventId], references: [id])
+  eventId String      @db.Uuid
+}

packages/api/src/controllers/foundEventsController.ts

@@ -0,0 +1,15 @@
+import { Controller, Post, Route, SuccessResponse } from "tsoa";
+import { inject } from "../infrastructure/di/index.js";
+import { FoundEventsService } from "../services/foundEvents/foundEventsService.js";
+
+@Route("found-events")
+export class FoundEventsController extends Controller {
+    private readonly foundEventsService = inject(FoundEventsService);
+
+    @Post("crawl")
+    @SuccessResponse("200", "OK")
+    //TODO add security
+    public async crawl() {
+        return await this.foundEventsService.crawl();
+    }
+}

packages/api/src/dtos/foundEvents.ts

@@ -0,0 +1,12 @@
+import type { FoundEvents } from "../generated/prisma/client.js";
+
+export class FoundEventCreateArgs implements Omit<
+    FoundEvents,
+    "id" | "foundDate"
+> {
+    source!: string;
+    url!: string;
+    eventDate!: string | null;
+    title!: string | null;
+    description!: string | null;
+}

packages/api/src/services/db/prisma.ts

@@ -56,10 +56,11 @@ export class DatabaseService {
 
     public async doRequest<T>(
         command: (prisma: SafeClient) => Promise<T>,
-        request: Request,
+        request: Request | null,
     ): Promise<T> {
         const transaction =
-            (request as any).transaction ?? (this.prismaClient as SafeClient);
+            (request as any | null)?.transaction ??
+            (this.prismaClient as SafeClient);
         return await command(transaction);
     }
 }

packages/api/src/services/foundEvents/foundEventsService.ts

@@ -0,0 +1,65 @@
+import { PlaywrightCrawler } from "crawlee";
+import { inject } from "../../infrastructure/di/index.js";
+import { Injectable } from "../../infrastructure/di/injectable-decorator.js";
+import { DatabaseService } from "../db/prisma.js";
+import { EnvironmentService } from "../environmentService.js";
+import { parseTUS, parseTUSDetail } from "./parseTUS.js";
+import { FoundEventSource } from "./sources.js";
+
+@Injectable()
+export class FoundEventsService {
+    private readonly environmnent = inject(EnvironmentService);
+    private readonly database = inject(DatabaseService);
+    private readonly crawler: PlaywrightCrawler;
+
+    public constructor() {
+        const top = this;
+
+        this.crawler = new PlaywrightCrawler({
+            async requestHandler({ request, page, log }) {
+                if (request.retryCount > 0) return;
+
+                const label = request.label;
+
+                switch (label) {
+                    case "TUS":
+                        const tusDetails = await parseTUS(page, top.database);
+                        await top.crawler.addRequests(
+                            tusDetails.map(url => ({
+                                url: url,
+                                label: "TUS_DETAIL",
+                            })),
+                        );
+                        break;
+                    case "TUS_DETAIL":
+                        await parseTUSDetail(
+                            request.loadedUrl,
+                            page,
+                            top.database,
+                        );
+                        break;
+                    // case "DTU":
+                    //     events = parseDTU(page);
+                    //     break;
+                    // case "BTU":
+                    //     events = parseBTU(page);
+                    //     break;
+                    // case "MAGOSCH":
+                    //     events = parseMAGOSCH(page);
+                    //     break;
+                    default:
+                        log.warning(`Unknown source ${label}`);
+                }
+            },
+            headless: !this.environmnent.isDev(),
+        });
+    }
+
+    public async crawl() {
+        const sources = Object.entries(FoundEventSource).map(([key, url]) => ({
+            url,
+            label: key,
+        }));
+        return await this.crawler.run(sources);
+    }
+}

packages/api/src/services/foundEvents/parseTUS.ts

@@ -0,0 +1,88 @@
+import type { Page } from "playwright";
+import { expect } from "playwright/test";
+import type { DatabaseService } from "../db/prisma.js";
+import { FoundEventSource } from "./sources.js";
+
+export async function parseTUS(
+    page: Page,
+    database: DatabaseService,
+): Promise<string[]> {
+    let rowLocators = await page.locator("table").locator("tr").all();
+    rowLocators = rowLocators.filter(
+        // skip --- and header rows
+        (_row, index) => index > 0 && index % 2 === 0,
+    );
+    const links = await Promise.all(
+        rowLocators.map(async row => {
+            return {
+                url: await row.locator("a").getAttribute("href"),
+                title: await row.locator("td").nth(1).textContent(),
+                date: await row.locator("td").nth(0).textContent(),
+            };
+        }),
+    );
+
+    const urls = links.map(link => link.url).filter(url => url != null);
+    const existingEvents = await database.doRequest(
+        async prisma =>
+            prisma.foundEvents.findMany({
+                where: {
+                    url: { in: urls },
+                },
+            }),
+        null,
+    );
+
+    const newLinks = links.filter(
+        newEvent =>
+            !existingEvents.some(
+                oldEvent =>
+                    oldEvent.url === newEvent.url ||
+                    (oldEvent.title === newEvent.title &&
+                        oldEvent.eventDate === newEvent.date),
+            ),
+    );
+
+    const datePages = newLinks.map(link => link.url).filter(url => url != null);
+    return datePages;
+}
+
+export async function parseTUSDetail(
+    url: string,
+    page: Page,
+    database: DatabaseService,
+): Promise<void> {
+    const content = page.locator("div#content");
+    const title = await content.locator("h1").textContent();
+    expect(content.locator("td").nth(0)).toHaveText("Datum:");
+    const dateString = await content.locator("td").nth(1).textContent();
+    const description = (await content.locator("span").allTextContents()).join(
+        " ",
+    );
+    const attachedFileUrls = (
+        await Promise.all(
+            (await content.locator("a").all()).map(async locator =>
+                locator.getAttribute("href"),
+            ),
+        )
+    ).filter(url => url != null);
+
+    await database.doRequest(async prisma => {
+        const newEvent = await prisma.foundEvents.create({
+            data: {
+                title: title?.trim() ?? null,
+                eventDate: dateString,
+                source: FoundEventSource.TUS,
+                url: url,
+                description: description?.trim() ?? null,
+            },
+        });
+
+        await prisma.foundEventAttachments.createMany({
+            data: attachedFileUrls.map(url => ({
+                url: url,
+                eventId: newEvent.id,
+            })),
+        });
+    }, null);
+}

packages/api/src/services/foundEvents/sources.ts

@@ -0,0 +1,6 @@
+export enum FoundEventSource {
+    TUS = "https://www.taekwondo-union-sachsen.de/index.php?page=Termine-TUS",
+    // DTU = "https://www.dtu.de/termine",
+    // BTU = "https://btu-online.de/termine/",
+    // MAGOSCH = "https://www.taekwondo-magosch.de/termine",
+}

packages/api/tsconfig.json

@@ -57,5 +57,5 @@
         "forceConsistentCasingInFileNames": true
     },
     "files": ["./src/index.ts"],
-    "include": ["src"]
+    "include": ["src", "./package.json"]
 }

scripts/api-ts.ps1

@@ -3,4 +3,4 @@ $env:TKD_DB_HOST="localhost"
 $env:TKD_DB_USER="webapi"
 $env:TKD_DB_PASSWORD="devpassword"
 $env:TKD_COOKIE_SECRET="devsecret"
-pnpx tsx --watch --tsconfig .\tsconfig.json .\src\index.ts
+pnpx tsx --watch --tsconfig .\tsconfig.json --clear-screen=false .\src\index.ts