toniwalter.green/Taekwondo-Admin
0
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

scraper

Browse Source
This commit is contained in:
toni
2026-03-09 20:40:53 +01:00
parent 10f4750755
commit 71ca4d81e1
17 changed files with 2157 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/api/.gitignore
+2
View File
@@ -8,3 +8,5 @@ keycloak/realm-export
src/cert.pem
src/key.pem
storage
packages/api/dev/postgres/initdb/webapi.sql
+16
View File
@@ -0,0 +1,16 @@
CREATE ROLE webapi WITH
LOGIN
NOSUPERUSER
INHERIT
NOCREATEDB
NOCREATEROLE
NOREPLICATION
NOBYPASSRLS
PASSWORD 'devpassword';
ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA public
GRANT DELETE, INSERT, SELECT, UPDATE ON TABLES TO webapi;
ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA public
GRANT USAGE ON TYPES TO webapi;
packages/api/package.json
+5 -2
View File
@@ -4,10 +4,11 @@
"scripts": {
"start": "node dist/index.js",
"build": "tsc -p .",
"dev": "pnpm -w run dev"
"dev": "pnpm -w run dev",
"postinstall": "playwright install"
},
"type": "module",
"packageManager": "pnpm@10.24.0+sha512.01ff8ae71b4419903b65c60fb2dc9d34cf8bb6e06d03bde112ef38f7a34d6904c424ba66bea5cdcf12890230bf39f9580473140ed9c946fef328b6e5238a345a",
"packageManager": "pnpm@10.31.0+sha512.e3927388bfaa8078ceb79b748ffc1e8274e84d75163e67bc22e06c0d3aed43dd153151cbf11d7f8301ff4acb98c68bdc5cadf6989532801ffafe3b3e4a63c268",
"private": true,
"devDependencies": {
"@types/connect-pg-simple": "^7.0.3",
@@ -27,12 +28,14 @@
"@tsoa/runtime": "^6.6.0",
"connect-pg-simple": "^10.0.0",
"cors": "^2.8.6",
"crawlee": "^3.16.0",
"dotenv": "^17.3.1",
"express": "^5.2.1",
"express-session": "^1.19.0",
"helmet": "^8.1.0",
"keycloak-connect": "^26.1.1",
"pg": "^8.20.0",
"playwright": "^1.58.2",
"rxjs": "~7.8.2",
"tsoa": "^6.6.0"
}
packages/api/prisma/migrations/20260308214122_scraper/migration.sql
+13
View File
@@ -0,0 +1,13 @@
-- CreateTable
CREATE TABLE "FoundEvents" (
"id" UUID NOT NULL DEFAULT uuidv7(),
"source" VARCHAR NOT NULL,
"url" VARCHAR NOT NULL,
"foundDate" DATE NOT NULL DEFAULT CURRENT_TIMESTAMP,
"eventDate" DATE NOT NULL,
"title" VARCHAR NOT NULL,
"description" VARCHAR NOT NULL,
"attachedFile" BYTEA NOT NULL,
CONSTRAINT "FoundEvents_pkey" PRIMARY KEY ("id")
);
packages/api/prisma/migrations/20260308234748_found_events2/migration.sql
+6
View File
@@ -0,0 +1,6 @@
-- AlterTable
ALTER TABLE "FoundEvents" ALTER COLUMN "eventDate" DROP NOT NULL,
ALTER COLUMN "title" DROP NOT NULL,
ALTER COLUMN "description" DROP NOT NULL,
ALTER COLUMN "attachedFile" DROP NOT NULL,
ALTER COLUMN "attachedFile" SET DATA TYPE VARCHAR;
packages/api/prisma/migrations/20260308235352_event_attachments/migration.sql
+20
View File
@@ -0,0 +1,20 @@
/*
Warnings:
- You are about to drop the column `attachedFile` on the `FoundEvents` table. All the data in the column will be lost.
*/
-- AlterTable
ALTER TABLE "FoundEvents" DROP COLUMN "attachedFile";
-- CreateTable
CREATE TABLE "FoundEventAttachments" (
"id" UUID NOT NULL DEFAULT uuidv7(),
"url" VARCHAR NOT NULL,
"eventId" UUID NOT NULL,
CONSTRAINT "FoundEventAttachments_pkey" PRIMARY KEY ("id")
);
-- AddForeignKey
ALTER TABLE "FoundEventAttachments" ADD CONSTRAINT "FoundEventAttachments_eventId_fkey" FOREIGN KEY ("eventId") REFERENCES "FoundEvents"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
packages/api/prisma/migrations/20260309001330_eventdaterange/migration.sql
+2
View File
@@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "FoundEvents" ALTER COLUMN "eventDate" SET DATA TYPE VARCHAR;
packages/api/prisma/schema.prisma
+18
View File
@@ -35,3 +35,21 @@ model session {
enum Topic {
TEST
}
model FoundEvents {
id String @id @default(dbgenerated("uuidv7()")) @db.Uuid
source String @db.VarChar
url String @db.VarChar
foundDate DateTime @db.Date @default(now())
eventDate String? @db.VarChar
title String? @db.VarChar
description String? @db.VarChar
attachedFiles FoundEventAttachments[]
}
model FoundEventAttachments {
id String @id @default(dbgenerated("uuidv7()")) @db.Uuid
url String @db.VarChar
event FoundEvents @relation(fields: [eventId], references: [id])
eventId String @db.Uuid
}
packages/api/src/controllers/foundEventsController.ts
+15
View File
@@ -0,0 +1,15 @@
import { Controller, Post, Route, SuccessResponse } from "tsoa";
import { inject } from "../infrastructure/di/index.js";
import { FoundEventsService } from "../services/foundEvents/foundEventsService.js";
@Route("found-events")
export class FoundEventsController extends Controller {
private readonly foundEventsService = inject(FoundEventsService);
@Post("crawl")
@SuccessResponse("200", "OK")
//TODO add security
public async crawl() {
return await this.foundEventsService.crawl();
}
}
packages/api/src/dtos/foundEvents.ts
+12
View File
@@ -0,0 +1,12 @@
import type { FoundEvents } from "../generated/prisma/client.js";
export class FoundEventCreateArgs implements Omit<
FoundEvents,
"id" | "foundDate"
> {
source!: string;
url!: string;
eventDate!: string | null;
title!: string | null;
description!: string | null;
}
packages/api/src/services/db/prisma.ts
+3 -2
View File
@@ -56,10 +56,11 @@ export class DatabaseService {
public async doRequest<T>(
command: (prisma: SafeClient) => Promise<T>,
request: Request,
request: Request | null,
): Promise<T> {
const transaction =
(request as any).transaction ?? (this.prismaClient as SafeClient);
(request as any | null)?.transaction ??
(this.prismaClient as SafeClient);
return await command(transaction);
}
}
packages/api/src/services/foundEvents/foundEventsService.ts
+65
View File
@@ -0,0 +1,65 @@
import { PlaywrightCrawler } from "crawlee";
import { inject } from "../../infrastructure/di/index.js";
import { Injectable } from "../../infrastructure/di/injectable-decorator.js";
import { DatabaseService } from "../db/prisma.js";
import { EnvironmentService } from "../environmentService.js";
import { parseTUS, parseTUSDetail } from "./parseTUS.js";
import { FoundEventSource } from "./sources.js";
@Injectable()
export class FoundEventsService {
private readonly environmnent = inject(EnvironmentService);
private readonly database = inject(DatabaseService);
private readonly crawler: PlaywrightCrawler;
public constructor() {
const top = this;
this.crawler = new PlaywrightCrawler({
async requestHandler({ request, page, log }) {
if (request.retryCount > 0) return;
const label = request.label;
switch (label) {
case "TUS":
const tusDetails = await parseTUS(page, top.database);
await top.crawler.addRequests(
tusDetails.map(url => ({
url: url,
label: "TUS_DETAIL",
})),
);
break;
case "TUS_DETAIL":
await parseTUSDetail(
request.loadedUrl,
page,
top.database,
);
break;
// case "DTU":
// events = parseDTU(page);
// break;
// case "BTU":
// events = parseBTU(page);
// break;
// case "MAGOSCH":
// events = parseMAGOSCH(page);
// break;
default:
log.warning(`Unknown source ${label}`);
}
},
headless: !this.environmnent.isDev(),
});
}
public async crawl() {
const sources = Object.entries(FoundEventSource).map(([key, url]) => ({
url,
label: key,
}));
return await this.crawler.run(sources);
}
}
packages/api/src/services/foundEvents/parseTUS.ts
+88
View File
@@ -0,0 +1,88 @@
import type { Page } from "playwright";
import { expect } from "playwright/test";
import type { DatabaseService } from "../db/prisma.js";
import { FoundEventSource } from "./sources.js";
export async function parseTUS(
page: Page,
database: DatabaseService,
): Promise<string[]> {
let rowLocators = await page.locator("table").locator("tr").all();
rowLocators = rowLocators.filter(
// skip --- and header rows
(_row, index) => index > 0 && index % 2 === 0,
);
const links = await Promise.all(
rowLocators.map(async row => {
return {
url: await row.locator("a").getAttribute("href"),
title: await row.locator("td").nth(1).textContent(),
date: await row.locator("td").nth(0).textContent(),
};
}),
);
const urls = links.map(link => link.url).filter(url => url != null);
const existingEvents = await database.doRequest(
async prisma =>
prisma.foundEvents.findMany({
where: {
url: { in: urls },
},
}),
null,
);
const newLinks = links.filter(
newEvent =>
!existingEvents.some(
oldEvent =>
oldEvent.url === newEvent.url ||
(oldEvent.title === newEvent.title &&
oldEvent.eventDate === newEvent.date),
),
);
const datePages = newLinks.map(link => link.url).filter(url => url != null);
return datePages;
}
export async function parseTUSDetail(
url: string,
page: Page,
database: DatabaseService,
): Promise<void> {
const content = page.locator("div#content");
const title = await content.locator("h1").textContent();
expect(content.locator("td").nth(0)).toHaveText("Datum:");
const dateString = await content.locator("td").nth(1).textContent();
const description = (await content.locator("span").allTextContents()).join(
" ",
);
const attachedFileUrls = (
await Promise.all(
(await content.locator("a").all()).map(async locator =>
locator.getAttribute("href"),
),
)
).filter(url => url != null);
await database.doRequest(async prisma => {
const newEvent = await prisma.foundEvents.create({
data: {
title: title?.trim() ?? null,
eventDate: dateString,
source: FoundEventSource.TUS,
url: url,
description: description?.trim() ?? null,
},
});
await prisma.foundEventAttachments.createMany({
data: attachedFileUrls.map(url => ({
url: url,
eventId: newEvent.id,
})),
});
}, null);
}
packages/api/src/services/foundEvents/sources.ts
+6
View File
@@ -0,0 +1,6 @@
export enum FoundEventSource {
TUS = "https://www.taekwondo-union-sachsen.de/index.php?page=Termine-TUS",
// DTU = "https://www.dtu.de/termine",
// BTU = "https://btu-online.de/termine/",
// MAGOSCH = "https://www.taekwondo-magosch.de/termine",
}
packages/api/tsconfig.json
+1 -1
View File
@@ -57,5 +57,5 @@
"forceConsistentCasingInFileNames": true
},
"files": ["./src/index.ts"],
"include": ["src"]
"include": ["src", "./package.json"]
}