scraper
This commit is contained in:
toni
@@ -8,3 +8,5 @@ keycloak/realm-export
|
||||
|
||||
src/cert.pem
|
||||
src/key.pem
|
||||
|
||||
storage
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE ROLE webapi WITH
|
||||
LOGIN
|
||||
NOSUPERUSER
|
||||
INHERIT
|
||||
NOCREATEDB
|
||||
NOCREATEROLE
|
||||
NOREPLICATION
|
||||
NOBYPASSRLS
|
||||
PASSWORD 'devpassword';
|
||||
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA public
|
||||
GRANT DELETE, INSERT, SELECT, UPDATE ON TABLES TO webapi;
|
||||
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA public
|
||||
GRANT USAGE ON TYPES TO webapi;
|
||||
|
||||
@@ -4,10 +4,11 @@
|
||||
"scripts": {
|
||||
"start": "node dist/index.js",
|
||||
"build": "tsc -p .",
|
||||
"dev": "pnpm -w run dev"
|
||||
"dev": "pnpm -w run dev",
|
||||
"postinstall": "playwright install"
|
||||
},
|
||||
"type": "module",
|
||||
"packageManager": "pnpm@10.24.0+sha512.01ff8ae71b4419903b65c60fb2dc9d34cf8bb6e06d03bde112ef38f7a34d6904c424ba66bea5cdcf12890230bf39f9580473140ed9c946fef328b6e5238a345a",
|
||||
"packageManager": "pnpm@10.31.0+sha512.e3927388bfaa8078ceb79b748ffc1e8274e84d75163e67bc22e06c0d3aed43dd153151cbf11d7f8301ff4acb98c68bdc5cadf6989532801ffafe3b3e4a63c268",
|
||||
"private": true,
|
||||
"devDependencies": {
|
||||
"@types/connect-pg-simple": "^7.0.3",
|
||||
@@ -27,12 +28,14 @@
|
||||
"@tsoa/runtime": "^6.6.0",
|
||||
"connect-pg-simple": "^10.0.0",
|
||||
"cors": "^2.8.6",
|
||||
"crawlee": "^3.16.0",
|
||||
"dotenv": "^17.3.1",
|
||||
"express": "^5.2.1",
|
||||
"express-session": "^1.19.0",
|
||||
"helmet": "^8.1.0",
|
||||
"keycloak-connect": "^26.1.1",
|
||||
"pg": "^8.20.0",
|
||||
"playwright": "^1.58.2",
|
||||
"rxjs": "~7.8.2",
|
||||
"tsoa": "^6.6.0"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
-- CreateTable
|
||||
CREATE TABLE "FoundEvents" (
|
||||
"id" UUID NOT NULL DEFAULT uuidv7(),
|
||||
"source" VARCHAR NOT NULL,
|
||||
"url" VARCHAR NOT NULL,
|
||||
"foundDate" DATE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"eventDate" DATE NOT NULL,
|
||||
"title" VARCHAR NOT NULL,
|
||||
"description" VARCHAR NOT NULL,
|
||||
"attachedFile" BYTEA NOT NULL,
|
||||
|
||||
CONSTRAINT "FoundEvents_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
@@ -0,0 +1,6 @@
|
||||
-- AlterTable
|
||||
ALTER TABLE "FoundEvents" ALTER COLUMN "eventDate" DROP NOT NULL,
|
||||
ALTER COLUMN "title" DROP NOT NULL,
|
||||
ALTER COLUMN "description" DROP NOT NULL,
|
||||
ALTER COLUMN "attachedFile" DROP NOT NULL,
|
||||
ALTER COLUMN "attachedFile" SET DATA TYPE VARCHAR;
|
||||
@@ -0,0 +1,20 @@
|
||||
/*
|
||||
Warnings:
|
||||
|
||||
- You are about to drop the column `attachedFile` on the `FoundEvents` table. All the data in the column will be lost.
|
||||
|
||||
*/
|
||||
-- AlterTable
|
||||
ALTER TABLE "FoundEvents" DROP COLUMN "attachedFile";
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "FoundEventAttachments" (
|
||||
"id" UUID NOT NULL DEFAULT uuidv7(),
|
||||
"url" VARCHAR NOT NULL,
|
||||
"eventId" UUID NOT NULL,
|
||||
|
||||
CONSTRAINT "FoundEventAttachments_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "FoundEventAttachments" ADD CONSTRAINT "FoundEventAttachments_eventId_fkey" FOREIGN KEY ("eventId") REFERENCES "FoundEvents"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
@@ -0,0 +1,2 @@
|
||||
-- AlterTable
|
||||
ALTER TABLE "FoundEvents" ALTER COLUMN "eventDate" SET DATA TYPE VARCHAR;
|
||||
@@ -35,3 +35,21 @@ model session {
|
||||
enum Topic {
|
||||
TEST
|
||||
}
|
||||
|
||||
model FoundEvents {
|
||||
id String @id @default(dbgenerated("uuidv7()")) @db.Uuid
|
||||
source String @db.VarChar
|
||||
url String @db.VarChar
|
||||
foundDate DateTime @db.Date @default(now())
|
||||
eventDate String? @db.VarChar
|
||||
title String? @db.VarChar
|
||||
description String? @db.VarChar
|
||||
attachedFiles FoundEventAttachments[]
|
||||
}
|
||||
|
||||
model FoundEventAttachments {
|
||||
id String @id @default(dbgenerated("uuidv7()")) @db.Uuid
|
||||
url String @db.VarChar
|
||||
event FoundEvents @relation(fields: [eventId], references: [id])
|
||||
eventId String @db.Uuid
|
||||
}
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import { Controller, Post, Route, SuccessResponse } from "tsoa";
|
||||
import { inject } from "../infrastructure/di/index.js";
|
||||
import { FoundEventsService } from "../services/foundEvents/foundEventsService.js";
|
||||
|
||||
@Route("found-events")
|
||||
export class FoundEventsController extends Controller {
|
||||
private readonly foundEventsService = inject(FoundEventsService);
|
||||
|
||||
@Post("crawl")
|
||||
@SuccessResponse("200", "OK")
|
||||
//TODO add security
|
||||
public async crawl() {
|
||||
return await this.foundEventsService.crawl();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import type { FoundEvents } from "../generated/prisma/client.js";
|
||||
|
||||
export class FoundEventCreateArgs implements Omit<
|
||||
FoundEvents,
|
||||
"id" | "foundDate"
|
||||
> {
|
||||
source!: string;
|
||||
url!: string;
|
||||
eventDate!: string | null;
|
||||
title!: string | null;
|
||||
description!: string | null;
|
||||
}
|
||||
@@ -56,10 +56,11 @@ export class DatabaseService {
|
||||
|
||||
public async doRequest<T>(
|
||||
command: (prisma: SafeClient) => Promise<T>,
|
||||
request: Request,
|
||||
request: Request | null,
|
||||
): Promise<T> {
|
||||
const transaction =
|
||||
(request as any).transaction ?? (this.prismaClient as SafeClient);
|
||||
(request as any | null)?.transaction ??
|
||||
(this.prismaClient as SafeClient);
|
||||
return await command(transaction);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { PlaywrightCrawler } from "crawlee";
|
||||
import { inject } from "../../infrastructure/di/index.js";
|
||||
import { Injectable } from "../../infrastructure/di/injectable-decorator.js";
|
||||
import { DatabaseService } from "../db/prisma.js";
|
||||
import { EnvironmentService } from "../environmentService.js";
|
||||
import { parseTUS, parseTUSDetail } from "./parseTUS.js";
|
||||
import { FoundEventSource } from "./sources.js";
|
||||
|
||||
@Injectable()
|
||||
export class FoundEventsService {
|
||||
private readonly environmnent = inject(EnvironmentService);
|
||||
private readonly database = inject(DatabaseService);
|
||||
private readonly crawler: PlaywrightCrawler;
|
||||
|
||||
public constructor() {
|
||||
const top = this;
|
||||
|
||||
this.crawler = new PlaywrightCrawler({
|
||||
async requestHandler({ request, page, log }) {
|
||||
if (request.retryCount > 0) return;
|
||||
|
||||
const label = request.label;
|
||||
|
||||
switch (label) {
|
||||
case "TUS":
|
||||
const tusDetails = await parseTUS(page, top.database);
|
||||
await top.crawler.addRequests(
|
||||
tusDetails.map(url => ({
|
||||
url: url,
|
||||
label: "TUS_DETAIL",
|
||||
})),
|
||||
);
|
||||
break;
|
||||
case "TUS_DETAIL":
|
||||
await parseTUSDetail(
|
||||
request.loadedUrl,
|
||||
page,
|
||||
top.database,
|
||||
);
|
||||
break;
|
||||
// case "DTU":
|
||||
// events = parseDTU(page);
|
||||
// break;
|
||||
// case "BTU":
|
||||
// events = parseBTU(page);
|
||||
// break;
|
||||
// case "MAGOSCH":
|
||||
// events = parseMAGOSCH(page);
|
||||
// break;
|
||||
default:
|
||||
log.warning(`Unknown source ${label}`);
|
||||
}
|
||||
},
|
||||
headless: !this.environmnent.isDev(),
|
||||
});
|
||||
}
|
||||
|
||||
public async crawl() {
|
||||
const sources = Object.entries(FoundEventSource).map(([key, url]) => ({
|
||||
url,
|
||||
label: key,
|
||||
}));
|
||||
return await this.crawler.run(sources);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
import type { Page } from "playwright";
|
||||
import { expect } from "playwright/test";
|
||||
import type { DatabaseService } from "../db/prisma.js";
|
||||
import { FoundEventSource } from "./sources.js";
|
||||
|
||||
export async function parseTUS(
|
||||
page: Page,
|
||||
database: DatabaseService,
|
||||
): Promise<string[]> {
|
||||
let rowLocators = await page.locator("table").locator("tr").all();
|
||||
rowLocators = rowLocators.filter(
|
||||
// skip --- and header rows
|
||||
(_row, index) => index > 0 && index % 2 === 0,
|
||||
);
|
||||
const links = await Promise.all(
|
||||
rowLocators.map(async row => {
|
||||
return {
|
||||
url: await row.locator("a").getAttribute("href"),
|
||||
title: await row.locator("td").nth(1).textContent(),
|
||||
date: await row.locator("td").nth(0).textContent(),
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
const urls = links.map(link => link.url).filter(url => url != null);
|
||||
const existingEvents = await database.doRequest(
|
||||
async prisma =>
|
||||
prisma.foundEvents.findMany({
|
||||
where: {
|
||||
url: { in: urls },
|
||||
},
|
||||
}),
|
||||
null,
|
||||
);
|
||||
|
||||
const newLinks = links.filter(
|
||||
newEvent =>
|
||||
!existingEvents.some(
|
||||
oldEvent =>
|
||||
oldEvent.url === newEvent.url ||
|
||||
(oldEvent.title === newEvent.title &&
|
||||
oldEvent.eventDate === newEvent.date),
|
||||
),
|
||||
);
|
||||
|
||||
const datePages = newLinks.map(link => link.url).filter(url => url != null);
|
||||
return datePages;
|
||||
}
|
||||
|
||||
export async function parseTUSDetail(
|
||||
url: string,
|
||||
page: Page,
|
||||
database: DatabaseService,
|
||||
): Promise<void> {
|
||||
const content = page.locator("div#content");
|
||||
const title = await content.locator("h1").textContent();
|
||||
expect(content.locator("td").nth(0)).toHaveText("Datum:");
|
||||
const dateString = await content.locator("td").nth(1).textContent();
|
||||
const description = (await content.locator("span").allTextContents()).join(
|
||||
" ",
|
||||
);
|
||||
const attachedFileUrls = (
|
||||
await Promise.all(
|
||||
(await content.locator("a").all()).map(async locator =>
|
||||
locator.getAttribute("href"),
|
||||
),
|
||||
)
|
||||
).filter(url => url != null);
|
||||
|
||||
await database.doRequest(async prisma => {
|
||||
const newEvent = await prisma.foundEvents.create({
|
||||
data: {
|
||||
title: title?.trim() ?? null,
|
||||
eventDate: dateString,
|
||||
source: FoundEventSource.TUS,
|
||||
url: url,
|
||||
description: description?.trim() ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
await prisma.foundEventAttachments.createMany({
|
||||
data: attachedFileUrls.map(url => ({
|
||||
url: url,
|
||||
eventId: newEvent.id,
|
||||
})),
|
||||
});
|
||||
}, null);
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export enum FoundEventSource {
|
||||
TUS = "https://www.taekwondo-union-sachsen.de/index.php?page=Termine-TUS",
|
||||
// DTU = "https://www.dtu.de/termine",
|
||||
// BTU = "https://btu-online.de/termine/",
|
||||
// MAGOSCH = "https://www.taekwondo-magosch.de/termine",
|
||||
}
|
||||
@@ -57,5 +57,5 @@
|
||||
"forceConsistentCasingInFileNames": true
|
||||
},
|
||||
"files": ["./src/index.ts"],
|
||||
"include": ["src"]
|
||||
"include": ["src", "./package.json"]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user