You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
93 lines
2.5 KiB
93 lines
2.5 KiB
/*
|
|
* vertretungsplan.io custom crawler
|
|
* Copyright (C) 2019 Jonas Lochmann
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU Affero General Public License as
|
|
* published by the Free Software Foundation, version 3 of the
|
|
* License.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU Affero General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Affero General Public License
|
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
import { config } from '../config'
|
|
import { getUrlInfo } from '../get-url-info'
|
|
import { getDates } from './date-parser'
|
|
import { getFileList } from './list-parser'
|
|
import { getSmallDates } from './smalldate-parser'
|
|
|
|
export const isConfigured = config.cwg.username.student && config.cwg.username.teacher
|
|
|
|
interface File {
|
|
title: string
|
|
mimeType: string
|
|
lastModified?: number
|
|
file: Array<{
|
|
url: string
|
|
sha512: string
|
|
size: number
|
|
}>
|
|
}
|
|
|
|
interface Message {
|
|
title: string
|
|
content: string
|
|
}
|
|
|
|
interface CrawlerResult {
|
|
studentPlans: Array<File>
|
|
teacherPlans: Array<File>
|
|
downloads: Array<File>
|
|
messages: Array<Message>
|
|
}
|
|
|
|
const addFileMetadata = (files: Array<{url: string, title: string}>) => Promise.all(files.map(async (file) => {
|
|
const info = await getUrlInfo(file.url)
|
|
|
|
return {
|
|
title: file.title,
|
|
lastModified: info.lastModified,
|
|
mimeType: info.mimeType,
|
|
file: [{
|
|
url: file.url,
|
|
sha512: info.sha512,
|
|
size: info.size
|
|
}]
|
|
}
|
|
}))
|
|
|
|
export async function crawl (): Promise<CrawlerResult> {
|
|
const [studentPlans, teacherPlans, downloads, dates, smallDates] = await Promise.all([
|
|
getFileList({
|
|
url: config.cwg.url.student,
|
|
username: config.cwg.username.student,
|
|
password: config.cwg.password.student
|
|
}).then((files) => addFileMetadata(files)),
|
|
getFileList({
|
|
url: config.cwg.url.teacher,
|
|
username: config.cwg.username.teacher,
|
|
password: config.cwg.password.teacher
|
|
}).then((files) => addFileMetadata(files)),
|
|
getFileList({
|
|
url: config.cwg.downloadUrl
|
|
}).then((files) => addFileMetadata(files)),
|
|
getDates(config.cwg.dateUrl),
|
|
getSmallDates(config.cwg.smallDateUrl)
|
|
])
|
|
|
|
return {
|
|
studentPlans,
|
|
teacherPlans,
|
|
downloads,
|
|
messages: [
|
|
...smallDates,
|
|
...dates
|
|
]
|
|
}
|
|
}
|