You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 

93 lines
2.5 KiB

/*
* vertretungsplan.io custom crawler
* Copyright (C) 2019 Jonas Lochmann
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, version 3 of the
* License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { config } from '../config'
import { getUrlInfo } from '../get-url-info'
import { getDates } from './date-parser'
import { getFileList } from './list-parser'
import { getSmallDates } from './smalldate-parser'
export const isConfigured = config.cwg.username.student && config.cwg.username.teacher
interface File {
title: string
mimeType: string
lastModified?: number
file: Array<{
url: string
sha512: string
size: number
}>
}
interface Message {
title: string
content: string
}
interface CrawlerResult {
studentPlans: Array<File>
teacherPlans: Array<File>
downloads: Array<File>
messages: Array<Message>
}
const addFileMetadata = (files: Array<{url: string, title: string}>) => Promise.all(files.map(async (file) => {
const info = await getUrlInfo(file.url)
return {
title: file.title,
lastModified: info.lastModified,
mimeType: info.mimeType,
file: [{
url: file.url,
sha512: info.sha512,
size: info.size
}]
}
}))
export async function crawl (): Promise<CrawlerResult> {
const [studentPlans, teacherPlans, downloads, dates, smallDates] = await Promise.all([
getFileList({
url: config.cwg.url.student,
username: config.cwg.username.student,
password: config.cwg.password.student
}).then((files) => addFileMetadata(files)),
getFileList({
url: config.cwg.url.teacher,
username: config.cwg.username.teacher,
password: config.cwg.password.teacher
}).then((files) => addFileMetadata(files)),
getFileList({
url: config.cwg.downloadUrl
}).then((files) => addFileMetadata(files)),
getDates(config.cwg.dateUrl),
getSmallDates(config.cwg.smallDateUrl)
])
return {
studentPlans,
teacherPlans,
downloads,
messages: [
...smallDates,
...dates
]
}
}