integration-custom/src/cwg/crawler.ts

/*
 * vertretungsplan.io custom crawler
 * Copyright (C) 2019 Jonas Lochmann
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, version 3 of the
 * License.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */

import { config } from '../config'
import { getUrlInfo } from '../get-url-info'
import { getDates } from './date-parser'
import { getFileList } from './list-parser'
import { getSmallDates } from './smalldate-parser'

export const isConfigured = config.cwg.username.student && config.cwg.username.teacher

interface File {
  title: string
  mimeType: string
  lastModified?: number
  file: Array<{
    url: string
    sha512: string
    size: number
  }>
}

interface Message {
  title: string
  content: string
}

interface CrawlerResult {
  studentPlans: Array<File>
  teacherPlans: Array<File>
  downloads: Array<File>
  messages: Array<Message>
}

const addFileMetadata = (files: Array<{url: string, title: string}>) => Promise.all(files.map(async (file) => {
  const info = await getUrlInfo(file.url)

  return {
    title: file.title,
    lastModified: info.lastModified,
    mimeType: info.mimeType,
    file: [{
      url: file.url,
      sha512: info.sha512,
      size: info.size
    }]
  }
}))

export async function crawl (): Promise<CrawlerResult> {
  const [studentPlans, teacherPlans, downloads, dates, smallDates] = await Promise.all([
    getFileList({
      url: config.cwg.url.student,
      username: config.cwg.username.student,
      password: config.cwg.password.student
    }).then((files) => addFileMetadata(files)),
    getFileList({
      url: config.cwg.url.teacher,
      username: config.cwg.username.teacher,
      password: config.cwg.password.teacher
    }).then((files) => addFileMetadata(files)),
    getFileList({
      url: config.cwg.downloadUrl
    }).then((files) => addFileMetadata(files)),
    getDates(config.cwg.dateUrl),
    getSmallDates(config.cwg.smallDateUrl)
  ])

  return {
    studentPlans,
    teacherPlans,
    downloads,
    messages: [
      ...smallDates,
      ...dates
    ]
  }
}