You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

94 lines
2.5KB

  1. /*
  2. * vertretungsplan.io custom crawler
  3. * Copyright (C) 2019 Jonas Lochmann
  4. *
  5. * This program is free software: you can redistribute it and/or modify
  6. * it under the terms of the GNU Affero General Public License as
  7. * published by the Free Software Foundation, version 3 of the
  8. * License.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU Affero General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU Affero General Public License
  16. * along with this program. If not, see <https://www.gnu.org/licenses/>.
  17. */
  18. import { config } from '../config'
  19. import { getUrlInfo } from '../get-url-info'
  20. import { getDates } from './date-parser'
  21. import { getFileList } from './list-parser'
  22. import { getSmallDates } from './smalldate-parser'
  23. export const isConfigured = config.cwg.username.student && config.cwg.username.teacher
  24. interface File {
  25. title: string
  26. mimeType: string
  27. lastModified?: number
  28. file: Array<{
  29. url: string
  30. sha512: string
  31. size: number
  32. }>
  33. }
  34. interface Message {
  35. title: string
  36. content: string
  37. }
  38. interface CrawlerResult {
  39. studentPlans: Array<File>
  40. teacherPlans: Array<File>
  41. downloads: Array<File>
  42. messages: Array<Message>
  43. }
  44. const addFileMetadata = (files: Array<{url: string, title: string}>) => Promise.all(files.map(async (file) => {
  45. const info = await getUrlInfo(file.url)
  46. return {
  47. title: file.title,
  48. lastModified: info.lastModified,
  49. mimeType: info.mimeType,
  50. file: [{
  51. url: file.url,
  52. sha512: info.sha512,
  53. size: info.size
  54. }]
  55. }
  56. }))
  57. export async function crawl (): Promise<CrawlerResult> {
  58. const [studentPlans, teacherPlans, downloads, dates, smallDates] = await Promise.all([
  59. getFileList({
  60. url: config.cwg.url.student,
  61. username: config.cwg.username.student,
  62. password: config.cwg.password.student
  63. }).then((files) => addFileMetadata(files)),
  64. getFileList({
  65. url: config.cwg.url.teacher,
  66. username: config.cwg.username.teacher,
  67. password: config.cwg.password.teacher
  68. }).then((files) => addFileMetadata(files)),
  69. getFileList({
  70. url: config.cwg.downloadUrl
  71. }).then((files) => addFileMetadata(files)),
  72. getDates(config.cwg.dateUrl),
  73. getSmallDates(config.cwg.smallDateUrl)
  74. ])
  75. return {
  76. studentPlans,
  77. teacherPlans,
  78. downloads,
  79. messages: [
  80. ...smallDates,
  81. ...dates
  82. ]
  83. }
  84. }