You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 

63 lines
1.7 KiB

/*
* vertretungsplan.io custom crawler
* Copyright (C) 2019 Jonas Lochmann
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, version 3 of the
* License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { config } from './config'
import { delay } from './delay'
export function wrapCrawler<T> (crawl: () => Promise<T>) {
let lastSuccessData = delay(Math.random() * config.startupDelay).then(() => crawl())
let lastPromise = lastSuccessData
lastSuccessData.catch((ex) => {
console.warn('inital query failed', ex)
})
;(async () => {
try {
await lastPromise
} catch (ex) {
// ignore
}
while (true) {
await delay(1000 * 60 * 5)
try {
const result = await crawl()
lastPromise = Promise.resolve(result)
lastSuccessData = lastPromise
} catch (ex) {
console.warn('crawling failed', ex)
lastPromise = Promise.reject(ex)
// required to prevent an unhandled Promise rejection
lastPromise.catch(() => null)
}
}
})().catch((ex) => {
console.warn('crawl loop crashed', ex)
process.exit(1)
})
return {
getLastPromise: () => lastPromise,
getLastSuccessPromise: () => lastSuccessData
}
}