Add timeout on youtube dl to cleaup files

Author: Chocobozzz

server/helpers/webtorrent.ts

@@ -5,7 +5,7 @@ import { createWriteStream, ensureDir, remove } from 'fs-extra'
 import { CONFIG } from '../initializers'
 import { dirname, join } from 'path'
 
-async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout?: number) {
+async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout: number) {
   const id = target.magnetUri || target.torrentName
   let timer
 
@@ -50,12 +50,10 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
 
     torrent.on('error', err => rej(err))
 
-    if (timeout) {
-      timer = setTimeout(async () => {
-        return safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName)
-          .then(() => rej(new Error('Webtorrent download timeout.')))
-      }, timeout)
-    }
+    timer = setTimeout(async () => {
+      return safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName)
+        .then(() => rej(new Error('Webtorrent download timeout.')))
+    }, timeout)
   })
 }
 

server/helpers/youtube-dl.ts

@@ -4,7 +4,7 @@ import { logger } from './logger'
 import { generateVideoTmpPath } from './utils'
 import { join } from 'path'
 import { root } from './core-utils'
-import { ensureDir, writeFile } from 'fs-extra'
+import { ensureDir, writeFile, remove } from 'fs-extra'
 import * as request from 'request'
 import { createWriteStream } from 'fs'
 
@@ -39,8 +39,9 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo>
   })
 }
 
-function downloadYoutubeDLVideo (url: string) {
+function downloadYoutubeDLVideo (url: string, timeout: number) {
   const path = generateVideoTmpPath(url)
+  let timer
 
   logger.info('Importing youtubeDL video %s', url)
 
@@ -49,10 +50,23 @@ function downloadYoutubeDLVideo (url: string) {
   return new Promise<string>(async (res, rej) => {
     const youtubeDL = await safeGetYoutubeDL()
     youtubeDL.exec(url, options, processOptions, err => {
-      if (err) return rej(err)
+      clearTimeout(timer)
+
+      if (err) {
+        remove(path)
+          .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err }))
+
+        return rej(err)
+      }
 
       return res(path)
     })
+
+    timer = setTimeout(async () => {
+      await remove(path)
+
+      return rej(new Error('YoutubeDL download timeout.'))
+    }, timeout)
   })
 }
 

server/initializers/constants.ts

@@ -119,7 +119,7 @@ const JOB_TTL: { [ id in JobType ]: number } = {
   'activitypub-follow': 60000 * 10, // 10 minutes
   'video-file-import': 1000 * 3600, // 1 hour
   'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long
-  'video-import': 1000 * 3600, // 1 hour
+  'video-import': 1000 * 3600 * 2, //  hours
   'email': 60000 * 10, // 10 minutes
   'videos-views': undefined // Unlimited
 }
@@ -133,6 +133,7 @@ const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activ
 const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...)
 const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds
 const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
+const VIDEO_IMPORT_TIMEOUT = 1000 * 3600 // 1 hour
 
 // 1 hour
 let SCHEDULER_INTERVALS_MS = {
@@ -700,6 +701,7 @@ export {
   TORRENT_MIMETYPE_EXT,
   STATIC_MAX_AGE,
   STATIC_PATHS,
+  VIDEO_IMPORT_TIMEOUT,
   ACTIVITY_PUB,
   ACTIVITY_PUB_ACTOR_TYPES,
   THUMBNAILS_SIZE,

server/lib/job-queue/handlers/video-import.ts

@@ -6,7 +6,7 @@ import { VideoImportState } from '../../../../shared/models/videos'
 import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
 import { extname, join } from 'path'
 import { VideoFileModel } from '../../../models/video/video-file'
-import { CONFIG, sequelizeTypescript } from '../../../initializers'
+import { CONFIG, sequelizeTypescript, VIDEO_IMPORT_TIMEOUT } from '../../../initializers'
 import { doRequestAndSaveToFile } from '../../../helpers/requests'
 import { VideoState } from '../../../../shared'
 import { JobQueue } from '../index'
@@ -65,7 +65,7 @@ async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentP
     torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined,
     magnetUri: videoImport.magnetUri
   }
-  return processFile(() => downloadWebTorrentVideo(target), videoImport, options)
+  return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options)
 }
 
 async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) {
@@ -83,7 +83,7 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub
     generatePreview: false
   }
 
-  return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl), videoImport, options)
+  return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl, VIDEO_IMPORT_TIMEOUT), videoImport, options)
 }
 
 async function getVideoImportOrDie (videoImportId: number) {

server/lib/schedulers/videos-redundancy-scheduler.ts

@@ -1,5 +1,5 @@
 import { AbstractScheduler } from './abstract-scheduler'
-import { CONFIG, JOB_TTL, REDUNDANCY } from '../../initializers'
+import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers'
 import { logger } from '../../helpers/logger'
 import { VideosRedundancy } from '../../../shared/models/redundancy'
 import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
@@ -9,7 +9,6 @@ import { join } from 'path'
 import { rename } from 'fs-extra'
 import { getServerActor } from '../../helpers/utils'
 import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
-import { VideoModel } from '../../models/video/video'
 import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
 import { removeVideoRedundancy } from '../redundancy'
 import { getOrCreateVideoAndAccountAndChannel } from '../activitypub'
@@ -142,7 +141,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
       const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
       const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
 
-      const tmpPath = await downloadWebTorrentVideo({ magnetUri }, JOB_TTL['video-import'])
+      const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
 
       const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file))
       await rename(tmpPath, destPath)