Merge branch 'master' into id3-tagging

backend/app.js

@@ -8,8 +8,11 @@ var https = require('https');
 var express = require("express");
 var bodyParser = require("body-parser");
 var archiver = require('archiver');
+var mergeFiles = require('merge-files');
 const low = require('lowdb')
 const NodeID3 = require('node-id3')
+const downloader = require('youtube-dl/lib/downloader')
+const fetch = require('node-fetch');
 var URL = require('url').URL;
 const shortid = require('shortid')
 const url_api = require('url');
@@ -19,7 +22,7 @@ var subscriptions_api = require('./subscriptions')
 var app = express();
 
 const FileSync = require('lowdb/adapters/FileSync')
-const adapter = new FileSync('db.json');
+const adapter = new FileSync('./appdata/db.json');
 const db = low(adapter)
 
 // Set some defaults
@@ -56,8 +59,17 @@ let debugMode = process.env.YTDL_MODE === 'debug';
 
 if (debugMode) console.log('YTDL-Material in debug mode!');
 
+// updates & starts youtubedl
+startYoutubeDL();
+
 var validDownloadingAgents = [
-    'aria2c'
+    'aria2c',
+    'avconv',
+    'axel',
+    'curl',
+    'ffmpeg',
+    'httpie',
+    'wget'
 ]
 
 // don't overwrite config if it already happened.. NOT
@@ -78,12 +90,17 @@ app.use(bodyParser.json());
 
 // objects
 
-function File(id, title, thumbnailURL, isAudio, duration) {
+function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date) {
     this.id = id;
     this.title = title;
     this.thumbnailURL = thumbnailURL;
     this.isAudio = isAudio;
     this.duration = duration;
+    this.url = url;
+    this.uploader = uploader;
+    this.size = size;
+    this.path = path;
+    this.upload_date = upload_date;
 }
 
 // actual functions
@@ -141,6 +158,8 @@ async function loadConfig() {
 
         if (!useDefaultDownloadingAgent && validDownloadingAgents.indexOf(customDownloadingAgent) !== -1 ) {
             console.log(`INFO: Using non-default downloading agent \'${customDownloadingAgent}\'`)
+        } else {
+            customDownloadingAgent = null;
         }
 
         if (usingEncryption)
@@ -345,10 +364,16 @@ function getVideoFormatID(name)
     }
 }
 
-async function createPlaylistZipFile(fileNames, type, outputName) {
+async function createPlaylistZipFile(fileNames, type, outputName, fullPathProvided = null) {
     return new Promise(async resolve => {
-        let zipFolderPath = path.join(__dirname, (type === 'audio') ? audioFolderPath : videoFolderPath);
-        // let name = fileNames[0].split(' ')[0] + fileNames[1].split(' ')[0];
+        let zipFolderPath = null;
+
+        if (!fullPathProvided) {
+            zipFolderPath = path.join(__dirname, (type === 'audio') ? audioFolderPath : videoFolderPath);
+        } else {
+            zipFolderPath = path.join(__dirname, config_api.getConfigItem('ytdl_subscriptions_base_path'));
+        }
+
         let ext = (type === 'audio') ? '.mp3' : '.mp4';
 
         let output = fs.createWriteStream(path.join(zipFolderPath, outputName + '.zip'));
@@ -368,7 +393,8 @@ async function createPlaylistZipFile(fileNames, type, outputName) {
 
         for (let i = 0; i < fileNames.length; i++) {
             let fileName = fileNames[i];
-            archive.file(zipFolderPath + fileName + ext, {name: fileName + ext})
+            let file_path = !fullPathProvided ? zipFolderPath + fileName + ext : fileName;
+            archive.file(file_path, {name: fileName + ext})
         }
 
         await archive.finalize();
@@ -383,7 +409,7 @@ async function createPlaylistZipFile(fileNames, type, outputName) {
 
 }
 
-function deleteAudioFile(name) {
+async function deleteAudioFile(name, blacklistMode = false) {
     return new Promise(resolve => {
         // TODO: split descriptors into audio and video descriptors, as deleting an audio file will close all video file streams
         var jsonPath = path.join(audioFolderPath,name+'.mp3.info.json');
@@ -404,7 +430,25 @@ function deleteAudioFile(name) {
             }
         } 
 
-        
+        let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+        if (useYoutubeDLArchive) {
+            const archive_path = audioFolderPath + 'archive.txt';
+
+            // get ID from JSON
+
+            var jsonobj = getJSONMp3(name);
+            let id = null;
+            if (jsonobj) id = jsonobj.id;
+
+            // use subscriptions API to remove video from the archive file, and write it to the blacklist
+            if (fs.existsSync(archive_path)) {
+                const line = id ? subscriptions_api.removeIDFromArchive(archive_path, id) : null;
+                if (blacklistMode && line) writeToBlacklist('audio', line);
+            } else {
+                console.log('Could not find archive file for audio files. Creating...');
+                fs.closeSync(fs.openSync(archive_path, 'w'));
+            }
+        }
 
         if (jsonExists) fs.unlinkSync(jsonPath);
         if (audioFileExists) {
@@ -423,7 +467,7 @@ function deleteAudioFile(name) {
     });
 }
 
-async function deleteVideoFile(name, customPath = null) {
+async function deleteVideoFile(name, customPath = null, blacklistMode = false) {
     return new Promise(resolve => {
         let filePath = customPath ? customPath : videoFolderPath;
         var jsonPath = path.join(filePath,name+'.info.json');
@@ -444,7 +488,25 @@ async function deleteVideoFile(name, customPath = null) {
             }
         } 
 
-        
+        let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+        if (useYoutubeDLArchive) {
+            const archive_path = videoFolderPath + 'archive.txt';
+
+            // get ID from JSON
+
+            var jsonobj = getJSONMp4(name);
+            let id = null;
+            if (jsonobj) id = jsonobj.id;
+
+            // use subscriptions API to remove video from the archive file, and write it to the blacklist
+            if (fs.existsSync(archive_path)) {
+                const line = id ? subscriptions_api.removeIDFromArchive(archive_path, id) : null;
+                if (blacklistMode && line) writeToBlacklist('video', line);
+            } else {
+                console.log('Could not find archive file for videos. Creating...');
+                fs.closeSync(fs.openSync(archive_path, 'w'));
+            }
+        }
 
         if (jsonExists) fs.unlinkSync(jsonPath);
         if (videoFileExists) {
@@ -550,6 +612,88 @@ async function getUrlInfos(urls) {
     });
 }
 
+function writeToBlacklist(type, line) {
+    let blacklistBasePath = (type === 'audio') ? audioFolderPath : videoFolderPath;
+    // adds newline to the beginning of the line
+    line = '\n' + line;
+    fs.appendFileSync(blacklistBasePath + 'blacklist.txt', line);
+}
+
+async function startYoutubeDL() {
+    // auto update youtube-dl
+    await autoUpdateYoutubeDL();
+}
+
+// auto updates the underlying youtube-dl binary, not YoutubeDL-Material
+async function autoUpdateYoutubeDL() {
+    return new Promise(resolve => {
+        // get current version
+        let current_app_details_path = 'node_modules/youtube-dl/bin/details';
+        let current_app_details_exists = fs.existsSync(current_app_details_path);
+        if (!current_app_details_exists) {
+            console.log(`Failed to get youtube-dl binary details at location: ${current_app_details_path}. Cancelling update check.`);
+            resolve(false);
+            return;
+        }
+        let current_app_details = JSON.parse(fs.readFileSync(current_app_details_path));
+        let current_version = current_app_details['version'];
+        let stored_binary_path = current_app_details['path'];
+
+        // got version, now let's check the latest version from the youtube-dl API
+        let youtubedl_api_path = 'https://api.github.com/repos/ytdl-org/youtube-dl/tags';
+        fetch(youtubedl_api_path, {method: 'Get'})
+        .then(async res => res.json())
+        .then(async (json) => {
+            // check if the versions are different
+            const latest_update_version = json[0]['name'];    
+            if (current_version !== latest_update_version) {
+                let binary_path = 'node_modules/youtube-dl/bin';
+                // versions different, download new update
+                console.log('INFO: Found new update for youtube-dl. Updating binary...');
+                await checkExistsWithTimeout(stored_binary_path, 10000);
+                downloader(binary_path, function error(err, done) {
+                    'use strict'
+                    if (err) {
+                        resolve(false);
+                        throw err;
+                    }
+                    console.log(`INFO: Binary successfully updated: ${current_version} -> ${latest_update_version}`);
+                    resolve(true);
+                });
+            }
+        
+        });
+    });
+}
+
+async function checkExistsWithTimeout(filePath, timeout) {
+    return new Promise(function (resolve, reject) {
+
+        var timer = setTimeout(function () {
+            watcher.close();
+            reject(new Error('File did not exists and was not created during the timeout.'));
+        }, timeout);
+
+        fs.access(filePath, fs.constants.R_OK, function (err) {
+            if (!err) {
+                clearTimeout(timer);
+                watcher.close();
+                resolve();
+            }
+        });
+
+        var dir = path.dirname(filePath);
+        var basename = path.basename(filePath);
+        var watcher = fs.watch(dir, function (eventType, filename) {
+            if (eventType === 'rename' && filename === basename) {
+                clearTimeout(timer);
+                watcher.close();
+                resolve();
+            }
+        });
+    });
+}
+
 app.use(function(req, res, next) {
     res.header("Access-Control-Allow-Origin", getOrigin());
     res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
@@ -584,7 +728,7 @@ app.get('/api/using-encryption', function(req, res) {
     res.send(usingEncryption);
 });
 
-app.post('/api/tomp3', function(req, res) {
+app.post('/api/tomp3', async function(req, res) {
     var url = req.body.url;
     var date = Date.now();
     var audiopath = '%(title)s';
@@ -597,10 +741,11 @@ app.post('/api/tomp3', function(req, res) {
     var youtubeUsername = req.body.youtubeUsername;
     var youtubePassword = req.body.youtubePassword;
 
-
     let downloadConfig = null;
     let qualityPath = '-f bestaudio';
 
+    let merged_string = null;
+
     if (customArgs) {
         downloadConfig = customArgs.split(' ');
     } else {
@@ -625,8 +770,32 @@ app.post('/api/tomp3', function(req, res) {
             downloadConfig.splice(3, 0, qualityPath);
         }
     
-        if (!useDefaultDownloadingAgent && customDownloadingAgent === 'aria2c') {
-            downloadConfig.splice(0, 0, '--external-downloader', 'aria2c');
+        if (!useDefaultDownloadingAgent && customDownloadingAgent) {
+            downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
+        }
+
+        let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+        if (useYoutubeDLArchive) {
+            let archive_path = audioFolderPath + 'archive.txt';
+            // create archive file if it doesn't exist
+            if (!fs.existsSync(archive_path)) {
+                fs.closeSync(fs.openSync(archive_path, 'w'));
+            }
+
+            let blacklist_path = audioFolderPath + 'blacklist.txt';
+            // create blacklist file if it doesn't exist
+            if (!fs.existsSync(blacklist_path)) {
+                fs.closeSync(fs.openSync(blacklist_path, 'w'));
+            }
+
+            let merged_path = audioFolderPath + 'merged.txt';
+            // merges blacklist and regular archive
+            let inputPathList = [archive_path, blacklist_path];
+            let status = await mergeFiles(inputPathList, merged_path);
+
+            merged_string = fs.readFileSync(merged_path, "utf8");
+
+            downloadConfig.push('--download-archive', merged_path);
         }
 
         if (globalArgs && globalArgs !== '') {
@@ -648,6 +817,10 @@ app.post('/api/tomp3', function(req, res) {
             throw err;
         } else if (output) {  
             var file_names = [];
+            if (output.length === 0 || output[0].length === 0) {
+                res.sendStatus(500);
+                return;
+            }
             for (let i = 0; i < output.length; i++) {
                 let output_json = null;
                 try {
@@ -680,7 +853,12 @@ app.post('/api/tomp3', function(req, res) {
             }
 
             let is_playlist = file_names.length > 1;
-            // if (!is_playlist) audiopath = file_names[0];
+
+            if (merged_string !== null) {
+                let current_merged_archive = fs.readFileSync(audioFolderPath + 'merged.txt', 'utf8');
+                let diff = current_merged_archive.replace(merged_string, '');
+                fs.appendFileSync(audioFolderPath + 'archive.txt', diff);
+            }
 
             var audiopathEncoded = encodeURIComponent(file_names[0]);
             res.send({
@@ -691,7 +869,7 @@ app.post('/api/tomp3', function(req, res) {
     });
 });
 
-app.post('/api/tomp4', function(req, res) {
+app.post('/api/tomp4', async function(req, res) {
     var url = req.body.url;
     var date = Date.now();
     var path = videoFolderPath;
@@ -705,6 +883,8 @@ app.post('/api/tomp4', function(req, res) {
     var youtubeUsername = req.body.youtubeUsername;
     var youtubePassword = req.body.youtubePassword;
 
+    let merged_string = null;
+
     let downloadConfig = null;
     let qualityPath = 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4';
 
@@ -727,14 +907,39 @@ app.post('/api/tomp4', function(req, res) {
             downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword);
         }
     
-        if (!useDefaultDownloadingAgent && customDownloadingAgent === 'aria2c') {
-            downloadConfig.splice(0, 0, '--external-downloader', 'aria2c');
+        if (!useDefaultDownloadingAgent && customDownloadingAgent) {
+            downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
+        }
+
+        let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+        if (useYoutubeDLArchive) {
+            let archive_path = videoFolderPath + 'archive.txt';
+            // create archive file if it doesn't exist
+            if (!fs.existsSync(archive_path)) {
+                fs.closeSync(fs.openSync(archive_path, 'w'));
+            }
+
+            let blacklist_path = videoFolderPath + 'blacklist.txt';
+            // create blacklist file if it doesn't exist
+            if (!fs.existsSync(blacklist_path)) {
+                fs.closeSync(fs.openSync(blacklist_path, 'w'));
+            }
+
+            let merged_path = videoFolderPath + 'merged.txt';
+            // merges blacklist and regular archive
+            let inputPathList = [archive_path, blacklist_path];
+            let status = await mergeFiles(inputPathList, merged_path);
+
+            merged_string = fs.readFileSync(merged_path, "utf8");
+
+            downloadConfig.push('--download-archive', merged_path);
         }
 
         if (globalArgs && globalArgs !== '') {
             // adds global args
             downloadConfig = downloadConfig.concat(globalArgs.split(' '));
         }
+
     }
 
     youtubedl.exec(url, downloadConfig, {}, function(err, output) {
@@ -749,7 +954,11 @@ app.post('/api/tomp4', function(req, res) {
             res.sendStatus(500);
             throw err;
         } else if (output) {
-            var file_names = [];
+            if (output.length === 0 || output[0].length === 0) {
+                res.sendStatus(500);
+                return;
+            }
+             var file_names = [];
             for (let i = 0; i < output.length; i++) {
                 let output_json = null;
                 try {
@@ -773,12 +982,19 @@ app.post('/api/tomp4', function(req, res) {
                 }
                 var alternate_file_name = file_name.substring(0, file_name.length-4);
                 var file_path = output_json['_filename'].substring(audioFolderPath.length, output_json['_filename'].length);
-                var alternate_file_path = file_path.substring(0, file_path.length-4);
+                // remove extension from file path
+                var alternate_file_path = file_path.replace(/\.[^/.]+$/, "")
                 if (alternate_file_name) file_names.push(alternate_file_path);
             }
 
             let is_playlist = file_names.length > 1;
             if (!is_playlist) audiopath = file_names[0];
+
+            if (merged_string !== null) {
+                let current_merged_archive = fs.readFileSync(videoFolderPath + 'merged.txt', 'utf8');
+                let diff = current_merged_archive.replace(merged_string, '');
+                fs.appendFileSync(videoFolderPath + 'archive.txt', diff);
+            }
             
             var videopathEncoded = encodeURIComponent(file_names[0]);
             res.send({
@@ -840,20 +1056,24 @@ app.post('/api/getMp3s', function(req, res) {
     for (let i = 0; i < files.length; i++) {
         let file = files[i];
         var file_path = file.substring(audioFolderPath.length, file.length);
+
+        var stats = fs.statSync(file);
+
         var id = file_path.substring(0, file_path.length-4);
         var jsonobj = getJSONMp3(id);
         if (!jsonobj) continue;
         var title = jsonobj.title;
+        var url = jsonobj.webpage_url;
+        var uploader = jsonobj.uploader;
+        var upload_date = jsonobj.upload_date;
+        upload_date = `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}`;
 
-        if (title.length > 14) // edits title if it's too long
-        {
-            title = title.substring(0,12) + "...";
-        }
+        var size = stats.size;
 
         var thumbnail = jsonobj.thumbnail;
         var duration = jsonobj.duration;
         var isaudio = true;
-        var file_obj = new File(id, title, thumbnail, isaudio, duration);
+        var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
         mp3s.push(file_obj);
     }
 
@@ -873,20 +1093,24 @@ app.post('/api/getMp4s', function(req, res) {
     for (let i = 0; i < files.length; i++) {
         let file = files[i];
         var file_path = file.substring(videoFolderPath.length, file.length);
+
+        var stats = fs.statSync(file);
+        
         var id = file_path.substring(0, file_path.length-4);
         var jsonobj = getJSONMp4(id);
         if (!jsonobj) continue;
         var title = jsonobj.title;
-
-        if (title.length > 14) // edits title if it's too long
-        {
-            title = title.substring(0,12) + "...";
-        }
-
+        var url = jsonobj.webpage_url;
+        var uploader = jsonobj.uploader;
+        var upload_date = jsonobj.upload_date;
+        upload_date = `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}`;
         var thumbnail = jsonobj.thumbnail;
         var duration = jsonobj.duration;
+
+        var size = stats.size;
+
         var isaudio = false;
-        var file_obj = new File(id, title, thumbnail, isaudio, duration);
+        var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
         mp4s.push(file_obj);
     }
 
@@ -990,6 +1214,8 @@ app.post('/api/getSubscription', async (req, res) => {
         for (let i = 0; i < files.length; i++) {
             let file = files[i];
             var file_path = file.substring(appended_base_path.length, file.length);
+            var stats = fs.statSync(file);
+
             var id = file_path.substring(0, file_path.length-4);
             var jsonobj = getJSONMp4(id, appended_base_path);
             if (!jsonobj) continue;
@@ -997,8 +1223,14 @@ app.post('/api/getSubscription', async (req, res) => {
 
             var thumbnail = jsonobj.thumbnail;
             var duration = jsonobj.duration;
+            var url = jsonobj.webpage_url;
+            var uploader = jsonobj.uploader;
+            var upload_date = jsonobj.upload_date;
+            upload_date = `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}`;
+            var size = stats.size;
+
             var isaudio = false;
-            var file_obj = new File(id, title, thumbnail, isaudio, duration);
+            var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date);
             parsed_files.push(file_obj);
         }
 
@@ -1009,9 +1241,6 @@ app.post('/api/getSubscription', async (req, res) => {
     } else {
         res.sendStatus(500);
     }
-    
-
-    
 });
 
 app.post('/api/downloadVideosForSubscription', async (req, res) => {
@@ -1105,11 +1334,12 @@ app.post('/api/deletePlaylist', async (req, res) => {
 // deletes mp3 file
 app.post('/api/deleteMp3', async (req, res) => {
     var name = req.body.name;
+    var blacklistMode = req.body.blacklistMode;
     var fullpath = audioFolderPath + name + ".mp3";
     var wasDeleted = false;
     if (fs.existsSync(fullpath))
     {
-        deleteAudioFile(name);
+        deleteAudioFile(name, blacklistMode);
         wasDeleted = true;
         res.send(wasDeleted);
         res.end("yes");
@@ -1125,11 +1355,12 @@ app.post('/api/deleteMp3', async (req, res) => {
 // deletes mp4 file
 app.post('/api/deleteMp4', async (req, res) => {
     var name = req.body.name;
+    var blacklistMode = req.body.blacklistMode;
     var fullpath = videoFolderPath + name + ".mp4";
     var wasDeleted = false;
     if (fs.existsSync(fullpath))
     {
-        wasDeleted = await deleteVideoFile(name);
+        wasDeleted = await deleteVideoFile(name, null, blacklistMode);
         // wasDeleted = true;
         res.send(wasDeleted);
         res.end("yes");
@@ -1144,11 +1375,12 @@ app.post('/api/deleteMp4', async (req, res) => {
 
 app.post('/api/downloadFile', async (req, res) => {
     let fileNames = req.body.fileNames;
-    let is_playlist = req.body.is_playlist;
+    let zip_mode = req.body.zip_mode;
     let type = req.body.type;
     let outputName = req.body.outputName;
+    let fullPathProvided = req.body.fullPathProvided;
     let file = null;
-    if (!is_playlist) {
+    if (!zip_mode) {
         fileNames = decodeURIComponent(fileNames);
         if (type === 'audio') {
             file = __dirname + '/' + audioFolderPath + fileNames + '.mp3';
@@ -1159,10 +1391,20 @@ app.post('/api/downloadFile', async (req, res) => {
         for (let i = 0; i < fileNames.length; i++) {
             fileNames[i] = decodeURIComponent(fileNames[i]);
         }
-        file = await createPlaylistZipFile(fileNames, type, outputName);
+        file = await createPlaylistZipFile(fileNames, type, outputName, fullPathProvided);
     }
 
-    res.sendFile(file);
+    res.sendFile(file, function (err) {
+        if (err) {
+          next(err);
+        } else if (fullPathProvided) {
+          try {
+            fs.unlinkSync(file); 
+          } catch(e) {
+            console.log("ERROR: Failed to remove file", file); 
+          }
+        }
+    });
 });
 
 app.post('/api/deleteFile', async (req, res) => {
@@ -1194,7 +1436,7 @@ app.get('/api/video/:id', function(req , res){
     var head;
     let optionalParams = url_api.parse(req.url,true).query;
     let id = decodeURIComponent(req.params.id);
-    let path = "video/" + id + '.mp4';
+    let path = videoFolderPath + id + '.mp4';
     if (optionalParams['subName']) {
         let basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
         const isPlaylist = optionalParams['subPlaylist'];

backend/appdata/default.json

@@ -12,6 +12,7 @@
       "Downloader": {
         "path-audio": "audio/",
         "path-video": "video/",
+        "use_youtubedl_archive": false,
         "custom_args": ""
       },
       "Extra": {

backend/appdata/encrypted.json

@@ -12,6 +12,7 @@
       "Downloader": {
         "path-audio": "audio/",
         "path-video": "video/",
+        "use_youtubedl_archive": false,
         "custom_args": ""
       },
       "Extra": {

backend/config.js

@@ -3,7 +3,7 @@ const fs = require('fs');
 let CONFIG_ITEMS = require('./consts.js')['CONFIG_ITEMS'];
 const debugMode = process.env.YTDL_MODE === 'debug';
 
-let configPath = debugMode ? '../src/assets/default.json' : 'config/default.json';
+let configPath = debugMode ? '../src/assets/default.json' : 'appdata/default.json';
 
 // https://stackoverflow.com/questions/6491463/accessing-nested-javascript-objects-with-string-key
 Object.byString = function(o, s) {

backend/consts.js

@@ -1,5 +1,3 @@
-var config = require('config');
-
 let CONFIG_ITEMS = {
     // Host
     'ytdl_url': {
@@ -34,6 +32,10 @@ let CONFIG_ITEMS = {
         'key': 'ytdl_video_folder_path',
         'path': 'YoutubeDLMaterial.Downloader.path-video'
     },
+    'ytdl_use_youtubedl_archive': {
+        'key': 'ytdl_use_youtubedl_archive',
+        'path': 'YoutubeDLMaterial.Downloader.use_youtubedl_archive'
+    },
     'ytdl_custom_args': {
         'key': 'ytdl_custom_args',
         'path': 'YoutubeDLMaterial.Downloader.custom_args'

backend/package.json

@@ -26,6 +26,8 @@
     "express": "^4.17.1",
     "lowdb": "^1.0.0",
     "node-id3": "^0.1.14",
+    "merge-files": "^0.1.2",
+    "node-fetch": "^2.6.0",
     "shortid": "^2.2.15",
     "uuidv4": "^6.0.6",
     "youtube-dl": "^3.0.2"

backend/subscriptions.js

@@ -8,7 +8,7 @@ var path = require('path');
 var youtubedl = require('youtube-dl');
 const config_api = require('./config');
 
-const adapter = new FileSync('db.json');
+const adapter = new FileSync('./appdata/db.json');
 const db = low(adapter)
 
 const debugMode = process.env.YTDL_MODE === 'debug';
@@ -279,30 +279,30 @@ const deleteFolderRecursive = function(folder_to_delete) {
   };
 
 function removeIDFromArchive(archive_path, id) {
-    fs.readFile(archive_path, {encoding: 'utf-8'}, function(err, data) {
-        if (err) throw error;
-    
-        let dataArray = data.split('\n'); // convert file data in an array
-        const searchKeyword = id; // we are looking for a line, contains, key word id in the file
-        let lastIndex = -1; // let say, we have not found the keyword
-    
-        for (let index=0; index<dataArray.length; index++) {
-            if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
-                lastIndex = index; // found a line includes a id keyword
-                break; 
-            }
+    let data = fs.readFileSync(archive_path, {encoding: 'utf-8'});
+    if (!data) {
+        console.log('Archive could not be found.');
+        return;
+    }
+
+    let dataArray = data.split('\n'); // convert file data in an array
+    const searchKeyword = id; // we are looking for a line, contains, key word id in the file
+    let lastIndex = -1; // let say, we have not found the keyword
+
+    for (let index=0; index<dataArray.length; index++) {
+        if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
+            lastIndex = index; // found a line includes a id keyword
+            break; 
         }
-    
-        dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
-    
-        // UPDATE FILE WITH NEW DATA
-        const updatedData = dataArray.join('\n');
-        fs.writeFile(archive_path, updatedData, (err) => {
-            if (err) throw err;
-            // console.log ('Successfully updated the file data');
-        });
-    
-    });
+    }
+
+    const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
+
+    // UPDATE FILE WITH NEW DATA
+    const updatedData = dataArray.join('\n');
+    fs.writeFileSync(archive_path, updatedData);
+    if (line) return line;
+    if (err) throw err;
 }
 
 module.exports = {
@@ -311,5 +311,6 @@ module.exports = {
     subscribe              : subscribe,
     unsubscribe            : unsubscribe,
     deleteSubscriptionFile : deleteSubscriptionFile,
-    getVideosForSub        : getVideosForSub
+    getVideosForSub        : getVideosForSub,
+    removeIDFromArchive    : removeIDFromArchive
 }