youtube-dl archive can now be used for regular downloads. a new blacklist system exists which subscriptions will shortly follow

Added new setting whether to use youtubedl archive

backend/app.js

@@ -8,6 +8,7 @@ var https = require('https');
 var express = require("express");
 var bodyParser = require("body-parser");
 var archiver = require('archiver');
+var mergeFiles = require('merge-files');
 const low = require('lowdb')
 var URL = require('url').URL;
 const shortid = require('shortid')
@@ -40,6 +41,7 @@ var usingEncryption = null;
 var basePath = null;
 var audioFolderPath = null;
 var videoFolderPath = null;
+var useYoutubeDLArchive = null;
 var downloadOnlyMode = null;
 var useDefaultDownloadingAgent = null;
 var customDownloadingAgent = null;
@@ -132,6 +134,7 @@ async function loadConfig() {
         usingEncryption = config_api.getConfigItem('ytdl_use_encryption');
         audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
         videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+        useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
         downloadOnlyMode = config_api.getConfigItem('ytdl_download_only_mode');
         useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent');
         customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent');
@@ -382,7 +385,7 @@ async function createPlaylistZipFile(fileNames, type, outputName) {
 
 }
 
-function deleteAudioFile(name) {
+async function deleteAudioFile(name, blacklistMode = false) {
     return new Promise(resolve => {
         // TODO: split descriptors into audio and video descriptors, as deleting an audio file will close all video file streams
         var jsonPath = path.join(audioFolderPath,name+'.mp3.info.json');
@@ -403,7 +406,19 @@ function deleteAudioFile(name) {
             }
         } 
 
-        
+        if (useYoutubeDLArchive) {
+            const archive_path = audioFolderPath + 'archive.txt';
+
+            // get ID from JSON
+
+            var jsonobj = getJSONMp3(name);
+            let id = null;
+            if (jsonobj) id = jsonobj.id;
+
+            // use subscriptions API to remove video from the archive file, and write it to the blacklist
+            const line = id ? subscriptions_api.removeIDFromArchive(archive_path, id) : null;
+            if (blacklistMode && line) writeToBlacklist('audio', line);
+        }
 
         if (jsonExists) fs.unlinkSync(jsonPath);
         if (audioFileExists) {
@@ -422,7 +437,7 @@ function deleteAudioFile(name) {
     });
 }
 
-async function deleteVideoFile(name, customPath = null) {
+async function deleteVideoFile(name, customPath = null, blacklistMode = false) {
     return new Promise(resolve => {
         let filePath = customPath ? customPath : videoFolderPath;
         var jsonPath = path.join(filePath,name+'.info.json');
@@ -443,7 +458,19 @@ async function deleteVideoFile(name, customPath = null) {
             }
         } 
 
-        
+        if (useYoutubeDLArchive) {
+            const archive_path = videoFolderPath + 'archive.txt';
+
+            // get ID from JSON
+
+            var jsonobj = getJSONMp4(name);
+            let id = null;
+            if (jsonobj) id = jsonobj.id;
+
+            // use subscriptions API to remove video from the archive file, and write it to the blacklist
+            const line = id ? subscriptions_api.removeIDFromArchive(archive_path, id) : null;
+            if (blacklistMode && line) writeToBlacklist('video', line);
+        }
 
         if (jsonExists) fs.unlinkSync(jsonPath);
         if (videoFileExists) {
@@ -549,6 +576,13 @@ async function getUrlInfos(urls) {
     });
 }
 
+function writeToBlacklist(type, line) {
+    let blacklistBasePath = (type === 'audio') ? audioFolderPath : videoFolderPath;
+    // adds newline to the beginning of the line
+    line = '\n' + line;
+    fs.appendFileSync(blacklistBasePath + 'blacklist.txt', line);
+}
+
 app.use(function(req, res, next) {
     res.header("Access-Control-Allow-Origin", getOrigin());
     res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
@@ -583,7 +617,7 @@ app.get('/api/using-encryption', function(req, res) {
     res.send(usingEncryption);
 });
 
-app.post('/api/tomp3', function(req, res) {
+app.post('/api/tomp3', async function(req, res) {
     var url = req.body.url;
     var date = Date.now();
     var audiopath = '%(title)s';
@@ -596,10 +630,11 @@ app.post('/api/tomp3', function(req, res) {
     var youtubeUsername = req.body.youtubeUsername;
     var youtubePassword = req.body.youtubePassword;
 
-
     let downloadConfig = null;
     let qualityPath = '-f bestaudio';
 
+    let merged_string = null;
+
     if (customArgs) {
         downloadConfig = customArgs.split(' ');
     } else {
@@ -628,6 +663,29 @@ app.post('/api/tomp3', function(req, res) {
             downloadConfig.splice(0, 0, '--external-downloader', 'aria2c');
         }
 
+        if (useYoutubeDLArchive) {
+            let archive_path = audioFolderPath + 'archive.txt';
+            // create archive file if it doesn't exist
+            if (!fs.existsSync(archive_path)) {
+                fs.closeSync(fs.openSync(archive_path, 'w'));
+            }
+
+            let blacklist_path = audioFolderPath + 'blacklist.txt';
+            // create blacklist file if it doesn't exist
+            if (!fs.existsSync(blacklist_path)) {
+                fs.closeSync(fs.openSync(blacklist_path, 'w'));
+            }
+
+            let merged_path = audioFolderPath + 'merged.txt';
+            // merges blacklist and regular archive
+            let inputPathList = [archive_path, blacklist_path];
+            let status = await mergeFiles(inputPathList, merged_path);
+
+            merged_string = fs.readFileSync(merged_path, "utf8");
+
+            downloadConfig.push('--download-archive', merged_path);
+        }
+
         if (globalArgs && globalArgs !== '') {
             // adds global args
             downloadConfig = downloadConfig.concat(globalArgs.split(' '));
@@ -647,6 +705,10 @@ app.post('/api/tomp3', function(req, res) {
             throw err;
         } else if (output) {  
             var file_names = [];
+            if (output.length === 0 || output[0].length === 0) {
+                res.sendStatus(500);
+                return;
+            }
             for (let i = 0; i < output.length; i++) {
                 let output_json = null;
                 try {
@@ -660,13 +722,19 @@ app.post('/api/tomp3', function(req, res) {
                 }
                 var file_name = output_json['_filename'].replace(/^.*[\\\/]/, '');
                 var file_path = output_json['_filename'].substring(audioFolderPath.length, output_json['_filename'].length);
-                var alternate_file_path = file_path.substring(0, file_path.length-4);
+                // remove extension from file path
+                var alternate_file_path = file_path.replace(/\.[^/.]+$/, "")
                 var alternate_file_name = file_name.substring(0, file_name.length-4);
                 if (alternate_file_path) file_names.push(alternate_file_path);
             }
 
             let is_playlist = file_names.length > 1;
-            // if (!is_playlist) audiopath = file_names[0];
+
+            if (merged_string !== null) {
+                let current_merged_archive = fs.readFileSync(audioFolderPath + 'merged.txt', 'utf8');
+                let diff = current_merged_archive.replace(merged_string, '');
+                fs.appendFileSync(audioFolderPath + 'archive.txt', diff);
+            }
 
             var audiopathEncoded = encodeURIComponent(file_names[0]);
             res.send({
@@ -677,7 +745,7 @@ app.post('/api/tomp3', function(req, res) {
     });
 });
 
-app.post('/api/tomp4', function(req, res) {
+app.post('/api/tomp4', async function(req, res) {
     var url = req.body.url;
     var date = Date.now();
     var path = videoFolderPath;
@@ -691,6 +759,8 @@ app.post('/api/tomp4', function(req, res) {
     var youtubeUsername = req.body.youtubeUsername;
     var youtubePassword = req.body.youtubePassword;
 
+    let merged_string = null;
+
     let downloadConfig = null;
     let qualityPath = 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4';
 
@@ -717,10 +787,34 @@ app.post('/api/tomp4', function(req, res) {
             downloadConfig.splice(0, 0, '--external-downloader', 'aria2c');
         }
 
+        if (useYoutubeDLArchive) {
+            let archive_path = videoFolderPath + 'archive.txt';
+            // create archive file if it doesn't exist
+            if (!fs.existsSync(archive_path)) {
+                fs.closeSync(fs.openSync(archive_path, 'w'));
+            }
+
+            let blacklist_path = videoFolderPath + 'blacklist.txt';
+            // create blacklist file if it doesn't exist
+            if (!fs.existsSync(blacklist_path)) {
+                fs.closeSync(fs.openSync(blacklist_path, 'w'));
+            }
+
+            let merged_path = videoFolderPath + 'merged.txt';
+            // merges blacklist and regular archive
+            let inputPathList = [archive_path, blacklist_path];
+            let status = await mergeFiles(inputPathList, merged_path);
+
+            merged_string = fs.readFileSync(merged_path, "utf8");
+
+            downloadConfig.push('--download-archive', merged_path);
+        }
+
         if (globalArgs && globalArgs !== '') {
             // adds global args
             downloadConfig = downloadConfig.concat(globalArgs.split(' '));
         }
+
     }
 
     youtubedl.exec(url, downloadConfig, {}, function(err, output) {
@@ -735,7 +829,11 @@ app.post('/api/tomp4', function(req, res) {
             res.sendStatus(500);
             throw err;
         } else if (output) {
-            var file_names = [];
+            if (output.length === 0 || output[0].length === 0) {
+                res.sendStatus(500);
+                return;
+            }
+             var file_names = [];
             for (let i = 0; i < output.length; i++) {
                 let output_json = null;
                 try {
@@ -759,12 +857,19 @@ app.post('/api/tomp4', function(req, res) {
                 }
                 var alternate_file_name = file_name.substring(0, file_name.length-4);
                 var file_path = output_json['_filename'].substring(audioFolderPath.length, output_json['_filename'].length);
-                var alternate_file_path = file_path.substring(0, file_path.length-4);
+                // remove extension from file path
+                var alternate_file_path = file_path.replace(/\.[^/.]+$/, "")
                 if (alternate_file_name) file_names.push(alternate_file_path);
             }
 
             let is_playlist = file_names.length > 1;
             if (!is_playlist) audiopath = file_names[0];
+
+            if (merged_string !== null) {
+                let current_merged_archive = fs.readFileSync(videoFolderPath + 'merged.txt', 'utf8');
+                let diff = current_merged_archive.replace(merged_string, '');
+                fs.appendFileSync(videoFolderPath + 'archive.txt', diff);
+            }
             
             var videopathEncoded = encodeURIComponent(file_names[0]);
             res.send({
@@ -1091,11 +1196,12 @@ app.post('/api/deletePlaylist', async (req, res) => {
 // deletes mp3 file
 app.post('/api/deleteMp3', async (req, res) => {
     var name = req.body.name;
+    var blacklistMode = req.body.blacklistMode;
     var fullpath = audioFolderPath + name + ".mp3";
     var wasDeleted = false;
     if (fs.existsSync(fullpath))
     {
-        deleteAudioFile(name);
+        deleteAudioFile(name, blacklistMode);
         wasDeleted = true;
         res.send(wasDeleted);
         res.end("yes");
@@ -1111,11 +1217,12 @@ app.post('/api/deleteMp3', async (req, res) => {
 // deletes mp4 file
 app.post('/api/deleteMp4', async (req, res) => {
     var name = req.body.name;
+    var blacklistMode = req.body.blacklistMode;
     var fullpath = videoFolderPath + name + ".mp4";
     var wasDeleted = false;
     if (fs.existsSync(fullpath))
     {
-        wasDeleted = await deleteVideoFile(name);
+        wasDeleted = await deleteVideoFile(name, null, blacklistMode);
         // wasDeleted = true;
         res.send(wasDeleted);
         res.end("yes");

backend/config/default.json

@@ -12,6 +12,7 @@
       "Downloader": {
         "path-audio": "audio/",
         "path-video": "video/",
+        "use_youtubedl_archive": false,
         "custom_args": ""
       },
       "Extra": {

backend/config/encrypted.json

@@ -12,6 +12,7 @@
       "Downloader": {
         "path-audio": "audio/",
         "path-video": "video/",
+        "use_youtubedl_archive": false,
         "custom_args": ""
       },
       "Extra": {

backend/consts.js

@@ -34,6 +34,10 @@ let CONFIG_ITEMS = {
         'key': 'ytdl_video_folder_path',
         'path': 'YoutubeDLMaterial.Downloader.path-video'
     },
+    'ytdl_use_youtubedl_archive': {
+        'key': 'ytdl_use_youtubedl_archive',
+        'path': 'YoutubeDLMaterial.Downloader.use_youtubedl_archive'
+    },
     'ytdl_custom_args': {
         'key': 'ytdl_custom_args',
         'path': 'YoutubeDLMaterial.Downloader.custom_args'

backend/package.json

@@ -25,6 +25,7 @@
     "exe": "^1.0.2",
     "express": "^4.17.1",
     "lowdb": "^1.0.0",
+    "merge-files": "^0.1.2",
     "shortid": "^2.2.15",
     "uuidv4": "^6.0.6",
     "youtube-dl": "^3.0.2"

backend/subscriptions.js

@@ -279,30 +279,30 @@ const deleteFolderRecursive = function(folder_to_delete) {
   };
 
 function removeIDFromArchive(archive_path, id) {
-    fs.readFile(archive_path, {encoding: 'utf-8'}, function(err, data) {
+    let data = fs.readFileSync(archive_path, {encoding: 'utf-8'});
-        if (err) throw error;
+    if (!data) {
-    
+        console.log('Archive could not be found.');
-        let dataArray = data.split('\n'); // convert file data in an array
+        return;
-        const searchKeyword = id; // we are looking for a line, contains, key word id in the file
+    }
-        let lastIndex = -1; // let say, we have not found the keyword
+
-    
+    let dataArray = data.split('\n'); // convert file data in an array
-        for (let index=0; index<dataArray.length; index++) {
+    const searchKeyword = id; // we are looking for a line, contains, key word id in the file
-            if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
+    let lastIndex = -1; // let say, we have not found the keyword
-                lastIndex = index; // found a line includes a id keyword
+
-                break; 
+    for (let index=0; index<dataArray.length; index++) {
-            }
+        if (dataArray[index].includes(searchKeyword)) { // check if a line contains the id keyword
+            lastIndex = index; // found a line includes a id keyword
+            break; 
         }
-    
+    }
-        dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
+
-    
+    const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array
-        // UPDATE FILE WITH NEW DATA
+
-        const updatedData = dataArray.join('\n');
+    // UPDATE FILE WITH NEW DATA
-        fs.writeFile(archive_path, updatedData, (err) => {
+    const updatedData = dataArray.join('\n');
-            if (err) throw err;
+    fs.writeFileSync(archive_path, updatedData);
-            // console.log ('Successfully updated the file data');
+    if (line) return line;
-        });
+    if (err) throw err;
-    
-    });
 }
 
 module.exports = {
@@ -311,5 +311,6 @@ module.exports = {
     subscribe              : subscribe,
     unsubscribe            : unsubscribe,
     deleteSubscriptionFile : deleteSubscriptionFile,
-    getVideosForSub        : getVideosForSub
+    getVideosForSub        : getVideosForSub,
+    removeIDFromArchive    : removeIDFromArchive
 }

src/app/posts.services.ts

@@ -98,11 +98,11 @@ export class PostsService {
         return this.http.post(this.path + 'setConfig', {new_config_file: config});
     }
 
-    deleteFile(name: string, isAudio: boolean) {
+    deleteFile(name: string, isAudio: boolean, blacklistMode = false) {
         if (isAudio) {
-            return this.http.post(this.path + 'deleteMp3', {name: name});
+            return this.http.post(this.path + 'deleteMp3', {name: name, blacklistMode: blacklistMode});
         } else {
-            return this.http.post(this.path + 'deleteMp4', {name: name});
+            return this.http.post(this.path + 'deleteMp4', {name: name, blacklistMode: blacklistMode});
         }
     }
 

src/app/settings/settings.component.html

@@ -93,6 +93,11 @@
                     <mat-hint><ng-container i18n="Custom args setting input hint">Global custom args for downloads on the home page.</ng-container></mat-hint>
                 </mat-form-field>
             </div>
+
+            <div class="col-12 mt-4">
+                <mat-checkbox color="accent" [(ngModel)]="new_config['Downloader']['use_youtubedl_archive']"><ng-container i18n="Use youtubedl archive setting">Use youtube-dl archive</ng-container></mat-checkbox>
+                <p>Note: This setting only applies to downloads on the Home page. If you would like to use youtube-dl archive functionality in subscriptions, head down to the Subscriptions section.</p>
+            </div>
         </div>
     </div>
 </mat-expansion-panel>

src/assets/default.json

@@ -1,44 +1,45 @@
 {
-    "YoutubeDLMaterial": {
+  "YoutubeDLMaterial": {
-      "Host": {
+    "Host": {
-        "url": "http://localhost",
+      "url": "http://localhost",
-        "port": "17442"
+      "port": "17442"
-      },
+    },
-      "Encryption": {
+    "Encryption": {
-        "use-encryption": false,
+      "use-encryption": false,
-        "cert-file-path": "/etc/letsencrypt/live/example.com/fullchain.pem",
+      "cert-file-path": "/etc/letsencrypt/live/example.com/fullchain.pem",
-        "key-file-path": "/etc/letsencrypt/live/example.com/privkey.pem"
+      "key-file-path": "/etc/letsencrypt/live/example.com/privkey.pem"
-      },
+    },
-      "Downloader": {
+    "Downloader": {
-        "path-audio": "audio/",
+      "path-audio": "audio/",
-        "path-video": "video/",
+      "path-video": "video/",
-        "custom_args": ""
+      "use_youtubedl_archive": true,
-      },
+      "custom_args": ""
-      "Extra": {
+    },
-        "title_top": "Youtube Downloader",
+    "Extra": {
-        "file_manager_enabled": true,
+      "title_top": "Youtube Downloader",
-        "allow_quality_select": true,
+      "file_manager_enabled": true,
-        "download_only_mode": false,
+      "allow_quality_select": true,
-        "allow_multi_download_mode": true
+      "download_only_mode": false,
-      },
+      "allow_multi_download_mode": true
-      "API": {
+    },
-        "use_youtube_API": false,
+    "API": {
-        "youtube_API_key": ""
+      "use_youtube_API": false,
-      },
+      "youtube_API_key": ""
-      "Themes": {
+    },
-        "default_theme": "default",
+    "Themes": {
-        "allow_theme_change": true
+      "default_theme": "default",
-      },
+      "allow_theme_change": true
-      "Subscriptions": {
+    },
-        "allow_subscriptions": true,
+    "Subscriptions": {
-        "subscriptions_base_path": "subscriptions/",
+      "allow_subscriptions": true,
-        "subscriptions_check_interval": "300",
+      "subscriptions_base_path": "subscriptions/",
-        "subscriptions_use_youtubedl_archive": true
+      "subscriptions_check_interval": "300",
-      },
+      "subscriptions_use_youtubedl_archive": true
-      "Advanced": {
+    },
-        "use_default_downloading_agent": true,
+    "Advanced": {
-        "custom_downloading_agent": "",
+      "use_default_downloading_agent": true,
-        "allow_advanced_download": true
+      "custom_downloading_agent": "",
-      }
+      "allow_advanced_download": true
     }
   }
+}