added basic subscriptions support for playlists and channels

update youtube-dl binary on windows

updated favicon to the new icon

backend/app.js

@@ -1,4 +1,5 @@
 var async = require('async');
+const { uuid } = require('uuidv4');
 var fs = require('fs');
 var path = require('path');
 var youtubedl = require('youtube-dl');
@@ -10,7 +11,9 @@ var archiver = require('archiver');
 const low = require('lowdb')
 var URL = require('url').URL;
 const shortid = require('shortid')
+const url_api = require('url');
 var config_api = require('./config.js'); 
+var subscriptions_api = require('./subscriptions')
 
 var app = express();
 
@@ -25,7 +28,8 @@ db.defaults(
             audio: [],
             video: []
         },
-        configWriteFlag: false
+        configWriteFlag: false,
+        subscriptions: []
 }).write();
 
 // config values
@@ -39,6 +43,8 @@ var videoFolderPath = null;
 var downloadOnlyMode = null;
 var useDefaultDownloadingAgent = null;
 var customDownloadingAgent = null;
+var allowSubscriptions = null;
+var subscriptionsCheckInterval = null;
 
 // other needed values
 var options = null; // encryption options
@@ -129,6 +135,9 @@ async function loadConfig() {
         downloadOnlyMode = config_api.getConfigItem('ytdl_download_only_mode');
         useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent');
         customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent');
+        allowSubscriptions = config_api.getConfigItem('ytdl_allow_subscriptions');
+        subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval');
+
         if (!useDefaultDownloadingAgent && validDownloadingAgents.indexOf(customDownloadingAgent) !== -1 ) {
             console.log(`INFO: Using non-default downloading agent \'${customDownloadingAgent}\'`)
         }
@@ -149,6 +158,11 @@ async function loadConfig() {
 
         url_domain = new URL(url);
 
+        // get subscriptions
+        if (allowSubscriptions) {
+            watchSubscriptions();
+        }
+
         // start the server here
         startServer();
 
@@ -157,6 +171,34 @@ async function loadConfig() {
     
 }
 
+function calculateSubcriptionRetrievalDelay(amount) {
+    // frequency is 5 mins
+    let frequency_in_ms = subscriptionsCheckInterval * 1000;
+    let minimum_frequency = 60 * 1000;
+    const first_frequency = frequency_in_ms/amount;
+    return (first_frequency < minimum_frequency) ? minimum_frequency : first_frequency;
+}
+
+function watchSubscriptions() { 
+    let subscriptions = subscriptions_api.getAllSubscriptions();
+
+    let subscriptions_amount = subscriptions.length;
+    let delay_interval = calculateSubcriptionRetrievalDelay(subscriptions_amount);
+
+    let current_delay = 0;
+    for (let i = 0; i < subscriptions.length; i++) {
+        let sub = subscriptions[i];
+        console.log('watching ' + sub.name + ' with delay interval of ' + delay_interval);
+        setTimeout(() => {
+            setInterval(() => {
+                subscriptions_api.getVideosForSub(sub);
+            }, subscriptionsCheckInterval * 1000);
+        }, current_delay);
+        current_delay += delay_interval;
+        if (current_delay >= subscriptionsCheckInterval * 1000) current_delay = 0;
+    }
+}
+
 function getOrigin() {
     return url_domain.origin;
 }
@@ -239,9 +281,14 @@ function getJSONMp3(name)
     return obj;
 }
 
-function getJSONMp4(name)
+function getJSONMp4(name, customPath = null)
 {
-    var jsonPath = videoFolderPath+name+".info.json";
+    let jsonPath = null;
+    if (!customPath) {
+        jsonPath = videoFolderPath+name+".info.json";
+    } else {
+        jsonPath = customPath + name + ".info.json";
+    }
     if (fs.existsSync(jsonPath))
     {
         var obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
@@ -802,6 +849,109 @@ app.post('/api/getMp4s', function(req, res) {
     res.end("yes");
 });
 
+app.post('/api/subscribe', async (req, res) => {
+    let name = req.body.name;
+    let url = req.body.url;
+    let timerange = req.body.timerange;
+
+    const new_sub = {
+                        name: name,
+                        url: url,
+                        id: uuid()
+                    };
+
+    // adds timerange if it exists, otherwise all videos will be downloaded
+    if (timerange) {
+        new_sub.timerange = timerange;
+    }
+
+    const result_obj = await subscriptions_api.subscribe(new_sub);
+
+    if (result_obj.success) {
+        res.send({
+            new_sub: new_sub
+        });
+    } else {
+        res.send({
+            new_sub: null,
+            error: result_obj.error
+        })
+    }
+});
+
+app.post('/api/unsubscribe', async (req, res) => {
+    let deleteMode = req.body.deleteMode
+    let sub = req.body.sub;
+
+    let result_obj = subscriptions_api.unsubscribe(sub, deleteMode);
+    if (result_obj.success) {
+        res.send({
+            success: result_obj.success
+        });
+    } else {
+        res.send({
+            success: false,
+            error: result_obj.error
+        });
+    }
+});
+
+app.post('/api/getSubscription', async (req, res) => {
+    let subID = req.body.id;
+
+    // get sub from db
+    let subscription = subscriptions_api.getSubscription(subID);
+
+    if (!subscription) {
+        // failed to get subscription from db, send 400 error
+        res.sendStatus(400);
+        return;
+    }
+
+    // get sub videos
+    let base_path = config_api.getConfigItem('ytdl_subscriptions_base_path');
+    let appended_base_path = path.join(base_path, subscription.isPlaylist ? 'playlists' : 'channels', subscription.name, '/');
+    var files = recFindByExt(appended_base_path, 'mp4');
+    var parsed_files = [];
+    for (let i = 0; i < files.length; i++) {
+        let file = files[i];
+        var file_path = file.substring(appended_base_path.length, file.length);
+        var id = file_path.substring(0, file_path.length-4);
+        var jsonobj = getJSONMp4(id, appended_base_path);
+        if (!jsonobj) continue;
+        var title = jsonobj.title;
+
+        var thumbnail = jsonobj.thumbnail;
+        var duration = jsonobj.duration;
+        var isaudio = false;
+        var file_obj = new File(id, title, thumbnail, isaudio, duration);
+        parsed_files.push(file_obj);
+    }
+
+    res.send({
+        subscription: subscription,
+        files: parsed_files
+    });
+});
+
+app.post('/api/downloadVideosForSubscription', async (req, res) => {
+    let subID = req.body.subID;
+    let sub = subscriptions_api.getSubscription(subID);
+    subscriptions_api.getVideosForSub(sub);
+    res.send({
+        success: true
+    });
+});
+
+app.post('/api/getAllSubscriptions', async (req, res) => {
+    // get subs from api
+    let subscriptions = subscriptions_api.getAllSubscriptions();
+
+    res.send({
+        subscriptions: subscriptions
+    });
+});
+
 app.post('/api/createPlaylist', async (req, res) => {
     let playlistName = req.body.playlistName;
     let fileNames = req.body.fileNames;
@@ -948,8 +1098,15 @@ app.post('/api/deleteFile', async (req, res) => {
 
 app.get('/api/video/:id', function(req , res){
     var head;
+    let optionalParams = url_api.parse(req.url,true).query;
     let id = decodeURIComponent(req.params.id);
-    const path = "video/" + id + '.mp4';
+    let path = "video/" + id + '.mp4';
+    if (optionalParams['subName']) {
+        let basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
+        const isPlaylist = optionalParams['subPlaylist'];
+        basePath += (isPlaylist === 'true' ? 'playlists/' : 'channels/');
+        path = basePath + optionalParams['subName'] + '/' + id + '.mp4'; 
+    }
     const stat = fs.statSync(path)
     const fileSize = stat.size
     const range = req.headers.range

backend/config.js

@@ -56,7 +56,10 @@ function setConfigFile(config) {
 
 function getConfigItem(key) {
     let config_json = getConfigFile();
-    if (!CONFIG_ITEMS[key]) console.log('cannot find config with key ' + key);
+    if (!CONFIG_ITEMS[key]) {
+        console.log('cannot find config with key ' + key);
+        return null;
+    }
     let path = CONFIG_ITEMS[key]['path'];
     return Object.byString(config_json, path);
 };

backend/config/default.json

@@ -28,6 +28,12 @@
         "default_theme": "default",
         "allow_theme_change": true
       },
+      "Subscriptions": {
+        "allow_subscriptions": true,
+        "subscriptions_base_path": "subscriptions/",
+        "subscriptions_check_interval": "300",
+        "subscriptions_use_youtubedl_archive": true
+      },
       "Advanced": {
         "use_default_downloading_agent": true,
         "custom_downloading_agent": "",

backend/config/encrypted.json

@@ -28,6 +28,12 @@
         "default_theme": "default",
         "allow_theme_change": true
       },
+      "Subscriptions": {
+        "allow_subscriptions": true,
+        "subscriptions_base_path": "subscriptions/",
+        "subscriptions_check_interval": "300",
+        "subscriptions_use_youtubedl_archive": true
+      },
       "Advanced": {
         "use_default_downloading_agent": true,
         "custom_downloading_agent": "",

backend/consts.js

@@ -56,7 +56,6 @@ let CONFIG_ITEMS = {
         'key': 'ytdl_allow_multi_download_mode',
         'path': 'YoutubeDLMaterial.Extra.allow_multi_download_mode'
     },
-    
 
     // API
     'ytdl_use_youtube_api': {
@@ -78,6 +77,28 @@ let CONFIG_ITEMS = {
         'path': 'YoutubeDLMaterial.Themes.allow_theme_change'
     },
 
+    // Subscriptions
+    'ytdl_allow_subscriptions': {
+        'key': 'ytdl_allow_subscriptions',
+        'path': 'YoutubeDLMaterial.Subscriptions.allow_subscriptions'
+    },
+    'ytdl_subscriptions_base_path': {
+        'key': 'ytdl_subscriptions_base_path',
+        'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_base_path'
+    },
+    'ytdl_subscriptions_check_interval': {
+        'key': 'ytdl_subscriptions_check_interval',
+        'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval'
+    },
+    'ytdl_subscriptions_check_interval': {
+        'key': 'ytdl_subscriptions_check_interval',
+        'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval'
+    },
+    'ytdl_subscriptions_use_youtubedl_archive': {
+        'key': 'ytdl_use_youtubedl_archive',
+        'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_use_youtubedl_archive'
+    },
+
     // Advanced
     'ytdl_use_default_downloading_agent': {
         'key': 'ytdl_use_default_downloading_agent',

backend/package.json

@@ -26,6 +26,7 @@
     "express": "^4.17.1",
     "lowdb": "^1.0.0",
     "shortid": "^2.2.15",
+    "uuidv4": "^6.0.6",
     "youtube-dl": "^3.0.2"
   }
 }

backend/subscriptions.js

@@ -0,0 +1,203 @@
+const low = require('lowdb')
+const FileSync = require('lowdb/adapters/FileSync')
+
+var fs = require('fs');
+const { uuid } = require('uuidv4');
+var path = require('path');
+
+var youtubedl = require('youtube-dl');
+const config_api = require('./config');
+
+const adapter = new FileSync('db.json');
+const db = low(adapter)
+
+let debugMode = process.env.YTDL_MODE === 'debug';
+
+async function subscribe(sub) {
+    const result_obj = {
+        success: false,
+        error: ''
+    };
+    return new Promise(async resolve => {
+        // sub should just have url and name. here we will get isPlaylist and path
+        sub.isPlaylist = sub.url.includes('playlist');
+
+        if (db.get('subscriptions').find({url: sub.url}).value()) {
+            console.log('Sub already exists');
+            result_obj.error = 'Subcription with URL ' + sub.url + ' already exists!';
+            resolve(result_obj);
+            return;
+        }
+
+        // add sub to db
+        db.get('subscriptions').push(sub).write();
+
+        await getVideosForSub(sub);
+        result_obj.success = true;
+        result_obj.sub = sub;
+        resolve(result_obj);
+    });
+    
+}
+
+async function unsubscribe(sub, deleteMode) {
+    return new Promise(async resolve => {
+        const basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
+        let result_obj = { success: false, error: '' };
+
+        let id = sub.id;
+        db.get('subscriptions').remove({id: id}).write();
+
+        const appendedBasePath = getAppendedBasePath(sub, basePath);
+        if (deleteMode && fs.existsSync(appendedBasePath)) {
+            if (sub.archive && fs.existsSync(sub.archive)) {
+                const archive_file_path = path.join(sub.archive, 'archive.txt');
+                // deletes archive if it exists
+                if (fs.existsSync(archive_file_path)) {
+                    fs.unlinkSync(archive_file_path);
+                }
+                fs.rmdirSync(sub.archive);
+            }
+            deleteFolderRecursive(appendedBasePath);
+        }
+    });
+
+}
+
+async function getVideosForSub(sub) {
+    return new Promise(resolve => {
+        const basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
+        const useArchive = config_api.getConfigItem('ytdl_subscriptions_use_youtubedl_archive');
+
+        const appendedBasePath = basePath + (sub.isPlaylist ? 'playlists/%(playlist_title)s' : 'channels/%(uploader)s');
+
+        let downloadConfig = ['-o', appendedBasePath + '/%(title)s.mp4', '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4', '-ciw', '--write-annotations', '--write-thumbnail', '--write-info-json', '--print-json'];
+
+        if (sub.timerange) {
+            downloadConfig.push('--dateafter', sub.timerange);
+        }
+
+        let archive_dir = null;
+        let archive_path = null;
+        let usingTempArchive = false;
+
+        if (useArchive) {
+            if (sub.archive) {
+                archive_dir = sub.archive;
+                archive_path = path.join(archive_dir, 'archive.txt')
+            } else {
+                usingTempArchive = true;
+
+                // set temporary archive
+                archive_dir = basePath + 'archives/' + sub.id;
+                archive_path = path.join(archive_dir, sub.id + '.txt');
+
+                // create temporary dir and archive txt
+                if (!fs.existsSync(archive_dir)) {
+                    fs.mkdirSync(archive_dir);
+                    fs.closeSync(fs.openSync(archive_path, 'w'));
+                }
+            }
+            downloadConfig.push('--download-archive', archive_path);
+        }
+
+        // get videos 
+        youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {
+            if (debugMode) {
+                console.log('Subscribe: got videos for subscription ' + sub.name);
+            }
+            if (err) {
+                console.log(err.stderr);
+                resolve(false);
+            } else if (output) {
+                if (output.length === 0) {
+                    if (debugMode) console.log('No additional videos to download for ' + sub.name);
+                }
+                for (let i = 0; i < output.length; i++) {
+                    let output_json = null;
+                    try {
+                        output_json = JSON.parse(output[i]);
+                    } catch(e) {
+                        output_json = null;
+                    }
+                    if (!output_json) {
+                        continue;
+                    }
+
+                    if (!sub.name && output_json) {
+                        sub.name = sub.isPlaylist ? output_json.playlist_title : output_json.uploader;
+                        // if it's now valid, update
+                        if (sub.name) {
+                            db.get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write();
+                        }
+                    }
+
+                    if (usingTempArchive && !sub.archive && sub.name) {
+                        let new_archive_dir = basePath + 'archives/' + sub.name;
+
+                        // TODO: clean up, code looks ugly
+                        if (fs.existsSync(new_archive_dir)) {
+                            if (fs.existsSync(new_archive_dir + '/archive.txt')) {
+                                console.log('INFO: Archive file already exists. Rewriting archive.');
+                                fs.unlinkSync(new_archive_dir + '/archive.txt')
+                            }
+                        } else {
+                            // creates archive directory for subscription
+                            fs.mkdirSync(new_archive_dir);
+                        }
+
+                        // moves archive
+                        fs.copyFileSync(archive_path, new_archive_dir + '/archive.txt');
+
+                        // updates subscription
+                        sub.archive = new_archive_dir;
+                        db.get('subscriptions').find({id: sub.id}).assign({archive: new_archive_dir}).write();
+
+                        // remove temporary archive directory
+                        fs.unlinkSync(archive_path);
+                        fs.rmdirSync(archive_dir);
+                    }
+                }
+                resolve(true);
+            }
+        });
+    });
+}
+
+function getAllSubscriptions() {
+    const subscriptions = db.get('subscriptions').value();
+    return subscriptions;
+}
+
+function getSubscription(subID) {
+    return db.get('subscriptions').find({id: subID}).value();
+}
+
+// helper functions
+
+function getAppendedBasePath(sub, base_path) {
+    return base_path + (sub.isPlaylist ? 'playlists/' : 'channels/') + sub.name;
+}
+
+// https://stackoverflow.com/a/32197381/8088021
+const deleteFolderRecursive = function(folder_to_delete) {
+    if (fs.existsSync(folder_to_delete)) {
+      fs.readdirSync(folder_to_delete).forEach((file, index) => {
+        const curPath = path.join(folder_to_delete, file);
+        if (fs.lstatSync(curPath).isDirectory()) { // recurse
+          deleteFolderRecursive(curPath);
+        } else { // delete file
+          fs.unlinkSync(curPath);
+        }
+      });
+      fs.rmdirSync(folder_to_delete);
+    }
+  };
+
+module.exports = {
+    getSubscription    : getSubscription,
+    getAllSubscriptions: getAllSubscriptions,
+    subscribe          : subscribe,
+    unsubscribe        : unsubscribe,
+    getVideosForSub    : getVideosForSub
+}