Added support for yt-dlp's --no-clean-infojson

MongoDB support, concurrent streams, player/backend file handling refactor, and more!

backend/app.js

@@ -1184,6 +1184,11 @@ async function generateArgs(url, type, options) {
             }
             downloadConfig = downloadConfig.concat(globalArgs.split(',,'));
         }
+        
+        const default_downloader = getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
+        if (default_downloader === 'yt-dlp') {
+            downloadConfig.push('--no-clean-infojson');
+        }
 
     }
 
@@ -1343,21 +1348,33 @@ async function startYoutubeDL() {
 
 // auto updates the underlying youtube-dl binary, not YoutubeDL-Material
 async function autoUpdateYoutubeDL() {
+    const download_sources = {
+        'youtube-dl': {
+            'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags',
+            'func': downloadLatestYoutubeDLBinary
+        },
+        'youtube-dlc': {
+            'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags',
+            'func': downloadLatestYoutubeDLCBinary
+        },
+        'yt-dlp': {
+            'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags',
+            'func': downloadLatestYoutubeDLPBinary
+        }
+    }
     return new Promise(async resolve => {
         const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
-        const using_youtube_dlc = default_downloader === 'youtube-dlc';
-        const youtube_dl_tags_url = 'https://api.github.com/repos/ytdl-org/youtube-dl/tags'
-        const youtube_dlc_tags_url = 'https://api.github.com/repos/blackjack4494/yt-dlc/tags'
+        const tags_url = download_sources[default_downloader]['tags_url'];
         // get current version
         let current_app_details_path = 'node_modules/youtube-dl/bin/details';
         let current_app_details_exists = fs.existsSync(current_app_details_path);
         if (!current_app_details_exists) {
-            logger.error(`Failed to get youtube-dl binary details at location '${current_app_details_path}'. Cancelling update check.`);
-            resolve(false);
-            return;
+            logger.warn(`Failed to get youtube-dl binary details at location '${current_app_details_path}'. Generating file...`);
+            fs.writeJSONSync('node_modules/youtube-dl/bin/details', {"version":"2020.00.00", "downloader": default_downloader});
         }
         let current_app_details = JSON.parse(fs.readFileSync(current_app_details_path));
         let current_version = current_app_details['version'];
+        let current_downloader = current_app_details['downloader'];
         let stored_binary_path = current_app_details['path'];
         if (!stored_binary_path || typeof stored_binary_path !== 'string') {
             // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${current_app_details_path}, attempting to guess actual path...`);
@@ -1374,15 +1391,9 @@ async function autoUpdateYoutubeDL() {
         }
 
         // got version, now let's check the latest version from the youtube-dl API
-        let youtubedl_api_path = using_youtube_dlc ? youtube_dlc_tags_url : youtube_dl_tags_url;
 
-        if (default_downloader === 'youtube-dl') {
-            await downloadLatestYoutubeDLBinary('unknown', 'unknown');
-            resolve(true);
-            return;
-        }
 
-        fetch(youtubedl_api_path, {method: 'Get'})
+        fetch(tags_url, {method: 'Get'})
         .then(async res => res.json())
         .then(async (json) => {
             // check if the versions are different
@@ -1392,16 +1403,16 @@ async function autoUpdateYoutubeDL() {
                 return false;
             }
             const latest_update_version = json[0]['name'];
-            if (current_version !== latest_update_version) {
-                // versions different, download new update
+            if (current_version !== latest_update_version || default_downloader !== current_downloader) {
+                // versions different or different downloader is being used, download new update
                 logger.info(`Found new update for ${default_downloader}. Updating binary...`);
                 try {
                     await checkExistsWithTimeout(stored_binary_path, 10000);
                 } catch(e) {
                     logger.error(`Failed to update ${default_downloader} - ${e}`);
                 }
-                if (using_youtube_dlc) await downloadLatestYoutubeDLCBinary(latest_update_version);
-                else await downloadLatestYoutubeDLBinary(current_version, latest_update_version);
+                
+                await download_sources[default_downloader]['func'](latest_update_version);
 
                 resolve(true);
             } else {
@@ -1415,19 +1426,15 @@ async function autoUpdateYoutubeDL() {
     });
 }
 
-async function downloadLatestYoutubeDLBinary(current_version, new_version) {
-    return new Promise(resolve => {
-        let binary_path = 'node_modules/youtube-dl/bin';
-        downloader(binary_path, function error(err, done) {
-            if (err) {
-                logger.error(`youtube-dl failed to update. Restart the server to try again.`);
-                logger.error(err);
-                resolve(false);
-            }
-            logger.info(`youtube-dl successfully updated!`);
-            resolve(true);
-        });
-    });
+async function downloadLatestYoutubeDLBinary(new_version) {
+    const file_ext = is_windows ? '.exe' : '';
+
+    const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`;
+    const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
+
+    await fetchFile(download_url, output_path, `youtube-dl ${new_version}`);
+
+    updateDetailsJSON(new_version, 'youtube-dl');
 }
 
 async function downloadLatestYoutubeDLCBinary(new_version) {
@@ -1438,13 +1445,34 @@ async function downloadLatestYoutubeDLCBinary(new_version) {
 
     await fetchFile(download_url, output_path, `youtube-dlc ${new_version}`);
 
-    const details_path = 'node_modules/youtube-dl/bin/details';
-    const details_json = fs.readJSONSync('node_modules/youtube-dl/bin/details');
-    details_json['version'] = new_version;
+    updateDetailsJSON(new_version, 'youtube-dlc');
+}
 
+async function downloadLatestYoutubeDLPBinary(new_version) {
+    const file_ext = is_windows ? '.exe' : '';
+
+    const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`;
+    const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
+
+    await fetchFile(download_url, output_path, `yt-dlp ${new_version}`);
+
+    updateDetailsJSON(new_version, 'yt-dlp');
+}
+
+function updateDetailsJSON(new_version, downloader) {
+    const details_path = 'node_modules/youtube-dl/bin/details';
+    const details_json = fs.readJSONSync(details_path);
+    if (new_version) details_json['version'] = new_version;
+    details_json['downloader'] = downloader;
     fs.writeJSONSync(details_path, details_json);
 }
 
+function getCurrentDownloader() {
+    const details_path = 'node_modules/youtube-dl/bin/details';
+    const details_json = fs.readJSONSync(details_path);
+    return details_json['downloader'];
+}
+
 async function checkExistsWithTimeout(filePath, timeout) {
     return new Promise(function (resolve, reject) {
 

backend/test/tests.js

@@ -76,24 +76,6 @@ describe('Database', async function() {
 
     });
 
-    describe('Import and Export', async function() {
-        it('Existing data', async function() {
-            const users_db_json = users_db.value();
-            const db_json = db.value();
-
-            const users_db_json_stringified = JSON.stringify(users_db_json);
-            const db_json_stringified = JSON.stringify(db_json);
-
-            const tables_obj = await db_api.importJSONtoDB(users_db_json, db_json);
-            const db_jsons = await db_api.exportDBToJSON(tables_obj);
-
-            const users_db_json_returned_stringified = db_jsons['users_db_json'];
-            const db_json_returned_stringified = db_jsons['db_json'];
-
-            assert(users_db_json_returned_stringified.length === users_db_json_stringified.length);
-            assert(db_json_returned_stringified.length === db_json_stringified.length);
-        });
-    });
 
     describe('Basic functions', async function() {
         beforeEach(async function() {
@@ -186,6 +168,35 @@ describe('Database', async function() {
             const stats = await db_api.getDBStats();
             assert(stats);
         });
+
+        it('Query speed', async function() {
+            this.timeout(120000); 
+            const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
+            const test_records = [];
+            let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
+            for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
+                const uid = uuid();
+                if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
+                test_records.push({"id":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","title":"A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J","thumbnailURL":"https://i.ytimg.com/vi/tt7gP_IW-1w/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=tt7gP_IW-1w","uploader":"asapmobVEVO","size":5060157,"path":"audio\\A$AP Mob - Yamborghini High (Official Music Video) ft. Juicy J.mp3","upload_date":"2016-05-11","description":"A$AP Mob ft. Juicy J  - \"Yamborghini High\" Get it now on:\niTunes: http://smarturl.it/iYAMH?IQid=yt\nListen on Spotify: http://smarturl.it/sYAMH?IQid=yt\nGoogle Play: http://smarturl.it/gYAMH?IQid=yt\nAmazon:  http://smarturl.it/aYAMH?IQid=yt\n\nFollow A$AP Mob:\nhttps://www.facebook.com/asapmobofficial\nhttps://twitter.com/ASAPMOB\nhttp://instagram.com/asapmob \nhttp://www.asapmob.com/\n\n#AsapMob #YamborghiniHigh #Vevo #HipHop #OfficialMusicVideo #JuicyJ","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
+            }
+            const insert_start = Date.now();
+            let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
+            const insert_end = Date.now();
+
+            console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
+
+            const query_start = Date.now();
+            const random_record = await db_api.getRecord('test', {uid: random_uid});
+            const query_end = Date.now();
+
+            console.log(random_record)
+
+            console.log(`Query time: ${(query_end - query_start)/1000}s`);
+
+            success = !!random_record;
+
+            assert(success);
+        });
     });
 });
 

src/app/settings/settings.component.html

@@ -326,8 +326,9 @@
                         <mat-form-field>
                             <mat-label><ng-container i18n="Default downloader select label">Select a downloader</ng-container></mat-label>
                             <mat-select color="accent" [(ngModel)]="new_config['Advanced']['default_downloader']">
-                                <mat-option value="youtube-dlc">youtube-dlc</mat-option>
                                 <mat-option value="youtube-dl">youtube-dl</mat-option>
+                                <mat-option value="youtube-dlc">youtube-dlc</mat-option>
+                                <mat-option value="yt-dlp">yt-dlp</mat-option>
                             </mat-select>
                         </mat-form-field>
                     </div>