From 8595864118634e463b4eedfa847b66260f858086 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Thu, 17 Sep 2020 03:14:24 -0400 Subject: [PATCH 01/14] Added basic categorization functionality in the server & UI --- backend/app.js | 68 ++++++++++- backend/categories.js | 112 ++++++++++++++++++ src/app/app.component.ts | 2 + src/app/app.module.ts | 4 +- .../edit-category-dialog.component.html | 47 ++++++++ .../edit-category-dialog.component.scss | 16 +++ .../edit-category-dialog.component.spec.ts | 25 ++++ .../edit-category-dialog.component.ts | 110 +++++++++++++++++ src/app/posts.services.ts | 26 ++++ src/app/settings/settings.component.html | 29 ++++- src/app/settings/settings.component.scss | 51 ++++++++ src/app/settings/settings.component.ts | 44 +++++++ 12 files changed, 526 insertions(+), 8 deletions(-) create mode 100644 backend/categories.js create mode 100644 src/app/dialogs/edit-category-dialog/edit-category-dialog.component.html create mode 100644 src/app/dialogs/edit-category-dialog/edit-category-dialog.component.scss create mode 100644 src/app/dialogs/edit-category-dialog/edit-category-dialog.component.spec.ts create mode 100644 src/app/dialogs/edit-category-dialog/edit-category-dialog.component.ts diff --git a/backend/app.js b/backend/app.js index 38ab93d..2d8bd7a 100644 --- a/backend/app.js +++ b/backend/app.js @@ -27,6 +27,7 @@ const shortid = require('shortid') const url_api = require('url'); var config_api = require('./config.js'); var subscriptions_api = require('./subscriptions') +var categories_api = require('./categories'); const CONSTS = require('./consts') const { spawn } = require('child_process') const read_last_lines = require('read-last-lines'); @@ -37,7 +38,7 @@ const is_windows = process.platform === 'win32'; var app = express(); // database setup -const FileSync = require('lowdb/adapters/FileSync') +const FileSync = require('lowdb/adapters/FileSync'); const adapter = new FileSync('./appdata/db.json'); const db = low(adapter) @@ -80,6 +81,15 @@ config_api.initialize(logger); auth_api.initialize(users_db, logger); db_api.initialize(db, users_db, logger); subscriptions_api.initialize(db, users_db, logger, db_api); +categories_api.initialize(db, users_db, logger, db_api); + + +async function test() { + const test_cat = await categories_api.categorize(fs.readJSONSync('video/Claire Lost Her First Tooth!.info.json')); + console.log(test_cat); +} + +test(); // var GithubContent = require('github-content'); @@ -1115,6 +1125,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { var is_audio = type === 'audio'; var ext = is_audio ? '.mp3' : '.mp4'; var fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; + let category = null; // prepend with user if needed let multiUserMode = null; @@ -1131,7 +1142,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { } options.downloading_method = 'exec'; - const downloadConfig = await generateArgs(url, type, options); + let downloadConfig = await generateArgs(url, type, options); // adds download to download helper const download_uid = uuid(); @@ -1153,11 +1164,22 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { updateDownloads(); // get video info prior to download - const info = await getVideoInfoByURL(url, downloadConfig, download); + let info = await getVideoInfoByURL(url, downloadConfig, download); if (!info) { resolve(false); return; } else { + // check if it fits into a category. If so, then get info again using new downloadConfig + category = await categories_api.categorize(info); + + // set custom output if the category has one and re-retrieve info so the download manager has the right file name + if (category && category['custom_output']) { + options.customOutput = category['custom_output']; + options.noRelativePath = true; + downloadConfig = await generateArgs(url, type, options); + info = await getVideoInfoByURL(url, downloadConfig, download); + } + // store info in download for future use download['_filename'] = info['_filename']; download['filesize'] = utils.getExpectedFileSize(info); @@ -1445,7 +1467,8 @@ async function generateArgs(url, type, options) { } if (customOutput) { - downloadConfig = ['-o', path.join(fileFolderPath, customOutput) + ".%(ext)s", '--write-info-json', '--print-json']; + customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput); + downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json']; } else { downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json']; } @@ -2131,6 +2154,43 @@ app.post('/api/disableSharing', optionalJwt, function(req, res) { }); }); +// categories + +app.post('/api/getAllCategories', optionalJwt, async (req, res) => { + const categories = db.get('categories').value(); + res.send({categories: categories}); +}); + +app.post('/api/createCategory', optionalJwt, async (req, res) => { + const name = req.body.name; + const new_category = { + name: name, + uid: uuid(), + rules: [] + }; + + db.get('categories').push(new_category).write(); + + res.send({ + new_category: new_category, + success: !!new_category + }); +}); + +app.post('/api/updateCategory', optionalJwt, async (req, res) => { + const category = req.body.category; + db.get('categories').find({uid: category.uid}).assign(category).write(); + res.send({success: true}); +}); + +app.post('/api/updateCategories', optionalJwt, async (req, res) => { + const categories = req.body.categories; + db.get('categories').assign(categories).write(); + res.send({success: true}); +}); + +// subscriptions + app.post('/api/subscribe', optionalJwt, async (req, res) => { let name = req.body.name; let url = req.body.url; diff --git a/backend/categories.js b/backend/categories.js new file mode 100644 index 0000000..ac3a3c7 --- /dev/null +++ b/backend/categories.js @@ -0,0 +1,112 @@ +const config_api = require('./config'); + +var logger = null; +var db = null; +var users_db = null; +var db_api = null; + +function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api } +function setLogger(input_logger) { logger = input_logger; } + +function initialize(input_db, input_users_db, input_logger, input_db_api) { + setDB(input_db, input_users_db, input_db_api); + setLogger(input_logger); +} + +/* + +Categories: + + Categories are a way to organize videos based on dynamic rules set by the user. Categories are universal (so not per-user). + + Categories, besides rules, have an optional custom output. This custom output can help users create their + desired directory structure. + +Rules: + A category rule consists of a property, a comparison, and a value. For example, "uploader includes 'VEVO'" + + Rules are stored as an object with the above fields. In addition to those fields, it also has a preceding_operator, which + is either OR or AND, and signifies whether the rule should be ANDed with the previous rules, or just ORed. For the first + rule, this field is null. + + Ex. (title includes 'Rihanna' OR title includes 'Beyonce' AND uploader includes 'VEVO') + +*/ + +async function categorize(file_json) { + return new Promise(resolve => { + let selected_category = null; + const categories = getCategories(); + if (!categories) { + logger.warn('Categories could not be found. Initializing categories...'); + db.assign({categories: []}).write(); + resolve(null); + return; + } + + for (let i = 0; i < categories.length; i++) { + const category = categories[i]; + const rules = category['rules']; + + // if rules for current category apply, then that is the selected category + if (applyCategoryRules(file_json, rules, category['name'])) { + selected_category = category; + logger.verbose(`Selected category ${category['name']} for ${file_json['webpage_url']}`); + } + } + + resolve(selected_category); + + }); +} + +function getCategories() { + const categories = db.get('categories').value(); + return categories ? categories : null; +} + +function applyCategoryRules(file_json, rules, category_name) { + let rules_apply = false; + for (let i = 0; i < rules.length; i++) { + const rule = rules[i]; + let rule_applies = null; + + let preceding_operator = rule['preceding_operator']; + + switch (rule['comparator']) { + case 'includes': + rule_applies = file_json[rule['property']].includes(rule['value']); + break; + case 'not_includes': + rule_applies = !(file_json[rule['property']].includes(rule['value'])); + break; + case 'equals': + rule_applies = file_json[rule['property']] === rule['value']; + break; + case 'not_equals': + rule_applies = file_json[rule['property']] !== rule['value']; + break; + default: + logger.warn(`Invalid comparison used for category ${category_name}`) + break; + } + + // OR the first rule with rules_apply, which will be initially false + if (i === 0) preceding_operator = 'or'; + + // update rules_apply based on current rule + if (preceding_operator === 'or') + rules_apply = rules_apply || rule_applies; + else + rules_apply = rules_apply && rule_applies; + } + + return rules_apply; +} + + + +module.exports = { + initialize: initialize, + categorize: categorize, +} \ No newline at end of file diff --git a/src/app/app.component.ts b/src/app/app.component.ts index d8540bb..a6c7630 100644 --- a/src/app/app.component.ts +++ b/src/app/app.component.ts @@ -116,6 +116,8 @@ export class AppComponent implements OnInit, AfterViewInit { if (this.allowSubscriptions) { this.postsService.reloadSubscriptions(); } + + this.postsService.reloadCategories(); } // theme stuff diff --git a/src/app/app.module.ts b/src/app/app.module.ts index 69cba76..423f714 100644 --- a/src/app/app.module.ts +++ b/src/app/app.module.ts @@ -79,6 +79,7 @@ import { UnifiedFileCardComponent } from './components/unified-file-card/unified import { RecentVideosComponent } from './components/recent-videos/recent-videos.component'; import { EditSubscriptionDialogComponent } from './dialogs/edit-subscription-dialog/edit-subscription-dialog.component'; import { CustomPlaylistsComponent } from './components/custom-playlists/custom-playlists.component'; +import { EditCategoryDialogComponent } from './dialogs/edit-category-dialog/edit-category-dialog.component'; registerLocaleData(es, 'es'); @@ -123,7 +124,8 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible UnifiedFileCardComponent, RecentVideosComponent, EditSubscriptionDialogComponent, - CustomPlaylistsComponent + CustomPlaylistsComponent, + EditCategoryDialogComponent ], imports: [ CommonModule, diff --git a/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.html b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.html new file mode 100644 index 0000000..fe6cec7 --- /dev/null +++ b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.html @@ -0,0 +1,47 @@ +

Editing category {{category['name']}}

+ + + + + + +
Rules
+ + + + + + OR + AND + + + + + {{propertyOption.label}} + + + + + {{comparatorOption.label}} + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.scss b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.scss new file mode 100644 index 0000000..53fcc70 --- /dev/null +++ b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.scss @@ -0,0 +1,16 @@ +.operator-select { + width: 55px; +} + +.property-select { + margin-left: 10px; + width: 110px; +} + +.comparator-select { + margin-left: 10px; +} + +.value-input { + margin-left: 10px; +} \ No newline at end of file diff --git a/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.spec.ts b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.spec.ts new file mode 100644 index 0000000..71d64a9 --- /dev/null +++ b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.spec.ts @@ -0,0 +1,25 @@ +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; + +import { EditCategoryDialogComponent } from './edit-category-dialog.component'; + +describe('EditCategoryDialogComponent', () => { + let component: EditCategoryDialogComponent; + let fixture: ComponentFixture; + + beforeEach(async(() => { + TestBed.configureTestingModule({ + declarations: [ EditCategoryDialogComponent ] + }) + .compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(EditCategoryDialogComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.ts b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.ts new file mode 100644 index 0000000..928ff1c --- /dev/null +++ b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.ts @@ -0,0 +1,110 @@ +import { Component, OnInit, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { PostsService } from 'app/posts.services'; + +@Component({ + selector: 'app-edit-category-dialog', + templateUrl: './edit-category-dialog.component.html', + styleUrls: ['./edit-category-dialog.component.scss'] +}) +export class EditCategoryDialogComponent implements OnInit { + + updating = false; + original_category = null; + category = null; + + propertyOptions = [ + { + value: 'fulltitle', + label: 'Title' + }, + { + value: 'id', + label: 'ID' + }, + { + value: 'webpage_url', + label: 'URL' + }, + { + value: 'view_count', + label: 'Views' + }, + { + value: 'uploader', + label: 'Uploader' + }, + { + value: '_filename', + label: 'File Name' + }, + { + value: 'tags', + label: 'Tags' + } + ]; + + comparatorOptions = [ + { + value: 'includes', + label: 'includes' + }, + { + value: 'not_includes', + label: 'not includes' + }, + { + value: 'equals', + label: 'equals' + }, + { + value: 'not_equals', + label: 'not equals' + }, + + ]; + + constructor(@Inject(MAT_DIALOG_DATA) public data: any, private postsService: PostsService) { + if (this.data) { + this.original_category = this.data.category; + this.category = JSON.parse(JSON.stringify(this.original_category)); + } + } + + ngOnInit(): void { + } + + addNewRule() { + this.category['rules'].push({ + preceding_operator: 'or', + property: 'fulltitle', + comparator: 'includes', + value: '' + }); + } + + saveClicked() { + this.updating = true; + this.postsService.updateCategory(this.category).subscribe(res => { + this.updating = false; + this.original_category = JSON.parse(JSON.stringify(this.category)); + }, err => { + this.updating = false; + console.error(err); + }); + } + + categoryChanged() { + return JSON.stringify(this.category) === JSON.stringify(this.original_category); + } + + swapRules(original_index, new_index) { + [this.category.rules[original_index], this.category.rules[new_index]] = [this.category.rules[new_index], + this.category.rules[original_index]]; + } + + removeRule(index) { + this.category['rules'].splice(index, 1); + } + +} diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 67e3bd1..3e45e81 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -52,6 +52,7 @@ export class PostsService implements CanActivate { // global vars config = null; subscriptions = null; + categories = null; sidenav = null; constructor(private http: HttpClient, private router: Router, @Inject(DOCUMENT) private document: Document, @@ -296,6 +297,31 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'deletePlaylist', {playlistID: playlistID, type: type}, this.httpOptions); } + // categories + + getAllCategories() { + return this.http.post(this.path + 'getAllCategories', {}, this.httpOptions); + } + + createCategory(name) { + console.log(name); + return this.http.post(this.path + 'createCategory', {name: name}, this.httpOptions); + } + + updateCategory(category) { + return this.http.post(this.path + 'updateCategory', {category: category}, this.httpOptions); + } + + updateCategories(categories) { + return this.http.post(this.path + 'updateCategories', {categories: categories}, this.httpOptions); + } + + reloadCategories() { + this.getAllCategories().subscribe(res => { + this.categories = res['categories']; + }); + } + createSubscription(url, name, timerange = null, streamingOnly = false, audioOnly = false, customArgs = null, customFileOutput = null) { return this.http.post(this.path + 'subscribe', {url: url, name: name, timerange: timerange, streamingOnly: streamingOnly, audioOnly: audioOnly, customArgs: customArgs, customFileOutput: customFileOutput}, this.httpOptions); diff --git a/src/app/settings/settings.component.html b/src/app/settings/settings.component.html index 719683b..8e6fc0d 100644 --- a/src/app/settings/settings.component.html +++ b/src/app/settings/settings.component.html @@ -115,15 +115,38 @@ -
+
Global custom args for downloads on the home page. Args are delimited using two commas like so: ,,
- -
+
+
+ +
+
+
+
Categories
+
+
+
+ {{category['name']}} + + + + +
+
+ +
+
+
+ +
+
+
Use youtube-dl archive
diff --git a/src/app/settings/settings.component.scss b/src/app/settings/settings.component.scss index 9ff0b70..f85952f 100644 --- a/src/app/settings/settings.component.scss +++ b/src/app/settings/settings.component.scss @@ -30,4 +30,55 @@ margin-left: 15px; margin-bottom: 12px; bottom: 4px; +} + +.category-list { + width: 500px; + max-width: 100%; + border: solid 1px #ccc; + min-height: 60px; + display: block; + // background: white; + border-radius: 4px; + overflow: hidden; +} + +.category-box { + padding: 20px 10px; + border-bottom: solid 1px #ccc; + display: flex; + flex-direction: row; + align-items: center; + justify-content: space-between; + box-sizing: border-box; + cursor: move; + // background: white; + font-size: 14px; +} + +.cdk-drag-preview { + box-sizing: border-box; + border-radius: 4px; + box-shadow: 0 5px 5px -3px rgba(0, 0, 0, 0.2), + 0 8px 10px 1px rgba(0, 0, 0, 0.14), + 0 3px 14px 2px rgba(0, 0, 0, 0.12); +} + +.cdk-drag-animating { + transition: transform 250ms cubic-bezier(0, 0, 0.2, 1); +} + +.category-box:last-child { + border: none; +} + +.category-list.cdk-drop-list-dragging .category-box:not(.cdk-drag-placeholder) { + transition: transform 250ms cubic-bezier(0, 0, 0.2, 1); +} + +.category-custom-placeholder { +background: #ccc; +border: dotted 3px #999; +min-height: 60px; +transition: transform 250ms cubic-bezier(0, 0, 0.2, 1); } \ No newline at end of file diff --git a/src/app/settings/settings.component.ts b/src/app/settings/settings.component.ts index 4a7684f..ec0c6d4 100644 --- a/src/app/settings/settings.component.ts +++ b/src/app/settings/settings.component.ts @@ -9,6 +9,9 @@ import { CURRENT_VERSION } from 'app/consts'; import { MatCheckboxChange } from '@angular/material/checkbox'; import { CookiesUploaderDialogComponent } from 'app/dialogs/cookies-uploader-dialog/cookies-uploader-dialog.component'; import { ConfirmDialogComponent } from 'app/dialogs/confirm-dialog/confirm-dialog.component'; +import { moveItemInArray, CdkDragDrop } from '@angular/cdk/drag-drop'; +import { InputDialogComponent } from 'app/input-dialog/input-dialog.component'; +import { EditCategoryDialogComponent } from 'app/dialogs/edit-category-dialog/edit-category-dialog.component'; @Component({ selector: 'app-settings', @@ -77,6 +80,47 @@ export class SettingsComponent implements OnInit { }) } + dropCategory(event: CdkDragDrop) { + moveItemInArray(this.postsService.categories, event.previousIndex, event.currentIndex); + this.postsService.updateCategories(this.postsService.categories); + } + + openAddCategoryDialog() { + const done = new EventEmitter(); + const dialogRef = this.dialog.open(InputDialogComponent, { + width: '300px', + data: { + inputTitle: 'Name the category', + inputPlaceholder: 'Name', + submitText: 'Add', + doneEmitter: done + } + }); + + done.subscribe(name => { + + // Eventually do additional checks on name + if (name) { + this.postsService.createCategory(name).subscribe(res => { + if (res['success']) { + this.postsService.reloadCategories(); + dialogRef.close(); + const new_category = res['new_category']; + this.openEditCategoryDialog(new_category); + } + }); + } + }); + } + + openEditCategoryDialog(category) { + this.dialog.open(EditCategoryDialogComponent, { + data: { + category: category + } + }); + } + generateAPIKey() { this.postsService.generateNewAPIKey().subscribe(res => { if (res['new_api_key']) { From fed0a54145da432490144ba5a052612fea10737e Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Mon, 12 Oct 2020 22:46:23 -0400 Subject: [PATCH 02/14] Updated styling on edit category dialog --- .../edit-category-dialog.component.html | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.html b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.html index fe6cec7..64aa2be 100644 --- a/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.html +++ b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.html @@ -1,11 +1,13 @@ -

Editing category {{category['name']}}

+

Editing category {{category['name']}}

- - - + + + -
Rules
+ + +
Rules
@@ -34,7 +36,18 @@ - + + + + + + + + + Documentation. + Path is relative to the config download path. Don't include extension. + + From dff4b141b0506b2664a3a838e5af24627002bdde Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Mon, 12 Oct 2020 22:47:11 -0400 Subject: [PATCH 03/14] Blobs are now only included in getAllFiles() if the config option for including thumbnail is set to true --- backend/app.js | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/backend/app.js b/backend/app.js index 2d8bd7a..f316c11 100644 --- a/backend/app.js +++ b/backend/app.js @@ -1936,11 +1936,13 @@ app.get('/api/getMp3s', optionalJwt, function(req, res) { mp3s = JSON.parse(JSON.stringify(mp3s)); - // add thumbnails if present - mp3s.forEach(mp3 => { - if (mp3['thumbnailPath'] && fs.existsSync(mp3['thumbnailPath'])) - mp3['thumbnailBlob'] = fs.readFileSync(mp3['thumbnailPath']); - }); + if (config_api.getConfigItem('ytdl_include_thumbnail')) { + // add thumbnails if present + mp3s.forEach(mp3 => { + if (mp3['thumbnailPath'] && fs.existsSync(mp3['thumbnailPath'])) + mp3['thumbnailBlob'] = fs.readFileSync(mp3['thumbnailPath']); + }); + } res.send({ mp3s: mp3s, @@ -1963,11 +1965,13 @@ app.get('/api/getMp4s', optionalJwt, function(req, res) { mp4s = JSON.parse(JSON.stringify(mp4s)); - // add thumbnails if present - mp4s.forEach(mp4 => { - if (mp4['thumbnailPath'] && fs.existsSync(mp4['thumbnailPath'])) - mp4['thumbnailBlob'] = fs.readFileSync(mp4['thumbnailPath']); - }); + if (config_api.getConfigItem('ytdl_include_thumbnail')) { + // add thumbnails if present + mp4s.forEach(mp4 => { + if (mp4['thumbnailPath'] && fs.existsSync(mp4['thumbnailPath'])) + mp4['thumbnailBlob'] = fs.readFileSync(mp4['thumbnailPath']); + }); + } res.send({ mp4s: mp4s, @@ -2055,12 +2059,14 @@ app.post('/api/getAllFiles', optionalJwt, function (req, res) { files = JSON.parse(JSON.stringify(files)); - // add thumbnails if present - files.forEach(file => { - if (file['thumbnailPath'] && fs.existsSync(file['thumbnailPath'])) - file['thumbnailBlob'] = fs.readFileSync(file['thumbnailPath']); - }); - + if (config_api.getConfigItem('ytdl_include_thumbnail')) { + // add thumbnails if present + files.forEach(file => { + if (file['thumbnailPath'] && fs.existsSync(file['thumbnailPath'])) + file['thumbnailBlob'] = fs.readFileSync(file['thumbnailPath']); + }); + } + res.send({ files: files, playlists: playlists @@ -2166,7 +2172,8 @@ app.post('/api/createCategory', optionalJwt, async (req, res) => { const new_category = { name: name, uid: uuid(), - rules: [] + rules: [], + custom_putput: '' }; db.get('categories').push(new_category).write(); From fe7303a1912563a072356cd72bbb994416a77924 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Thu, 15 Oct 2020 16:57:45 -0400 Subject: [PATCH 04/14] Replaced /audio and /video APIs with /stream that now requires a type parameter to simplify future code changes getSubscription can now accept a subscription name instead of just an ID Added API call to delete a category Categories can now have a custom path Minor code cleanup --- backend/app.js | 127 +++++++++++++-------------------------- backend/categories.js | 43 ++++++------- backend/db.js | 6 +- backend/subscriptions.js | 8 +++ 4 files changed, 74 insertions(+), 110 deletions(-) diff --git a/backend/app.js b/backend/app.js index f316c11..acbb45e 100644 --- a/backend/app.js +++ b/backend/app.js @@ -83,16 +83,6 @@ db_api.initialize(db, users_db, logger); subscriptions_api.initialize(db, users_db, logger, db_api); categories_api.initialize(db, users_db, logger, db_api); - -async function test() { - const test_cat = await categories_api.categorize(fs.readJSONSync('video/Claire Lost Her First Tooth!.info.json')); - console.log(test_cat); -} - -test(); - -// var GithubContent = require('github-content'); - // Set some defaults db.defaults( { @@ -184,7 +174,6 @@ const subscription_timeouts = {}; // don't overwrite config if it already happened.. NOT // let alreadyWritten = db.get('configWriteFlag').value(); let writeConfigMode = process.env.write_ytdl_config; -var config = null; // checks if config exists, if not, a config is auto generated config_api.configExistsCheck(); @@ -1221,7 +1210,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { } catch(e) { output_json = null; } - var modified_file_name = output_json ? output_json['title'] : null; + if (!output_json) { continue; } @@ -1250,8 +1239,11 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']); } + const file_path = options.noRelativePath ? path.basename(full_file_path) : full_file_path.substring(fileFolderPath.length, full_file_path.length); + const customPath = options.noRelativePath ? path.dirname(full_file_path).split(path.sep).pop() : null; + // registers file in DB - file_uid = db_api.registerFileDB(full_file_path.substring(fileFolderPath.length, full_file_path.length), type, multiUserMode); + file_uid = db_api.registerFileDB(file_path, type, multiUserMode, null, customPath); if (file_name) file_names.push(file_name); } @@ -1795,7 +1787,7 @@ app.use(function(req, res, next) { next(); } else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key') && req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) { next(); - } else if (req.path.includes('/api/video/') || req.path.includes('/api/audio/')) { + } else if (req.path.includes('/api/stream/')) { next(); } else { logger.verbose(`Rejecting request - invalid API use for endpoint: ${req.path}. API key received: ${req.query.apiKey}`); @@ -1808,15 +1800,14 @@ app.use(compression()); const optionalJwt = function (req, res, next) { const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); if (multiUserMode && ((req.body && req.body.uuid) || (req.query && req.query.uuid)) && (req.path.includes('/api/getFile') || - req.path.includes('/api/audio') || - req.path.includes('/api/video') || + req.path.includes('/api/stream') || req.path.includes('/api/downloadFile'))) { // check if shared video const using_body = req.body && req.body.uuid; const uuid = using_body ? req.body.uuid : req.query.uuid; const uid = using_body ? req.body.uid : req.query.uid; const type = using_body ? req.body.type : req.query.type; - const file = !req.query.id ? auth_api.getUserVideo(uuid, uid, type, true, req.body) : auth_api.getUserPlaylist(uuid, req.query.id, null, true); + const file = !req.query.id ? auth_api.getUserVideo(uuid, uid, type, true, !!req.body) : auth_api.getUserPlaylist(uuid, req.query.id, null, true); const is_shared = file ? file['sharingEnabled'] : false; if (is_shared) { req.can_watch = true; @@ -2184,6 +2175,16 @@ app.post('/api/createCategory', optionalJwt, async (req, res) => { }); }); +app.post('/api/deleteCategory', optionalJwt, async (req, res) => { + const category_uid = req.body.category_uid; + + db.get('categories').remove({uid: category_uid}).write(); + + res.send({ + success: true + }); +}); + app.post('/api/updateCategory', optionalJwt, async (req, res) => { const category = req.body.category; db.get('categories').find({uid: category.uid}).assign(category).write(); @@ -2282,10 +2283,17 @@ app.post('/api/deleteSubscriptionFile', optionalJwt, async (req, res) => { app.post('/api/getSubscription', optionalJwt, async (req, res) => { let subID = req.body.id; + let subName = req.body.name; // if included, subID is optional + let user_uid = req.isAuthenticated() ? req.user.uid : null; // get sub from db - let subscription = subscriptions_api.getSubscription(subID, user_uid); + let subscription = null; + if (subID) { + subscription = subscriptions_api.getSubscription(subID, user_uid) + } else if (subName) { + subscription = subscriptions_api.getSubscriptionByName(subName, user_uid) + } if (!subscription) { // failed to get subscription from db, send 400 error @@ -2708,25 +2716,33 @@ app.post('/api/generateNewAPIKey', function (req, res) { // Streaming API calls -app.get('/api/video/:id', optionalJwt, function(req , res){ +app.get('/api/stream/:id', optionalJwt, (req, res) => { + const type = req.query.type; + const ext = type === 'audio' ? '.mp3' : '.mp4'; + const mimetype = type === 'audio' ? 'audio/mp3' : 'video/mp4'; var head; let optionalParams = url_api.parse(req.url,true).query; let id = decodeURIComponent(req.params.id); - let file_path = videoFolderPath + id + '.mp4'; - if (req.isAuthenticated() || req.can_watch) { + let file_path = req.query.file_path ? decodeURIComponent(req.query.file_path) : null; + if (!file_path && (req.isAuthenticated() || req.can_watch)) { let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); if (optionalParams['subName']) { const isPlaylist = optionalParams['subPlaylist']; - file_path = path.join(usersFileFolder, req.user.uid, 'subscriptions', (isPlaylist === 'true' ? 'playlists/' : 'channels/'),optionalParams['subName'], id + '.mp4') + file_path = path.join(usersFileFolder, req.user.uid, 'subscriptions', (isPlaylist === 'true' ? 'playlists/' : 'channels/'),optionalParams['subName'], id + ext) } else { - file_path = path.join(usersFileFolder, req.query.uuid ? req.query.uuid : req.user.uid, 'video', id + '.mp4'); + file_path = path.join(usersFileFolder, req.query.uuid ? req.query.uuid : req.user.uid, type, id + ext); } - } else if (optionalParams['subName']) { + } else if (!file_path && optionalParams['subName']) { let basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); const isPlaylist = optionalParams['subPlaylist']; basePath += (isPlaylist === 'true' ? 'playlists/' : 'channels/'); - file_path = basePath + optionalParams['subName'] + '/' + id + '.mp4'; + file_path = basePath + optionalParams['subName'] + '/' + id + ext; } + + if (!file_path) { + file_path = path.join(videoFolderPath, id + ext); + } + const stat = fs.statSync(file_path) const fileSize = stat.size const range = req.headers.range @@ -2749,77 +2765,20 @@ app.get('/api/video/:id', optionalJwt, function(req , res){ 'Content-Range': `bytes ${start}-${end}/${fileSize}`, 'Accept-Ranges': 'bytes', 'Content-Length': chunksize, - 'Content-Type': 'video/mp4', + 'Content-Type': mimetype, } res.writeHead(206, head); file.pipe(res); } else { head = { 'Content-Length': fileSize, - 'Content-Type': 'video/mp4', + 'Content-Type': mimetype, } res.writeHead(200, head) fs.createReadStream(file_path).pipe(res) } }); -app.get('/api/audio/:id', optionalJwt, function(req , res){ - var head; - let id = decodeURIComponent(req.params.id); - let file_path = "audio/" + id + '.mp3'; - let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); - let optionalParams = url_api.parse(req.url,true).query; - if (req.isAuthenticated()) { - if (optionalParams['subName']) { - const isPlaylist = optionalParams['subPlaylist']; - file_path = path.join(usersFileFolder, req.user.uid, 'subscriptions', (isPlaylist === 'true' ? 'playlists/' : 'channels/'),optionalParams['subName'], id + '.mp3') - } else { - let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); - file_path = path.join(usersFileFolder, req.user.uid, 'audio', id + '.mp3'); - } - } else if (optionalParams['subName']) { - let basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); - const isPlaylist = optionalParams['subPlaylist']; - basePath += (isPlaylist === 'true' ? 'playlists/' : 'channels/'); - file_path = basePath + optionalParams['subName'] + '/' + id + '.mp3'; - } - file_path = file_path.replace(/\"/g, '\''); - const stat = fs.statSync(file_path) - const fileSize = stat.size - const range = req.headers.range - if (range) { - const parts = range.replace(/bytes=/, "").split("-") - const start = parseInt(parts[0], 10) - const end = parts[1] - ? parseInt(parts[1], 10) - : fileSize-1 - const chunksize = (end-start)+1 - const file = fs.createReadStream(file_path, {start, end}); - if (config_api.descriptors[id]) config_api.descriptors[id].push(file); - else config_api.descriptors[id] = [file]; - file.on('close', function() { - let index = config_api.descriptors[id].indexOf(file); - config_api.descriptors[id].splice(index, 1); - logger.debug('Successfully closed stream and removed file reference.'); - }); - head = { - 'Content-Range': `bytes ${start}-${end}/${fileSize}`, - 'Accept-Ranges': 'bytes', - 'Content-Length': chunksize, - 'Content-Type': 'audio/mp3', - } - res.writeHead(206, head); - file.pipe(res); - } else { - head = { - 'Content-Length': fileSize, - 'Content-Type': 'audio/mp3', - } - res.writeHead(200, head) - fs.createReadStream(file_path).pipe(res) - } - }); - // Downloads management app.get('/api/downloads', async (req, res) => { diff --git a/backend/categories.js b/backend/categories.js index ac3a3c7..545a6a1 100644 --- a/backend/categories.js +++ b/backend/categories.js @@ -34,30 +34,27 @@ Rules: */ async function categorize(file_json) { - return new Promise(resolve => { - let selected_category = null; - const categories = getCategories(); - if (!categories) { - logger.warn('Categories could not be found. Initializing categories...'); - db.assign({categories: []}).write(); - resolve(null); - return; + let selected_category = null; + const categories = getCategories(); + if (!categories) { + logger.warn('Categories could not be found. Initializing categories...'); + db.assign({categories: []}).write(); + return null; + return; + } + + for (let i = 0; i < categories.length; i++) { + const category = categories[i]; + const rules = category['rules']; + + // if rules for current category apply, then that is the selected category + if (applyCategoryRules(file_json, rules, category['name'])) { + selected_category = category; + logger.verbose(`Selected category ${category['name']} for ${file_json['webpage_url']}`); + return selected_category; } - - for (let i = 0; i < categories.length; i++) { - const category = categories[i]; - const rules = category['rules']; - - // if rules for current category apply, then that is the selected category - if (applyCategoryRules(file_json, rules, category['name'])) { - selected_category = category; - logger.verbose(`Selected category ${category['name']} for ${file_json['webpage_url']}`); - } - } - - resolve(selected_category); - - }); + } + return selected_category; } function getCategories() { diff --git a/backend/db.js b/backend/db.js index c46cfd8..f625807 100644 --- a/backend/db.js +++ b/backend/db.js @@ -15,10 +15,10 @@ function initialize(input_db, input_users_db, input_logger) { setLogger(input_logger); } -function registerFileDB(file_path, type, multiUserMode = null, sub = null) { +function registerFileDB(file_path, type, multiUserMode = null, sub = null, customPath = null) { let db_path = null; const file_id = file_path.substring(0, file_path.length-4); - const file_object = generateFileObject(file_id, type, multiUserMode && multiUserMode.file_path, sub); + const file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub); if (!file_object) { logger.error(`Could not find associated JSON file for ${type} file ${file_id}`); return false; @@ -27,7 +27,7 @@ function registerFileDB(file_path, type, multiUserMode = null, sub = null) { utils.fixVideoMetadataPerms(file_id, type, multiUserMode && multiUserMode.file_path); // add thumbnail path - file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_id, type, multiUserMode && multiUserMode.file_path); + file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_id, type, customPath || multiUserMode && multiUserMode.file_path); if (!sub) { if (multiUserMode) { diff --git a/backend/subscriptions.js b/backend/subscriptions.js index e65043e..f1c20a7 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -433,6 +433,13 @@ function getSubscription(subID, user_uid = null) { return db.get('subscriptions').find({id: subID}).value(); } +function getSubscriptionByName(subName, user_uid = null) { + if (user_uid) + return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({name: subName}).value(); + else + return db.get('subscriptions').find({name: subName}).value(); +} + function updateSubscription(sub, user_uid = null) { if (user_uid) { users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign(sub).write(); @@ -500,6 +507,7 @@ function removeIDFromArchive(archive_path, id) { module.exports = { getSubscription : getSubscription, + getSubscriptionByName : getSubscriptionByName, getAllSubscriptions : getAllSubscriptions, updateSubscription : updateSubscription, subscribe : subscribe, From 0a38b0197120ff4016dd83edeeef4bec0a2a5483 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Thu, 15 Oct 2020 16:59:14 -0400 Subject: [PATCH 05/14] Updated posts service to allow for category deletion and subscription retrieval based on name --- src/app/posts.services.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 3e45e81..f65af2e 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -304,10 +304,13 @@ export class PostsService implements CanActivate { } createCategory(name) { - console.log(name); return this.http.post(this.path + 'createCategory', {name: name}, this.httpOptions); } + deleteCategory(category_uid) { + return this.http.post(this.path + 'deleteCategory', {category_uid: category_uid}, this.httpOptions); + } + updateCategory(category) { return this.http.post(this.path + 'updateCategory', {category: category}, this.httpOptions); } @@ -340,8 +343,8 @@ export class PostsService implements CanActivate { file_uid: file_uid}, this.httpOptions) } - getSubscription(id) { - return this.http.post(this.path + 'getSubscription', {id: id}, this.httpOptions); + getSubscription(id, name = null) { + return this.http.post(this.path + 'getSubscription', {id: id, name: name}, this.httpOptions); } getAllSubscriptions() { From 6f089491a5be49ff6c995c627f43ec6d07a65dc2 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Thu, 15 Oct 2020 16:59:33 -0400 Subject: [PATCH 06/14] Updated player component to support categories --- src/app/player/player.component.ts | 48 ++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/src/app/player/player.component.ts b/src/app/player/player.component.ts index 6582cd7..28ce1cc 100644 --- a/src/app/player/player.component.ts +++ b/src/app/player/player.component.ts @@ -124,6 +124,8 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { this.getFile(); } else if (this.id) { this.getPlaylistFiles(); + } else if (this.subscriptionName) { + this.getSubscription(); } if (this.url) { @@ -139,7 +141,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { this.currentItem = this.playlist[0]; this.currentIndex = 0; this.show_player = true; - } else if (this.subscriptionName || this.fileNames) { + } else if (this.fileNames && !this.subscriptionName) { this.show_player = true; this.parseFileNames(); } @@ -171,6 +173,25 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { }); } + getSubscription() { + this.postsService.getSubscription(null, this.subscriptionName).subscribe(res => { + const subscription = res['subscription']; + if (this.fileNames) { + subscription.videos.forEach(video => { + if (video['id'] === this.fileNames[0]) { + this.db_file = video; + this.show_player = true; + this.parseFileNames(); + } + }); + } else { + console.log('no file name specified'); + } + }, err => { + this.openSnackBar(`Failed to find subscription ${this.subscriptionName}`, 'Dismiss'); + }); + } + getPlaylistFiles() { this.postsService.getPlaylist(this.id, null, this.uuid).subscribe(res => { if (res['playlist']) { @@ -202,23 +223,26 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { const fileName = this.fileNames[i]; let baseLocation = null; let fullLocation = null; - if (!this.subscriptionName) { - baseLocation = this.type + '/'; - fullLocation = this.baseStreamPath + baseLocation + encodeURIComponent(fileName); - } else { - // default to video but include subscription name param - baseLocation = this.type === 'audio' ? 'audio/' : 'video/'; - fullLocation = this.baseStreamPath + baseLocation + encodeURIComponent(fileName) + '?subName=' + this.subscriptionName + - '&subPlaylist=' + this.subPlaylist; - } // adds user token if in multi-user-mode const uuid_str = this.uuid ? `&uuid=${this.uuid}` : ''; const uid_str = (this.id || !this.db_file) ? '' : `&uid=${this.db_file.uid}`; - const type_str = (this.id || !this.db_file) ? '' : `&type=${this.db_file.type}` + const type_str = (this.type || !this.db_file) ? `&type=${this.type}` : `&type=${this.db_file.type}` const id_str = this.id ? `&id=${this.id}` : ''; + const file_path_str = (!this.db_file) ? '' : `&file_path=${encodeURIComponent(this.db_file.path)}`; + + if (!this.subscriptionName) { + baseLocation = 'stream/'; + fullLocation = this.baseStreamPath + baseLocation + encodeURIComponent(fileName) + `?test=test${type_str}${file_path_str}`; + } else { + // default to video but include subscription name param + baseLocation = 'stream/'; + fullLocation = this.baseStreamPath + baseLocation + encodeURIComponent(fileName) + '?subName=' + this.subscriptionName + + '&subPlaylist=' + this.subPlaylist + `${file_path_str}${type_str}`; + } + if (this.postsService.isLoggedIn) { - fullLocation += (this.subscriptionName ? '&' : '?') + `jwt=${this.postsService.token}`; + fullLocation += (this.subscriptionName ? '&' : '&') + `jwt=${this.postsService.token}`; if (this.is_shared) { fullLocation += `${uuid_str}${uid_str}${type_str}${id_str}`; } } else if (this.is_shared) { fullLocation += (this.subscriptionName ? '&' : '?') + `test=test${uuid_str}${uid_str}${type_str}${id_str}`; From d4e5082039dbe05b4801c5897267d331a96e921c Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Thu, 15 Oct 2020 17:00:48 -0400 Subject: [PATCH 07/14] Confirm dialog can now optionally use warn colors (used for deletion or breaking changes) Category re-ordering is fixed Category deletion in settings is now functional --- .../logs-viewer/logs-viewer.component.ts | 3 +- .../confirm-dialog.component.html | 2 +- .../confirm-dialog.component.ts | 4 ++- src/app/settings/settings.component.html | 2 +- src/app/settings/settings.component.ts | 32 +++++++++++++++++-- 5 files changed, 37 insertions(+), 6 deletions(-) diff --git a/src/app/components/logs-viewer/logs-viewer.component.ts b/src/app/components/logs-viewer/logs-viewer.component.ts index fbeeb46..5a218cc 100644 --- a/src/app/components/logs-viewer/logs-viewer.component.ts +++ b/src/app/components/logs-viewer/logs-viewer.component.ts @@ -61,7 +61,8 @@ export class LogsViewerComponent implements OnInit { data: { dialogTitle: 'Clear logs', dialogText: 'Would you like to clear your logs? This will delete all your current logs, permanently.', - submitText: 'Clear' + submitText: 'Clear', + warnSubmitColor: true } }); dialogRef.afterClosed().subscribe(confirmed => { diff --git a/src/app/dialogs/confirm-dialog/confirm-dialog.component.html b/src/app/dialogs/confirm-dialog/confirm-dialog.component.html index 5c6ef6b..ddfde17 100644 --- a/src/app/dialogs/confirm-dialog/confirm-dialog.component.html +++ b/src/app/dialogs/confirm-dialog/confirm-dialog.component.html @@ -6,7 +6,7 @@ - + diff --git a/src/app/dialogs/confirm-dialog/confirm-dialog.component.ts b/src/app/dialogs/confirm-dialog/confirm-dialog.component.ts index 36892b8..31ec18a 100644 --- a/src/app/dialogs/confirm-dialog/confirm-dialog.component.ts +++ b/src/app/dialogs/confirm-dialog/confirm-dialog.component.ts @@ -15,12 +15,14 @@ export class ConfirmDialogComponent implements OnInit { doneEmitter: EventEmitter = null; onlyEmitOnDone = false; - + + warnSubmitColor = false; constructor(@Inject(MAT_DIALOG_DATA) public data: any, public dialogRef: MatDialogRef) { if (this.data.dialogTitle) { this.dialogTitle = this.data.dialogTitle }; if (this.data.dialogText) { this.dialogText = this.data.dialogText }; if (this.data.submitText) { this.submitText = this.data.submitText }; + if (this.data.warnSubmitColor) { this.warnSubmitColor = this.data.warnSubmitColor }; // checks if emitter exists, if so don't autoclose as it should be handled by caller if (this.data.doneEmitter) { diff --git a/src/app/settings/settings.component.html b/src/app/settings/settings.component.html index 8e6fc0d..1342457 100644 --- a/src/app/settings/settings.component.html +++ b/src/app/settings/settings.component.html @@ -135,7 +135,7 @@ {{category['name']}} - +
diff --git a/src/app/settings/settings.component.ts b/src/app/settings/settings.component.ts index ec0c6d4..26c6eaa 100644 --- a/src/app/settings/settings.component.ts +++ b/src/app/settings/settings.component.ts @@ -82,7 +82,11 @@ export class SettingsComponent implements OnInit { dropCategory(event: CdkDragDrop) { moveItemInArray(this.postsService.categories, event.previousIndex, event.currentIndex); - this.postsService.updateCategories(this.postsService.categories); + this.postsService.updateCategories(this.postsService.categories).subscribe(res => { + + }, err => { + this.postsService.openSnackBar('Failed to update categories!'); + }); } openAddCategoryDialog() { @@ -113,6 +117,29 @@ export class SettingsComponent implements OnInit { }); } + deleteCategory(category) { + const dialogRef = this.dialog.open(ConfirmDialogComponent, { + data: { + dialogTitle: 'Delete category', + dialogText: `Would you like to delete ${category['name']}?`, + submitText: 'Delete', + warnSubmitColor: true + } + }); + dialogRef.afterClosed().subscribe(confirmed => { + if (confirmed) { + this.postsService.deleteCategory(category['uid']).subscribe(res => { + if (res['success']) { + this.postsService.openSnackBar(`Successfully deleted ${category['name']}!`); + this.postsService.reloadCategories(); + } + }, err => { + this.postsService.openSnackBar(`Failed to delete ${category['name']}!`); + }); + } + }); + } + openEditCategoryDialog(category) { this.dialog.open(EditCategoryDialogComponent, { data: { @@ -206,7 +233,8 @@ export class SettingsComponent implements OnInit { dialogTitle: 'Kill downloads', dialogText: 'Are you sure you want to kill all downloads? Any subscription and non-subscription downloads will end immediately, though this operation may take a minute or so to complete.', submitText: 'Kill all downloads', - doneEmitter: done + doneEmitter: done, + warnSubmitColor: true } }); done.subscribe(confirmed => { From deac54e8d62c75539b035543b4767567e78999b1 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Thu, 15 Oct 2020 17:00:56 -0400 Subject: [PATCH 08/14] Fixed bug in goToPlaylist --- .../components/custom-playlists/custom-playlists.component.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/components/custom-playlists/custom-playlists.component.ts b/src/app/components/custom-playlists/custom-playlists.component.ts index 1f2549f..5668e32 100644 --- a/src/app/components/custom-playlists/custom-playlists.component.ts +++ b/src/app/components/custom-playlists/custom-playlists.component.ts @@ -50,7 +50,8 @@ export class CustomPlaylistsComponent implements OnInit { }); } - goToPlaylist(playlist) { + goToPlaylist(event_info) { + const playlist = event_info.file; const playlistID = playlist.id; const type = playlist.type; From 0189d292a8e7516ab3325f58604e2a0042550ee3 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sun, 18 Oct 2020 02:20:06 -0400 Subject: [PATCH 09/14] Fixed bug that prevented categorized files from being deletes and simplified the two delete file API calls into one --- backend/app.js | 58 +++---------------- .../recent-videos/recent-videos.component.ts | 4 +- src/app/file-card/file-card.component.ts | 2 +- src/app/main/main.component.ts | 4 +- src/app/posts.services.ts | 8 +-- 5 files changed, 15 insertions(+), 61 deletions(-) diff --git a/backend/app.js b/backend/app.js index acbb45e..50af52e 100644 --- a/backend/app.js +++ b/backend/app.js @@ -2523,56 +2523,25 @@ app.post('/api/deletePlaylist', optionalJwt, async (req, res) => { }) }); -// deletes mp3 file -app.post('/api/deleteMp3', optionalJwt, async (req, res) => { - // var name = req.body.name; +// deletes non-subscription files +app.post('/api/deleteFile', optionalJwt, async (req, res) => { var uid = req.body.uid; + var type = req.body.type; var blacklistMode = req.body.blacklistMode; if (req.isAuthenticated()) { - let success = auth_api.deleteUserFile(req.user.uid, uid, 'audio', blacklistMode); + let success = auth_api.deleteUserFile(req.user.uid, uid, type, blacklistMode); res.send(success); return; } - var audio_obj = db.get('files.audio').find({uid: uid}).value(); - var name = audio_obj.id; - var fullpath = audioFolderPath + name + ".mp3"; + var file_obj = db.get(`files.${type}`).find({uid: uid}).value(); + var name = file_obj.id; + var fullpath = file_obj ? file_obj.path : null; var wasDeleted = false; if (fs.existsSync(fullpath)) { - deleteAudioFile(name, null, blacklistMode); - db.get('files.audio').remove({uid: uid}).write(); - wasDeleted = true; - res.send(wasDeleted); - } else if (audio_obj) { - db.get('files.audio').remove({uid: uid}).write(); - wasDeleted = true; - res.send(wasDeleted); - } else { - wasDeleted = false; - res.send(wasDeleted); - } -}); - -// deletes mp4 file -app.post('/api/deleteMp4', optionalJwt, async (req, res) => { - var uid = req.body.uid; - var blacklistMode = req.body.blacklistMode; - - if (req.isAuthenticated()) { - let success = auth_api.deleteUserFile(req.user.uid, uid, 'video', blacklistMode); - res.send(success); - return; - } - - var video_obj = db.get('files.video').find({uid: uid}).value(); - var name = video_obj.id; - var fullpath = videoFolderPath + name + ".mp4"; - var wasDeleted = false; - if (fs.existsSync(fullpath)) - { - wasDeleted = await deleteVideoFile(name, null, blacklistMode); + wasDeleted = type === 'audio' ? await deleteAudioFile(name, path.basename(fullpath), blacklistMode) : await deleteVideoFile(name, path.basename(fullpath), blacklistMode); db.get('files.video').remove({uid: uid}).write(); // wasDeleted = true; res.send(wasDeleted); @@ -2638,17 +2607,6 @@ app.post('/api/downloadFile', optionalJwt, async (req, res) => { }); }); -app.post('/api/deleteFile', async (req, res) => { - let fileName = req.body.fileName; - let type = req.body.type; - if (type === 'audio') { - deleteAudioFile(fileName); - } else if (type === 'video') { - deleteVideoFile(fileName); - } - res.send({}); -}); - app.post('/api/downloadArchive', async (req, res) => { let sub = req.body.sub; let archive_dir = sub.archive; diff --git a/src/app/components/recent-videos/recent-videos.component.ts b/src/app/components/recent-videos/recent-videos.component.ts index 8333f79..2eb7f36 100644 --- a/src/app/components/recent-videos/recent-videos.component.ts +++ b/src/app/components/recent-videos/recent-videos.component.ts @@ -210,7 +210,7 @@ export class RecentVideosComponent implements OnInit { if (!this.postsService.config.Extra.file_manager_enabled) { // tell server to delete the file once downloaded - this.postsService.deleteFile(name, false).subscribe(delRes => { + this.postsService.deleteFile(name, type).subscribe(delRes => { // reload mp4s this.getAllFiles(); }); @@ -233,7 +233,7 @@ export class RecentVideosComponent implements OnInit { } deleteNormalFile(file, index, blacklistMode = false) { - this.postsService.deleteFile(file.uid, file.isAudio, blacklistMode).subscribe(result => { + this.postsService.deleteFile(file.uid, file.isAudio ? 'audio' : 'video', blacklistMode).subscribe(result => { if (result) { this.postsService.openSnackBar('Delete success!', 'OK.'); this.files.splice(index, 1); diff --git a/src/app/file-card/file-card.component.ts b/src/app/file-card/file-card.component.ts index 280d131..68a8453 100644 --- a/src/app/file-card/file-card.component.ts +++ b/src/app/file-card/file-card.component.ts @@ -56,7 +56,7 @@ export class FileCardComponent implements OnInit { deleteFile(blacklistMode = false) { if (!this.playlist) { - this.postsService.deleteFile(this.uid, this.isAudio, blacklistMode).subscribe(result => { + this.postsService.deleteFile(this.uid, this.isAudio ? 'audio' : 'video', blacklistMode).subscribe(result => { if (result) { this.openSnackBar('Delete success!', 'OK.'); this.removeFile.emit(this.name); diff --git a/src/app/main/main.component.ts b/src/app/main/main.component.ts index ef4f929..b50337a 100644 --- a/src/app/main/main.component.ts +++ b/src/app/main/main.component.ts @@ -740,7 +740,7 @@ export class MainComponent implements OnInit { if (!this.fileManagerEnabled) { // tell server to delete the file once downloaded - this.postsService.deleteFile(name, true).subscribe(delRes => { + this.postsService.deleteFile(name, 'video').subscribe(delRes => { // reload mp3s this.getMp3s(); }); @@ -757,7 +757,7 @@ export class MainComponent implements OnInit { if (!this.fileManagerEnabled) { // tell server to delete the file once downloaded - this.postsService.deleteFile(name, false).subscribe(delRes => { + this.postsService.deleteFile(name, 'audio').subscribe(delRes => { // reload mp4s this.getMp4s(); }); diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index f65af2e..8ee9117 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -199,12 +199,8 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'setConfig', {new_config_file: config}, this.httpOptions); } - deleteFile(uid: string, isAudio: boolean, blacklistMode = false) { - if (isAudio) { - return this.http.post(this.path + 'deleteMp3', {uid: uid, blacklistMode: blacklistMode}, this.httpOptions); - } else { - return this.http.post(this.path + 'deleteMp4', {uid: uid, blacklistMode: blacklistMode}, this.httpOptions); - } + deleteFile(uid: string, type: string, blacklistMode = false) { + return this.http.post(this.path + 'deleteFile', {uid: uid, type: type, blacklistMode: blacklistMode}, this.httpOptions); } getMp3s() { From d659a7614fa0c1cb6f01467bf6f80a94194e9322 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Fri, 23 Oct 2020 01:49:27 -0400 Subject: [PATCH 10/14] updated default.json --- src/assets/default.json | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/assets/default.json b/src/assets/default.json index e03372b..893896f 100644 --- a/src/assets/default.json +++ b/src/assets/default.json @@ -7,9 +7,11 @@ "Downloader": { "path-audio": "audio/", "path-video": "video/", - "use_youtubedl_archive": true, + "use_youtubedl_archive": false, "custom_args": "", - "safe_download_override": false + "safe_download_override": false, + "include_thumbnail": false, + "include_metadata": true }, "Extra": { "title_top": "YoutubeDL-Material", @@ -33,12 +35,20 @@ "Subscriptions": { "allow_subscriptions": true, "subscriptions_base_path": "subscriptions/", - "subscriptions_check_interval": "30", + "subscriptions_check_interval": "300", "subscriptions_use_youtubedl_archive": true }, "Users": { "base_path": "users/", - "allow_registration": true + "allow_registration": true, + "auth_method": "internal", + "ldap_config": { + "url": "ldap://localhost:389", + "bindDN": "cn=root", + "bindCredentials": "secret", + "searchBase": "ou=passport-ldapauth", + "searchFilter": "(uid={{username}})" + } }, "Advanced": { "use_default_downloading_agent": true, @@ -47,7 +57,7 @@ "allow_advanced_download": true, "jwt_expiration": 86400, "logger_level": "debug", - "use_cookies": true + "use_cookies": false } } } \ No newline at end of file From 3aa08e18176a839aeb38f39749da7e79cbc44670 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Fri, 23 Oct 2020 02:44:24 -0400 Subject: [PATCH 11/14] Added scaffolding for tags --- backend/categories.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/backend/categories.js b/backend/categories.js index 545a6a1..d7df916 100644 --- a/backend/categories.js +++ b/backend/categories.js @@ -101,6 +101,26 @@ function applyCategoryRules(file_json, rules, category_name) { return rules_apply; } +async function addTagToVideo(tag, video, user_uid) { + +} + +async function removeTagFromVideo(tag, video, user_uid) { + +} + +async function incrementTagCount(tag) { + const current_value = db.get(`stats.tags.${tag}`).value(); + if (!current_value) { + db.set(`stats.tags.${tag}`, 1).write(); + } else { + db.update(`stats.tags.${tag}`, n => n + 1).write(); + } +} + +async function decrementTagCount(tag) { + db.update(`stats.tags.${tag}`, n => n - 1).write(); +} module.exports = { From 6ea4176d637a65006e8d83e3656d543414eeb918 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 24 Oct 2020 00:15:47 -0400 Subject: [PATCH 12/14] Added missing code that makes category paths relative to the root dir --- backend/app.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/app.js b/backend/app.js index 19331f0..f5138a6 100644 --- a/backend/app.js +++ b/backend/app.js @@ -1420,7 +1420,8 @@ async function generateArgs(url, type, options) { } if (customOutput) { - downloadConfig = ['-o', path.join(fileFolderPath, customOutput) + ".%(ext)s", '--write-info-json', '--print-json']; + customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput); + downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json']; } else { downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json']; } From 1ce85813fbe6f289d9f165991e9a748f5377cbe2 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 24 Oct 2020 00:20:39 -0400 Subject: [PATCH 13/14] Saving a category will now cause the UI to refresh the cache of categories --- .../edit-category-dialog/edit-category-dialog.component.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.ts b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.ts index 928ff1c..294414e 100644 --- a/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.ts +++ b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.ts @@ -88,6 +88,7 @@ export class EditCategoryDialogComponent implements OnInit { this.postsService.updateCategory(this.category).subscribe(res => { this.updating = false; this.original_category = JSON.parse(JSON.stringify(this.category)); + this.postsService.reloadCategories(); }, err => { this.updating = false; console.error(err); From 3318ac364d8587964b33d744ce74492c11a0101e Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 24 Oct 2020 00:29:42 -0400 Subject: [PATCH 14/14] Code cleanup and changed proposed handling of existing tags for suggestions --- backend/categories.js | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/backend/categories.js b/backend/categories.js index d7df916..d0b249a 100644 --- a/backend/categories.js +++ b/backend/categories.js @@ -102,27 +102,21 @@ function applyCategoryRules(file_json, rules, category_name) { } async function addTagToVideo(tag, video, user_uid) { - + // TODO: Implement } async function removeTagFromVideo(tag, video, user_uid) { - + // TODO: Implement } -async function incrementTagCount(tag) { - const current_value = db.get(`stats.tags.${tag}`).value(); - if (!current_value) { - db.set(`stats.tags.${tag}`, 1).write(); - } else { - db.update(`stats.tags.${tag}`, n => n + 1).write(); +// adds tag to list of existing tags (used for tag suggestions) +async function addTagToExistingTags(tag) { + const existing_tags = db.get('tags').value(); + if (!existing_tags.includes(tag)) { + db.get('tags').push(tag).write(); } } -async function decrementTagCount(tag) { - db.update(`stats.tags.${tag}`, n => n - 1).write(); -} - - module.exports = { initialize: initialize, categorize: categorize,