diff --git a/backend/app.js b/backend/app.js index 38ab93d..2d8bd7a 100644 --- a/backend/app.js +++ b/backend/app.js @@ -27,6 +27,7 @@ const shortid = require('shortid') const url_api = require('url'); var config_api = require('./config.js'); var subscriptions_api = require('./subscriptions') +var categories_api = require('./categories'); const CONSTS = require('./consts') const { spawn } = require('child_process') const read_last_lines = require('read-last-lines'); @@ -37,7 +38,7 @@ const is_windows = process.platform === 'win32'; var app = express(); // database setup -const FileSync = require('lowdb/adapters/FileSync') +const FileSync = require('lowdb/adapters/FileSync'); const adapter = new FileSync('./appdata/db.json'); const db = low(adapter) @@ -80,6 +81,15 @@ config_api.initialize(logger); auth_api.initialize(users_db, logger); db_api.initialize(db, users_db, logger); subscriptions_api.initialize(db, users_db, logger, db_api); +categories_api.initialize(db, users_db, logger, db_api); + + +async function test() { + const test_cat = await categories_api.categorize(fs.readJSONSync('video/Claire Lost Her First Tooth!.info.json')); + console.log(test_cat); +} + +test(); // var GithubContent = require('github-content'); @@ -1115,6 +1125,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { var is_audio = type === 'audio'; var ext = is_audio ? '.mp3' : '.mp4'; var fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; + let category = null; // prepend with user if needed let multiUserMode = null; @@ -1131,7 +1142,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { } options.downloading_method = 'exec'; - const downloadConfig = await generateArgs(url, type, options); + let downloadConfig = await generateArgs(url, type, options); // adds download to download helper const download_uid = uuid(); @@ -1153,11 +1164,22 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { updateDownloads(); // get video info prior to download - const info = await getVideoInfoByURL(url, downloadConfig, download); + let info = await getVideoInfoByURL(url, downloadConfig, download); if (!info) { resolve(false); return; } else { + // check if it fits into a category. If so, then get info again using new downloadConfig + category = await categories_api.categorize(info); + + // set custom output if the category has one and re-retrieve info so the download manager has the right file name + if (category && category['custom_output']) { + options.customOutput = category['custom_output']; + options.noRelativePath = true; + downloadConfig = await generateArgs(url, type, options); + info = await getVideoInfoByURL(url, downloadConfig, download); + } + // store info in download for future use download['_filename'] = info['_filename']; download['filesize'] = utils.getExpectedFileSize(info); @@ -1445,7 +1467,8 @@ async function generateArgs(url, type, options) { } if (customOutput) { - downloadConfig = ['-o', path.join(fileFolderPath, customOutput) + ".%(ext)s", '--write-info-json', '--print-json']; + customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput); + downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json']; } else { downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json']; } @@ -2131,6 +2154,43 @@ app.post('/api/disableSharing', optionalJwt, function(req, res) { }); }); +// categories + +app.post('/api/getAllCategories', optionalJwt, async (req, res) => { + const categories = db.get('categories').value(); + res.send({categories: categories}); +}); + +app.post('/api/createCategory', optionalJwt, async (req, res) => { + const name = req.body.name; + const new_category = { + name: name, + uid: uuid(), + rules: [] + }; + + db.get('categories').push(new_category).write(); + + res.send({ + new_category: new_category, + success: !!new_category + }); +}); + +app.post('/api/updateCategory', optionalJwt, async (req, res) => { + const category = req.body.category; + db.get('categories').find({uid: category.uid}).assign(category).write(); + res.send({success: true}); +}); + +app.post('/api/updateCategories', optionalJwt, async (req, res) => { + const categories = req.body.categories; + db.get('categories').assign(categories).write(); + res.send({success: true}); +}); + +// subscriptions + app.post('/api/subscribe', optionalJwt, async (req, res) => { let name = req.body.name; let url = req.body.url; diff --git a/backend/categories.js b/backend/categories.js new file mode 100644 index 0000000..ac3a3c7 --- /dev/null +++ b/backend/categories.js @@ -0,0 +1,112 @@ +const config_api = require('./config'); + +var logger = null; +var db = null; +var users_db = null; +var db_api = null; + +function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api } +function setLogger(input_logger) { logger = input_logger; } + +function initialize(input_db, input_users_db, input_logger, input_db_api) { + setDB(input_db, input_users_db, input_db_api); + setLogger(input_logger); +} + +/* + +Categories: + + Categories are a way to organize videos based on dynamic rules set by the user. Categories are universal (so not per-user). + + Categories, besides rules, have an optional custom output. This custom output can help users create their + desired directory structure. + +Rules: + A category rule consists of a property, a comparison, and a value. For example, "uploader includes 'VEVO'" + + Rules are stored as an object with the above fields. In addition to those fields, it also has a preceding_operator, which + is either OR or AND, and signifies whether the rule should be ANDed with the previous rules, or just ORed. For the first + rule, this field is null. + + Ex. (title includes 'Rihanna' OR title includes 'Beyonce' AND uploader includes 'VEVO') + +*/ + +async function categorize(file_json) { + return new Promise(resolve => { + let selected_category = null; + const categories = getCategories(); + if (!categories) { + logger.warn('Categories could not be found. Initializing categories...'); + db.assign({categories: []}).write(); + resolve(null); + return; + } + + for (let i = 0; i < categories.length; i++) { + const category = categories[i]; + const rules = category['rules']; + + // if rules for current category apply, then that is the selected category + if (applyCategoryRules(file_json, rules, category['name'])) { + selected_category = category; + logger.verbose(`Selected category ${category['name']} for ${file_json['webpage_url']}`); + } + } + + resolve(selected_category); + + }); +} + +function getCategories() { + const categories = db.get('categories').value(); + return categories ? categories : null; +} + +function applyCategoryRules(file_json, rules, category_name) { + let rules_apply = false; + for (let i = 0; i < rules.length; i++) { + const rule = rules[i]; + let rule_applies = null; + + let preceding_operator = rule['preceding_operator']; + + switch (rule['comparator']) { + case 'includes': + rule_applies = file_json[rule['property']].includes(rule['value']); + break; + case 'not_includes': + rule_applies = !(file_json[rule['property']].includes(rule['value'])); + break; + case 'equals': + rule_applies = file_json[rule['property']] === rule['value']; + break; + case 'not_equals': + rule_applies = file_json[rule['property']] !== rule['value']; + break; + default: + logger.warn(`Invalid comparison used for category ${category_name}`) + break; + } + + // OR the first rule with rules_apply, which will be initially false + if (i === 0) preceding_operator = 'or'; + + // update rules_apply based on current rule + if (preceding_operator === 'or') + rules_apply = rules_apply || rule_applies; + else + rules_apply = rules_apply && rule_applies; + } + + return rules_apply; +} + + + +module.exports = { + initialize: initialize, + categorize: categorize, +} \ No newline at end of file diff --git a/src/app/app.component.ts b/src/app/app.component.ts index d8540bb..a6c7630 100644 --- a/src/app/app.component.ts +++ b/src/app/app.component.ts @@ -116,6 +116,8 @@ export class AppComponent implements OnInit, AfterViewInit { if (this.allowSubscriptions) { this.postsService.reloadSubscriptions(); } + + this.postsService.reloadCategories(); } // theme stuff diff --git a/src/app/app.module.ts b/src/app/app.module.ts index 69cba76..423f714 100644 --- a/src/app/app.module.ts +++ b/src/app/app.module.ts @@ -79,6 +79,7 @@ import { UnifiedFileCardComponent } from './components/unified-file-card/unified import { RecentVideosComponent } from './components/recent-videos/recent-videos.component'; import { EditSubscriptionDialogComponent } from './dialogs/edit-subscription-dialog/edit-subscription-dialog.component'; import { CustomPlaylistsComponent } from './components/custom-playlists/custom-playlists.component'; +import { EditCategoryDialogComponent } from './dialogs/edit-category-dialog/edit-category-dialog.component'; registerLocaleData(es, 'es'); @@ -123,7 +124,8 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible UnifiedFileCardComponent, RecentVideosComponent, EditSubscriptionDialogComponent, - CustomPlaylistsComponent + CustomPlaylistsComponent, + EditCategoryDialogComponent ], imports: [ CommonModule, diff --git a/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.html b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.html new file mode 100644 index 0000000..fe6cec7 --- /dev/null +++ b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.html @@ -0,0 +1,47 @@ +

Editing category {{category['name']}}

+ + + + + + +
Rules
+ + + + + + OR + AND + + + + + {{propertyOption.label}} + + + + + {{comparatorOption.label}} + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.scss b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.scss new file mode 100644 index 0000000..53fcc70 --- /dev/null +++ b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.scss @@ -0,0 +1,16 @@ +.operator-select { + width: 55px; +} + +.property-select { + margin-left: 10px; + width: 110px; +} + +.comparator-select { + margin-left: 10px; +} + +.value-input { + margin-left: 10px; +} \ No newline at end of file diff --git a/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.spec.ts b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.spec.ts new file mode 100644 index 0000000..71d64a9 --- /dev/null +++ b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.spec.ts @@ -0,0 +1,25 @@ +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; + +import { EditCategoryDialogComponent } from './edit-category-dialog.component'; + +describe('EditCategoryDialogComponent', () => { + let component: EditCategoryDialogComponent; + let fixture: ComponentFixture; + + beforeEach(async(() => { + TestBed.configureTestingModule({ + declarations: [ EditCategoryDialogComponent ] + }) + .compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(EditCategoryDialogComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.ts b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.ts new file mode 100644 index 0000000..928ff1c --- /dev/null +++ b/src/app/dialogs/edit-category-dialog/edit-category-dialog.component.ts @@ -0,0 +1,110 @@ +import { Component, OnInit, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { PostsService } from 'app/posts.services'; + +@Component({ + selector: 'app-edit-category-dialog', + templateUrl: './edit-category-dialog.component.html', + styleUrls: ['./edit-category-dialog.component.scss'] +}) +export class EditCategoryDialogComponent implements OnInit { + + updating = false; + original_category = null; + category = null; + + propertyOptions = [ + { + value: 'fulltitle', + label: 'Title' + }, + { + value: 'id', + label: 'ID' + }, + { + value: 'webpage_url', + label: 'URL' + }, + { + value: 'view_count', + label: 'Views' + }, + { + value: 'uploader', + label: 'Uploader' + }, + { + value: '_filename', + label: 'File Name' + }, + { + value: 'tags', + label: 'Tags' + } + ]; + + comparatorOptions = [ + { + value: 'includes', + label: 'includes' + }, + { + value: 'not_includes', + label: 'not includes' + }, + { + value: 'equals', + label: 'equals' + }, + { + value: 'not_equals', + label: 'not equals' + }, + + ]; + + constructor(@Inject(MAT_DIALOG_DATA) public data: any, private postsService: PostsService) { + if (this.data) { + this.original_category = this.data.category; + this.category = JSON.parse(JSON.stringify(this.original_category)); + } + } + + ngOnInit(): void { + } + + addNewRule() { + this.category['rules'].push({ + preceding_operator: 'or', + property: 'fulltitle', + comparator: 'includes', + value: '' + }); + } + + saveClicked() { + this.updating = true; + this.postsService.updateCategory(this.category).subscribe(res => { + this.updating = false; + this.original_category = JSON.parse(JSON.stringify(this.category)); + }, err => { + this.updating = false; + console.error(err); + }); + } + + categoryChanged() { + return JSON.stringify(this.category) === JSON.stringify(this.original_category); + } + + swapRules(original_index, new_index) { + [this.category.rules[original_index], this.category.rules[new_index]] = [this.category.rules[new_index], + this.category.rules[original_index]]; + } + + removeRule(index) { + this.category['rules'].splice(index, 1); + } + +} diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 67e3bd1..3e45e81 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -52,6 +52,7 @@ export class PostsService implements CanActivate { // global vars config = null; subscriptions = null; + categories = null; sidenav = null; constructor(private http: HttpClient, private router: Router, @Inject(DOCUMENT) private document: Document, @@ -296,6 +297,31 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'deletePlaylist', {playlistID: playlistID, type: type}, this.httpOptions); } + // categories + + getAllCategories() { + return this.http.post(this.path + 'getAllCategories', {}, this.httpOptions); + } + + createCategory(name) { + console.log(name); + return this.http.post(this.path + 'createCategory', {name: name}, this.httpOptions); + } + + updateCategory(category) { + return this.http.post(this.path + 'updateCategory', {category: category}, this.httpOptions); + } + + updateCategories(categories) { + return this.http.post(this.path + 'updateCategories', {categories: categories}, this.httpOptions); + } + + reloadCategories() { + this.getAllCategories().subscribe(res => { + this.categories = res['categories']; + }); + } + createSubscription(url, name, timerange = null, streamingOnly = false, audioOnly = false, customArgs = null, customFileOutput = null) { return this.http.post(this.path + 'subscribe', {url: url, name: name, timerange: timerange, streamingOnly: streamingOnly, audioOnly: audioOnly, customArgs: customArgs, customFileOutput: customFileOutput}, this.httpOptions); diff --git a/src/app/settings/settings.component.html b/src/app/settings/settings.component.html index 719683b..8e6fc0d 100644 --- a/src/app/settings/settings.component.html +++ b/src/app/settings/settings.component.html @@ -115,15 +115,38 @@ -
+
Global custom args for downloads on the home page. Args are delimited using two commas like so: ,,
- -
+
+
+ +
+
+
+
Categories
+
+
+
+ {{category['name']}} + + + + +
+
+ +
+
+
+ +
+
+
Use youtube-dl archive
diff --git a/src/app/settings/settings.component.scss b/src/app/settings/settings.component.scss index 9ff0b70..f85952f 100644 --- a/src/app/settings/settings.component.scss +++ b/src/app/settings/settings.component.scss @@ -30,4 +30,55 @@ margin-left: 15px; margin-bottom: 12px; bottom: 4px; +} + +.category-list { + width: 500px; + max-width: 100%; + border: solid 1px #ccc; + min-height: 60px; + display: block; + // background: white; + border-radius: 4px; + overflow: hidden; +} + +.category-box { + padding: 20px 10px; + border-bottom: solid 1px #ccc; + display: flex; + flex-direction: row; + align-items: center; + justify-content: space-between; + box-sizing: border-box; + cursor: move; + // background: white; + font-size: 14px; +} + +.cdk-drag-preview { + box-sizing: border-box; + border-radius: 4px; + box-shadow: 0 5px 5px -3px rgba(0, 0, 0, 0.2), + 0 8px 10px 1px rgba(0, 0, 0, 0.14), + 0 3px 14px 2px rgba(0, 0, 0, 0.12); +} + +.cdk-drag-animating { + transition: transform 250ms cubic-bezier(0, 0, 0.2, 1); +} + +.category-box:last-child { + border: none; +} + +.category-list.cdk-drop-list-dragging .category-box:not(.cdk-drag-placeholder) { + transition: transform 250ms cubic-bezier(0, 0, 0.2, 1); +} + +.category-custom-placeholder { +background: #ccc; +border: dotted 3px #999; +min-height: 60px; +transition: transform 250ms cubic-bezier(0, 0, 0.2, 1); } \ No newline at end of file diff --git a/src/app/settings/settings.component.ts b/src/app/settings/settings.component.ts index 4a7684f..ec0c6d4 100644 --- a/src/app/settings/settings.component.ts +++ b/src/app/settings/settings.component.ts @@ -9,6 +9,9 @@ import { CURRENT_VERSION } from 'app/consts'; import { MatCheckboxChange } from '@angular/material/checkbox'; import { CookiesUploaderDialogComponent } from 'app/dialogs/cookies-uploader-dialog/cookies-uploader-dialog.component'; import { ConfirmDialogComponent } from 'app/dialogs/confirm-dialog/confirm-dialog.component'; +import { moveItemInArray, CdkDragDrop } from '@angular/cdk/drag-drop'; +import { InputDialogComponent } from 'app/input-dialog/input-dialog.component'; +import { EditCategoryDialogComponent } from 'app/dialogs/edit-category-dialog/edit-category-dialog.component'; @Component({ selector: 'app-settings', @@ -77,6 +80,47 @@ export class SettingsComponent implements OnInit { }) } + dropCategory(event: CdkDragDrop) { + moveItemInArray(this.postsService.categories, event.previousIndex, event.currentIndex); + this.postsService.updateCategories(this.postsService.categories); + } + + openAddCategoryDialog() { + const done = new EventEmitter(); + const dialogRef = this.dialog.open(InputDialogComponent, { + width: '300px', + data: { + inputTitle: 'Name the category', + inputPlaceholder: 'Name', + submitText: 'Add', + doneEmitter: done + } + }); + + done.subscribe(name => { + + // Eventually do additional checks on name + if (name) { + this.postsService.createCategory(name).subscribe(res => { + if (res['success']) { + this.postsService.reloadCategories(); + dialogRef.close(); + const new_category = res['new_category']; + this.openEditCategoryDialog(new_category); + } + }); + } + }); + } + + openEditCategoryDialog(category) { + this.dialog.open(EditCategoryDialogComponent, { + data: { + category: category + } + }); + } + generateAPIKey() { this.postsService.generateNewAPIKey().subscribe(res => { if (res['new_api_key']) {