TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-26 06:30:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

Added basic categorization functionality in the server & UI

Browse Source
This commit is contained in:
Isaac Abadi
2020-09-17 03:14:24 -04:00
parent 851bfb81ba
commit 8595864118
12 changed files with 526 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
backend/app.js
View File

@@ -27,6 +27,7 @@ const shortid = require('shortid')
const url_api = require('url');
var config_api = require('./config.js');
var subscriptions_api = require('./subscriptions')
var categories_api = require('./categories');
const CONSTS = require('./consts')
const { spawn } = require('child_process')
const read_last_lines = require('read-last-lines');
@@ -37,7 +38,7 @@ const is_windows = process.platform === 'win32';
var app = express();
// database setup
const FileSync = require('lowdb/adapters/FileSync')
const FileSync = require('lowdb/adapters/FileSync');
const adapter = new FileSync('./appdata/db.json');
const db = low(adapter)
@@ -80,6 +81,15 @@ config_api.initialize(logger);
auth_api.initialize(users_db, logger);
db_api.initialize(db, users_db, logger);
subscriptions_api.initialize(db, users_db, logger, db_api);
categories_api.initialize(db, users_db, logger, db_api);
async function test() {
const test_cat = await categories_api.categorize(fs.readJSONSync('video/Claire Lost Her First Tooth!.info.json'));
console.log(test_cat);
}
test();
// var GithubContent = require('github-content');
@@ -1115,6 +1125,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
var is_audio = type === 'audio';
var ext = is_audio ? '.mp3' : '.mp4';
var fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath;
let category = null;
// prepend with user if needed
let multiUserMode = null;
@@ -1131,7 +1142,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
}
options.downloading_method = 'exec';
const downloadConfig = await generateArgs(url, type, options);
let downloadConfig = await generateArgs(url, type, options);
// adds download to download helper
const download_uid = uuid();
@@ -1153,11 +1164,22 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
updateDownloads();
// get video info prior to download
const info = await getVideoInfoByURL(url, downloadConfig, download);
let info = await getVideoInfoByURL(url, downloadConfig, download);
if (!info) {
resolve(false);
return;
} else {
// check if it fits into a category. If so, then get info again using new downloadConfig
category = await categories_api.categorize(info);
// set custom output if the category has one and re-retrieve info so the download manager has the right file name
if (category && category['custom_output']) {
options.customOutput = category['custom_output'];
options.noRelativePath = true;
downloadConfig = await generateArgs(url, type, options);
info = await getVideoInfoByURL(url, downloadConfig, download);
}
// store info in download for future use
download['_filename'] = info['_filename'];
download['filesize'] = utils.getExpectedFileSize(info);
@@ -1445,7 +1467,8 @@ async function generateArgs(url, type, options) {
}
if (customOutput) {
downloadConfig = ['-o', path.join(fileFolderPath, customOutput) + ".%(ext)s", '--write-info-json', '--print-json'];
customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput);
downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json'];
} else {
downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json'];
}
@@ -2131,6 +2154,43 @@ app.post('/api/disableSharing', optionalJwt, function(req, res) {
});
});
// categories
app.post('/api/getAllCategories', optionalJwt, async (req, res) => {
const categories = db.get('categories').value();
res.send({categories: categories});
});
app.post('/api/createCategory', optionalJwt, async (req, res) => {
const name = req.body.name;
const new_category = {
name: name,
uid: uuid(),
rules: []
};
db.get('categories').push(new_category).write();
res.send({
new_category: new_category,
success: !!new_category
});
});
app.post('/api/updateCategory', optionalJwt, async (req, res) => {
const category = req.body.category;
db.get('categories').find({uid: category.uid}).assign(category).write();
res.send({success: true});
});
app.post('/api/updateCategories', optionalJwt, async (req, res) => {
const categories = req.body.categories;
db.get('categories').assign(categories).write();
res.send({success: true});
});
// subscriptions
app.post('/api/subscribe', optionalJwt, async (req, res) => {
let name = req.body.name;
let url = req.body.url;

112
backend/categories.js Normal file
View File

@@ -0,0 +1,112 @@
const config_api = require('./config');
var logger = null;
var db = null;
var users_db = null;
var db_api = null;
function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api }
function setLogger(input_logger) { logger = input_logger; }
function initialize(input_db, input_users_db, input_logger, input_db_api) {
setDB(input_db, input_users_db, input_db_api);
setLogger(input_logger);
}
/*
Categories:
Categories are a way to organize videos based on dynamic rules set by the user. Categories are universal (so not per-user).
Categories, besides rules, have an optional custom output. This custom output can help users create their
desired directory structure.
Rules:
A category rule consists of a property, a comparison, and a value. For example, "uploader includes 'VEVO'"
Rules are stored as an object with the above fields. In addition to those fields, it also has a preceding_operator, which
is either OR or AND, and signifies whether the rule should be ANDed with the previous rules, or just ORed. For the first
rule, this field is null.
Ex. (title includes 'Rihanna' OR title includes 'Beyonce' AND uploader includes 'VEVO')
*/
async function categorize(file_json) {
return new Promise(resolve => {
let selected_category = null;
const categories = getCategories();
if (!categories) {
logger.warn('Categories could not be found. Initializing categories...');
db.assign({categories: []}).write();
resolve(null);
return;
}
for (let i = 0; i < categories.length; i++) {
const category = categories[i];
const rules = category['rules'];
// if rules for current category apply, then that is the selected category
if (applyCategoryRules(file_json, rules, category['name'])) {
selected_category = category;
logger.verbose(`Selected category ${category['name']} for ${file_json['webpage_url']}`);
}
}
resolve(selected_category);
});
}
function getCategories() {
const categories = db.get('categories').value();
return categories ? categories : null;
}
function applyCategoryRules(file_json, rules, category_name) {
let rules_apply = false;
for (let i = 0; i < rules.length; i++) {
const rule = rules[i];
let rule_applies = null;
let preceding_operator = rule['preceding_operator'];
switch (rule['comparator']) {
case 'includes':
rule_applies = file_json[rule['property']].includes(rule['value']);
break;
case 'not_includes':
rule_applies = !(file_json[rule['property']].includes(rule['value']));
break;
case 'equals':
rule_applies = file_json[rule['property']] === rule['value'];
break;
case 'not_equals':
rule_applies = file_json[rule['property']] !== rule['value'];
break;
default:
logger.warn(`Invalid comparison used for category ${category_name}`)
break;
}
// OR the first rule with rules_apply, which will be initially false
if (i === 0) preceding_operator = 'or';
// update rules_apply based on current rule
if (preceding_operator === 'or')
rules_apply = rules_apply || rule_applies;
else
rules_apply = rules_apply && rule_applies;
}
return rules_apply;
}
module.exports = {
initialize: initialize,
categorize: categorize,
}

2
src/app/app.component.ts
View File

@@ -116,6 +116,8 @@ export class AppComponent implements OnInit, AfterViewInit {
if (this.allowSubscriptions) {
this.postsService.reloadSubscriptions();
}
this.postsService.reloadCategories();
}
// theme stuff

4
src/app/app.module.ts
View File

@@ -79,6 +79,7 @@ import { UnifiedFileCardComponent } from './components/unified-file-card/unified
import { RecentVideosComponent } from './components/recent-videos/recent-videos.component';
import { EditSubscriptionDialogComponent } from './dialogs/edit-subscription-dialog/edit-subscription-dialog.component';
import { CustomPlaylistsComponent } from './components/custom-playlists/custom-playlists.component';
import { EditCategoryDialogComponent } from './dialogs/edit-category-dialog/edit-category-dialog.component';
registerLocaleData(es, 'es');
@@ -123,7 +124,8 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible
UnifiedFileCardComponent,
RecentVideosComponent,
EditSubscriptionDialogComponent,
CustomPlaylistsComponent
CustomPlaylistsComponent,
EditCategoryDialogComponent
],
imports: [
CommonModule,

47
src/app/dialogs/edit-category-dialog/edit-category-dialog.component.html Normal file
View File

@@ -0,0 +1,47 @@
<h4 mat-dialog-title>Editing category {{category['name']}}</h4>
<mat-dialog-content>
<mat-form-field>
<input matInput placeholder="Name" i18n-placeholder="Name" [(ngModel)]="category['name']">
</mat-form-field>
<h6 i18n="Rules">Rules</h6>
<mat-list>
<mat-list-item *ngFor="let rule of category['rules']; let i = index">
<mat-form-field [style.visibility]="i === 0 ? 'hidden' : null" class="operator-select">
<mat-select [disabled]="i === 0" [(ngModel)]="rule['preceding_operator']">
<mat-option value="or">OR</mat-option>
<mat-option value="and">AND</mat-option>
</mat-select>
</mat-form-field>
<mat-form-field class="property-select">
<mat-select [(ngModel)]="rule['property']">
<mat-option *ngFor="let propertyOption of propertyOptions" [value]="propertyOption.value">{{propertyOption.label}}</mat-option>
</mat-select>
</mat-form-field>
<mat-form-field class="comparator-select">
<mat-select [(ngModel)]="rule['comparator']">
<mat-option *ngFor="let comparatorOption of comparatorOptions" [value]="comparatorOption.value">{{comparatorOption.label}}</mat-option>
</mat-select>
</mat-form-field>
<mat-form-field class="value-input">
<input matInput [(ngModel)]="rule['value']">
</mat-form-field>
<button [disabled]="i === category['rules'].length-1" (click)="swapRules(i, i+1)" mat-icon-button><mat-icon>arrow_downward</mat-icon></button>
<button [disabled]="i === 0" (click)="swapRules(i, i-1)" mat-icon-button><mat-icon>arrow_upward</mat-icon></button>
<button (click)="removeRule(i)" mat-icon-button><mat-icon>cancel</mat-icon></button>
</mat-list-item>
</mat-list>
<button mat-icon-button (click)="addNewRule()" matTooltip="Add new rule" i18n-matTooltip="Add new rule tooltip"><mat-icon>add</mat-icon></button>
</mat-dialog-content>
<mat-dialog-actions>
<button mat-button mat-dialog-close><ng-container i18n="Cancel">Cancel</ng-container></button>
<button mat-button [disabled]="categoryChanged()" type="submit" (click)="saveClicked()"><ng-container i18n="Save button">Save</ng-container></button>
<div class="mat-spinner" *ngIf="updating">
<mat-spinner [diameter]="25"></mat-spinner>
</div>
</mat-dialog-actions>

16
src/app/dialogs/edit-category-dialog/edit-category-dialog.component.scss Normal file
View File

@@ -0,0 +1,16 @@
.operator-select {
width: 55px;
}
.property-select {
margin-left: 10px;
width: 110px;
}
.comparator-select {
margin-left: 10px;
}
.value-input {
margin-left: 10px;
}

25
src/app/dialogs/edit-category-dialog/edit-category-dialog.component.spec.ts Normal file
View File

@@ -0,0 +1,25 @@
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { EditCategoryDialogComponent } from './edit-category-dialog.component';
describe('EditCategoryDialogComponent', () => {
let component: EditCategoryDialogComponent;
let fixture: ComponentFixture<EditCategoryDialogComponent>;
beforeEach(async(() => {
TestBed.configureTestingModule({
declarations: [ EditCategoryDialogComponent ]
})
.compileComponents();
}));
beforeEach(() => {
fixture = TestBed.createComponent(EditCategoryDialogComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
});

110
src/app/dialogs/edit-category-dialog/edit-category-dialog.component.ts Normal file
View File

@@ -0,0 +1,110 @@
import { Component, OnInit, Inject } from '@angular/core';
import { MAT_DIALOG_DATA } from '@angular/material/dialog';
import { PostsService } from 'app/posts.services';
@Component({
selector: 'app-edit-category-dialog',
templateUrl: './edit-category-dialog.component.html',
styleUrls: ['./edit-category-dialog.component.scss']
})
export class EditCategoryDialogComponent implements OnInit {
updating = false;
original_category = null;
category = null;
propertyOptions = [
{
value: 'fulltitle',
label: 'Title'
},
{
value: 'id',
label: 'ID'
},
{
value: 'webpage_url',
label: 'URL'
},
{
value: 'view_count',
label: 'Views'
},
{
value: 'uploader',
label: 'Uploader'
},
{
value: '_filename',
label: 'File Name'
},
{
value: 'tags',
label: 'Tags'
}
];
comparatorOptions = [
{
value: 'includes',
label: 'includes'
},
{
value: 'not_includes',
label: 'not includes'
},
{
value: 'equals',
label: 'equals'
},
{
value: 'not_equals',
label: 'not equals'
},
];
constructor(@Inject(MAT_DIALOG_DATA) public data: any, private postsService: PostsService) {
if (this.data) {
this.original_category = this.data.category;
this.category = JSON.parse(JSON.stringify(this.original_category));
}
}
ngOnInit(): void {
}
addNewRule() {
this.category['rules'].push({
preceding_operator: 'or',
property: 'fulltitle',
comparator: 'includes',
value: ''
});
}
saveClicked() {
this.updating = true;
this.postsService.updateCategory(this.category).subscribe(res => {
this.updating = false;
this.original_category = JSON.parse(JSON.stringify(this.category));
}, err => {
this.updating = false;
console.error(err);
});
}
categoryChanged() {
return JSON.stringify(this.category) === JSON.stringify(this.original_category);
}
swapRules(original_index, new_index) {
[this.category.rules[original_index], this.category.rules[new_index]] = [this.category.rules[new_index],
this.category.rules[original_index]];
}
removeRule(index) {
this.category['rules'].splice(index, 1);
}
}

26
src/app/posts.services.ts
View File

@@ -52,6 +52,7 @@ export class PostsService implements CanActivate {
// global vars
config = null;
subscriptions = null;
categories = null;
sidenav = null;
constructor(private http: HttpClient, private router: Router, @Inject(DOCUMENT) private document: Document,
@@ -296,6 +297,31 @@ export class PostsService implements CanActivate {
return this.http.post(this.path + 'deletePlaylist', {playlistID: playlistID, type: type}, this.httpOptions);
}
// categories
getAllCategories() {
return this.http.post(this.path + 'getAllCategories', {}, this.httpOptions);
}
createCategory(name) {
console.log(name);
return this.http.post(this.path + 'createCategory', {name: name}, this.httpOptions);
}
updateCategory(category) {
return this.http.post(this.path + 'updateCategory', {category: category}, this.httpOptions);
}
updateCategories(categories) {
return this.http.post(this.path + 'updateCategories', {categories: categories}, this.httpOptions);
}
reloadCategories() {
this.getAllCategories().subscribe(res => {
this.categories = res['categories'];
});
}
createSubscription(url, name, timerange = null, streamingOnly = false, audioOnly = false, customArgs = null, customFileOutput = null) {
return this.http.post(this.path + 'subscribe', {url: url, name: name, timerange: timerange, streamingOnly: streamingOnly,
audioOnly: audioOnly, customArgs: customArgs, customFileOutput: customFileOutput}, this.httpOptions);

29
src/app/settings/settings.component.html
View File

@@ -115,15 +115,38 @@
</mat-form-field>
</div>
<div class="col-12 mt-4">
<div class="col-12 mt-4 mb-5">
<mat-form-field class="text-field" style="margin-right: 12px;" color="accent">
<textarea matInput [(ngModel)]="new_config['Downloader']['custom_args']" placeholder="Custom args" i18n-placeholder="Custom args input placeholder"></textarea>
<mat-hint><ng-container i18n="Custom args setting input hint">Global custom args for downloads on the home page. Args are delimited using two commas like so: ,,</ng-container></mat-hint>
<button class="args-edit-button" (click)="openArgsModifierDialog()" mat-icon-button><mat-icon>edit</mat-icon></button>
</mat-form-field>
</div>
<div class="col-12 mt-5">
</div>
</div>
<mat-divider></mat-divider>
<div *ngIf="new_config" class="container-fluid">
<div class="row">
<div class="col-12 mt-3 mb-2">
<h6 i18n="Categories">Categories</h6>
<div cdkDropList class="category-list" (cdkDropListDropped)="dropCategory($event)">
<div class="category-box" *ngFor="let category of postsService.categories" cdkDrag>
<div class="category-custom-placeholder" *cdkDragPlaceholder></div>
{{category['name']}}
<span style="float: right">
<button mat-icon-button (click)="openEditCategoryDialog(category)"><mat-icon>edit</mat-icon></button>
<button mat-icon-button><mat-icon>cancel</mat-icon></button>
</span>
</div>
</div>
<button style="margin-top: 10px;" mat-mini-fab (click)="openAddCategoryDialog()"><mat-icon>add</mat-icon></button>
</div>
</div>
</div>
<mat-divider></mat-divider>
<div *ngIf="new_config" class="container-fluid">
<div class="row">
<div class="col-12 mt-3">
<mat-checkbox color="accent" [(ngModel)]="new_config['Downloader']['use_youtubedl_archive']"><ng-container i18n="Use youtubedl archive setting">Use youtube-dl archive</ng-container></mat-checkbox>
</div>

51
src/app/settings/settings.component.scss
View File

@@ -30,4 +30,55 @@
margin-left: 15px;
margin-bottom: 12px;
bottom: 4px;
}
.category-list {
width: 500px;
max-width: 100%;
border: solid 1px #ccc;
min-height: 60px;
display: block;
// background: white;
border-radius: 4px;
overflow: hidden;
}
.category-box {
padding: 20px 10px;
border-bottom: solid 1px #ccc;
display: flex;
flex-direction: row;
align-items: center;
justify-content: space-between;
box-sizing: border-box;
cursor: move;
// background: white;
font-size: 14px;
}
.cdk-drag-preview {
box-sizing: border-box;
border-radius: 4px;
box-shadow: 0 5px 5px -3px rgba(0, 0, 0, 0.2),
0 8px 10px 1px rgba(0, 0, 0, 0.14),
0 3px 14px 2px rgba(0, 0, 0, 0.12);
}
.cdk-drag-animating {
transition: transform 250ms cubic-bezier(0, 0, 0.2, 1);
}
.category-box:last-child {
border: none;
}
.category-list.cdk-drop-list-dragging .category-box:not(.cdk-drag-placeholder) {
transition: transform 250ms cubic-bezier(0, 0, 0.2, 1);
}
.category-custom-placeholder {
background: #ccc;
border: dotted 3px #999;
min-height: 60px;
transition: transform 250ms cubic-bezier(0, 0, 0.2, 1);
}

44
src/app/settings/settings.component.ts
View File

@@ -9,6 +9,9 @@ import { CURRENT_VERSION } from 'app/consts';
import { MatCheckboxChange } from '@angular/material/checkbox';
import { CookiesUploaderDialogComponent } from 'app/dialogs/cookies-uploader-dialog/cookies-uploader-dialog.component';
import { ConfirmDialogComponent } from 'app/dialogs/confirm-dialog/confirm-dialog.component';
import { moveItemInArray, CdkDragDrop } from '@angular/cdk/drag-drop';
import { InputDialogComponent } from 'app/input-dialog/input-dialog.component';
import { EditCategoryDialogComponent } from 'app/dialogs/edit-category-dialog/edit-category-dialog.component';
@Component({
selector: 'app-settings',
@@ -77,6 +80,47 @@ export class SettingsComponent implements OnInit {
})
}
dropCategory(event: CdkDragDrop<string[]>) {
moveItemInArray(this.postsService.categories, event.previousIndex, event.currentIndex);
this.postsService.updateCategories(this.postsService.categories);
}
openAddCategoryDialog() {
const done = new EventEmitter<any>();
const dialogRef = this.dialog.open(InputDialogComponent, {
width: '300px',
data: {
inputTitle: 'Name the category',
inputPlaceholder: 'Name',
submitText: 'Add',
doneEmitter: done
}
});
done.subscribe(name => {
// Eventually do additional checks on name
if (name) {
this.postsService.createCategory(name).subscribe(res => {
if (res['success']) {
this.postsService.reloadCategories();
dialogRef.close();
const new_category = res['new_category'];
this.openEditCategoryDialog(new_category);
}
});
}
});
}
openEditCategoryDialog(category) {
this.dialog.open(EditCategoryDialogComponent, {
data: {
category: category
}
});
}
generateAPIKey() {
this.postsService.generateNewAPIKey().subscribe(res => {
if (res['new_api_key']) {