TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-29 16:10:56 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/Tzahi12345/YoutubeDL-Material into subscriptions-custom-path

Browse Source
This commit is contained in:
Tzahi12345
2020-06-02 23:24:54 -04:00
parent 9e5ad66a9d b37d912e04
commit d3d0f92ea5
39 changed files with 526 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
backend/Dockerfile
View File

@@ -1,18 +1,25 @@
FROM alpine:3.11
FROM alpine:3.12
RUN \
apk add --no-cache npm python ffmpeg && \
apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
ENV UID=1000 GID=1000
RUN export user=youtube \
&& addgroup -S $user -g $GID && adduser -D -S $user -G $user -u $UID
USER $user
RUN apk add --no-cache \
ffmpeg \
npm \
python2 \
su-exec \
&& apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
atomicparsley
WORKDIR /app
COPY package.json /app/
COPY --chown=$UID:$GID [ "package.json", "package-lock.json", "/app/" ]
RUN npm install
COPY ./ /app/
COPY --chown=$UID:$GID [ "./", "/app/" ]
EXPOSE 17442
ENTRYPOINT [ "/app/entrypoint.sh" ]
CMD [ "node", "app.js" ]

110
backend/app.js
View File

@@ -8,6 +8,7 @@ var youtubedl = require('youtube-dl');
var ffmpeg = require('fluent-ffmpeg');
var compression = require('compression');
var https = require('https');
var multer = require('multer');
var express = require("express");
var bodyParser = require("body-parser");
var archiver = require('archiver');
@@ -164,7 +165,9 @@ var validDownloadingAgents = [
'ffmpeg',
'httpie',
'wget'
]
];
const subscription_timeouts = {};
// don't overwrite config if it already happened.. NOT
// let alreadyWritten = db.get('configWriteFlag').value();
@@ -615,15 +618,14 @@ function loadConfigValues() {
logger.transports[2].level = logger_level;
}
function calculateSubcriptionRetrievalDelay(amount) {
// frequency is 5 mins
let frequency_in_ms = subscriptionsCheckInterval * 1000;
let minimum_frequency = 60 * 1000;
const first_frequency = frequency_in_ms/amount;
return (first_frequency < minimum_frequency) ? minimum_frequency : first_frequency;
function calculateSubcriptionRetrievalDelay(subscriptions_amount) {
// frequency is once every 5 mins by default
let interval_in_ms = subscriptionsCheckInterval * 1000;
const subinterval_in_ms = interval_in_ms/subscriptions_amount;
return subinterval_in_ms;
}
function watchSubscriptions() {
async function watchSubscriptions() {
let subscriptions = null;
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
@@ -645,10 +647,19 @@ function watchSubscriptions() {
let current_delay = 0;
for (let i = 0; i < subscriptions.length; i++) {
let sub = subscriptions[i];
// don't check the sub if the last check for the same subscription has not completed
if (subscription_timeouts[sub.id]) {
logger.verbose(`Subscription: skipped checking ${sub.name} as the last check for ${sub.name} has not completed.`);
continue;
}
logger.verbose('Watching ' + sub.name + ' with delay interval of ' + delay_interval);
setTimeout(() => {
subscriptions_api.getVideosForSub(sub, sub.user_uid);
setTimeout(async () => {
await subscriptions_api.getVideosForSub(sub, sub.user_uid);
subscription_timeouts[sub.id] = false;
}, current_delay);
subscription_timeouts[sub.id] = true;
current_delay += delay_interval;
if (current_delay >= subscriptionsCheckInterval * 1000) current_delay = 0;
}
@@ -1237,6 +1248,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
} else if (output) {
if (output.length === 0 || output[0].length === 0) {
download['error'] = 'No output. Check if video already exists in your archive.';
logger.warn(`No output received for video download, check if it exists in your archive.`)
updateDownloads();
resolve(false);
@@ -1287,10 +1299,10 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
let is_playlist = file_names.length > 1;
if (options.merged_string) {
let current_merged_archive = fs.readFileSync(fileFolderPath + 'merged.txt', 'utf8');
if (options.merged_string !== null && options.merged_string !== undefined) {
let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8');
let diff = current_merged_archive.replace(options.merged_string, '');
const archive_path = path.join(archivePath, `archive_${type}.txt`);
const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`);
fs.appendFileSync(archive_path, diff);
}
@@ -1318,7 +1330,7 @@ async function downloadFileByURL_normal(url, type, options, sessionID = null) {
const ext = is_audio ? '.mp3' : '.mp4';
var fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
if (is_audio) options.skip_audio_args = true;
if (is_audio && url.includes('youtu')) { options.skip_audio_args = true; }
// prepend with user if needed
let multiUserMode = null;
@@ -1421,10 +1433,10 @@ async function downloadFileByURL_normal(url, type, options, sessionID = null) {
const base_file_name = video_info._filename.substring(fileFolderPath.length, video_info._filename.length);
file_uid = registerFileDB(base_file_name, type, multiUserMode);
if (options.merged_string) {
let current_merged_archive = fs.readFileSync(fileFolderPath + 'merged.txt', 'utf8');
if (options.merged_string !== null && options.merged_string !== undefined) {
let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8');
let diff = current_merged_archive.replace(options.merged_string, '');
const archive_path = req.isAuthenticated() ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`);
const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`);
fs.appendFileSync(archive_path, diff);
}
@@ -1453,6 +1465,7 @@ async function generateArgs(url, type, options) {
return new Promise(async resolve => {
var videopath = '%(title)s';
var globalArgs = config_api.getConfigItem('ytdl_custom_args');
let useCookies = config_api.getConfigItem('ytdl_use_cookies');
var is_audio = type === 'audio';
var fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
@@ -1474,10 +1487,12 @@ async function generateArgs(url, type, options) {
let downloadConfig = null;
let qualityPath = (is_audio && !options.skip_audio_args) ? '-f bestaudio' :'-f best[ext=mp4]';
if (!is_audio && (url.includes('tiktok') || url.includes('pscp.tv'))) {
const is_youtube = url.includes('youtu');
if (!is_audio && !is_youtube) {
// tiktok videos fail when using the default format
qualityPath = '-f best';
qualityPath = null;
} else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) {
qualityPath = '-f bestvideo+bestaudio'
}
if (customArgs) {
@@ -1492,11 +1507,13 @@ async function generateArgs(url, type, options) {
}
if (customOutput) {
downloadConfig = ['-o', path.join(fileFolderPath, customOutput) + ".%(ext)s", qualityPath, '--write-info-json', '--print-json'];
downloadConfig = ['-o', path.join(fileFolderPath, customOutput) + ".%(ext)s", '--write-info-json', '--print-json'];
} else {
downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), qualityPath, '--write-info-json', '--print-json'];
downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json'];
}
if (qualityPath) downloadConfig.push(qualityPath);
if (is_audio && !options.skip_audio_args) {
downloadConfig.push('-x');
downloadConfig.push('--audio-format', 'mp3');
@@ -1505,6 +1522,14 @@ async function generateArgs(url, type, options) {
if (youtubeUsername && youtubePassword) {
downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword);
}
if (useCookies) {
if (fs.existsSync(path.join(__dirname, 'appdata', 'cookies.txt'))) {
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
} else {
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
}
}
if (!useDefaultDownloadingAgent && customDownloadingAgent) {
downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
@@ -1512,7 +1537,11 @@ async function generateArgs(url, type, options) {
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`);
const archive_folder = options.user ? path.join(fileFolderPath, 'archives') : archivePath;
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
fs.ensureDirSync(archive_folder);
// create archive file if it doesn't exist
if (!fs.existsSync(archive_path)) {
fs.closeSync(fs.openSync(archive_path, 'w'));
@@ -1524,7 +1553,7 @@ async function generateArgs(url, type, options) {
fs.closeSync(fs.openSync(blacklist_path, 'w'));
}
let merged_path = fileFolderPath + 'merged.txt';
let merged_path = path.join(fileFolderPath, `merged_${type}.txt`);
fs.ensureFileSync(merged_path);
// merges blacklist and regular archive
let inputPathList = [archive_path, blacklist_path];
@@ -1546,6 +1575,7 @@ async function generateArgs(url, type, options) {
}
}
logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
// downloadConfig.map((arg) => `"${arg}"`);
resolve(downloadConfig);
});
@@ -1815,7 +1845,7 @@ const optionalJwt = function (req, res, next) {
res.sendStatus(401);
return;
}
} else if (multiUserMode && !(req.path.includes('/api/auth/register') && !req.query.jwt)) { // registration should get passed through
} else if (multiUserMode && !(req.path.includes('/api/auth/register') && !(req.path.includes('/api/config')) && !req.query.jwt)) { // registration should get passed through
if (!req.query.jwt) {
res.sendStatus(401);
return;
@@ -1865,8 +1895,11 @@ app.post('/api/tomp3', optionalJwt, async function(req, res) {
user: req.isAuthenticated() ? req.user.uid : null
}
const safeDownloadOverride = config_api.getConfigItem('ytdl_safe_download_override');
const is_playlist = url.includes('playlist');
if (is_playlist || options.customQualityConfiguration || options.customArgs || options.maxBitrate)
let result_obj = null;
if (safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.maxBitrate)
result_obj = await downloadFileByURL_exec(url, 'audio', options, req.query.sessionID);
else
result_obj = await downloadFileByURL_normal(url, 'audio', options, req.query.sessionID);
@@ -1892,9 +1925,11 @@ app.post('/api/tomp4', optionalJwt, async function(req, res) {
user: req.isAuthenticated() ? req.user.uid : null
}
const safeDownloadOverride = config_api.getConfigItem('ytdl_safe_download_override');
const is_playlist = url.includes('playlist');
let result_obj = null;
if (is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight)
if (safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight || !url.includes('youtu'))
result_obj = await downloadFileByURL_exec(url, 'video', options, req.query.sessionID);
else
result_obj = await downloadFileByURL_normal(url, 'video', options, req.query.sessionID);
@@ -2202,7 +2237,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
else
base_path = config_api.getConfigItem('ytdl_subscriptions_base_path');
let appended_base_path = path.join(base_path, subscription.isPlaylist ? 'playlists' : 'channels', subscription.name, '/');
let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/');
let files;
try {
files = recFindByExt(appended_base_path, 'mp4');
@@ -2532,6 +2567,25 @@ app.post('/api/downloadArchive', async (req, res) => {
});
var upload_multer = multer({ dest: __dirname + '/appdata/' });
app.post('/api/uploadCookies', upload_multer.single('cookies'), async (req, res) => {
const new_path = path.join(__dirname, 'appdata', 'cookies.txt');
if (fs.existsSync(req.file.path)) {
fs.renameSync(req.file.path, new_path);
} else {
res.sendStatus(500);
return;
}
if (fs.existsSync(new_path)) {
res.send({success: true});
} else {
res.sendStatus(500);
}
});
// Updater API calls
app.get('/api/updaterStatus', async (req, res) => {

4
backend/appdata/default.json
View File

@@ -13,7 +13,8 @@
"path-audio": "audio/",
"path-video": "video/",
"use_youtubedl_archive": false,
"custom_args": ""
"custom_args": "",
"safe_download_override": false
},
"Extra": {
"title_top": "YoutubeDL-Material",
@@ -49,6 +50,7 @@
"custom_downloading_agent": "",
"multi_user_mode": false,
"allow_advanced_download": false,
"use_cookies": false,
"logger_level": "info"
}
}

4
backend/appdata/encrypted.json
View File

@@ -13,7 +13,8 @@
"path-audio": "audio/",
"path-video": "video/",
"use_youtubedl_archive": false,
"custom_args": ""
"custom_args": "",
"safe_download_override": false
},
"Extra": {
"title_top": "YoutubeDL-Material",
@@ -49,6 +50,7 @@
"custom_downloading_agent": "",
"multi_user_mode": false,
"allow_advanced_download": false,
"use_cookies": false,
"logger_level": "info"
}
}

4
backend/authentication/auth.js
View File

@@ -430,8 +430,8 @@ exports.deleteUserFile = function(user_uid, file_uid, type, blacklistMode = fals
fs.appendFileSync(blacklistPath, line);
}
} else {
logger.info('Could not find archive file for audio files. Creating...');
fs.closeSync(fs.openSync(archive_path, 'w'));
logger.info(`Could not find archive file for ${type} files. Creating...`);
fs.ensureFileSync(archive_path);
}
}
}

4
backend/config.js
View File

@@ -182,7 +182,8 @@ DEFAULT_CONFIG = {
"path-audio": "audio/",
"path-video": "video/",
"use_youtubedl_archive": false,
"custom_args": ""
"custom_args": "",
"safe_download_override": false
},
"Extra": {
"title_top": "YoutubeDL-Material",
@@ -218,6 +219,7 @@ DEFAULT_CONFIG = {
"custom_downloading_agent": "",
"multi_user_mode": false,
"allow_advanced_download": false,
"use_cookies": false,
"logger_level": "info"
}
}

8
backend/consts.js
View File

@@ -40,6 +40,10 @@ let CONFIG_ITEMS = {
'key': 'ytdl_custom_args',
'path': 'YoutubeDLMaterial.Downloader.custom_args'
},
'ytdl_safe_download_override': {
'key': 'ytdl_safe_download_override',
'path': 'YoutubeDLMaterial.Downloader.safe_download_override'
},
// Extra
'ytdl_title_top': {
@@ -148,6 +152,10 @@ let CONFIG_ITEMS = {
'key': 'ytdl_allow_advanced_download',
'path': 'YoutubeDLMaterial.Advanced.allow_advanced_download'
},
'ytdl_use_cookies': {
'key': 'ytdl_use_cookies',
'path': 'YoutubeDLMaterial.Advanced.use_cookies'
},
'ytdl_logger_level': {
'key': 'ytdl_logger_level',
'path': 'YoutubeDLMaterial.Advanced.logger_level'

17
backend/entrypoint.sh Executable file
View File

@@ -0,0 +1,17 @@
#!/bin/sh
set -eu
CMD="node app.js"
# if the first arg starts with "-" pass it to program
if [ "${1#-}" != "$1" ]; then
set -- "$CMD" "$@"
fi
# chown current working directory to current user
if [ "$@" = "$CMD" ] && [ "$(id -u)" = "0" ]; then
find . \! -user "$UID" -exec chown "$UID:$GID" -R '{}' +
exec su-exec "$UID:$GID" "$0" "$@"
fi
exec "$@"

131
backend/package-lock.json generated
View File

@@ -59,6 +59,11 @@
"picomatch": "^2.0.4"
}
},
"append-field": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
"integrity": "sha1-HjRA6RXwsSA9I3SOeO3XubW0PlY="
},
"aproba": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
@@ -324,6 +329,11 @@
"resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
"integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk="
},
"buffer-from": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
},
"buffer-indexof-polyfill": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.1.tgz",
@@ -334,6 +344,38 @@
"resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
"integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s="
},
"busboy": {
"version": "0.2.14",
"resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz",
"integrity": "sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=",
"requires": {
"dicer": "0.2.5",
"readable-stream": "1.1.x"
},
"dependencies": {
"isarray": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
},
"readable-stream": {
"version": "1.1.14",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
"integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.1",
"isarray": "0.0.1",
"string_decoder": "~0.10.x"
}
},
"string_decoder": {
"version": "0.10.31",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
"integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
}
}
},
"bytes": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
@@ -524,6 +566,33 @@
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
},
"concat-stream": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
"requires": {
"buffer-from": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^2.2.2",
"typedarray": "^0.0.6"
},
"dependencies": {
"readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
}
}
},
"config": {
"version": "3.3.1",
"resolved": "https://registry.npmjs.org/config/-/config-3.3.1.tgz",
@@ -679,6 +748,38 @@
"kuler": "1.0.x"
}
},
"dicer": {
"version": "0.2.5",
"resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz",
"integrity": "sha1-WZbAhrszIYyBLAkL3cCc0S+stw8=",
"requires": {
"readable-stream": "1.1.x",
"streamsearch": "0.1.2"
},
"dependencies": {
"isarray": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
},
"readable-stream": {
"version": "1.1.14",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
"integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.1",
"isarray": "0.0.1",
"string_decoder": "~0.10.x"
}
},
"string_decoder": {
"version": "0.10.31",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
"integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
}
}
},
"dot-prop": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz",
@@ -1718,6 +1819,21 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"multer": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/multer/-/multer-1.4.2.tgz",
"integrity": "sha512-xY8pX7V+ybyUpbYMxtjM9KAiD9ixtg5/JkeKUTD6xilfDv0vzzOFcCp4Ljb1UU3tSOM3VTZtKo63OmzOrGi3Cg==",
"requires": {
"append-field": "^1.0.0",
"busboy": "^0.2.11",
"concat-stream": "^1.5.2",
"mkdirp": "^0.5.1",
"object-assign": "^4.1.1",
"on-finished": "^2.3.0",
"type-is": "^1.6.4",
"xtend": "^4.0.0"
}
},
"multistream": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/multistream/-/multistream-2.1.1.tgz",
@@ -2434,6 +2550,11 @@
"hashish": "~0.0.4"
}
},
"streamsearch": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz",
"integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo="
},
"string-width": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
@@ -2670,6 +2791,11 @@
"mime-types": "~2.1.24"
}
},
"typedarray": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
"integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
},
"undefsafe": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.3.tgz",
@@ -2905,6 +3031,11 @@
"resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-3.0.0.tgz",
"integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ="
},
"xtend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
},
"yallist": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",

1
backend/package.json
View File

@@ -41,6 +41,7 @@
"lowdb": "^1.0.0",
"md5": "^2.2.1",
"merge-files": "^0.1.2",
"multer": "^1.4.2",
"node-fetch": "^2.6.0",
"node-id3": "^0.1.14",
"nodemon": "^2.0.2",

1
backend/public/1-es2015.19816864d8bf40ef8134.js Normal file
View File

File diff suppressed because one or more lines are too long

1
backend/public/1-es2015.d61dc0b722bb5a6d7e07.js
View File

File diff suppressed because one or more lines are too long

1
backend/public/1-es5.19816864d8bf40ef8134.js Normal file
View File

File diff suppressed because one or more lines are too long

1
backend/public/1-es5.d61dc0b722bb5a6d7e07.js
View File

File diff suppressed because one or more lines are too long

3
backend/public/3rdpartylicenses.txt
View File

@@ -215,6 +215,9 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
ngx-file-drop
MIT
ngx-videogular
MIT

2
backend/public/index.html
View File

@@ -14,5 +14,5 @@
<link rel="stylesheet" href="styles.5112d6db78cf21541598.css"></head>
<body>
<app-root></app-root>
<script src="runtime-es2015.adce6980ecb528c465bb.js" type="module"></script><script src="runtime-es5.adce6980ecb528c465bb.js" nomodule defer></script><script src="polyfills-es5.7f923c8f5afda210edd3.js" nomodule defer></script><script src="polyfills-es2015.5b408f108bcea938a7e2.js" type="module"></script><script src="main-es2015.38c23f6efbbfa0797d6d.js" type="module"></script><script src="main-es5.38c23f6efbbfa0797d6d.js" nomodule defer></script></body>
<script src="runtime-es2015.6ca29b87fe1fc2ebfd61.js" type="module"></script><script src="runtime-es5.6ca29b87fe1fc2ebfd61.js" nomodule defer></script><script src="polyfills-es5.7f923c8f5afda210edd3.js" nomodule defer></script><script src="polyfills-es2015.5b408f108bcea938a7e2.js" type="module"></script><script src="main-es2015.0cbc545a4a3bee376826.js" type="module"></script><script src="main-es5.0cbc545a4a3bee376826.js" nomodule defer></script></body>
</html>

1
backend/public/main-es2015.0cbc545a4a3bee376826.js Normal file
View File

File diff suppressed because one or more lines are too long

1
backend/public/main-es2015.38c23f6efbbfa0797d6d.js
View File

File diff suppressed because one or more lines are too long

1
backend/public/main-es5.0cbc545a4a3bee376826.js Normal file
View File

File diff suppressed because one or more lines are too long

1
backend/public/main-es5.38c23f6efbbfa0797d6d.js
View File

File diff suppressed because one or more lines are too long

2
backend/public/runtime-es2015.adce6980ecb528c465bb.js → backend/public/runtime-es2015.6ca29b87fe1fc2ebfd61.js
View File

@@ -1 +1 @@
!function(e){function r(r){for(var n,i,a=r[0],c=r[1],l=r[2],p=0,s=[];p<a.length;p++)i=a[p],Object.prototype.hasOwnProperty.call(o,i)&&o[i]&&s.push(o[i][0]),o[i]=0;for(n in c)Object.prototype.hasOwnProperty.call(c,n)&&(e[n]=c[n]);for(f&&f(r);s.length;)s.shift()();return u.push.apply(u,l||[]),t()}function t(){for(var e,r=0;r<u.length;r++){for(var t=u[r],n=!0,a=1;a<t.length;a++)0!==o[t[a]]&&(n=!1);n&&(u.splice(r--,1),e=i(i.s=t[0]))}return e}var n={},o={0:0},u=[];function i(r){if(n[r])return n[r].exports;var t=n[r]={i:r,l:!1,exports:{}};return e[r].call(t.exports,t,t.exports,i),t.l=!0,t.exports}i.e=function(e){var r=[],t=o[e];if(0!==t)if(t)r.push(t[2]);else{var n=new Promise((function(r,n){t=o[e]=[r,n]}));r.push(t[2]=n);var u,a=document.createElement("script");a.charset="utf-8",a.timeout=120,i.nc&&a.setAttribute("nonce",i.nc),a.src=function(e){return i.p+""+({}[e]||e)+"-es2015."+{1:"d61dc0b722bb5a6d7e07"}[e]+".js"}(e);var c=new Error;u=function(r){a.onerror=a.onload=null,clearTimeout(l);var t=o[e];if(0!==t){if(t){var n=r&&("load"===r.type?"missing":r.type),u=r&&r.target&&r.target.src;c.message="Loading chunk "+e+" failed.\n("+n+": "+u+")",c.name="ChunkLoadError",c.type=n,c.request=u,t[1](c)}o[e]=void 0}};var l=setTimeout((function(){u({type:"timeout",target:a})}),12e4);a.onerror=a.onload=u,document.head.appendChild(a)}return Promise.all(r)},i.m=e,i.c=n,i.d=function(e,r,t){i.o(e,r)||Object.defineProperty(e,r,{enumerable:!0,get:t})},i.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.t=function(e,r){if(1&r&&(e=i(e)),8&r)return e;if(4&r&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(i.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&r&&"string"!=typeof e)for(var n in e)i.d(t,n,(function(r){return e[r]}).bind(null,n));return t},i.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return i.d(r,"a",r),r},i.o=function(e,r){return Object.prototype.hasOwnProperty.call(e,r)},i.p="",i.oe=function(e){throw console.error(e),e};var a=window.webpackJsonp=window.webpackJsonp||[],c=a.push.bind(a);a.push=r,a=a.slice();for(var l=0;l<a.length;l++)r(a[l]);var f=c;t()}([]);
!function(e){function r(r){for(var n,i,a=r[0],c=r[1],l=r[2],p=0,s=[];p<a.length;p++)i=a[p],Object.prototype.hasOwnProperty.call(o,i)&&o[i]&&s.push(o[i][0]),o[i]=0;for(n in c)Object.prototype.hasOwnProperty.call(c,n)&&(e[n]=c[n]);for(f&&f(r);s.length;)s.shift()();return u.push.apply(u,l||[]),t()}function t(){for(var e,r=0;r<u.length;r++){for(var t=u[r],n=!0,a=1;a<t.length;a++)0!==o[t[a]]&&(n=!1);n&&(u.splice(r--,1),e=i(i.s=t[0]))}return e}var n={},o={0:0},u=[];function i(r){if(n[r])return n[r].exports;var t=n[r]={i:r,l:!1,exports:{}};return e[r].call(t.exports,t,t.exports,i),t.l=!0,t.exports}i.e=function(e){var r=[],t=o[e];if(0!==t)if(t)r.push(t[2]);else{var n=new Promise((function(r,n){t=o[e]=[r,n]}));r.push(t[2]=n);var u,a=document.createElement("script");a.charset="utf-8",a.timeout=120,i.nc&&a.setAttribute("nonce",i.nc),a.src=function(e){return i.p+""+({}[e]||e)+"-es2015."+{1:"19816864d8bf40ef8134"}[e]+".js"}(e);var c=new Error;u=function(r){a.onerror=a.onload=null,clearTimeout(l);var t=o[e];if(0!==t){if(t){var n=r&&("load"===r.type?"missing":r.type),u=r&&r.target&&r.target.src;c.message="Loading chunk "+e+" failed.\n("+n+": "+u+")",c.name="ChunkLoadError",c.type=n,c.request=u,t[1](c)}o[e]=void 0}};var l=setTimeout((function(){u({type:"timeout",target:a})}),12e4);a.onerror=a.onload=u,document.head.appendChild(a)}return Promise.all(r)},i.m=e,i.c=n,i.d=function(e,r,t){i.o(e,r)||Object.defineProperty(e,r,{enumerable:!0,get:t})},i.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.t=function(e,r){if(1&r&&(e=i(e)),8&r)return e;if(4&r&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(i.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&r&&"string"!=typeof e)for(var n in e)i.d(t,n,(function(r){return e[r]}).bind(null,n));return t},i.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return i.d(r,"a",r),r},i.o=function(e,r){return Object.prototype.hasOwnProperty.call(e,r)},i.p="",i.oe=function(e){throw console.error(e),e};var a=window.webpackJsonp=window.webpackJsonp||[],c=a.push.bind(a);a.push=r,a=a.slice();for(var l=0;l<a.length;l++)r(a[l]);var f=c;t()}([]);

2
backend/public/runtime-es5.adce6980ecb528c465bb.js → backend/public/runtime-es5.6ca29b87fe1fc2ebfd61.js
View File

@@ -1 +1 @@
!function(e){function r(r){for(var n,i,a=r[0],c=r[1],l=r[2],p=0,s=[];p<a.length;p++)i=a[p],Object.prototype.hasOwnProperty.call(o,i)&&o[i]&&s.push(o[i][0]),o[i]=0;for(n in c)Object.prototype.hasOwnProperty.call(c,n)&&(e[n]=c[n]);for(f&&f(r);s.length;)s.shift()();return u.push.apply(u,l||[]),t()}function t(){for(var e,r=0;r<u.length;r++){for(var t=u[r],n=!0,a=1;a<t.length;a++)0!==o[t[a]]&&(n=!1);n&&(u.splice(r--,1),e=i(i.s=t[0]))}return e}var n={},o={0:0},u=[];function i(r){if(n[r])return n[r].exports;var t=n[r]={i:r,l:!1,exports:{}};return e[r].call(t.exports,t,t.exports,i),t.l=!0,t.exports}i.e=function(e){var r=[],t=o[e];if(0!==t)if(t)r.push(t[2]);else{var n=new Promise((function(r,n){t=o[e]=[r,n]}));r.push(t[2]=n);var u,a=document.createElement("script");a.charset="utf-8",a.timeout=120,i.nc&&a.setAttribute("nonce",i.nc),a.src=function(e){return i.p+""+({}[e]||e)+"-es5."+{1:"d61dc0b722bb5a6d7e07"}[e]+".js"}(e);var c=new Error;u=function(r){a.onerror=a.onload=null,clearTimeout(l);var t=o[e];if(0!==t){if(t){var n=r&&("load"===r.type?"missing":r.type),u=r&&r.target&&r.target.src;c.message="Loading chunk "+e+" failed.\n("+n+": "+u+")",c.name="ChunkLoadError",c.type=n,c.request=u,t[1](c)}o[e]=void 0}};var l=setTimeout((function(){u({type:"timeout",target:a})}),12e4);a.onerror=a.onload=u,document.head.appendChild(a)}return Promise.all(r)},i.m=e,i.c=n,i.d=function(e,r,t){i.o(e,r)||Object.defineProperty(e,r,{enumerable:!0,get:t})},i.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.t=function(e,r){if(1&r&&(e=i(e)),8&r)return e;if(4&r&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(i.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&r&&"string"!=typeof e)for(var n in e)i.d(t,n,(function(r){return e[r]}).bind(null,n));return t},i.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return i.d(r,"a",r),r},i.o=function(e,r){return Object.prototype.hasOwnProperty.call(e,r)},i.p="",i.oe=function(e){throw console.error(e),e};var a=window.webpackJsonp=window.webpackJsonp||[],c=a.push.bind(a);a.push=r,a=a.slice();for(var l=0;l<a.length;l++)r(a[l]);var f=c;t()}([]);
!function(e){function r(r){for(var n,i,a=r[0],c=r[1],l=r[2],p=0,s=[];p<a.length;p++)i=a[p],Object.prototype.hasOwnProperty.call(o,i)&&o[i]&&s.push(o[i][0]),o[i]=0;for(n in c)Object.prototype.hasOwnProperty.call(c,n)&&(e[n]=c[n]);for(f&&f(r);s.length;)s.shift()();return u.push.apply(u,l||[]),t()}function t(){for(var e,r=0;r<u.length;r++){for(var t=u[r],n=!0,a=1;a<t.length;a++)0!==o[t[a]]&&(n=!1);n&&(u.splice(r--,1),e=i(i.s=t[0]))}return e}var n={},o={0:0},u=[];function i(r){if(n[r])return n[r].exports;var t=n[r]={i:r,l:!1,exports:{}};return e[r].call(t.exports,t,t.exports,i),t.l=!0,t.exports}i.e=function(e){var r=[],t=o[e];if(0!==t)if(t)r.push(t[2]);else{var n=new Promise((function(r,n){t=o[e]=[r,n]}));r.push(t[2]=n);var u,a=document.createElement("script");a.charset="utf-8",a.timeout=120,i.nc&&a.setAttribute("nonce",i.nc),a.src=function(e){return i.p+""+({}[e]||e)+"-es5."+{1:"19816864d8bf40ef8134"}[e]+".js"}(e);var c=new Error;u=function(r){a.onerror=a.onload=null,clearTimeout(l);var t=o[e];if(0!==t){if(t){var n=r&&("load"===r.type?"missing":r.type),u=r&&r.target&&r.target.src;c.message="Loading chunk "+e+" failed.\n("+n+": "+u+")",c.name="ChunkLoadError",c.type=n,c.request=u,t[1](c)}o[e]=void 0}};var l=setTimeout((function(){u({type:"timeout",target:a})}),12e4);a.onerror=a.onload=u,document.head.appendChild(a)}return Promise.all(r)},i.m=e,i.c=n,i.d=function(e,r,t){i.o(e,r)||Object.defineProperty(e,r,{enumerable:!0,get:t})},i.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.t=function(e,r){if(1&r&&(e=i(e)),8&r)return e;if(4&r&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(i.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&r&&"string"!=typeof e)for(var n in e)i.d(t,n,(function(r){return e[r]}).bind(null,n));return t},i.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return i.d(r,"a",r),r},i.o=function(e,r){return Object.prototype.hasOwnProperty.call(e,r)},i.p="",i.oe=function(e){throw console.error(e),e};var a=window.webpackJsonp=window.webpackJsonp||[],c=a.push.bind(a);a.push=r,a=a.slice();for(var l=0;l<a.length;l++)r(a[l]);var f=c;t()}([]);

62
backend/subscriptions.js
View File

@@ -240,10 +240,10 @@ async function getVideosForSub(sub, user_uid = null) {
if (sub.name) {
appendedBasePath = getAppendedBasePath(sub, basePath);
} else {
appendedBasePath = basePath + (sub.isPlaylist ? 'playlists/%(playlist_title)s' : 'channels/%(uploader)s');
appendedBasePath = path.join(basePath, (sub.isPlaylist ? 'playlists/%(playlist_title)s' : 'channels/%(uploader)s'));
}
let downloadConfig = ['-o', appendedBasePath + '/%(title)s.mp4', '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4', '-ciw', '--write-annotations', '--write-thumbnail', '--write-info-json', '--print-json'];
let downloadConfig = ['-o', appendedBasePath + '/%(title)s.mp4', '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4', '-ciw', '--write-info-json', '--print-json'];
let archive_dir = null;
let archive_path = null;
@@ -265,11 +265,38 @@ async function getVideosForSub(sub, user_uid = null) {
downloadConfig.push('--dateafter', sub.timerange);
}
let useCookies = config_api.getConfigItem('ytdl_use_cookies');
if (useCookies) {
if (fs.existsSync(path.join(__dirname, 'appdata', 'cookies.txt'))) {
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
} else {
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
}
}
// get videos
logger.verbose('Subscribe: getting videos for subscription ' + sub.name);
logger.verbose('Subscription: getting videos for subscription ' + sub.name);
youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {
if (err) {
logger.verbose('Subscription: finished check for ' + sub.name);
if (err && !output) {
logger.error(err.stderr);
if (err.stderr.includes('This video is unavailable')) {
logger.info('An error was encountered with at least one video, backup method will be used.')
try {
const outputs = err.stdout.split(/\r\n|\r|\n/);
for (let i = 0; i < outputs.length; i++) {
const output = JSON.parse(outputs[i]);
handleOutputJSON(sub, sub_db, output, i === 0)
if (err.stderr.includes(output['id']) && archive_path) {
// we found a video that errored! add it to the archive to prevent future errors
fs.appendFileSync(archive_path, output['id']);
}
}
} catch(e) {
logger.error('Backup method failed. See error below:');
logger.error(e);
}
}
resolve(false);
} else if (output) {
if (output.length === 0 || (output.length === 1 && output[0] === '')) {
@@ -287,17 +314,8 @@ async function getVideosForSub(sub, user_uid = null) {
continue;
}
if (sub.streamingOnly) {
if (i === 0) {
sub_db.assign({videos: []}).write();
}
// remove unnecessary info
output_json.formats = null;
// add to db
sub_db.get('videos').push(output_json).write();
}
const reset_videos = i === 0;
handleOutputJSON(sub, sub_db, output_json, reset_videos);
// TODO: Potentially store downloaded files in db?
@@ -308,6 +326,20 @@ async function getVideosForSub(sub, user_uid = null) {
});
}
function handleOutputJSON(sub, sub_db, output_json, reset_videos = false) {
if (sub.streamingOnly) {
if (reset_videos) {
sub_db.assign({videos: []}).write();
}
// remove unnecessary info
output_json.formats = null;
// add to db
sub_db.get('videos').push(output_json).write();
}
}
function getAllSubscriptions(user_uid = null) {
if (user_uid)
return users_db.get('users').find({uid: user_uid}).get('subscriptions').value();