Fixed docker build issues

Fixing DNS issues caused by outdated musl version

Dockerfile

@@ -1,12 +1,16 @@
-FROM alpine:latest AS ffmpeg
+FROM ubuntu:20.04 AS ffmpeg
 
+RUN apt-get update && apt-get -y install curl xz-utils 
 COPY docker-build.sh .
 RUN sh ./docker-build.sh
 
-FROM alpine:latest as frontend
+FROM ubuntu:20.04 as frontend
 
-RUN apk add --no-cache \
-  npm
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update
+RUN apt-get -y install curl gnupg
+RUN curl -sL https://deb.nodesource.com/setup_12.x  | bash -
+RUN apt-get -y install nodejs
 
 RUN npm install -g @angular/cli
 
@@ -20,35 +24,35 @@ RUN npm run build
 
 #--------------#
 
-FROM alpine:latest
+FROM ubuntu:20.04
 
 ENV UID=1000 \
   GID=1000 \
-  USER=youtube
+  USER=youtube \
+  NO_UPDATE_NOTIFIER=true
 
-ENV NO_UPDATE_NOTIFIER=true
+RUN groupadd -g $GID $USER && useradd --system -g $USER --uid $UID $USER
 
-RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID
+RUN apt-get update && apt-get -y install curl
 
-RUN apk add --no-cache \
-  npm \
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
+RUN apt-get update && apt-get -y install \
+  nodejs \
   python2 \
   python3 \
-  su-exec \
-  && apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
-    atomicparsley
+  atomicparsley
 
 WORKDIR /app
 COPY --from=ffmpeg /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
 COPY --from=ffmpeg /usr/local/bin/ffprobe /usr/local/bin/ffprobe 
 COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ]
 ENV PM2_HOME=/app/pm2
-RUN npm install pm2 -g
-RUN npm install && chown -R $UID:$GID ./
+RUN npm install pm2 -g && \
+  npm install && chown -R $UID:$GID ./
 
 COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
 COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
 
 EXPOSE 17442
-ENTRYPOINT [ "/app/entrypoint.sh" ]
+# ENTRYPOINT [ "/app/entrypoint.sh" ]
 CMD [ "pm2-runtime", "pm2.config.js" ]

Public API v1.yaml

@@ -16,7 +16,7 @@ paths:
         - downloader
       summary: Download video file
       description: |-
-        Downloads a video file with the given URL. Will include global args if they exist.
+        Downloads a file with the given URL. Will include global args if they exist.
 
 
         HTTP requests will return once the video file download completes. In the future, it will (by default) return once the download starts, and a separate API call will be used for checking the download status.
@@ -41,7 +41,7 @@ paths:
     post:
       tags:
         - downloader
-      summary: Download video file
+      summary: Generates arguments used to download file
       description: Generates args, used for checking what args would run if you ran downloadFile
       operationId: post-generateArgs
       requestBody:

SECURITY.md

@@ -0,0 +1,21 @@
+# Security Policy
+
+## Supported Versions
+
+Currently all work on this project goes into the nightly builds.
+4.2's RELEASE build is now quite old and should be considered legacy.
+We urge users to use the nightly releases, because the project
+constantly sees fixes.
+
+| Version       | Supported          |
+| ------------- | ------------------ |
+| 4.2 Nightlies | :white_check_mark: |
+| 4.2 Release   | :x:                |
+| < 4.2         | :x:                |
+
+## Reporting a Vulnerability
+
+Please file an issue in our GitHub's repo, because this app
+isn't meant to be safe to run as public instance yet, but rather as a LAN facing app.
+
+We welcome PRs and help in general in making YTDL-M more secure, but it's not a priority as of now.

backend/app.js

@@ -803,7 +803,7 @@ app.post('/api/testConnectionString', optionalJwt, async (req, res) => {
 app.post('/api/downloadFile', optionalJwt, async function(req, res) {
     req.setTimeout(0); // remove timeout in case of long videos
     const url = req.body.url;
-    const type = req.body.type;
+    const type = req.body.type ? req.body.type : 'video';
     const user_uid = req.isAuthenticated() ? req.user.uid : null;
     const options = {
         customArgs: req.body.customArgs,

backend/authentication/auth.js

@@ -18,10 +18,19 @@ let JWT_EXPIRATION = null;
 let opts = null;
 let saltRounds = null;
 
-exports.initialize = function() {
+exports.initialize = function () {
   /*************************
    * Authentication module
    ************************/
+
+  if (db_api.database_initialized) {
+    setupRoles();
+  } else {
+      db_api.database_initialized_bs.subscribe(init => {
+          if (init) setupRoles();
+      });
+  }
+
   saltRounds = 10;
 
   JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
@@ -49,6 +58,41 @@ exports.initialize = function() {
   }));
 }
 
+const setupRoles = async () => {
+  const required_roles = {
+    admin: {
+        permissions: [
+            'filemanager',
+            'settings',
+            'subscriptions',
+            'sharing',
+            'advanced_download',
+            'downloads_manager'
+        ]
+    },
+    user: {
+        permissions: [
+            'filemanager',
+            'subscriptions',
+            'sharing'
+        ]
+    }
+  }
+
+  const role_keys = Object.keys(required_roles);
+  for (let i = 0; i < role_keys.length; i++) {
+    const role_key = role_keys[i];
+    const role_in_db = await db_api.getRecord('roles', {key: role_key});
+    if (!role_in_db) {
+      // insert task metadata into table if missing
+      await db_api.insertRecordIntoTable('roles', {
+          key: role_key,
+          permissions: required_roles[role_key]['permissions']
+      });
+    }
+  }
+}
+
 exports.passport = require('passport');
 
 exports.passport.serializeUser(function(user, done) {

backend/db.js

@@ -85,6 +85,7 @@ exports.initialize = (input_db, input_users_db) => {
 }
 
 exports.connectToDB = async (retries = 5, no_fallback = false, custom_connection_string = null) => {
+    using_local_db = config_api.getConfigItem('ytdl_use_local_db'); // verify
     if (using_local_db && !custom_connection_string) return;
     const success = await exports._connectToDB(custom_connection_string);
     if (success) return true;

backend/downloader.js

@@ -3,7 +3,6 @@ const { uuid } = require('uuidv4');
 const path = require('path');
 const mergeFiles = require('merge-files');
 const NodeID3 = require('node-id3')
-const glob = require('glob')
 const Mutex = require('async-mutex').Mutex;
 
 const youtubedl = require('youtube-dl');
@@ -583,20 +582,26 @@ async function checkDownloadPercent(download_uid) {
     if (!resulting_file_size) return;
 
     let sum_size = 0;
-    glob(`{${files_to_check_for_progress.join(',')}, }*`, async (err, files) => {
-        files.forEach(async file => {
-            try {
-                const file_stats = fs.statSync(file);
-                if (file_stats && file_stats.size) {
-                    sum_size += file_stats.size;
-                }
-            } catch (e) {
-
+    for (let i = 0; i < files_to_check_for_progress.length; i++) {
+        const file_to_check_for_progress = files_to_check_for_progress[i];
+        const dir = path.dirname(file_to_check_for_progress);
+        if (!fs.existsSync(dir)) continue;
+        fs.readdir(dir, async (err, files) => {
+            for (let j = 0; j < files.length; j++) {
+                const file = files[j];
+                if (!file.includes(path.basename(file_to_check_for_progress))) continue;
+                try {
+                    const file_stats = fs.statSync(path.join(dir, file));
+                    if (file_stats && file_stats.size) {
+                        sum_size += file_stats.size;
+                    }
+                } catch (e) {}
             }
+            
+            const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2);
+            await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete});
         });
-        const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2);
-        await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete});
-    });
+    }
 }
 
 exports.generateNFOFile = (info, output_path) => {

backend/package-lock.json

@@ -2567,9 +2567,9 @@
       "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="
     },
     "passport": {
-      "version": "0.5.2",
-      "resolved": "https://registry.npmjs.org/passport/-/passport-0.5.2.tgz",
-      "integrity": "sha512-w9n/Ot5I7orGD4y+7V3EFJCQEznE5RxHamUxcqLT2QoJY0f2JdN8GyHonYFvN0Vz+L6lUJfVhrk2aZz2LbuREw==",
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/passport/-/passport-0.4.1.tgz",
+      "integrity": "sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg==",
       "requires": {
         "passport-strategy": "1.x.x",
         "pause": "0.0.1"

backend/package.json

@@ -40,7 +40,6 @@
     "express": "^4.17.3",
     "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^9.0.0",
-    "glob": "^7.1.6",
     "jsonwebtoken": "^8.5.1",
     "lowdb": "^1.0.0",
     "md5": "^2.2.1",
@@ -53,7 +52,7 @@
     "node-id3": "^0.1.14",
     "node-schedule": "^2.1.0",
     "nodemon": "^2.0.7",
-    "passport": "^0.5.2",
+    "passport": "^0.4.1",
     "passport-http": "^0.3.0",
     "passport-jwt": "^4.0.0",
     "passport-ldapauth": "^3.0.1",

backend/subscriptions.js

@@ -266,11 +266,17 @@ async function getVideosForSub(sub, user_uid = null) {
                 }
                 resolve(false);
             } else if (output) {
+                if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
+                    await setFreshUploads(sub, user_uid);
+                    checkVideosForFreshUploads(sub, user_uid);
+                }
+
                 if (output.length === 0 || (output.length === 1 && output[0] === '')) {
                     logger.verbose('No additional videos to download for ' + sub.name);
                     resolve(true);
                     return;
                 }
+
                 const output_jsons = [];
                 for (let i = 0; i < output.length; i++) {
                     let output_json = null;
@@ -294,14 +300,7 @@ async function getVideosForSub(sub, user_uid = null) {
                 }
 
                 resolve(files_to_download);
-
-                if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
-                    await setFreshUploads(sub, user_uid);
-                    checkVideosForFreshUploads(sub, user_uid);
-                }
-
-                resolve(true);
-            }
+           }
         });
     }, err => {
         logger.error(err);
@@ -473,22 +472,24 @@ async function updateSubscriptionProperty(sub, assignment_obj) {
     return true;
 }
 
-async function setFreshUploads(sub, user_uid) {
+async function setFreshUploads(sub) {
+    const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
-    sub.videos.forEach(async video => {
-        if (current_date === video['upload_date'].replace(/-/g, '')) {
+    sub_files.forEach(async file => {
+        if (current_date === file['upload_date'].replace(/-/g, '')) {
             // set upload as fresh
-            const video_uid = video['uid'];
-            await db_api.setVideoProperty(video_uid, {'fresh_upload': true}, user_uid, sub['id']);
+            const file_uid = file['uid'];
+            await db_api.setVideoProperty(file_uid, {'fresh_upload': true});
         }
     });
 }
 
 async function checkVideosForFreshUploads(sub, user_uid) {
+    const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
-    sub.videos.forEach(async video => {
-        if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) {
-            await checkVideoIfBetterExists(video, sub, user_uid)
+    sub_files.forEach(async file => {
+        if (file['fresh_upload'] && current_date > file['upload_date'].replace(/-/g, '')) {
+            await checkVideoIfBetterExists(file, sub, user_uid)
         }
     });
 }
@@ -510,13 +511,13 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
                         logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
                     } else if (output) {
                         logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`);
-                        await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}, user_uid, sub['id']);
+                        await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]});
                     }
                 });
             } 
         }
     });
-    await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false}, user_uid, sub['id']);
+    await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false});
 }
 
 // helper functions

backend/tasks.js

@@ -42,9 +42,9 @@ function scheduleJob(task_key, schedule) {
     if (schedule['type'] === 'timestamp') {
         converted_schedule = new Date(schedule['data']['timestamp']);
     } else if (schedule['type'] === 'recurring') {
-        const dayOfWeek = schedule['data']['dayOfWeek'] ? schedule['data']['dayOfWeek'] : null;
-        const hour = schedule['data']['hour']           ? schedule['data']['hour']      : null;
-        const minute = schedule['data']['minute']       ? schedule['data']['minute']    : null;
+        const dayOfWeek = schedule['data']['dayOfWeek'] != null       ? schedule['data']['dayOfWeek'] : null;
+        const hour = schedule['data']['hour']           != null       ? schedule['data']['hour']      : null;
+        const minute = schedule['data']['minute']       != null       ? schedule['data']['minute']    : null;
         converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute);
     } else {
         logger.error(`Failed to schedule job '${task_key}' as the type '${schedule['type']}' is invalid.`)

docker-build.sh

@@ -4,7 +4,7 @@
 # and also optimizing some code with this commit.
 # xoxo :D
 
-set -xeuo pipefail
+# set -xeuo pipefail
 
 case $(uname -m) in
   x86_64)
@@ -23,7 +23,12 @@ case $(uname -m) in
 esac
 
 echo "Architecture: $ARCH"
-wget "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz" -O ffmpeg.txz
+curl --connect-timeout 5 \
+    --max-time 10 \
+    --retry 5 \
+    --retry-delay 0 \
+    --retry-max-time 40 \
+    "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz" -o ffmpeg.txz
 mkdir /tmp/ffmpeg
 tar xf ffmpeg.txz -C /tmp/ffmpeg
 cp /tmp/ffmpeg/*/ffmpeg /usr/local/bin/ffmpeg

src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.ts

@@ -1,6 +1,6 @@
 import { Component, Inject, OnInit } from '@angular/core';
 import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog';
-import { Schedule } from 'api-types';
+import { Schedule, Task } from 'api-types';
 import { PostsService } from 'app/posts.services';
 
 @Component({
@@ -18,7 +18,7 @@ export class UpdateTaskScheduleDialogComponent implements OnInit {
   date = null;
   today = new Date();
 
-  constructor(@Inject(MAT_DIALOG_DATA) public data: any, private dialogRef: MatDialogRef<UpdateTaskScheduleDialogComponent>, private postsService: PostsService) {
+  constructor(@Inject(MAT_DIALOG_DATA) public data: {task: Task}, private dialogRef: MatDialogRef<UpdateTaskScheduleDialogComponent>, private postsService: PostsService) {
     this.processTask(this.data.task);
     this.postsService.getTask(this.data.task.key).subscribe(res => {
       this.processTask(res['task']);
@@ -28,7 +28,7 @@ export class UpdateTaskScheduleDialogComponent implements OnInit {
   ngOnInit(): void {
   }
 
-  processTask(task) {
+  processTask(task: Task): void {
     if (!task['schedule']) {
       this.enabled = false;
       return;
@@ -39,7 +39,11 @@ export class UpdateTaskScheduleDialogComponent implements OnInit {
     this.recurring = schedule['type'] === Schedule.type.RECURRING;
 
     if (this.recurring) {
-      this.time = `${schedule['data']['hour']}:${schedule['data']['minute']}`;
+      const hour = schedule['data']['hour'];
+      const minute = schedule['data']['minute'];
+
+      // add padding 0s if necessary to hours and minutes
+      this.time = (hour < 10 ? '0' : '') + hour + ':' + (minute < 10 ? '0' : '') + minute;
 
       if (schedule['data']['dayOfWeek']) {
         this.days_of_week = schedule['data']['dayOfWeek'];
@@ -75,7 +79,6 @@ export class UpdateTaskScheduleDialogComponent implements OnInit {
       }
     } else {
       this.date.setHours(hours, minutes);
-      console.log(this.date);
       schedule['data'] = {timestamp: this.date.getTime()};
     }
     this.dialogRef.close(schedule);