Added github actions to test PR with a docker build

Set strict-ssl to false for npm in Dockerfile

.github/workflows/docker-pr.yml

@@ -0,0 +1,42 @@
+name: docker-pr
+
+on:
+  pull_request:
+    branches: [master]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: Set hash
+        id: vars
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+      - name: create-json
+        id: create-json
+        uses: jsdaniell/create-json@1.1.2
+        with:
+          name: "version.json"
+          json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
+          dir: 'backend/'
+      - name: setup platform emulator
+        uses: docker/setup-qemu-action@v1
+      - name: setup multi-arch docker build
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: build & push images
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm,linux/arm64/v8
+          push: false
+          tags: tzahi12345/youtubedl-material:nightly-pr

Dockerfile

@@ -1,13 +1,18 @@
-FROM alpine:latest AS ffmpeg
+FROM ubuntu:20.04 AS ffmpeg
+
+ENV DEBIAN_FRONTEND=noninteractive
 
 COPY docker-build.sh .
 RUN sh ./docker-build.sh
 
-FROM alpine:latest as frontend
-
-RUN apk add --no-cache \
-  npm
+FROM ubuntu:20.04 as frontend
 
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get -y install curl
+RUN curl -sL https://deb.nodesource.com/setup_12.x  | bash -
+RUN apt-get -y install nodejs \
+  yarn
+RUN npm config set strict-ssl false
 RUN npm install -g @angular/cli
 
 WORKDIR /build
@@ -20,35 +25,42 @@ RUN npm run build
 
 #--------------#
 
-FROM alpine:latest
+FROM ubuntu:20.04
 
 ENV UID=1000 \
   GID=1000 \
-  USER=youtube
+  USER=youtube \
+  NO_UPDATE_NOTIFIER=true
 
-ENV NO_UPDATE_NOTIFIER=true
+ENV DEBIAN_FRONTEND=noninteractive
 
-RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID
+RUN groupadd -g $GID $USER && useradd --system -g $USER --uid $UID $USER
 
-RUN apk add --no-cache \
-  npm \
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
+RUN apt-get update && apt-get -y install \
+  nodejs \
+  yarn \
   python2 \
   python3 \
-  su-exec \
-  && apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
-    atomicparsley
+  atomicparsley && \
+  apt-get autoremove --purge && \
+  apt-get autoremove && \
+  apt-get clean && \
+  rm -rf /var/lib/apt
+
+RUN npm config set strict-ssl false
 
 WORKDIR /app
 COPY --from=ffmpeg /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
 COPY --from=ffmpeg /usr/local/bin/ffprobe /usr/local/bin/ffprobe 
 COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ]
 ENV PM2_HOME=/app/pm2
-RUN npm install pm2 -g
-RUN npm install && chown -R $UID:$GID ./
+RUN npm install pm2 -g && \
+  npm install && chown -R $UID:$GID ./
 
 COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
 COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
 
 EXPOSE 17442
-ENTRYPOINT [ "/app/entrypoint.sh" ]
+# ENTRYPOINT [ "/app/entrypoint.sh" ]
 CMD [ "pm2-runtime", "pm2.config.js" ]

Public API v1.yaml

@@ -16,7 +16,7 @@ paths:
         - downloader
       summary: Download video file
       description: |-
-        Downloads a video file with the given URL. Will include global args if they exist.
+        Downloads a file with the given URL. Will include global args if they exist.
 
 
         HTTP requests will return once the video file download completes. In the future, it will (by default) return once the download starts, and a separate API call will be used for checking the download status.
@@ -41,7 +41,7 @@ paths:
     post:
       tags:
         - downloader
-      summary: Download video file
+      summary: Generates arguments used to download file
       description: Generates args, used for checking what args would run if you ran downloadFile
       operationId: post-generateArgs
       requestBody:

SECURITY.md

@@ -0,0 +1,21 @@
+# Security Policy
+
+## Supported Versions
+
+Currently all work on this project goes into the nightly builds.
+4.2's RELEASE build is now quite old and should be considered legacy.
+We urge users to use the nightly releases, because the project
+constantly sees fixes.
+
+| Version       | Supported          |
+| ------------- | ------------------ |
+| 4.2 Nightlies | :white_check_mark: |
+| 4.2 Release   | :x:                |
+| < 4.2         | :x:                |
+
+## Reporting a Vulnerability
+
+Please file an issue in our GitHub's repo, because this app
+isn't meant to be safe to run as public instance yet, but rather as a LAN facing app.
+
+We welcome PRs and help in general in making YTDL-M more secure, but it's not a priority as of now.

backend/app.js

@@ -803,7 +803,7 @@ app.post('/api/testConnectionString', optionalJwt, async (req, res) => {
 app.post('/api/downloadFile', optionalJwt, async function(req, res) {
     req.setTimeout(0); // remove timeout in case of long videos
     const url = req.body.url;
-    const type = req.body.type;
+    const type = req.body.type ? req.body.type : 'video';
     const user_uid = req.isAuthenticated() ? req.user.uid : null;
     const options = {
         customArgs: req.body.customArgs,

backend/authentication/auth.js

@@ -18,10 +18,19 @@ let JWT_EXPIRATION = null;
 let opts = null;
 let saltRounds = null;
 
-exports.initialize = function() {
+exports.initialize = function () {
   /*************************
    * Authentication module
    ************************/
+
+  if (db_api.database_initialized) {
+    setupRoles();
+  } else {
+      db_api.database_initialized_bs.subscribe(init => {
+          if (init) setupRoles();
+      });
+  }
+
   saltRounds = 10;
 
   JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
@@ -49,6 +58,41 @@ exports.initialize = function() {
   }));
 }
 
+const setupRoles = async () => {
+  const required_roles = {
+    admin: {
+        permissions: [
+            'filemanager',
+            'settings',
+            'subscriptions',
+            'sharing',
+            'advanced_download',
+            'downloads_manager'
+        ]
+    },
+    user: {
+        permissions: [
+            'filemanager',
+            'subscriptions',
+            'sharing'
+        ]
+    }
+  }
+
+  const role_keys = Object.keys(required_roles);
+  for (let i = 0; i < role_keys.length; i++) {
+    const role_key = role_keys[i];
+    const role_in_db = await db_api.getRecord('roles', {key: role_key});
+    if (!role_in_db) {
+      // insert task metadata into table if missing
+      await db_api.insertRecordIntoTable('roles', {
+          key: role_key,
+          permissions: required_roles[role_key]['permissions']
+      });
+    }
+  }
+}
+
 exports.passport = require('passport');
 
 exports.passport.serializeUser(function(user, done) {

backend/db.js

@@ -85,6 +85,7 @@ exports.initialize = (input_db, input_users_db) => {
 }
 
 exports.connectToDB = async (retries = 5, no_fallback = false, custom_connection_string = null) => {
+    using_local_db = config_api.getConfigItem('ytdl_use_local_db'); // verify
     if (using_local_db && !custom_connection_string) return;
     const success = await exports._connectToDB(custom_connection_string);
     if (success) return true;

backend/downloader.js

@@ -3,7 +3,6 @@ const { uuid } = require('uuidv4');
 const path = require('path');
 const mergeFiles = require('merge-files');
 const NodeID3 = require('node-id3')
-const glob = require('glob')
 const Mutex = require('async-mutex').Mutex;
 
 const youtubedl = require('youtube-dl');
@@ -583,20 +582,26 @@ async function checkDownloadPercent(download_uid) {
     if (!resulting_file_size) return;
 
     let sum_size = 0;
-    glob(`{${files_to_check_for_progress.join(',')}, }*`, async (err, files) => {
-        files.forEach(async file => {
-            try {
-                const file_stats = fs.statSync(file);
-                if (file_stats && file_stats.size) {
-                    sum_size += file_stats.size;
-                }
-            } catch (e) {
-
+    for (let i = 0; i < files_to_check_for_progress.length; i++) {
+        const file_to_check_for_progress = files_to_check_for_progress[i];
+        const dir = path.dirname(file_to_check_for_progress);
+        if (!fs.existsSync(dir)) continue;
+        fs.readdir(dir, async (err, files) => {
+            for (let j = 0; j < files.length; j++) {
+                const file = files[j];
+                if (!file.includes(path.basename(file_to_check_for_progress))) continue;
+                try {
+                    const file_stats = fs.statSync(path.join(dir, file));
+                    if (file_stats && file_stats.size) {
+                        sum_size += file_stats.size;
+                    }
+                } catch (e) {}
             }
+            
+            const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2);
+            await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete});
         });
-        const percent_complete = (sum_size/resulting_file_size * 100).toFixed(2);
-        await db_api.updateRecord('download_queue', {uid: download_uid}, {percent_complete: percent_complete});
-    });
+    }
 }
 
 exports.generateNFOFile = (info, output_path) => {

backend/package-lock.json

@@ -2567,9 +2567,9 @@
       "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="
     },
     "passport": {
-      "version": "0.5.2",
-      "resolved": "https://registry.npmjs.org/passport/-/passport-0.5.2.tgz",
-      "integrity": "sha512-w9n/Ot5I7orGD4y+7V3EFJCQEznE5RxHamUxcqLT2QoJY0f2JdN8GyHonYFvN0Vz+L6lUJfVhrk2aZz2LbuREw==",
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/passport/-/passport-0.4.1.tgz",
+      "integrity": "sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg==",
       "requires": {
         "passport-strategy": "1.x.x",
         "pause": "0.0.1"

backend/package.json

@@ -40,7 +40,6 @@
     "express": "^4.17.3",
     "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^9.0.0",
-    "glob": "^7.1.6",
     "jsonwebtoken": "^8.5.1",
     "lowdb": "^1.0.0",
     "md5": "^2.2.1",
@@ -53,7 +52,7 @@
     "node-id3": "^0.1.14",
     "node-schedule": "^2.1.0",
     "nodemon": "^2.0.7",
-    "passport": "^0.5.2",
+    "passport": "^0.4.1",
     "passport-http": "^0.3.0",
     "passport-jwt": "^4.0.0",
     "passport-ldapauth": "^3.0.1",

backend/subscriptions.js

@@ -141,6 +141,7 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
         if (sub.archive && (await fs.pathExists(sub.archive))) {
             const archive_file_path = path.join(sub.archive, 'archive.txt');
             // deletes archive if it exists
+            // TODO: Keep entries in blacklist_video.txt by moving them to a global blacklist
             if (await fs.pathExists(archive_file_path)) {
                 await fs.unlink(archive_file_path);
             }
@@ -266,11 +267,17 @@ async function getVideosForSub(sub, user_uid = null) {
                 }
                 resolve(false);
             } else if (output) {
+                if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
+                    await setFreshUploads(sub, user_uid);
+                    checkVideosForFreshUploads(sub, user_uid);
+                }
+
                 if (output.length === 0 || (output.length === 1 && output[0] === '')) {
                     logger.verbose('No additional videos to download for ' + sub.name);
                     resolve(true);
                     return;
                 }
+
                 const output_jsons = [];
                 for (let i = 0; i < output.length; i++) {
                     let output_json = null;
@@ -294,14 +301,7 @@ async function getVideosForSub(sub, user_uid = null) {
                 }
 
                 resolve(files_to_download);
-
-                if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
-                    await setFreshUploads(sub, user_uid);
-                    checkVideosForFreshUploads(sub, user_uid);
-                }
-
-                resolve(true);
-            }
+           }
         });
     }, err => {
         logger.error(err);
@@ -380,7 +380,11 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
     if (useArchive && !redownload) {
         if (sub.archive) {
             archive_dir = sub.archive;
-            archive_path = path.join(archive_dir, 'archive.txt')
+            if (sub.type && sub.type === 'audio') {
+                archive_path = path.join(archive_dir, 'merged_audio.txt');
+            } else {
+                archive_path = path.join(archive_dir, 'merged_video.txt');
+            }            
         }
         downloadConfig.push('--download-archive', archive_path);
     }
@@ -473,22 +477,24 @@ async function updateSubscriptionProperty(sub, assignment_obj) {
     return true;
 }
 
-async function setFreshUploads(sub, user_uid) {
+async function setFreshUploads(sub) {
+    const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
-    sub.videos.forEach(async video => {
-        if (current_date === video['upload_date'].replace(/-/g, '')) {
+    sub_files.forEach(async file => {
+        if (current_date === file['upload_date'].replace(/-/g, '')) {
             // set upload as fresh
-            const video_uid = video['uid'];
-            await db_api.setVideoProperty(video_uid, {'fresh_upload': true}, user_uid, sub['id']);
+            const file_uid = file['uid'];
+            await db_api.setVideoProperty(file_uid, {'fresh_upload': true});
         }
     });
 }
 
 async function checkVideosForFreshUploads(sub, user_uid) {
+    const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
-    sub.videos.forEach(async video => {
-        if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) {
-            await checkVideoIfBetterExists(video, sub, user_uid)
+    sub_files.forEach(async file => {
+        if (file['fresh_upload'] && current_date > file['upload_date'].replace(/-/g, '')) {
+            await checkVideoIfBetterExists(file, sub, user_uid)
         }
     });
 }
@@ -510,13 +516,13 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
                         logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
                     } else if (output) {
                         logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`);
-                        await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}, user_uid, sub['id']);
+                        await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]});
                     }
                 });
             } 
         }
     });
-    await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false}, user_uid, sub['id']);
+    await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false});
 }
 
 // helper functions

backend/tasks.js

@@ -42,9 +42,9 @@ function scheduleJob(task_key, schedule) {
     if (schedule['type'] === 'timestamp') {
         converted_schedule = new Date(schedule['data']['timestamp']);
     } else if (schedule['type'] === 'recurring') {
-        const dayOfWeek = schedule['data']['dayOfWeek'] ? schedule['data']['dayOfWeek'] : null;
-        const hour = schedule['data']['hour']           ? schedule['data']['hour']      : null;
-        const minute = schedule['data']['minute']       ? schedule['data']['minute']    : null;
+        const dayOfWeek = schedule['data']['dayOfWeek'] != null       ? schedule['data']['dayOfWeek'] : null;
+        const hour = schedule['data']['hour']           != null       ? schedule['data']['hour']      : null;
+        const minute = schedule['data']['minute']       != null       ? schedule['data']['minute']    : null;
         converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute);
     } else {
         logger.error(`Failed to schedule job '${task_key}' as the type '${schedule['type']}' is invalid.`)

docker-build.sh

@@ -4,8 +4,6 @@
 # and also optimizing some code with this commit.
 # xoxo :D
 
-set -xeuo pipefail
-
 case $(uname -m) in
   x86_64)
     ARCH=amd64;;
@@ -22,10 +20,24 @@ case $(uname -m) in
     exit 1
 esac
 
-echo "Architecture: $ARCH"
-wget "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz" -O ffmpeg.txz
+echo "(INFO) Architecture detected: $ARCH"
+echo "(1/5) READY - Acquire temp dependencies in ffmpeg obtain layer"
+apt-get update && apt-get -y install curl xz-utils
+echo "(2/5) DOWNLOAD - Acquire latest ffmpeg and ffprobe from John van Sickle's master-sourced builds in ffmpeg obtain layer"
+curl -o ffmpeg.txz \
+    --connect-timeout 5 \
+    --max-time 10 \
+    --retry 5 \
+    --retry-delay 0 \
+    --retry-max-time 40 \
+    "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz"
 mkdir /tmp/ffmpeg
 tar xf ffmpeg.txz -C /tmp/ffmpeg
+echo "(3/5) CLEANUP - Remove temp dependencies from ffmpeg obtain layer"
+apt-get -y remove curl xz-utils
+apt-get -y autoremove
+echo "(4/5) PROVISION - Provide ffmpeg and ffprobe from ffmpeg obtain layer"
 cp /tmp/ffmpeg/*/ffmpeg /usr/local/bin/ffmpeg
 cp /tmp/ffmpeg/*/ffprobe /usr/local/bin/ffprobe
+echo "(5/5) CLEANUP - Remove temporary downloads from ffmpeg obtain layer"
 rm -rf /tmp/ffmpeg ffmpeg.txz

package-lock.json

@@ -3367,9 +3367,9 @@
       "dev": true
     },
     "async": {
-      "version": "2.6.4",
-      "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz",
-      "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==",
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz",
+      "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==",
       "dev": true,
       "requires": {
         "lodash": "^4.17.14"

src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.ts

@@ -1,6 +1,6 @@
 import { Component, Inject, OnInit } from '@angular/core';
 import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog';
-import { Schedule } from 'api-types';
+import { Schedule, Task } from 'api-types';
 import { PostsService } from 'app/posts.services';
 
 @Component({
@@ -18,7 +18,7 @@ export class UpdateTaskScheduleDialogComponent implements OnInit {
   date = null;
   today = new Date();
 
-  constructor(@Inject(MAT_DIALOG_DATA) public data: any, private dialogRef: MatDialogRef<UpdateTaskScheduleDialogComponent>, private postsService: PostsService) {
+  constructor(@Inject(MAT_DIALOG_DATA) public data: {task: Task}, private dialogRef: MatDialogRef<UpdateTaskScheduleDialogComponent>, private postsService: PostsService) {
     this.processTask(this.data.task);
     this.postsService.getTask(this.data.task.key).subscribe(res => {
       this.processTask(res['task']);
@@ -28,7 +28,7 @@ export class UpdateTaskScheduleDialogComponent implements OnInit {
   ngOnInit(): void {
   }
 
-  processTask(task) {
+  processTask(task: Task): void {
     if (!task['schedule']) {
       this.enabled = false;
       return;
@@ -39,7 +39,11 @@ export class UpdateTaskScheduleDialogComponent implements OnInit {
     this.recurring = schedule['type'] === Schedule.type.RECURRING;
 
     if (this.recurring) {
-      this.time = `${schedule['data']['hour']}:${schedule['data']['minute']}`;
+      const hour = schedule['data']['hour'];
+      const minute = schedule['data']['minute'];
+
+      // add padding 0s if necessary to hours and minutes
+      this.time = (hour < 10 ? '0' : '') + hour + ':' + (minute < 10 ? '0' : '') + minute;
 
       if (schedule['data']['dayOfWeek']) {
         this.days_of_week = schedule['data']['dayOfWeek'];
@@ -75,7 +79,6 @@ export class UpdateTaskScheduleDialogComponent implements OnInit {
       }
     } else {
       this.date.setHours(hours, minutes);
-      console.log(this.date);
       schedule['data'] = {timestamp: this.date.getTime()};
     }
     this.dialogRef.close(schedule);