Fixed frontend build error

removing strict SSL from npm config

.github/workflows/docker-pr.yml

@@ -0,0 +1,27 @@
+name: docker-pr
+
+on:
+  pull_request:
+    branches: [master]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: Set hash
+        id: vars
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+      - name: create-json
+        id: create-json
+        uses: jsdaniell/create-json@1.1.2
+        with:
+          name: "version.json"
+          json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
+          dir: 'backend/'
+      - name: Build docker images
+        run: docker build . -t tzahi12345/youtubedl-material:nightly-pr

.github/workflows/docker.yml

@@ -21,7 +21,7 @@ jobs:
         uses: jsdaniell/create-json@1.1.2
         with:
           name: "version.json"
-          json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
+          json: '{"type": "docker", "tag": "${{secrets.DOCKERHUB_MASTER_TAG}}", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
           dir: 'backend/'
       - name: setup platform emulator
         uses: docker/setup-qemu-action@v1
@@ -39,4 +39,8 @@ jobs:
           file: ./Dockerfile
           platforms: linux/amd64,linux/arm,linux/arm64/v8
           push: true
-          tags: tzahi12345/youtubedl-material:nightly
+          # Defaults:
+          #     DOCKERHUB_USERNAME  : tzahi12345
+          #     DOCKERHUB_REPO      : youtubedl-material
+          #     DOCKERHUB_MASTER_TAG: nightly
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ secrets.DOCKERHUB_REPO }}:${{secrets.DOCKERHUB_MASTER_TAG}}

Dockerfile

@@ -1,14 +1,25 @@
-FROM alpine:latest AS ffmpeg
+FROM ubuntu:20.04 AS ffmpeg
+
+ENV DEBIAN_FRONTEND=noninteractive
 
 COPY docker-build.sh .
 RUN sh ./docker-build.sh
 
-FROM alpine:latest as frontend
+FROM ubuntu:20.04 as frontend
 
-RUN apk add --no-cache \
+ENV DEBIAN_FRONTEND=noninteractive
-  npm
+RUN apt-get update && apt-get -y install \
-
+  curl \
-RUN npm install -g @angular/cli
+  gnupg && \
+  curl -sL https://deb.nodesource.com/setup_12.x  | bash - && \
+  apt-get -y install \
+  nodejs \
+  # YARN: brings along npm, solves dependency conflicts,
+  # spares us this spaghetti approach: https://stackoverflow.com/a/60547197
+  yarn && \
+  apt-get install -f && \
+  npm config set strict-ssl false && \
+  npm install -g @angular/cli
 
 WORKDIR /build
 COPY [ "package.json", "package-lock.json", "/build/" ]
@@ -20,35 +31,41 @@ RUN npm run build
 
 #--------------#
 
-FROM alpine:latest
+FROM ubuntu:20.04
 
 ENV UID=1000 \
   GID=1000 \
-  USER=youtube
+  USER=youtube \
+  NO_UPDATE_NOTIFIER=true
 
-ENV NO_UPDATE_NOTIFIER=true
+ENV DEBIAN_FRONTEND=noninteractive
 
-RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID
+RUN groupadd -g $GID $USER && useradd --system -g $USER --uid $UID $USER
 
-RUN apk add --no-cache \
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
+RUN apt-get update && apt-get -y install \
   npm \
   python2 \
   python3 \
-  su-exec \
+  atomicparsley && \
-  && apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
+  apt-get install -f && \
-    atomicparsley
+  apt-get autoremove --purge && \
+  apt-get autoremove && \
+  apt-get clean && \
+  rm -rf /var/lib/apt
 
 WORKDIR /app
 COPY --from=ffmpeg /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
 COPY --from=ffmpeg /usr/local/bin/ffprobe /usr/local/bin/ffprobe 
 COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ]
 ENV PM2_HOME=/app/pm2
-RUN npm install pm2 -g
+RUN npm config set strict-ssl false && \
-RUN npm install && chown -R $UID:$GID ./
+  npm install pm2 -g && \
+  npm install && chown -R $UID:$GID ./
 
 COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
 COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
 
 EXPOSE 17442
-ENTRYPOINT [ "/app/entrypoint.sh" ]
+# ENTRYPOINT [ "/app/entrypoint.sh" ]
 CMD [ "pm2-runtime", "pm2.config.js" ]

README.md

@@ -6,7 +6,7 @@
 [![GitHub issues badge](https://img.shields.io/github/issues/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/issues)
 [![License badge](https://img.shields.io/github/license/Tzahi12345/YoutubeDL-Material)](https://github.com/Tzahi12345/YoutubeDL-Material/blob/master/LICENSE.md)
 
-YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 11](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
+YoutubeDL-Material is a Material Design frontend for [youtube-dl](https://rg3.github.io/youtube-dl/). It's coded using [Angular 13](https://angular.io/) for the frontend, and [Node.js](https://nodejs.org/) on the backend.
 
 Now with [Docker](#Docker) support!
 

armhf.Dockerfile

@@ -1,49 +0,0 @@
-FROM alpine:3.12 as frontend
-
-RUN apk add --no-cache \
-  npm \
-  curl
-
-RUN npm install -g @angular/cli
-
-WORKDIR /build
-
-RUN curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static .
-
-COPY [ "package.json", "package-lock.json", "/build/" ]
-RUN npm install
-
-COPY [ "angular.json", "tsconfig.json", "/build/" ]
-COPY [ "src/", "/build/src/" ]
-RUN ng build --prod
-
-#--------------#
-
-FROM arm32v7/alpine:3.12
-
-COPY --from=frontend /build/qemu-arm-static /usr/bin
-
-ENV UID=1000 \
-  GID=1000 \
-  USER=youtube
-
-RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID
-
-RUN apk add --no-cache \
-  ffmpeg \
-  npm \
-  python2 \
-  su-exec \
-  && apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
-    atomicparsley
-
-WORKDIR /app
-COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ]
-RUN npm install && chown -R $UID:$GID ./
-
-COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
-COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
-
-EXPOSE 17442
-ENTRYPOINT [ "/app/entrypoint.sh" ]
-CMD [ "node", "app.js" ]

backend/authentication/auth.js

@@ -18,10 +18,19 @@ let JWT_EXPIRATION = null;
 let opts = null;
 let saltRounds = null;
 
-exports.initialize = function() {
+exports.initialize = function () {
   /*************************
    * Authentication module
    ************************/
+
+  if (db_api.database_initialized) {
+    setupRoles();
+  } else {
+      db_api.database_initialized_bs.subscribe(init => {
+          if (init) setupRoles();
+      });
+  }
+
   saltRounds = 10;
 
   JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
@@ -49,6 +58,41 @@ exports.initialize = function() {
   }));
 }
 
+const setupRoles = async () => {
+  const required_roles = {
+    admin: {
+        permissions: [
+            'filemanager',
+            'settings',
+            'subscriptions',
+            'sharing',
+            'advanced_download',
+            'downloads_manager'
+        ]
+    },
+    user: {
+        permissions: [
+            'filemanager',
+            'subscriptions',
+            'sharing'
+        ]
+    }
+  }
+
+  const role_keys = Object.keys(required_roles);
+  for (let i = 0; i < role_keys.length; i++) {
+    const role_key = role_keys[i];
+    const role_in_db = await db_api.getRecord('roles', {key: role_key});
+    if (!role_in_db) {
+      // insert task metadata into table if missing
+      await db_api.insertRecordIntoTable('roles', {
+          key: role_key,
+          permissions: required_roles[role_key]['permissions']
+      });
+    }
+  }
+}
+
 exports.passport = require('passport');
 
 exports.passport.serializeUser(function(user, done) {

backend/db.js

@@ -85,6 +85,7 @@ exports.initialize = (input_db, input_users_db) => {
 }
 
 exports.connectToDB = async (retries = 5, no_fallback = false, custom_connection_string = null) => {
+    using_local_db = config_api.getConfigItem('ytdl_use_local_db'); // verify
     if (using_local_db && !custom_connection_string) return;
     const success = await exports._connectToDB(custom_connection_string);
     if (success) return true;

backend/subscriptions.js

@@ -141,6 +141,7 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
         if (sub.archive && (await fs.pathExists(sub.archive))) {
             const archive_file_path = path.join(sub.archive, 'archive.txt');
             // deletes archive if it exists
+            // TODO: Keep entries in blacklist_video.txt by moving them to a global blacklist
             if (await fs.pathExists(archive_file_path)) {
                 await fs.unlink(archive_file_path);
             }
@@ -266,11 +267,17 @@ async function getVideosForSub(sub, user_uid = null) {
                 }
                 resolve(false);
             } else if (output) {
+                if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
+                    await setFreshUploads(sub, user_uid);
+                    checkVideosForFreshUploads(sub, user_uid);
+                }
+
                 if (output.length === 0 || (output.length === 1 && output[0] === '')) {
                     logger.verbose('No additional videos to download for ' + sub.name);
                     resolve(true);
                     return;
                 }
+
                 const output_jsons = [];
                 for (let i = 0; i < output.length; i++) {
                     let output_json = null;
@@ -294,14 +301,7 @@ async function getVideosForSub(sub, user_uid = null) {
                 }
 
                 resolve(files_to_download);
-
+           }
-                if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
-                    await setFreshUploads(sub, user_uid);
-                    checkVideosForFreshUploads(sub, user_uid);
-                }
-
-                resolve(true);
-            }
         });
     }, err => {
         logger.error(err);
@@ -380,7 +380,11 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
     if (useArchive && !redownload) {
         if (sub.archive) {
             archive_dir = sub.archive;
-            archive_path = path.join(archive_dir, 'archive.txt')
+            if (sub.type && sub.type === 'audio') {
+                archive_path = path.join(archive_dir, 'merged_audio.txt');
+            } else {
+                archive_path = path.join(archive_dir, 'merged_video.txt');
+            }            
         }
         downloadConfig.push('--download-archive', archive_path);
     }
@@ -473,22 +477,24 @@ async function updateSubscriptionProperty(sub, assignment_obj) {
     return true;
 }
 
-async function setFreshUploads(sub, user_uid) {
+async function setFreshUploads(sub) {
+    const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
-    sub.videos.forEach(async video => {
+    sub_files.forEach(async file => {
-        if (current_date === video['upload_date'].replace(/-/g, '')) {
+        if (current_date === file['upload_date'].replace(/-/g, '')) {
             // set upload as fresh
-            const video_uid = video['uid'];
+            const file_uid = file['uid'];
-            await db_api.setVideoProperty(video_uid, {'fresh_upload': true}, user_uid, sub['id']);
+            await db_api.setVideoProperty(file_uid, {'fresh_upload': true});
         }
     });
 }
 
 async function checkVideosForFreshUploads(sub, user_uid) {
+    const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
-    sub.videos.forEach(async video => {
+    sub_files.forEach(async file => {
-        if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) {
+        if (file['fresh_upload'] && current_date > file['upload_date'].replace(/-/g, '')) {
-            await checkVideoIfBetterExists(video, sub, user_uid)
+            await checkVideoIfBetterExists(file, sub, user_uid)
         }
     });
 }
@@ -510,13 +516,13 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
                         logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
                     } else if (output) {
                         logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`);
-                        await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}, user_uid, sub['id']);
+                        await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]});
                     }
                 });
             } 
         }
     });
-    await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false}, user_uid, sub['id']);
+    await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false});
 }
 
 // helper functions

docker-build.sh

@@ -4,8 +4,6 @@
 # and also optimizing some code with this commit.
 # xoxo :D
 
-set -xeuo pipefail
-
 case $(uname -m) in
   x86_64)
     ARCH=amd64;;
@@ -22,10 +20,24 @@ case $(uname -m) in
     exit 1
 esac
 
-echo "Architecture: $ARCH"
+echo "(INFO) Architecture detected: $ARCH"
-wget "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz" -O ffmpeg.txz
+echo "(1/5) READY - Acquire temp dependencies in ffmpeg obtain layer"
+apt-get update && apt-get -y install curl xz-utils
+echo "(2/5) DOWNLOAD - Acquire latest ffmpeg and ffprobe from John van Sickle's master-sourced builds in ffmpeg obtain layer"
+curl -o ffmpeg.txz \
+    --connect-timeout 5 \
+    --max-time 10 \
+    --retry 5 \
+    --retry-delay 0 \
+    --retry-max-time 40 \
+    "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz"
 mkdir /tmp/ffmpeg
 tar xf ffmpeg.txz -C /tmp/ffmpeg
+echo "(3/5) CLEANUP - Remove temp dependencies from ffmpeg obtain layer"
+apt-get -y remove curl xz-utils
+apt-get -y autoremove
+echo "(4/5) PROVISION - Provide ffmpeg and ffprobe from ffmpeg obtain layer"
 cp /tmp/ffmpeg/*/ffmpeg /usr/local/bin/ffmpeg
 cp /tmp/ffmpeg/*/ffprobe /usr/local/bin/ffprobe
+echo "(5/5) CLEANUP - Remove temporary downloads from ffmpeg obtain layer"
 rm -rf /tmp/ffmpeg ffmpeg.txz

src/app/app.component.html

@@ -51,7 +51,7 @@
                     </ng-container>
                     <ng-container *ngIf="postsService.config && allowSubscriptions && postsService.subscriptions && postsService.hasPermission('subscriptions')">
                         <mat-divider *ngIf="postsService.subscriptions.length > 0"></mat-divider>
-                        <a *ngFor="let subscription of postsService.subscriptions" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" [routerLink]="['/subscription', { id: subscription.id }]"><ngx-avatar [style.margin-right]="'10px'" size="32" [name]="subscription.name"></ngx-avatar>{{subscription.name}}</a>
+                        <a *ngFor="let subscription of postsService.subscriptions" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" [routerLink]="['/subscription', { id: subscription.id }]"><ngx-avatars [style.margin-right]="'10px'" size="32" [name]="subscription.name"></ngx-avatars>{{subscription.name}}</a>
                     </ng-container>
                 </mat-nav-list>
             </mat-sidenav>

src/app/components/concurrent-stream/concurrent-stream.component.ts

@@ -42,6 +42,11 @@ export class ConcurrentStreamComponent implements OnInit {
 
   }
 
+  ngOnDestroy(): void {
+    if (this.check_timeout) { clearInterval(this.check_timeout); }
+    if (this.update_timeout) { clearInterval(this.update_timeout); }
+  }
+
   startServer() {
     this.started = true;
     this.server_started = true;

src/app/components/twitch-chat/twitch-chat.component.ts

@@ -1,4 +1,4 @@
-import { AfterViewInit, Component, ElementRef, Input, OnInit, QueryList, ViewChild, ViewChildren } from '@angular/core';
+import { Component, ElementRef, Input, OnDestroy, OnInit, QueryList, ViewChild, ViewChildren } from '@angular/core';
 import { PostsService } from 'app/posts.services';
 
 @Component({
@@ -6,7 +6,7 @@ import { PostsService } from 'app/posts.services';
   templateUrl: './twitch-chat.component.html',
   styleUrls: ['./twitch-chat.component.scss']
 })
-export class TwitchChatComponent implements OnInit, AfterViewInit {
+export class TwitchChatComponent implements OnInit, OnDestroy {
 
   full_chat = null;
   visible_chat = null;
@@ -33,7 +33,8 @@ export class TwitchChatComponent implements OnInit, AfterViewInit {
     this.getFullChat();
   }
 
-  ngAfterViewInit() {
+  ngOnDestroy(): void {
+    if (this.chat_check_interval_obj) { clearInterval(this.chat_check_interval_obj); }
   }
 
   private isUserNearBottom(): boolean {

src/app/subscription/subscription/subscription.component.ts

@@ -52,6 +52,8 @@ export class SubscriptionComponent implements OnInit, OnDestroy {
     this.route.params.subscribe(params => {
       this.id = params['id'];
 
+      if (this.sub_interval) { clearInterval(this.sub_interval); }
+
       this.postsService.service_initialized.subscribe(init => {
         if (init) {
           this.getConfig();