Added github actions to test PR with a docker build

Set strict-ssl to false for npm in Dockerfile

.github/workflows/docker-pr.yml

@@ -0,0 +1,42 @@
+name: docker-pr
+
+on:
+  pull_request:
+    branches: [master]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: Set hash
+        id: vars
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+      - name: create-json
+        id: create-json
+        uses: jsdaniell/create-json@1.1.2
+        with:
+          name: "version.json"
+          json: '{"type": "docker", "tag": "nightly", "commit": "${{ steps.vars.outputs.sha_short }}", "date": "${{ steps.date.outputs.date }}"}'
+          dir: 'backend/'
+      - name: setup platform emulator
+        uses: docker/setup-qemu-action@v1
+      - name: setup multi-arch docker build
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: build & push images
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm,linux/arm64/v8
+          push: false
+          tags: tzahi12345/youtubedl-material:nightly-pr

Dockerfile

@@ -1,13 +1,18 @@
-FROM alpine:latest AS ffmpeg
+FROM ubuntu:20.04 AS ffmpeg
+
+ENV DEBIAN_FRONTEND=noninteractive
 
 COPY docker-build.sh .
 RUN sh ./docker-build.sh
 
-FROM alpine:latest as frontend
-
-RUN apk add --no-cache \
-  npm
+FROM ubuntu:20.04 as frontend
 
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get -y install curl
+RUN curl -sL https://deb.nodesource.com/setup_12.x  | bash -
+RUN apt-get -y install nodejs \
+  yarn
+RUN npm config set strict-ssl false
 RUN npm install -g @angular/cli
 
 WORKDIR /build
@@ -20,35 +25,42 @@ RUN npm run build
 
 #--------------#
 
-FROM alpine:latest
+FROM ubuntu:20.04
 
 ENV UID=1000 \
   GID=1000 \
-  USER=youtube
+  USER=youtube \
+  NO_UPDATE_NOTIFIER=true
 
-ENV NO_UPDATE_NOTIFIER=true
+ENV DEBIAN_FRONTEND=noninteractive
 
-RUN addgroup -S $USER -g $GID && adduser -D -S $USER -G $USER -u $UID
+RUN groupadd -g $GID $USER && useradd --system -g $USER --uid $UID $USER
 
-RUN apk add --no-cache \
-  npm \
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
+RUN apt-get update && apt-get -y install \
+  nodejs \
+  yarn \
   python2 \
   python3 \
-  su-exec \
-  && apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
-    atomicparsley
+  atomicparsley && \
+  apt-get autoremove --purge && \
+  apt-get autoremove && \
+  apt-get clean && \
+  rm -rf /var/lib/apt
+
+RUN npm config set strict-ssl false
 
 WORKDIR /app
 COPY --from=ffmpeg /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
 COPY --from=ffmpeg /usr/local/bin/ffprobe /usr/local/bin/ffprobe 
 COPY --chown=$UID:$GID [ "backend/package.json", "backend/package-lock.json", "/app/" ]
 ENV PM2_HOME=/app/pm2
-RUN npm install pm2 -g
-RUN npm install && chown -R $UID:$GID ./
+RUN npm install pm2 -g && \
+  npm install && chown -R $UID:$GID ./
 
 COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
 COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
 
 EXPOSE 17442
-ENTRYPOINT [ "/app/entrypoint.sh" ]
+# ENTRYPOINT [ "/app/entrypoint.sh" ]
 CMD [ "pm2-runtime", "pm2.config.js" ]

backend/authentication/auth.js

@@ -18,10 +18,19 @@ let JWT_EXPIRATION = null;
 let opts = null;
 let saltRounds = null;
 
-exports.initialize = function() {
+exports.initialize = function () {
   /*************************
    * Authentication module
    ************************/
+
+  if (db_api.database_initialized) {
+    setupRoles();
+  } else {
+      db_api.database_initialized_bs.subscribe(init => {
+          if (init) setupRoles();
+      });
+  }
+
   saltRounds = 10;
 
   JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration');
@@ -49,6 +58,41 @@ exports.initialize = function() {
   }));
 }
 
+const setupRoles = async () => {
+  const required_roles = {
+    admin: {
+        permissions: [
+            'filemanager',
+            'settings',
+            'subscriptions',
+            'sharing',
+            'advanced_download',
+            'downloads_manager'
+        ]
+    },
+    user: {
+        permissions: [
+            'filemanager',
+            'subscriptions',
+            'sharing'
+        ]
+    }
+  }
+
+  const role_keys = Object.keys(required_roles);
+  for (let i = 0; i < role_keys.length; i++) {
+    const role_key = role_keys[i];
+    const role_in_db = await db_api.getRecord('roles', {key: role_key});
+    if (!role_in_db) {
+      // insert task metadata into table if missing
+      await db_api.insertRecordIntoTable('roles', {
+          key: role_key,
+          permissions: required_roles[role_key]['permissions']
+      });
+    }
+  }
+}
+
 exports.passport = require('passport');
 
 exports.passport.serializeUser(function(user, done) {

backend/db.js

@@ -85,6 +85,7 @@ exports.initialize = (input_db, input_users_db) => {
 }
 
 exports.connectToDB = async (retries = 5, no_fallback = false, custom_connection_string = null) => {
+    using_local_db = config_api.getConfigItem('ytdl_use_local_db'); // verify
     if (using_local_db && !custom_connection_string) return;
     const success = await exports._connectToDB(custom_connection_string);
     if (success) return true;

backend/subscriptions.js

@@ -141,6 +141,7 @@ async function unsubscribe(sub, deleteMode, user_uid = null) {
         if (sub.archive && (await fs.pathExists(sub.archive))) {
             const archive_file_path = path.join(sub.archive, 'archive.txt');
             // deletes archive if it exists
+            // TODO: Keep entries in blacklist_video.txt by moving them to a global blacklist
             if (await fs.pathExists(archive_file_path)) {
                 await fs.unlink(archive_file_path);
             }
@@ -266,11 +267,17 @@ async function getVideosForSub(sub, user_uid = null) {
                 }
                 resolve(false);
             } else if (output) {
+                if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
+                    await setFreshUploads(sub, user_uid);
+                    checkVideosForFreshUploads(sub, user_uid);
+                }
+
                 if (output.length === 0 || (output.length === 1 && output[0] === '')) {
                     logger.verbose('No additional videos to download for ' + sub.name);
                     resolve(true);
                     return;
                 }
+
                 const output_jsons = [];
                 for (let i = 0; i < output.length; i++) {
                     let output_json = null;
@@ -294,14 +301,7 @@ async function getVideosForSub(sub, user_uid = null) {
                 }
 
                 resolve(files_to_download);
-
-                if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
-                    await setFreshUploads(sub, user_uid);
-                    checkVideosForFreshUploads(sub, user_uid);
-                }
-
-                resolve(true);
-            }
+           }
         });
     }, err => {
         logger.error(err);
@@ -380,7 +380,11 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
     if (useArchive && !redownload) {
         if (sub.archive) {
             archive_dir = sub.archive;
-            archive_path = path.join(archive_dir, 'archive.txt')
+            if (sub.type && sub.type === 'audio') {
+                archive_path = path.join(archive_dir, 'merged_audio.txt');
+            } else {
+                archive_path = path.join(archive_dir, 'merged_video.txt');
+            }            
         }
         downloadConfig.push('--download-archive', archive_path);
     }
@@ -473,22 +477,24 @@ async function updateSubscriptionProperty(sub, assignment_obj) {
     return true;
 }
 
-async function setFreshUploads(sub, user_uid) {
+async function setFreshUploads(sub) {
+    const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
-    sub.videos.forEach(async video => {
-        if (current_date === video['upload_date'].replace(/-/g, '')) {
+    sub_files.forEach(async file => {
+        if (current_date === file['upload_date'].replace(/-/g, '')) {
             // set upload as fresh
-            const video_uid = video['uid'];
-            await db_api.setVideoProperty(video_uid, {'fresh_upload': true}, user_uid, sub['id']);
+            const file_uid = file['uid'];
+            await db_api.setVideoProperty(file_uid, {'fresh_upload': true});
         }
     });
 }
 
 async function checkVideosForFreshUploads(sub, user_uid) {
+    const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
     const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
-    sub.videos.forEach(async video => {
-        if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) {
-            await checkVideoIfBetterExists(video, sub, user_uid)
+    sub_files.forEach(async file => {
+        if (file['fresh_upload'] && current_date > file['upload_date'].replace(/-/g, '')) {
+            await checkVideoIfBetterExists(file, sub, user_uid)
         }
     });
 }
@@ -510,13 +516,13 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
                         logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
                     } else if (output) {
                         logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`);
-                        await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}, user_uid, sub['id']);
+                        await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]});
                     }
                 });
             } 
         }
     });
-    await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false}, user_uid, sub['id']);
+    await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false});
 }
 
 // helper functions

docker-build.sh

@@ -4,8 +4,6 @@
 # and also optimizing some code with this commit.
 # xoxo :D
 
-set -xeuo pipefail
-
 case $(uname -m) in
   x86_64)
     ARCH=amd64;;
@@ -22,10 +20,24 @@ case $(uname -m) in
     exit 1
 esac
 
-echo "Architecture: $ARCH"
-wget "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz" -O ffmpeg.txz
+echo "(INFO) Architecture detected: $ARCH"
+echo "(1/5) READY - Acquire temp dependencies in ffmpeg obtain layer"
+apt-get update && apt-get -y install curl xz-utils
+echo "(2/5) DOWNLOAD - Acquire latest ffmpeg and ffprobe from John van Sickle's master-sourced builds in ffmpeg obtain layer"
+curl -o ffmpeg.txz \
+    --connect-timeout 5 \
+    --max-time 10 \
+    --retry 5 \
+    --retry-delay 0 \
+    --retry-max-time 40 \
+    "https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-${ARCH}-static.tar.xz"
 mkdir /tmp/ffmpeg
 tar xf ffmpeg.txz -C /tmp/ffmpeg
+echo "(3/5) CLEANUP - Remove temp dependencies from ffmpeg obtain layer"
+apt-get -y remove curl xz-utils
+apt-get -y autoremove
+echo "(4/5) PROVISION - Provide ffmpeg and ffprobe from ffmpeg obtain layer"
 cp /tmp/ffmpeg/*/ffmpeg /usr/local/bin/ffmpeg
 cp /tmp/ffmpeg/*/ffprobe /usr/local/bin/ffprobe
+echo "(5/5) CLEANUP - Remove temporary downloads from ffmpeg obtain layer"
 rm -rf /tmp/ffmpeg ffmpeg.txz