Sort orphaned videos by added date, add queue functionality

.dockerignore

@@ -0,0 +1,8 @@
+# Ignore everything
+**
+
+# Add required files and folders
+!ayta
+!README.md
+!LICENCE
+!requirements.txt

.gitea/workflows/demo.yaml

@@ -1,19 +0,0 @@
-name: Gitea Actions Demo
-run-name: ${{ gitea.actor }} is testing out Gitea Actions 🚀
-on: [push]
-
-jobs:
-  Explore-Gitea-Actions:
-    runs-on: ubuntu-latest
-    steps:
-      - run: echo "🎉 The job was automatically triggered by a ${{ gitea.event_name }} event."
-      - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!"
-      - run: echo "🔎 The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}."
-      - name: Check out repository code
-        uses: actions/checkout@v3
-      - run: echo "💡 The ${{ gitea.repository }} repository has been cloned to the runner."
-      - run: echo "🖥️ The workflow is now ready to test your code on the runner."
-      - name: List files in the repository
-        run: |
-          ls ${{ gitea.workspace }}
-      - run: echo "🍏 This job's status is ${{ job.status }}."

.gitea/workflows/release.yaml

@@ -0,0 +1,34 @@
+name: Generate release
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v2
+
+    - name: Log in to Gitea Packages
+      uses: docker/login-action@v1
+      with:
+        registry: git.ventilaar.nl
+        username: ${{ secrets.PACKAGEREG_USERNAME }}
+        password: ${{ secrets.PACKAGEREG_PASSWORD }}
+
+    - name: Build and push
+      uses: docker/build-push-action@v5
+      with:
+        push: true
+        tags: git.ventilaar.nl/ventilaar/ayta:latest
+        
+    - name: Update worker server
+      uses: appleboy/ssh-action@v1.0.3
+      with:
+        host: 192.168.66.109
+        username: root
+        key: ${{ secrets.SERVER_KEY }}
+        port: 22
+        script: /root/update_worker.sh

.gitignore

@@ -142,3 +142,6 @@ cython_debug/
 .idea/
 /learning/api testing/youtube_api_key.py
 /learning/api testing/youtube_api_key.py
+
+# secrets
+client_secrets.json

Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3-alpine
+WORKDIR /app
+COPY requirements.txt /app
+RUN pip install --no-cache-dir -r requirements.txt
+COPY . /app
+EXPOSE 8000
+CMD ["gunicorn", "--bind", "0.0.0.0:8000", "ayta:create_app()"]

ayta/__init__.py

@@ -1,35 +1,52 @@
-import os
-from flask import Flask
-from flask_caching import Cache
-from .filters import pretty_time
-
-
-
 def create_app(test_config=None):
+    import os, secrets
+    from flask import Flask
+    from ayta.extensions import limiter, caching, celery_init_app
+    from werkzeug.middleware.proxy_fix import ProxyFix
+
+    from . import filters
+
     config = {'MONGO_CONNECTION': os.environ.get('AYTA_MONGOCONNECTION', 'mongodb://root:example@192.168.66.140:27017'),
               'S3_CONNECTION': os.environ.get('AYTA_S3CONNECTION', '192.168.66.111:9001'),
               'S3_ACCESSKEY': os.environ.get('AYTA_S3ACCESSKEY', 'lnUiGClFVXVuZbsr'),
               'S3_SECRETKEY': os.environ.get('AYTA_S3SECRETKEY', 'Qz9NG7rpcOWdK2WL'),
               'CACHE_TYPE': os.environ.get('AYTA_CACHETYPE', 'SimpleCache'),
-              'CACHE_DEFAULT_TIMEOUT': os.environ.get('AYTA_CACHETIMEOUT', 300)
+              'CACHE_DEFAULT_TIMEOUT': int(os.environ.get('AYTA_CACHETIMEOUT', 6)),
+              'SECRET_KEY': os.environ.get('AYTA_SECRETKEY', secrets.token_hex(32)),
+              'DEBUG': bool(os.environ.get('AYTA_DEBUG', False)),
+              'DOMAIN': os.environ.get('AYTA_DOMAIN', 'testing.mashallah.nl'),
+              'CELERY': dict(broker_url=str(os.environ.get('AYTA_CELERYBROKER', 'amqp://guest:guest@192.168.66.140:5672/')), 
+                             task_ignore_result=True,)
              }
-
+    
     app = Flask(__name__)
     app.config.from_mapping(config)
     
-    app.jinja_env.filters['pretty_time'] = pretty_time
+    limiter.init_app(app)
+    caching.init_app(app)
+    celery_init_app(app)
+    
+    app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1)
+    
+    app.jinja_env.filters['pretty_duration'] = filters.pretty_duration
+    app.jinja_env.filters['pretty_time'] = filters.pretty_time
+    app.jinja_env.filters['current_time'] = filters.current_time
+    app.jinja_env.filters['epoch_time'] = filters.epoch_time
 
     from .blueprints import watch
     from .blueprints import index
     from .blueprints import admin
     from .blueprints import search
     from .blueprints import channel
+    from .blueprints import auth
+    from .blueprints import api
 
     app.register_blueprint(watch.bp)
     app.register_blueprint(index.bp)
     app.register_blueprint(admin.bp)
     app.register_blueprint(search.bp)
     app.register_blueprint(channel.bp)
-    app.add_url_rule("/", endpoint="base")
+    app.register_blueprint(auth.bp)
+    app.register_blueprint(api.bp)
 
     return app

ayta/blueprints/admin.py

@@ -1,23 +1,31 @@
-import functools
+from flask import Blueprint, render_template, request, redirect, url_for, flash
-from datetime import datetime
-
-from flask import Blueprint
-from flask import g
-from flask import redirect
-from flask import render_template
-from flask import request
-from flask import url_for
 from ..nosql import get_nosql
 from ..s3 import get_s3
 from ..dlp import checkChannelId, getChannelInfo
+from ..decorators import login_required
+from ..tasks import subscribe_websub_callback, unsubscribe_websub_callback
+from datetime import datetime
+from secrets import token_urlsafe
 
 bp = Blueprint('admin', __name__, url_prefix='/admin')
 
 @bp.route('')
+@login_required
 def base():
     return render_template('admin/index.html')
 
-@bp.route('/channels', methods=['GET', 'POST'])
+@bp.route('/system', methods=['GET', 'POST'])
+@login_required
+def system():
+    if request.method == 'POST':
+        task = request.form.get('task', None)
+        if task == 'update-value':
+            pass
+        
+    return render_template('admin/system.html')
+
+@bp.route('/channel', methods=['GET', 'POST'])
+@login_required
 def channels():
     channels = {}
     generic = {}
@@ -35,7 +43,8 @@ def channels():
             channelId, originalName = getChannelInfo(channelId, ('channel_id', 'uploader'))
         
         if not get_nosql().insert_new_channel(channelId, originalName, addedDate):
-            return 'Error inserting new channel, you probably made a mistake somewhere'
+            flash('Error inserting new channel, you probably made a mistake somewhere')
+            return redirect(url_for('admin.channels'))
 
         return redirect(url_for('admin.channel', channelId=channelId))
 
@@ -49,52 +58,144 @@ def channels():
     return render_template('admin/channels.html', channels=channels, generic=generic)
 
 @bp.route('/channel/<channelId>', methods=['GET', 'POST'])
+@login_required
 def channel(channelId):
+    channelInfo = get_nosql().get_channel_info(channelId)
+
+    if not channelInfo:
+        flash('That channel ID does not exist in the system')
+        return redirect(url_for('admin.channels'))
+
     if request.method == 'POST':
         task = request.form.get('task', None)
         key = request.form.get('key', None)
         value = request.form.get('value', None)
         
+        if task == 'subscribe-websub':
+            task = subscribe_websub_callback.delay(channelId)
+            flash(f"Started task {task.id}")
+            return redirect(url_for('admin.channel', channelId=channelId))
+        
         if task == 'update-value':
-            if key == 'active' and value is None: # html checkbox is not present if not checked
+            if key == 'active':
-                value = False
+                value = True if value else False
-            elif key == 'active' and value is not None:  # html checkbox is False if checked
+
-                value = True
-                
             if key == 'added_date':
                 value = datetime.strptime(value, '%Y-%m-%d')
               
             get_nosql().update_channel_key(channelId, key, value)
-
+            return redirect(url_for('admin.channel', channelId=channelId))
-    channelInfo = get_nosql().get_channel_info(channelId)
-
-    if not channelInfo:
-        return 'That channel ID does not exist in the system'
-        
-    #if channelInfo.get('added_date'):
-    #  channelInfo['added_date'] = channelInfo['added_date'].strftime("%Y-%m-%d")
 
     return render_template('admin/channel.html', channelInfo=channelInfo)
 
-@bp.route('/runs', methods=['GET', 'POST'])
+@bp.route('/run', methods=['GET', 'POST'])
+@login_required
 def runs():
     if request.method == 'POST':
         task = request.form.get('task', None)
         if task == 'clean_runs':
             get_nosql().clean_runs()
-        else:
+            return redirect(url_for('admin.runs'))
-            pass
+
-            
 
     runs = reversed(list(get_nosql().get_runs()))
     return render_template('admin/runs.html', runs=runs)
 
 @bp.route('/run/<runId>', methods=['GET', 'POST'])
+@login_required
 def run(runId):
     run = get_nosql().get_run(runId)
     return render_template('admin/run.html', run=run)
 
+@bp.route('/websub', methods=['GET', 'POST'])
+@login_required
+def websub():
+    if request.method == 'POST':
+        task = request.form.get('task', None)
+        value = request.form.get('value', None)
+        
+        if task == 'unsubscribe':
+            channelId = get_nosql().websub_getCallback(value).get('channel')
+            
+            task = unsubscribe_websub_callback.delay(value, channelId)
+            
+            flash(f"Started task {task.id}")
+            return redirect(url_for('admin.websub'))
+            
+        elif task == 'clean-retired':
+            get_nosql().websub_cleanRetired()
+            return redirect(url_for('admin.websub'))
+    
+    callbackIds = get_nosql().websub_getCallbacks()
+    callbacks = {}
+    
+    for callbackId in callbackIds:
+        callbacks[callbackId] = get_nosql().websub_getCallback(callbackId)
+    
+    return render_template('admin/websub.html', callbacks=callbacks)
+
+@bp.route('/reports', methods=['GET', 'POST'])
+@login_required
+def reports():
+    if request.method == 'POST':
+        task = request.form.get('task', None)
+        value = request.form.get('value', None)
+        
+        if task == 'close':
+            get_nosql().close_report(value)
+            flash(f'Report closed {value}')
+            return redirect(url_for('admin.reports'))
+
+    reports = get_nosql().list_reports()
+   
+    return render_template('admin/reports.html', reports=reports)
+
+@bp.route('/posters', methods=['GET', 'POST'])
+@login_required
+def posters():
+    if request.method == 'POST':
+        task = request.form.get('task', None)
+        value = request.form.get('value', None)
+        
+        if task == 'add-endpoint':
+            description = request.form.get('description', None)
+            if not description or len(description) <= 7:
+                flash('Description must be at least 8 characters long')
+            
+            if value and len(value) >= 12:
+                get_nosql().poster_newEndpoint(value, description)
+                flash(f'Created endpoint ID: {value}')
+            else:
+                value = token_urlsafe(16)
+                get_nosql().poster_newEndpoint(value, description)
+                flash(f'Created endpoint ID: {value}')
+        elif task == 'retire':
+            get_nosql().poster_retireEndpoint(value)
+            flash(f'Endpoint retired: {value}')
+            
+        elif task == 'clean-retired':
+            get_nosql().poster_cleanRetired()
+            flash(f'Cleaned retired endpoints')
+            
+        elif task == 'manual-queue':
+            get_nosql().poster_insertQueue('manual', value)
+            flash(f'Added to queue: {value}')
+            
+        elif task == 'delete-queue':
+            get_nosql().poster_deleteQueue(value)
+            flash(f'Deleted from queue: {value}')
+        
+        return redirect(url_for('admin.posters'))
+        
+    endpoints = get_nosql().poster_getEndpoints()
+    queue = get_nosql().poster_getQueue()
+   
+    return render_template('admin/posters.html', endpoints=endpoints, queue=queue)
+
+
+
 @bp.route('/files', methods=['GET', 'POST'])
+@login_required
 def files():
     run = get_s3().list_objects()
     return str(run)

ayta/blueprints/api.py

@@ -0,0 +1,66 @@
+from flask import Blueprint, render_template, request, redirect, url_for, flash, abort
+from ..nosql import get_nosql
+from ..extensions import caching, caching_unless
+
+import re
+
+bp = Blueprint('api', __name__, url_prefix='/api')
+
+@bp.route('/websub/<cap>', methods=['GET', 'POST'])
+def websub(cap):
+    if request.method == 'GET':
+        topic = request.args.get('hub.topic')
+        challenge = request.args.get('hub.challenge')
+        mode = request.args.get('hub.mode')
+        lease_seconds = request.args.get('hub.lease_seconds')
+        
+        if mode not in ['subscribe', 'unsubscribe']:
+            return abort(400)
+        
+        if not get_nosql().websub_existsCallback(cap):
+            return abort(404)
+
+        if mode == 'unsubscribe':
+            get_nosql().websub_retireCallback(cap)
+            return challenge
+        
+        if not all([topic, challenge, mode, lease_seconds]):
+            return abort(400)
+        
+        if not get_nosql().websub_activateCallback(cap, lease_seconds):
+            return abort(500)
+            
+        return challenge
+    
+    if get_nosql().websub_existsCallback(cap):
+        if not get_nosql().websub_savePost(cap, str(request.data)):
+            return abort(500)
+        return '', 202
+    
+    return abort(404)
+    
+@bp.route('/poster/<cap>', methods=['POST'])
+def poster(cap):
+    # if endpoint does not exist
+    if not get_nosql().poster_isActive(cap):
+        return abort(404)
+    
+    videoId = request.form.get('v')
+    
+    # if request is not valid
+    if not videoId:
+        return abort(400)
+        
+    # if requested string is not correct
+    if not re.match(r"^[a-zA-Z0-9_-]{11}$", videoId):
+        return abort(422)
+        
+    # if given string is already in the archive
+    if get_nosql().check_exists(videoId):
+        return abort(409)
+    
+    # try to insert
+    if get_nosql().poster_insertQueue(cap, videoId):
+        return '', 202
+    else:
+        return abort(409)

ayta/blueprints/auth.py

@@ -0,0 +1,53 @@
+from flask import Blueprint, redirect, url_for, render_template, request, session, flash, current_app
+from ..extensions import limiter, caching, caching_unless
+
+from argon2 import PasswordHasher
+from argon2.exceptions import VerifyMismatchError
+
+corr = '$argon2id$v=19$m=65536,t=3,p=4$XzX9K2MKRrGWEf/0iHf2AA$m6Q/aHoj1/uct+8a00QTS5xVWnANeMPKVUg4P822sbM'
+
+bp = Blueprint('auth', __name__, url_prefix='/auth')
+
+@bp.route('')
+@caching.cached()
+def base():
+    return redirect(url_for('auth.login'))
+    
+@bp.route('/logout')
+def logout():
+    session.pop('username', None)
+    flash('You have been logged out')
+    return redirect(url_for('auth.login'))
+    
+@bp.route('/login', methods=['GET', 'POST'])
+@limiter.limit('10 per day', override_defaults=False)
+@caching.cached(unless=caching_unless)
+def login():
+    if request.method == 'POST':
+        password = request.form.get('password', None)
+        
+        if current_app.config.get('DEBUG'):
+            session['username'] = 'admin'
+            flash('You have been logged in')
+            return redirect(request.args.get('next', url_for('admin.base')))
+        
+        if not password:
+            flash('Password was empty')
+            return redirect(url_for('auth.login'))
+        
+        try:
+            ph = PasswordHasher()
+            if ph.verify(corr, password):
+                session['username'] = 'admin'
+                flash('You have been logged in')
+                
+                return redirect(request.args.get('next', url_for('admin.base')))
+                    
+        except VerifyMismatchError:
+            flash('Wrong password')
+            return redirect(url_for('auth.login'))
+        except:
+            flash('Something went wrong')
+            return redirect(url_for('auth.login'))
+            
+    return render_template('login.html')

ayta/blueprints/channel.py

@@ -1,18 +1,12 @@
-import functools
+from flask import Blueprint, render_template, flash, url_for, redirect
-from datetime import datetime
-
-from flask import Blueprint
-from flask import g
-from flask import redirect
-from flask import render_template
-from flask import request
-from flask import url_for
 from ..nosql import get_nosql
 from ..s3 import get_s3
+from ..extensions import caching, caching_unless
 
 bp = Blueprint('channel', __name__, url_prefix='/channel')
 
 @bp.route('')
+@caching.cached(unless=caching_unless)
 def base():
     channels = {}
     channelIds = get_nosql().list_all_channels()
@@ -24,16 +18,33 @@ def base():
     return render_template('channel/index.html', channels=channels)
     
 @bp.route('/<channelId>')
+@caching.cached(unless=caching_unless)
 def channel(channelId):
     channelInfo = get_nosql().get_channel_info(channelId)
 
     if not channelInfo:
-        return 'That channel ID does not exist in the system'
+        flash('That channel ID does not exist in the system')
+        return redirect(url_for('channel.base'))
         
     videoIds = get_nosql().get_channel_videoIds(channelId)
     
-    videos = {}
+    videos = []
     for videoId in videoIds:
-        videos[videoId] = get_nosql().get_video_info(videoId, limited=True)
+        videos.append(get_nosql().get_video_info(videoId, limited=True))
+        
+    videos = sorted(videos, key=lambda x: x.get('upload_date'), reverse=True)
 
-    return render_template('channel/channel.html', channel=channelInfo, videos=videos)
+    return render_template('channel/channel.html', channel=channelInfo, videos=videos)
+    
+@bp.route('/orphaned')
+@caching.cached(unless=caching_unless)
+def orphaned():
+    videoIds = get_nosql().get_orphaned_videos()
+    
+    videos = []
+    for videoId in videoIds:
+        videos.append(get_nosql().get_video_info(videoId, limited=True))
+        
+    videos = sorted(videos, key=lambda x: x.get('epoch', 0), reverse=True)
+
+    return render_template('channel/orphaned.html', videos=videos)

ayta/blueprints/index.py

@@ -1,12 +1,19 @@
-from flask import Blueprint
+from flask import Blueprint, render_template, send_from_directory
-from flask import render_template
+from ..extensions import caching, caching_unless
-from flask import request
-from flask import url_for
-from werkzeug.security import check_password_hash
-from werkzeug.security import generate_password_hash
 
 bp = Blueprint('index', __name__, url_prefix='/')
 
-@bp.route('/', methods=['GET'])
+@bp.route('', methods=['GET'])
+@caching.cached(unless=caching_unless)
 def base():
     return render_template('index.html')
+
+@bp.route('help', methods=['GET'])
+@caching.cached(unless=caching_unless)
+def help():
+    return render_template('help.html')
+    
+@bp.route('robots.txt', methods=['GET'])
+@caching.cached(unless=caching_unless)
+def robots():
+    return render_template('robots.txt')

ayta/blueprints/search.py

@@ -1,16 +1,26 @@
-import functools
+from flask import Blueprint, render_template, request, flash, redirect, url_for
-from datetime import datetime
-
-from flask import Blueprint
-from flask import g
-from flask import redirect
-from flask import render_template
-from flask import request
-from flask import url_for
 from ..nosql import get_nosql
+from ..extensions import limiter, caching, caching_unless
 
 bp = Blueprint('search', __name__, url_prefix='/search')
 
-@bp.route('/')
+@bp.route('', methods=['GET', 'POST'])
+@limiter.limit('50 per day', override_defaults=False)
+@caching.cached(unless=caching_unless)
 def base():
-    return render_template('search/index.html', stats=get_nosql().gen_stats())
+    if request.method == 'POST':
+        task = request.form.get('task')
+        
+        if task == 'search':
+            query = request.form.get('query')
+            
+            if not 3 <= len(query) <= 64:
+                flash('Query too short or too long, must be between 3 and 64')
+                return redirect(url_for('search.base'))
+                
+            results = get_nosql().search_videos(query)
+            
+            return render_template('search/index.html', results=results, query=query)
+                
+        
+    return render_template('search/index.html', stats=get_nosql().gen_stats())

ayta/blueprints/watch.py

@@ -1,27 +1,39 @@
-import functools
+from flask import Blueprint, render_template, request, flash, redirect, url_for
-
-from flask import Blueprint
-from flask import g
-from flask import redirect
-from flask import render_template
-from flask import request
-from flask import url_for
-from werkzeug.security import check_password_hash
-from werkzeug.security import generate_password_hash
 from ..nosql import get_nosql
+from ..extensions import caching, caching_v_parameter, caching_unless
 
 bp = Blueprint('watch', __name__, url_prefix='/watch')
 
-@bp.route('', methods=['GET'])
+@bp.route('', methods=['GET', 'POST'])
+@caching.cached(make_cache_key=caching_v_parameter, unless=caching_unless)
 def base():
-    render = {}
-    
     vGet = request.args.get('v')
     
-    if not get_nosql().check_exists(vGet):
+    if not vGet:
-        return render_template('watch/404.html')
+        flash('Thats not how it works pal')
-
+        return redirect(url_for('index.base'))
     
+    if not get_nosql().check_exists(vGet):
+        flash('The requested video is not in the archive')
+        return redirect(url_for('index.base'))
+        
+    render = {}
+
+    if request.method == 'POST':
+        reason = request.form.get('reason')
+        
+        if reason not in ['auto-video', 'metadata', 'illegal']:
+            flash('Invalid report reason')
+            return redirect(url_for('watch.base', v=vGet))
+        else:
+            reportId = get_nosql().insert_report(vGet, reason)
+            if reportId:
+                flash(f'Report has been received: {reportId}')
+                return redirect(url_for('watch.base', v=vGet))
+            else:
+                flash('Something went wrong with reporting')
+                return redirect(url_for('watch.base', v=vGet))
+
     render['info'] = get_nosql().get_video_info(vGet)
     render['params'] = request.args.get('v')
     return render_template('watch/index.html', render=render)

ayta/decorators.py

@@ -0,0 +1,11 @@
+from functools import wraps
+from flask import session, request, redirect, url_for, flash
+
+def login_required(f):
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        if 'username' in session:
+            return f(*args, **kwargs)
+        flash('Login required')
+        return redirect(url_for('auth.login', next=request.url))
+    return decorated_function

ayta/dlp.py

@@ -2,10 +2,10 @@ import yt_dlp
 
 
 def checkChannelId(channelId):
-    if len(channelId) < 24:  # channelId lengths are 24 characters
+    if len(channelId) <= 23:  # channelId lengths are 24 characters
         return False
         
-    if len(channelId) > 25:  # But some are 25, idk why
+    if len(channelId) >= 26:  # But some are 25, idk why
         return False
         
     if channelId[0:2] not in ['UC', 'UU']:

ayta/extensions.py

@@ -0,0 +1,48 @@
+from flask_limiter import Limiter
+from flask_limiter.util import get_remote_address
+
+from flask_caching import Cache
+
+from flask import Flask, request, session
+
+from celery import Celery, Task
+
+def celery_init_app(app: Flask) -> Celery:
+    class FlaskTask(Task):
+        def __call__(self, *args: object, **kwargs: object) -> object:
+            with app.app_context():
+                return self.run(*args, **kwargs)
+
+    celery_app = Celery(app.name, task_cls=FlaskTask)
+    celery_app.config_from_object(app.config["CELERY"])
+    celery_app.set_default()
+    app.extensions["celery"] = celery_app
+    return celery_app
+
+def caching_unless(*args, **kwargs):
+    # if it is not a get request
+    if request.method != 'GET':
+        return True
+    
+    # if username is defined in session cookie
+    if session.get('username'):
+        return True
+    
+    # in the case that a user is not logged in but a message needs to be flashed, do not cache page
+    if session.get('_flashes'):
+            return True
+        
+    # do cache page
+    return False
+    
+def caching_v_parameter(*args, **kwargs):
+    return request.args.get('v')
+
+limiter = Limiter(
+        get_remote_address,
+        default_limits=['86400 per day', '3600 per hour'],
+        storage_uri="memory://",
+    )
+    
+caching = Cache()
+

ayta/filters.py

@@ -1,3 +1,28 @@
-def pretty_time(seconds):
+from datetime import datetime
-    minutes, seconds = divmod(seconds, 60)
+
-    return '{:3}:{:02} minutes'.format(int(minutes), int(seconds))
+def pretty_duration(seconds):
+    if seconds is None:
+        return None
+        
+    minutes, seconds = divmod(seconds, 60)  # split total seconds to minute and remaining seconds
+    return f'{minutes}:{seconds:>02} minutes'  # return mm:ss format including padding in seconds
+    
+def pretty_time(time):
+    try:
+        return time.strftime('%d %b %Y')  # try to return pretty time if given object is datetime
+    except:
+        try:
+            return datetime.strptime(time, '%Y%m%d').strftime('%d %b %Y')  # try to parse str and give back pretty time
+        except:
+            return time  # return given time
+
+def epoch_time(time):
+    try:
+        return datetime.fromtimestamp(time).strftime('%d %b %Y')
+    except:
+        return None
+
+def current_time(null=None, object=False):
+    if object:
+        return datetime.utcnow().replace(microsecond=0)
+    return datetime.utcnow().isoformat(sep=" ", timespec="seconds")  # return time in iso format without milliseconds

ayta/static/master.js

@@ -1,44 +0,0 @@
-function channelSort() {
-	const sortOption = document.querySelector(".sort").value;
-	const [sortBy, direction] = sortOption.split("-");
-	const isInt = sortBy !== "search";
-	const container = document.querySelector(".channels.flex-grid");
-	[...container.children]
-		.sort((a,b)=>{
-			const dir = direction ? 1 : -1;
-			let valA = a.dataset[sortBy];
-			let valB = b.dataset[sortBy];
-			if (isInt) {
-				valA = parseInt(valA);
-				valB = parseInt(valB);
-			}
-			return (valA>valB?1:-1)*dir;
-		})
-		.forEach(node=>container.appendChild(node));
-}
-
-function channelSearch() {
-	let searchTerm = document.querySelector(".search").value.toLowerCase();
-	const allowedClasses = [];
-	const filteredClasses = [];
-
-	document.querySelectorAll('.searchable').forEach((e) => {
-		let filtered = false;
-		for (const c of allowedClasses) {
-			if (!e.querySelector(`.${c}`)) filtered = true;
-		}
-		for (const c of filteredClasses) {
-			if (e.querySelector(`.${c}`)) filtered = true;
-		}
-
-    	if (!filtered && (searchTerm === "" || e.dataset.search.toLowerCase().includes(searchTerm))) {
-    		e.classList.remove("hide");
-    	} else {
-    		e.classList.add("hide");
-    	}
-	});
-}
-
-window.addEventListener("load", () => {
-	channelSearch();
-});

ayta/tasks.py

@@ -0,0 +1,47 @@
+from celery import shared_task
+from flask import current_app
+
+@shared_task()
+def subscribe_websub_callback(channelId):
+    import requests
+    from .nosql import get_nosql
+    
+    callbackId = get_nosql().websub_newCallback(channelId)
+    
+    url = 'https://pubsubhubbub.appspot.com/subscribe'
+    data = {
+        'hub.callback': f'https://{current_app.config["DOMAIN"]}/api/websub//{callbackId}',
+        'hub.topic': f'https://www.youtube.com/xml/feeds/videos.xml?channel_id={channelId}',
+        'hub.verify': 'async',
+        'hub.mode': 'subscribe',
+        'hub.verify_token': '',
+        'hub.secret': '',
+        'hub.lease_numbers': '86400',
+    }
+    
+    get_nosql().websub_requestingCallback(callbackId)
+    response = requests.post(url, data=data)
+    if response.status_code == 202:
+        return True
+        
+    return False
+    
+@shared_task()
+def unsubscribe_websub_callback(callbackId, channelId):
+    import requests
+    from .nosql import get_nosql
+    
+    url = 'https://pubsubhubbub.appspot.com/subscribe'
+    data = {'hub.callback': f'https://{current_app.config["DOMAIN"]}/api/websub/{callbackId}',
+            'hub.topic': f'https://www.youtube.com/xml/feeds/videos.xml?channel_id={channelId}',
+            'hub.verify': 'async',
+            'hub.mode': 'unsubscribe'
+           }
+    
+    get_nosql().websub_retiringCallback(callbackId)
+    response = requests.post(url, data=data)
+    
+    if response.status_code == 202:
+        return True
+        
+    return False

ayta/templates/admin/channel.html

@@ -1,13 +1,18 @@
 {% extends 'material_base.html' %}
-{% block title %}Channel administration page | AYTA{% endblock %}
+{% block title %}Channel administration page{% endblock %}
 {% block description %}Channel administration page of the AYTA system{% endblock %}
 
 {% block content %}
 <div class="row">
-  <div class="col s12">
+  <div class="col s12 l11">
     <h4>{{ channelInfo.original_name }} administration page</h4>
 	<p>The update actions below directly apply to the database!</p>
   </div>
+  <div class="col s12 l1 m-5">
+    <form method="POST">
+	  <input title="Requests callback URL from youtube API" type="submit" value="subscribe-websub" name="task">
+    </form>
+  </div>
 </div>
 <div class="row">
   <div class="col s12 l4">

ayta/templates/admin/channels.html

@@ -1,5 +1,5 @@
 {% extends 'material_base.html' %}
-{% block title %}Channels administration page | AYTA{% endblock %}
+{% block title %}Channels administration page{% endblock %}
 {% block description %}Channels administration page of the AYTA system{% endblock %}
 
 {% block content %}
@@ -43,30 +43,6 @@
       </div>
     </div>
   </div>
-  <div class="col s12 l4 m-4">
-	<div class="card green">
-      <div class="card-content white-text">
-        <span class="card-title">Placeholder</span>
-        <p>I am a very simple card. I am good at containing small bits of information. I am convenient because I require little markup to use effectively.</p>
-      </div>
-      <div class="card-action">
-        <a href="#">This is a link</a>
-        <a href="#">This is a link</a>
-      </div>
-    </div>
-  </div>
-  <div class="col s12 l4 m-4">
-	<div class="card green">
-      <div class="card-content white-text">
-        <span class="card-title">Placeholder</span>
-        <p>I am a very simple card. I am good at containing small bits of information. I am convenient because I require little markup to use effectively.</p>
-      </div>
-      <div class="card-action">
-        <a href="#">This is a link</a>
-        <a href="#">This is a link</a>
-      </div>
-    </div>
-  </div>
 </div>
 <div class="divider"></div>
 <div class="row">

ayta/templates/admin/index.html

@@ -1,46 +1,79 @@
-{% extends 'base.html' %}
+{% extends 'material_base.html' %}
-{% block title %}Admin page | AYTA{% endblock %}
+{% block title %}Administration page{% endblock %}
 {% block description %}Administration page of the AYTA system{% endblock %}
 
 {% block content %}
-<div class="container channels">
+<div class="row">
-  <div class="head">
+  <div class="col s12">
-    <div class="title">
+    <h4>Administration page</h4>
-      <h1 style="display: inline-block;">Administration page</h1>
-    </div>
-	<div>
-        <select name="sort" class="sort{sort}" oninput="channelSort()">
-          <option value="name-a">Name (asc)</option>
-          <option value="name-d">Name (desc)</option>
-        </select>
-        <input type="text" class="search" oninput="channelSearch()" placeholder="Search...">
-    </div>
   </div>
-  <div class="channels flex-grid">
+</div>
-	 <div class="card" data-search="system">
+<div class="divider"></div>
-       <a href="{{ url_for('admin.channels') }}" class="inner">
+<div class="row">
-         <div class="content">
+  <div class="col s12">
-           <div class="title">System</div>
+	<h5>Global channel options</h5>
-           <div class="meta">Internal system settings</div>
+  </div>
+</div>
+<div class="row">
+  <div class="col s6 l4 m-4">
+	<a href="{{ url_for('admin.system') }}">
+	  <div class="card black-text">
+        <div class="card-content">
+          <span class="card-title">System</span>
+		  <p class="grey-text">Internal system settings</p>
         </div>
-       </a>
+      </div>
-     </div>
+	</a>
-	 <div class="card" data-search="channels">
+  </div>
-       <a href="{{ url_for('admin.channels') }}" class="inner">
+  <div class="col s6 l4 m-4">
-         <div class="content">
+	<a href="{{ url_for('admin.channels') }}">
-           <div class="title">Channels</div>
+	  <div class="card black-text">
-           <div class="meta">Manage channels in the system</div>
+        <div class="card-content">
+          <span class="card-title">Channels</span>
+		  <p class="grey-text">Manage channels in the system</p>
         </div>
-       </a>
+      </div>
-     </div>
+	</a>
-	 <div class="card" data-search="runs">
+  </div>
-       <a href="{{ url_for('admin.runs') }}" class="inner">
+  <div class="col s6 l4 m-4">
-         <div class="content">
+	<a href="{{ url_for('admin.runs') }}">
-           <div class="title">Archive runs</div>
+	  <div class="card black-text">
-           <div class="meta">Look at the cron run logs</div>
+        <div class="card-content">
+          <span class="card-title">Archive runs</span>
+		  <p class="grey-text">Look at the cron run logs</p>
         </div>
-       </a>
+      </div>
-     </div>
+	</a>
+  </div>
+  <div class="col s6 l4 m-4">
+	<a href="{{ url_for('admin.websub') }}">
+	  <div class="card black-text">
+        <div class="card-content">
+          <span class="card-title">WebSub</span>
+		  <p class="grey-text">Edit WebSub YouTube links</p>
+        </div>
+      </div>
+	</a>
+  </div>
+  <div class="col s6 l4 m-4">
+	<a href="{{ url_for('admin.reports') }}">
+	  <div class="card black-text">
+        <div class="card-content">
+          <span class="card-title">Reports</span>
+		  <p class="grey-text">View user reports</p>
+        </div>
+      </div>
+	</a>
+  </div>
+  <div class="col s6 l4 m-4">
+	<a href="{{ url_for('admin.posters') }}">
+	  <div class="card black-text">
+        <div class="card-content">
+          <span class="card-title">Posters</span>
+		  <p class="grey-text">User extension posters</p>
+        </div>
+      </div>
+	</a>
   </div>
 </div>
 {% endblock %}

ayta/templates/admin/posters.html

@@ -0,0 +1,150 @@
+{% extends 'material_base.html' %}
+{% block title %}Posters administration page{% endblock %}
+{% block description %}Posters administration page of the AYTA system{% endblock %}
+
+{% block content %}
+<div class="row">
+  <div class="col s12 l11">
+    <h4>Posters administration page</h4>
+  </div>
+  <div class="col s12 l1 m-5">
+    <form method="POST">
+      <input title="Prunes all deleted endpoints, but keeps last 3 days" type="submit" value="clean-retired" name="task">
+    </form>
+  </div>
+</div>
+<div class="divider"></div>
+<div class="row">
+  <div class="col s12">
+	<h5>Poster options</h5>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12 l4 m-4">
+	<div class="card">
+      <div class="card-content">
+        <span class="card-title">Create new endpoint</span>
+	    <form method="post">
+		  <div class="row">
+            <div class="col s12 input-field">
+              <input placeholder="Custom endpoint" name="value" type="text" class="validate" minlength="12">
+			  <span class="supporting-text">Leaving this empty will create a random secure string</span>
+	        </div>
+			<div class="col s12 input-field">
+              <input placeholder="Description" name="description" type="text" class="validate" minlength="8" maxlength="64" required>
+			  <span class="supporting-text">Description for the endpoint for better administration</span>
+	        </div>
+          <button class="btn mt-4" type="submit" name="task" value="add-endpoint">Create</button>
+		  </div>
+	    </form>
+      </div>
+    </div>
+  </div>
+  <div class="col s12 l4 m-4">
+	<div class="card">
+      <div class="card-content">
+        <span class="card-title">Queue manually</span>
+	    <form method="post">
+		  <div class="row">
+            <div class="col s12 input-field">
+              <input placeholder="Youtube video ID" name="value" type="text" class="validate" minlength="11" maxlength="11" required>
+			  <span class="supporting-text">Must be a valid Youtube video ID</span>
+	        </div>
+			<div class="col s12 mt-5 input-field">
+              <div class="switch">
+		        <label>Queue<input type="checkbox" value="direct" name="value" disabled><span class="lever"></span>Direct</label>
+				<span class="supporting-text">Queue up or start directly</span>
+		      </div>
+	        </div>
+          <button class="btn mt-4" type="submit" name="task" value="manual-queue">Queue</button>
+		  </div>
+	    </form>
+      </div>
+    </div>
+  </div>
+</div>
+<div class="divider"></div>
+<div class="row">
+  <div class="col s6 l9">
+	<h5>Registered endpoints</h5>
+  </div>
+  <div class="col s6 l3 m-4 input-field">
+	<input id="filter_query" type="text">
+    <label for="filter_query">Filter results</label>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12">
+    <table class="striped highlight responsive-table">
+      <thead>
+        <tr>
+		  <th>Actions</th>
+		  <th>id</th>
+		  <th>description</th>
+		  <th>status</th>
+		  <th>created_time</th>
+		  <th>retired_time</th>
+        </tr>
+      </thead>
+      <tbody>
+	    {% for endpoint in endpoints %}
+        <tr class="filterable">
+		  <td>
+		    <form method="post">
+			  <input type="text" value="{{ endpoint.get('id') }}" name="value" hidden>
+		      <button class="btn-small waves-effect waves-light" type="submit" name="task" value="retire" title="Retire endpoint" {% if endpoint.get('status') != 'active' %}disabled{% endif %}>🗑️</button>
+			</form>
+		  </td>
+		  <td>{{ endpoint.get('id') }}</td>
+		  <td>{{ endpoint.get('description') }}</td>
+		  <td>{{ endpoint.get('status') }}</td>
+		  <td>{{ endpoint.get('created_time') }}</td>
+		  <td>{{ endpoint.get('retired_time') }}</td>
+        </tr>
+		{% endfor %}
+      </tbody>
+    </table>
+  </div>
+</div>
+<div class="divider"></div>
+<div class="row">
+  <div class="col s6 l9">
+	<h5>Queued ID's</h5>
+  </div>
+  <div class="col s6 l3 m-4 input-field">
+	<input id="filter_query" type="text">
+    <label for="filter_query">Filter results</label>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12">
+    <table class="striped highlight responsive-table">
+      <thead>
+        <tr>
+		  <th>Actions</th>
+		  <th>id</th>
+		  <th>endpoint</th>
+		  <th>status</th>
+		  <th>created_time</th>
+        </tr>
+      </thead>
+      <tbody>
+	    {% for id in queue %}
+        <tr class="filterable">
+		  <td>
+		    <form method="post">
+			  <input type="text" value="{{ id.get('id') }}" name="value" hidden>
+		      <button class="btn-small waves-effect waves-light" type="submit" name="task" value="delete-queue" title="Delete from queue" {% if id.get('status') != 'queued' %}disabled{% endif %}>🗑️</button>
+			</form>
+		  </td>
+		  <td>{{ id.get('id') }}</td>
+		  <td>{{ id.get('endpoint') }}</td>
+		  <td>{{ id.get('status') }}</td>
+		  <td>{{ id.get('created_time') }}</td>
+        </tr>
+		{% endfor %}
+      </tbody>
+    </table>
+  </div>
+</div>
+{% endblock %}

ayta/templates/admin/reports.html

@@ -0,0 +1,71 @@
+{% extends 'material_base.html' %}
+{% block title %}Reports administration page{% endblock %}
+{% block description %}Reports administration page of the AYTA system{% endblock %}
+
+{% block content %}
+<div class="row">
+  <div class="col s12 l11">
+    <h4>Reports administration page</h4>
+  </div>
+  <div class="col s12 l1 m-5">
+    <form method="POST">
+	  <input title="Prunes all closed reports, but keeps last 30 days" type="submit" value="clean-closed" name="task">
+    </form>
+  </div>
+</div>
+<div class="divider"></div>
+<div class="row">
+  <div class="col s12">
+	<h5>Report options</h5>
+  </div>
+</div>
+<div class="divider"></div>
+<div class="row">
+  <div class="col s6 l9">
+	<h5>All reports</h5>
+  </div>
+  <div class="col s6 l3 m-4 input-field">
+	<input id="filter_query" type="text">
+    <label for="filter_query">Filter results</label>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12">
+    {% if reports is not defined %}
+	<p>No reports!</p>
+	{% else %}
+    <table class="striped highlight responsive-table">
+      <thead>
+        <tr>
+		  <th>Actions</th>
+		  <th>_id</th>
+		  <th>videoId</th>
+		  <th>status</th>
+		  <th>reason</th>
+		  <th>reporting_time</th>
+		  <th>closing_time</th>
+        </tr>
+      </thead>
+      <tbody>
+	    {% for report in reports %}
+        <tr class="filterable">
+		  <td>
+		    <form method="post">
+			  <input type="text" value="{{ report.get('_id') }}" name="value" hidden>
+		      <button class="btn-small waves-effect waves-light" type="submit" name="task" value="close" title="Close the report" {% if report.get('status') != 'open' %}disabled{% endif %}>✅</button>
+			</form>
+		  </td>
+		  <td>{{ report.get('_id') }}</td>
+		  <td><a href="{{ url_for('watch.base') }}?v={{ report.get('videoId') }}">{{ report.get('videoId') }}</a></td>
+		  <td>{{ report.get('status') }}</td>
+		  <td>{{ report.get('reason') }}</td>
+		  <td>{{ report.get('reporting_time') }}</td>
+		  <td>{{ report.get('closing_time') }}</td>
+        </tr>
+		{% endfor %}
+      </tbody>
+    </table>
+	{% endif %}
+  </div>
+</div>
+{% endblock %}