Add base WebSub support (not finished). Add orphaned videos view. Implement video reporting and managing. Some small changes

ayta/__init__.py

@@ -25,6 +25,7 @@ def create_app(test_config=None):
     app.jinja_env.filters['pretty_duration'] = filters.pretty_duration
     app.jinja_env.filters['pretty_time'] = filters.pretty_time
     app.jinja_env.filters['current_time'] = filters.current_time
+    app.jinja_env.filters['epoch_time'] = filters.epoch_time
 
     from .blueprints import watch
     from .blueprints import index
@@ -32,6 +33,7 @@ def create_app(test_config=None):
     from .blueprints import search
     from .blueprints import channel
     from .blueprints import auth
+    from .blueprints import websub
 
     app.register_blueprint(watch.bp)
     app.register_blueprint(index.bp)
@@ -39,7 +41,6 @@ def create_app(test_config=None):
     app.register_blueprint(search.bp)
     app.register_blueprint(channel.bp)
     app.register_blueprint(auth.bp)
-    
-    app.add_url_rule("/", endpoint="base")
+    app.register_blueprint(websub.bp)
 
     return app

ayta/blueprints/admin.py

@@ -1,9 +1,10 @@
-from flask import Blueprint, render_template, request, redirect, url_for
+from flask import Blueprint, render_template, request, redirect, url_for, flash
 from ..nosql import get_nosql
 from ..s3 import get_s3
 from ..dlp import checkChannelId, getChannelInfo
 from ..decorators import login_required
 from datetime import datetime
+import requests
 
 bp = Blueprint('admin', __name__, url_prefix='/admin')
 
@@ -12,6 +13,14 @@ bp = Blueprint('admin', __name__, url_prefix='/admin')
 def base():
     return render_template('admin/index.html')
 
+@bp.route('/system', methods=['GET', 'POST'])
+@login_required
+def system():
+    if request.method == 'POST':
+        pass
+        
+    return render_template('admin/system.html')
+
 @bp.route('/channel', methods=['GET', 'POST'])
 @login_required
 def channels():
@@ -91,6 +100,51 @@ def run(runId):
     run = get_nosql().get_run(runId)
     return render_template('admin/run.html', run=run)
 
+@bp.route('/websub', methods=['GET', 'POST'])
+@login_required
+def websub():
+    if request.method == 'POST':
+        task = request.form.get('task', None)
+        value = request.form.get('value', None)
+        
+        if task == 'unsubscribe':
+            channelId = get_nosql().websub_getCallback(value).get('channel')
+            data = {'hub.callback': f'https://testing.ventilaar.net/websub/c/{value}',
+                    'hub.topic': f'https://www.youtube.com/xml/feeds/videos.xml?channel_id={channelId}',
+                    'hub.verify': 'async',
+                    'hub.mode': 'unsubscribe'
+                   }
+            requests.post('https://pubsubhubbub.appspot.com/subscribe', data=data)
+            
+        elif task == 'clean-retired':
+            get_nosql().websub_cleanRetired()
+    
+    callbackIds = get_nosql().websub_getCallbacks()
+    callbacks = {}
+    
+    for callbackId in callbackIds:
+        callbacks[callbackId] = get_nosql().websub_getCallback(callbackId)
+    
+    return render_template('admin/websub.html', callbacks=callbacks)
+
+@bp.route('/reports', methods=['GET', 'POST'])
+@login_required
+def reports():
+    if request.method == 'POST':
+        task = request.form.get('task', None)
+        value = request.form.get('value', None)
+        
+        if task == 'close':
+            get_nosql().close_report(value)
+            flash(f'Report closed {value}')
+
+    reports = get_nosql().list_reports()
+   
+    for x in reports:
+        print(x, flush=True)
+   
+    return render_template('admin/reports.html', reports=reports)
+
 @bp.route('/files', methods=['GET', 'POST'])
 @login_required
 def files():

ayta/blueprints/auth.py

@@ -1,5 +1,5 @@
 from flask import Blueprint, redirect, url_for, render_template, request, session, flash, current_app
-from ..extensions import limiter, caching, caching_only_get
+from ..extensions import limiter, caching, caching_unless
 
 from argon2 import PasswordHasher
 from argon2.exceptions import VerifyMismatchError
@@ -21,7 +21,7 @@ def logout():
     
 @bp.route('/login', methods=['GET', 'POST'])
 @limiter.limit('10 per day', override_defaults=False)
-@caching.cached(unless=caching_only_get)
+@caching.cached(unless=caching_unless)
 def login():
     if request.method == 'POST':
         password = request.form.get('password', None)

ayta/blueprints/channel.py

@@ -1,12 +1,12 @@
 from flask import Blueprint, render_template
 from ..nosql import get_nosql
 from ..s3 import get_s3
-from ..extensions import caching
+from ..extensions import caching, caching_unless
 
 bp = Blueprint('channel', __name__, url_prefix='/channel')
 
 @bp.route('')
-@caching.cached()
+@caching.cached(unless=caching_unless)
 def base():
     channels = {}
     channelIds = get_nosql().list_all_channels()
@@ -18,7 +18,7 @@ def base():
     return render_template('channel/index.html', channels=channels)
     
 @bp.route('/<channelId>')
-@caching.cached()
+@caching.cached(unless=caching_unless)
 def channel(channelId):
     channelInfo = get_nosql().get_channel_info(channelId)
 
@@ -31,4 +31,15 @@ def channel(channelId):
     for videoId in videoIds:
         videos[videoId] = get_nosql().get_video_info(videoId, limited=True)
 
-    return render_template('channel/channel.html', channel=channelInfo, videos=videos)
+    return render_template('channel/channel.html', channel=channelInfo, videos=videos)
+    
+@bp.route('/orphaned')
+@caching.cached(unless=caching_unless)
+def orphaned():
+    videoIds = get_nosql().get_orphaned_videos()
+    
+    videos = {}
+    for videoId in videoIds:
+        videos[videoId] = get_nosql().get_video_info(videoId, limited=True)
+
+    return render_template('channel/orphaned.html', videos=videos)

ayta/blueprints/index.py

@@ -1,14 +1,19 @@
-from flask import Blueprint, render_template
-from ..extensions import caching
+from flask import Blueprint, render_template, send_from_directory, request
+from ..extensions import caching, caching_unless
 
 bp = Blueprint('index', __name__, url_prefix='/')
 
 @bp.route('', methods=['GET'])
-@caching.cached()
+@caching.cached(unless=caching_unless)
 def base():
     return render_template('index.html')
 
 @bp.route('help', methods=['GET'])
-@caching.cached()
+@caching.cached(unless=caching_unless)
 def help():
-    return render_template('help.html')
+    return render_template('help.html')
+    
+@bp.route('robots.txt', methods=['GET'])
+@caching.cached(unless=caching_unless)
+def robots():
+    return send_from_directory('static', 'robots.txt')

ayta/blueprints/search.py

@@ -1,10 +1,10 @@
 from flask import Blueprint, render_template
 from ..nosql import get_nosql
-from ..extensions import caching
+from ..extensions import caching, caching_unless
 
 bp = Blueprint('search', __name__, url_prefix='/search')
 
 @bp.route('')
-@caching.cached()
+@caching.cached(unless=caching_unless)
 def base():
     return render_template('search/index.html', stats=get_nosql().gen_stats())

ayta/blueprints/watch.py

@@ -1,11 +1,11 @@
-from flask import Blueprint, render_template, request
+from flask import Blueprint, render_template, request, flash
 from ..nosql import get_nosql
-from ..extensions import caching, caching_v_parameter
+from ..extensions import caching, caching_v_parameter, caching_unless
 
 bp = Blueprint('watch', __name__, url_prefix='/watch')
 
-@bp.route('', methods=['GET'])
-@caching.cached(make_cache_key=caching_v_parameter)
+@bp.route('', methods=['GET', 'POST'])
+@caching.cached(make_cache_key=caching_v_parameter, unless=caching_unless)
 def base():
     render = {}
     
@@ -14,6 +14,18 @@ def base():
     if not get_nosql().check_exists(vGet):
         return render_template('watch/404.html'), 404
 
+    if request.method == 'POST':
+        reason = request.form.get('reason')
+        
+        if reason not in ['auto-video', 'metadata', 'illegal']:
+            flash('Invalid report reason')
+        else:
+            reportId = get_nosql().insert_report(vGet, reason)
+            if reportId:
+                flash(f'Report has been received: {reportId}')
+            else:
+                flash('Something went wrong with reporting')
+
     render['info'] = get_nosql().get_video_info(vGet)
     render['params'] = request.args.get('v')
     return render_template('watch/index.html', render=render)

ayta/blueprints/websub.py

@@ -0,0 +1,40 @@
+from flask import Blueprint, render_template, request, redirect, url_for, flash, abort
+from ..nosql import get_nosql
+from ..extensions import caching, caching_unless
+
+bp = Blueprint('websub', __name__, url_prefix='/websub')
+
+@bp.route('/c/<cap>', methods=['GET', 'POST'])
+# Caching GET requests should be save since this endpoint is used as a capability URL
+@caching.cached(unless=caching_unless)
+def callback(cap):
+    if request.method == 'GET':
+        topic = request.args.get('hub.topic')
+        challenge = request.args.get('hub.challenge')
+        mode = request.args.get('hub.mode')
+        lease_seconds = request.args.get('hub.lease_seconds')
+        
+        if mode not in ['subscribe', 'unsubscribe']:
+            return abort(400)
+        
+        if not get_nosql().websub_existsCallback(cap):
+            return abort(404)
+
+        if mode == 'unsubscribe':
+            get_nosql().websub_retireCallback(cap)
+            return challenge
+        
+        if not all([topic, challenge, mode, lease_seconds]):
+            return abort(400)
+        
+        if not get_nosql().websub_activateCallback(cap, lease_seconds):
+            return abort(500)
+            
+        return challenge
+    
+    if get_nosql().websub_existsCallback(cap):
+        if not get_nosql().websub_savePost(cap, str(request.data)):
+            return abort(500)
+        return '', 204
+    
+    return abort(404)

ayta/extensions.py

@@ -3,14 +3,24 @@ from flask_limiter.util import get_remote_address
 
 from flask_caching import Cache
 
-from flask import request
+from flask import request, session
 
 
-def caching_only_get(*args, **kwargs):
-    if request.method == 'GET':
-        return False
+def caching_unless(*args, **kwargs):
+    # if it is not a get request
+    if request.method != 'GET':
+        return True
+    
+    # if username is defined in session cookie
+    if session.get('username'):
+        return True
+    
+    # in the case that a user is not logged in but a message needs to be flashed, do not cache page
+    if session.get('_flashes'):
+            return True
         
-    return True
+    # do cache page
+    return False
     
 def caching_v_parameter(*args, **kwargs):
     return request.args.get('v')

ayta/filters.py

@@ -16,6 +16,13 @@ def pretty_time(time):
         except:
             return time  # return given time
 
-    
-def current_time(null):
+def epoch_time(time):
+    try:
+        return datetime.fromtimestamp(time).strftime('%d %b %Y')
+    except:
+        return None
+
+def current_time(null=None, object=False):
+    if object:
+        return datetime.utcnow().replace(microsecond=0)
     return datetime.utcnow().isoformat(sep=" ", timespec="seconds")  # return time in iso format without milliseconds

ayta/nosql.py

@@ -1,10 +1,13 @@
 import json
 import pymongo
+import secrets
 from bson.objectid import ObjectId
 
 from flask import current_app
 from flask import g
 
+from .filters import current_time
+
 ##########################################
 #              SETUP FLASK               #
 ##########################################
@@ -35,24 +38,62 @@ def init_app(app):
 ##########################################
 class Mango:
     def __init__(self, connect):
+        from datetime import datetime, timedelta
+        
+        self.datetime = datetime
+        self.timedelta = timedelta
+        
         try:
             self.client = pymongo.MongoClient(connect)
-            self.channels = self.client['ayta']['channels']
-            self.info_json = self.client['ayta']['info_json']
-            self.download_queue = self.client['ayta']['download_queue']
-            self.run_log = self.client['ayta']['run_log']
-            self.channel_log = self.client['ayta']['channel_log']
-            
-            #self.channels.create_index([('id', pymongo.ASCENDING)], unique=True)
         except ConnectionError:
             print('MongoDB connection error')
-            
-    def check_exists(self, vId):
-        """ Returns BOOL; Given positional argument which is a valid or invalid YouTube video ID STR"""
-        if self.info_json.count_documents({'id': vId}) >= 1:
-            return True
-        return False
         
+        self.db = self.client['ayta']
+        self.channels = self.db['channels']
+        self.info_json = self.db['info_json']
+        self.download_queue = self.db['download_queue']
+        self.run_log = self.db['run_log']
+        self.channel_log = self.db['channel_log']
+        self.websub_callbacks = self.db['websub_callbacks']
+        self.websub_data = self.db['websub_data']
+        self.reports = self.db['reports']
+            
+        self.ensure_indexes()
+   
+    def ensure_indexes(self):
+        required = {
+            'channels': [
+                ('id_1', True)
+            ],
+            'info_json': [
+                ('id_1', True),
+                ('channel_id_1', False),
+                ('uploader_1', False)
+            ],
+            'websub_callbacks': [
+                ('id', True)
+            ],
+            'websub_data': [
+                ('callback_id', False)
+            ],
+            'channel_log': [
+                ('run_id', False),
+                ('id', False)
+            ]
+        }
+
+        # cursed but works
+        for collection in self.db.list_collection_names():
+            if collection in required:
+                indexes = self.db[collection].index_information()
+                for index in required[collection]:
+                    if index[0] not in indexes:
+                        self.db[collection].create_index(index[0], unique=index[1])
+    
+    ##########################################
+    #             general functions          #
+    ##########################################
+    
     def gen_stats(self):
         """ Returns DICT; Channel statistics given the dict key """
         stats = {}
@@ -62,7 +103,15 @@ class Mango:
         stats['queue'] = self.download_queue.count_documents({})
         
         return stats
-        
+    
+    def insert_download_queue(self, video):
+        if not self.download_queue.count_documents({'id': video}) >= 1:
+            return self.download_queue.insert_one({'id': video}).inserted_id
+    
+    ##########################################
+    #            channel operations          #
+    ##########################################
+    
     def list_all_channels(self, active=False):
         """ Returns a SET of YouTube channel ID's; Depending on given positional BOOL only active channels or everything"""
         search_terms = {}
@@ -75,32 +124,12 @@ class Mango:
             channels.append(channel['id'])
         return tuple(channels)
         
-        
-    def get_last_etag(self, channelId):
-        """ string of channel etag if exists or None """
-        data = self.channels.find_one({'id': channelId}, {'fast.etag': 1})
-        try:
-            return data['fast']['etag']
-        except KeyError:
-            return None
-            
     def get_channel_videos_count(self, channelId):
         """
         Returns int of total video count on specific channel
         """
         return self.info_json.count_documents({'channel_id': channelId})
         
-    def get_video_info(self, videoId, limited=False):
-        """
-        """
-        
-        if limited:
-            projection = {'_id': 1, 'id': 1, 'title': 1, 'upload_date': 1, 'description': 1}
-        else:
-            projection = {}
-        
-        return self.info_json.find_one({'id': videoId}, projection)
-    
     def get_channel_videoIds(self, channelId):
         ids = []
         for video in self.info_json.find({'channel_id': channelId}, {'id': 1}):
@@ -110,6 +139,72 @@ class Mango:
     def get_channel_info(self, channelId):
         return self.channels.find_one({'id': channelId})
         
+    def update_channel_state(self, channelId, state):
+        self.channels.update_one({'id': channelId}, {"$set": {"active": bool(state)}})
+        return True
+        
+    def update_channel_key(self, channelId, key, value):
+        self.channels.update_one({'id': channelId}, {"$set": {key: value}})
+        return True
+    
+    def insert_new_channel(self, channelId, originalName, addedDate):
+        if self.channels.count_documents({'id': channelId}) >= 1:
+            return False
+    
+        return self.channels.insert_one({'id': channelId, 'original_name': originalName, 'added_date': addedDate, 'active': False, 'websub': False}).inserted_id
+    
+    ##########################################
+    #            video operations          #
+    ##########################################
+    
+    def check_exists(self, vId):
+        """ Returns BOOL; Given positional argument which is a valid or invalid YouTube video ID STR"""
+        if self.info_json.count_documents({'id': vId}) >= 1:
+            return True
+        return False
+        
+    def get_orphaned_videos(self):
+        """ Returns a SET of YouTube video ID's which have info_jsons in the collection but no permanent channel is defined. SLOW OPERATION """
+        # Ok lemme explain. Perform inner join from channel collection on channel_id key. match only the fields which are empty. return video id
+        pipeline = [{'$lookup': {'from': 'channels', 'localField': 'channel_id', 'foreignField': 'id', 'as': 'channel'}}, {'$match': {'channel': {'$size': 0}}},{'$project': {'id': 1}}]
+
+        results = self.info_json.aggregate(pipeline)
+        ids = [result['id'] for result in results]
+        
+        return tuple(ids)
+        
+    def get_video_info(self, videoId, limited=False):
+        if limited:
+            projection = {'_id': 1, 'id': 1, 'title': 1, 'upload_date': 1, 'description': 1}
+        else:
+            projection = {}
+        
+        return self.info_json.find_one({'id': videoId}, projection)
+    
+    ##########################################
+    #          REPORTING FUNCTIONS           #
+    ##########################################
+        
+    def insert_report(self, videoId, reason):
+        if reason not in ['auto-video', 'metadata', 'illegal']:
+            return False
+    
+        return self.reports.insert_one({'videoId': videoId, 'status': 'open', 'reason': reason, 'reporting_time': current_time(object=True)}).inserted_id
+        
+    def list_reports(self):
+        reports = []
+        for report in self.reports.find({}):
+            reports.append(report)
+        return tuple(reports)
+        
+    def close_report(self, _id):
+        _id = ObjectId(_id)
+        return self.reports.update_one({'_id': _id}, {'$set': {'status': 'closed', 'closing_time': current_time(object=True)}})
+
+    ##########################################
+    #           RUNLOG FUNCTIONS             #
+    ##########################################
+    
     def get_runs(self):
         return self.run_log.find({})
     
@@ -118,28 +213,6 @@ class Mango:
         run['channel_runs'] = list(self.channel_log.find({'run_id': ObjectId(runId)}))
         
         return run
-    
-    def update_fast_etag(self, channel, etag):
-        self.channels.update_one({'id': channel}, {"$set": {"fast.etag": etag, "fast.lastrun": get_utc()}})
-        return etag
-        
-    def update_channel_state(self, channelId, state):
-        self.channels.update_one({'id': channelId}, {"$set": {"active": bool(state)}})
-        return True
-        
-    def update_channel_key(self, channelId, key, value):
-        self.channels.update_one({'id': channelId}, {"$set": {key: value}})
-        return True
-        
-    def insert_download_queue(self, video):
-        if not self.download_queue.count_documents({'id': video}) >= 1:
-            return self.download_queue.insert_one({'id': video}).inserted_id
-            
-    def insert_new_channel(self, channelId, originalName, addedDate):
-        if self.channels.count_documents({'id': channelId}) >= 1:
-            return False
-    
-        return self.channels.insert_one({'id': channelId, 'original_name': originalName, 'added_date': addedDate, 'active': False}).inserted_id
         
     def clean_runs(self, keep=3):
         runs = list(self.run_log.find({}, {'_id': 1}).sort('time', pymongo.ASCENDING))
@@ -156,6 +229,87 @@ class Mango:
             self.run_log.delete_one({'_id': run['_id']})
         
         return True
+        
+    ##########################################
+    #           WEBSUB FUNCTIONS             #
+    ##########################################
+    
+    def websub_newCallback(self, channelId):
+        callbackId = secrets.token_hex(8)
+        
+        self.websub_callbacks.insert_one({'id': callbackId, 'channel': channelId, 'status': 'new', 'created_time': current_time(object=True)})
+        
+        return callbackId
+
+    def websub_requestingCallback(self, callbackId):
+        return self.websub_callbacks.update_one({'id': callbackId}, {'$set': {'status': 'requesting', 'requesting_time': current_time(object=True)}})
+
+    def websub_activateCallback(self, callbackId, lease):
+        status = self.websub_callbacks.find_one({'id': callbackId}, {'status': 1})
+        
+        if not status:
+            return False
+            
+        status = status.get('status')
+        
+        if status in ['requesting']:
+            self.websub_callbacks.update_one({'id': callbackId}, {'$set': {'status': 'active', 'activation_time': current_time(object=True), 'lease': lease}})
+            return True
+        
+        return False
+
+    def websub_existsCallback(self, callbackId):
+        status = self.websub_callbacks.find_one({'id': callbackId}, {'status': 1})
+
+        if not status:
+            return False
+            
+        status = status.get('status')
+        
+        if status in ['requesting', 'active']:
+            return True
+            
+        return False
+        
+    def websub_retireCallback(self, callbackId):
+        return self.websub_callbacks.update_one({'id': callbackId}, {'$set': {'status': 'retired', 'retiring_time': current_time(object=True)}})
+        
+    def websub_deleteCallback(self, callbackId):
+        return self.websub_callbacks.delete_one({'id': callbackId})
+        
+    def websub_getChannels(self, active=False):
+        channels = []
+        for channel in self.channels.find({'websub': True}, {'id': 1}):
+            channels.append(channel['id'])
+        return tuple(channels)
+        
+    def websub_getCallback(self, callbackId):
+        return self.websub_callbacks.find_one({'id': callbackId})
+        
+    def websub_getCallbacks(self, channelId=''):
+        callbacks = []
+        
+        if channelId:
+            filter = {'channel': channelId}
+        else:
+            filter = {}
+        
+        
+        for callback in self.websub_callbacks.find(filter, {'id': 1}):
+            callbacks.append(callback['id'])
+            
+        return tuple(callbacks)
+    
+    def websub_savePost(self, callbackId, data):
+        return self.websub_data.insert_one({'callback_id': callbackId, 'state': 'unprocessed', 'received_time': current_time(object=True), 'raw_data': data}).inserted_id
+        
+    def websub_cleanRetired(self, days=3):
+        days = self.datetime.utcnow() - self.timedelta(days=days)
+
+        self.websub_callbacks.delete_many({'status': 'retired', 'retiring_time': {'$lt': days}})
+        
+        return True
+
 
 ##########################################
 #           HELPER FUNCTIONS             #

ayta/static/robots.txt

@@ -0,0 +1,2 @@
+user-agent: *
+disallow: /_h5ai

ayta/templates/admin/index.html

@@ -16,7 +16,7 @@
 </div>
 <div class="row">
   <div class="col s6 l4 m-4">
-	<a href="{{ url_for('admin.channels') }}">
+	<a href="{{ url_for('admin.system') }}">
 	  <div class="card black-text">
         <div class="card-content">
           <span class="card-title">System</span>
@@ -45,5 +45,25 @@
       </div>
 	</a>
   </div>
+  <div class="col s6 l4 m-4">
+	<a href="{{ url_for('admin.websub') }}">
+	  <div class="card black-text">
+        <div class="card-content">
+          <span class="card-title">WebSub</span>
+		  <p class="grey-text">Edit WebSub YouTube links</p>
+        </div>
+      </div>
+	</a>
+  </div>
+  <div class="col s6 l4 m-4">
+	<a href="{{ url_for('admin.reports') }}">
+	  <div class="card black-text">
+        <div class="card-content">
+          <span class="card-title">Reports</span>
+		  <p class="grey-text">View user reports</p>
+        </div>
+      </div>
+	</a>
+  </div>
 </div>
 {% endblock %}

ayta/templates/admin/reports.html

@@ -0,0 +1,97 @@
+{% extends 'material_base.html' %}
+{% block title %}Reports administration page{% endblock %}
+{% block description %}Reports administration page of the AYTA system{% endblock %}
+
+{% block content %}
+<div class="row">
+  <div class="col s12 l11">
+    <h4>Reports administration page</h4>
+  </div>
+  <div class="col s12 l1 m-5">
+    <form method="POST">
+	  <input title="Prunes all closed reports, but keeps last 30 days" type="submit" value="clean-closed" name="task">
+    </form>
+  </div>
+</div>
+<div class="divider"></div>
+<div class="row">
+  <div class="col s12">
+	<h5>Report options</h5>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12 l4 m-4">
+	<div class="card green">
+      <div class="card-content white-text">
+        <span class="card-title">Placeholder</span>
+        <p>I am a very simple card. I am good at containing small bits of information. I am convenient because I require little markup to use effectively.</p>
+      </div>
+      <div class="card-action">
+        <a href="#">This is a link</a>
+        <a href="#">This is a link</a>
+      </div>
+    </div>
+  </div>
+  <div class="col s12 l4 m-4">
+	<div class="card green">
+      <div class="card-content white-text">
+        <span class="card-title">Placeholder</span>
+        <p>I am a very simple card. I am good at containing small bits of information. I am convenient because I require little markup to use effectively.</p>
+      </div>
+      <div class="card-action">
+        <a href="#">This is a link</a>
+        <a href="#">This is a link</a>
+      </div>
+    </div>
+  </div>
+</div>
+<div class="divider"></div>
+<div class="row">
+  <div class="col s6 l9">
+	<h5>All reports</h5>
+  </div>
+  <div class="col s6 l3 m-4 input-field">
+	<input id="filter_query" type="text">
+    <label for="filter_query">Filter results</label>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12">
+    {% if reports is not defined %}
+	<p>No reports!</p>
+	{% else %}
+    <table class="striped highlight responsive-table">
+      <thead>
+        <tr>
+		  <th>Actions</th>
+		  <th>_id</th>
+		  <th>videoId</th>
+		  <th>status</th>
+		  <th>reason</th>
+		  <th>reporting_time</th>
+		  <th>closing_time</th>
+        </tr>
+      </thead>
+      <tbody>
+	    {% for report in reports %}
+        <tr class="filterable">
+		  <td>
+		    <form method="post">
+			  <input type="text" value="{{ report.get('_id') }}" name="value" hidden>
+		      <button class="btn-small waves-effect waves-light" type="submit" name="task" value="close" title="Close the report" {% if report.get('status') != 'open' %}disabled{% endif %}>✅</button>
+			</form>
+		  </td>
+		  <td>{{ report.get('_id') }}</td>
+		  <td><a href="{{ url_for('watch.base') }}?v={{ report.get('videoId') }}">{{ report.get('videoId') }}</a></td>
+		  <td>{{ report.get('status') }}</td>
+		  <td>{{ report.get('reason') }}</td>
+		  <td>{{ report.get('reporting_time') }}</td>
+		  <td>{{ report.get('closing_time') }}</td>
+        </tr>
+		{% endfor %}
+      </tbody>
+    </table>
+	{% endif %}
+  </div>
+</div>
+{% endblock %}

ayta/templates/admin/runs.html

@@ -8,7 +8,7 @@
     <h4>Runs administration page</h4>
   </div>
   <div class="col s12 l1 m-5">
-    <form class="center" method="POST">
+    <form method="POST">
 	  <input title="Prunes all runs, but keeps last 3" type="submit" value="clean_runs" name="task">
     </form>
   </div>

ayta/templates/admin/system.html

@@ -0,0 +1,37 @@
+{% extends 'material_base.html' %}
+{% block title %}Administration page{% endblock %}
+{% block description %}Administration page of the AYTA system{% endblock %}
+
+{% block content %}
+<div class="row">
+  <div class="col s12">
+    <h4>Administration page</h4>
+  </div>
+</div>
+<div class="divider"></div>
+<div class="row">
+  <div class="col s12">
+	<h5>AYTA system settings</h5>
+  </div>
+</div>
+<div class="row">
+  <div class="col s6 l4 m-4">
+	<div class="card black-text">
+      <div class="card-content">
+        <form method="POST">
+	      <div class="input-field">
+		    <span class="supporting-text">Enable WebSub</span>
+		    <input class="validate" type="text" value="{{ item }}" name="key" hidden>
+		  </div>
+		  <div class="input-field m-4">
+		    <div class="switch">
+		      <label>Off<input type="checkbox" value="None" name="value"><span class="lever"></span>On</label>
+		    </div>
+	      </div>
+	      <button class="btn icon-right waves-effect waves-light" type="submit" name="task" value="update-value">Set</button>
+	    </form>
+      </div>
+    </div>
+  </div>
+</div>
+{% endblock %}

ayta/templates/admin/websub.html

@@ -0,0 +1,97 @@
+{% extends 'material_base.html' %}
+{% block title %}WebSub administration page{% endblock %}
+{% block description %}WebSub administration page of the AYTA system{% endblock %}
+
+{% block content %}
+<div class="row">
+  <div class="col s12 l11">
+    <h4>WebSub administration page</h4>
+  </div>
+  <div class="col s12 l1 m-5">
+    <form method="POST">
+	  <input title="Prunes all retired callbacks, but keeps last 3 days" type="submit" value="clean-retired" name="task">
+    </form>
+  </div>
+</div>
+<div class="divider"></div>
+<div class="row">
+  <div class="col s12">
+	<h5>WebSub options</h5>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12 l4 m-4">
+	<div class="card green">
+      <div class="card-content white-text">
+        <span class="card-title">Placeholder</span>
+        <p>I am a very simple card. I am good at containing small bits of information. I am convenient because I require little markup to use effectively.</p>
+      </div>
+      <div class="card-action">
+        <a href="#">This is a link</a>
+        <a href="#">This is a link</a>
+      </div>
+    </div>
+  </div>
+  <div class="col s12 l4 m-4">
+	<div class="card green">
+      <div class="card-content white-text">
+        <span class="card-title">Placeholder</span>
+        <p>I am a very simple card. I am good at containing small bits of information. I am convenient because I require little markup to use effectively.</p>
+      </div>
+      <div class="card-action">
+        <a href="#">This is a link</a>
+        <a href="#">This is a link</a>
+      </div>
+    </div>
+  </div>
+</div>
+<div class="divider"></div>
+<div class="row">
+  <div class="col s6 l9">
+	<h5>Registered callbacks</h5>
+  </div>
+  <div class="col s6 l3 m-4 input-field">
+	<input id="filter_query" type="text">
+    <label for="filter_query">Filter results</label>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12">
+    <table class="striped highlight responsive-table">
+      <thead>
+        <tr>
+		  <th>Actions</th>
+		  <th>id</th>
+		  <th>channel</th>
+		  <th>status</th>
+		  <th>created_time</th>
+		  <th>requesting_time</th>
+		  <th>activation_time</th>
+		  <th>retiring_time</th>
+		  <th>lease</th>
+        </tr>
+      </thead>
+      <tbody>
+	    {% for callback in callbacks %}
+        <tr class="filterable">
+		  <td>
+		    <form method="post">
+			  <input type="text" value="{{ callbacks[callback].get('id') }}" name="value" hidden>
+		      <button class="btn-small waves-effect waves-light" type="submit" name="task" value="unsubscribe" title="Send unsubscribe request to hub" {% if callbacks[callback].get('status') != 'active' %}disabled{% endif %}>🗑️</button>
+			</form>
+		  </td>
+		  <td>{{ callbacks[callback].get('id') }}</td>
+		  <td><a href="{{ url_for('admin.channel', channelId=callbacks[callback].get('channel')) }}">{{ callbacks[callback].get('channel') }}</a></td>
+		  <td>{{ callbacks[callback].get('status') }}</td>
+		  <td>{{ callbacks[callback].get('created_time') }}</td>
+		  <td>{{ callbacks[callback].get('requesting_time') }}</td>
+		  <td>{{ callbacks[callback].get('activation_time') }}</td>
+		  <td>{{ callbacks[callback].get('retiring_time') }}</td>
+		  <td>{{ callbacks[callback].get('lease') }}</td>
+        </tr>
+		{% endfor %}
+      </tbody>
+    </table>
+  </div>
+</div>
+{% endblock %}

ayta/templates/channel/channel.html

@@ -33,7 +33,7 @@
 		<p class="grey-text">{{ videos[video].get('id') }} | {{ videos[video].get('upload_date')|pretty_time }}</p>
       </div>
       <div class="card-reveal">
-        <span class="card-title truncate"><i class="material-icons right">close</i>{{ videos[video].get('title') }}</span>
+        <span class="card-title truncate">{{ videos[video].get('title') }}</span>
         <p style="white-space: pre-wrap;">{{ videos[video].get('description') }}</p>
       </div>
 	</div>

ayta/templates/channel/index.html

@@ -19,6 +19,16 @@
   </div>
 </div>
 <div class="row">
+  <div class="col s12 m-4 filterable">
+	<a href="{{ url_for('channel.orphaned') }}">
+	  <div class="card black-text">
+        <div class="card-content center">
+          <span class="card-title">Orphaned videos</span>
+		  <p class="grey-text">Individual videos which do not have a permanent channel linked</p>
+        </div>
+      </div>
+	</a>
+  </div>
   {% for channel in channels %}
   <div class="col s6 l4 m-4 filterable">
 	<a href="{{ url_for('channel.channel', channelId=channel) }}">
@@ -26,7 +36,7 @@
         <div class="card-content">
           <span class="card-title">{{ channels[channel].get('original_name') }}</span>
 		  <p class="grey-text">{{ channels[channel].get('id') }}</p>
-          <p>{{ channels[channel].get('added_date') }} | <b>Active:</b> {{ channels[channel].get('active') }} | <b>Videos:</b> {{ channels[channel].get('video_count') }}</p>
+          <p><b>Added:</b> {{ channels[channel].get('added_date')|pretty_time }} | <b>Active:</b> {{ channels[channel].get('active') }} | <b>Videos:</b> {{ channels[channel].get('video_count') }}</p>
         </div>
       </div>
 	</a>

ayta/templates/channel/orphaned.html

@@ -0,0 +1,43 @@
+{% extends 'material_base.html' %}
+{% block title %}Orphaned videos{% endblock %}
+{% block description %}Videos in the archive which do not have a permanent channel linked.{% endblock %}
+
+{% block content %}
+<div class="row">
+  <div class="col s12">
+    <h4>Channels lising page</h4>
+  </div>
+</div>
+<div class="divider"></div>
+<div class="row">
+  <div class="col s6 l9">
+	<h5>Orphaned videos</h5>
+	<p>Videos in the archive which do not have a permanent channel linked. There is a high chance that the videos are manually downloaded.</p>
+  </div>
+  <div class="col s6 l3 m-4 input-field">
+	<input id="filter_query" type="text">
+    <label for="filter_query">Filter results</label>
+  </div>
+</div>
+<div class="row">
+  {% for video in videos %}
+  <div class="col s6 l4 m-4 filterable">
+	<div class="card medium black-text">
+	  <a href="{{ url_for('watch.base') }}?v={{ video }}">
+	    <div class="card-image">
+		  <img loading="lazy" src="{{ url_for('static', filename='img/nothumb.jpg') }}">
+        </div>
+	  </a>
+      <div class="card-content activator">
+        <span class="card-title">{{ videos[video].get('title') }}</span>
+		<p class="grey-text">{{ videos[video].get('id') }} | {{ videos[video].get('upload_date')|pretty_time }}</p>
+      </div>
+      <div class="card-reveal">
+        <span class="card-title truncate">{{ videos[video].get('title') }}</span>
+        <p style="white-space: pre-wrap;">{{ videos[video].get('description') }}</p>
+      </div>
+	</div>
+  </div>
+  {% endfor %}
+</div>
+{% endblock %}

ayta/templates/material_base.html

@@ -12,16 +12,18 @@
     <meta name="description" content="{% block description %}{% endblock %}">
 	<meta name="viewport" content="width=device-width, initial-scale=1.0"> 
   </head>
-  <body class="indigo lighten-5">
+  <body class="grey lighten-2">
     <header>
       <nav>
-        <div class="nav-wrapper red accent-4">
+        <div class="nav-wrapper deep-orange">
           <ul id="nav-mobile" class="left">
             <li><a href="{{ url_for('channel.base') }}">Channels</a></li>
             <li><a href="{{ url_for('admin.base') }}">Admin</a></li>
+			{% if config.get('DEBUG') %}<li><span class="new badge mt-5" data-badge-caption="True">Debug mode is</span></li>{% endif %}
           </ul>
 	      <a href="{{ url_for('index.base') }}" class="brand-logo center">AYTA</a>
 		  <ul id="nav-mobile" class="right">
+		    {% if 'username' in session %}<li><a href="{{ url_for('auth.logout') }}"><span class="new badge" data-badge-caption="{{ session.username }}">Logged in as</span></a></li>{% endif %}
             <li><a href="{{ url_for('search.base') }}">Search</a></li>
             <li><a href="{{ url_for('index.help') }}">Help</a></li>
           </ul>
@@ -41,16 +43,14 @@
 		  {% block content %}{% endblock %}
 	  </div>
 	</main>
-    <footer class="page-footer red accent-4">
+    <footer class="page-footer deep-orange">
       <div class="container">
         <div class="row">
-          <div class="s6">
+          <div class="s12 l6">
             <h5>Awesome YouTube Archive</h5>
             <p>A custom content management system for archived YouTube videos!</p>
           </div>
-		  <div class="s6">
-		    {% if 'username' in session %}<a href="{{ url_for('auth.logout') }}"><span class="new badge" data-badge-caption="{{ session.username }}">Logged in as</span></a>{% endif %}
-			{% if config.get('DEBUG') %}<span class="new badge" data-badge-caption="True">Debug mode is</span>{% endif %}
+		  <div class="s12 l6">
 			<span class="new badge" data-badge-caption="{{ null|current_time }}">Page generated on</span>
             <h6>Still in development, slowly...</h6>
 			<h6>This is not a streaming website! Videos may buffer (a lot)!</h6>

ayta/templates/watch/index.html

@@ -11,10 +11,10 @@
     <p><b>Video by:</b> <a href="{{ url_for('channel.channel', channelId=render.get('info').get('channel_id')) }}">{{ render.get('info').get('uploader') }}</a></p>
   </div>
   <div class="col s3">
-    <p><b>Upload date:</b> {{ render.get('info').get('upload_date') }}</p>
+    <p><b>Upload date:</b> {{ render.get('info').get('upload_date')|pretty_time }}</p>
   </div>
   <div class="col s3">
-    <p><b>Archive date:</b> {{ render.get('info').get('archive_date') }}</p>
+    <p><b>Archive date:</b> {{ render.get('info').get('epoch')|epoch_time }}</p>
   </div>
   <div class="col s3">
     <p><b>Video length:</b> {{ render.get('info').get('duration')|pretty_duration }}</p>
@@ -34,23 +34,25 @@
     <div class="section">
 	  <div class="row">
 	    <div class="col s12 m3">
-	      <p><a href="https://youtu.be/{{ render.get('info').get('id') }}" target="_blank" rel="noopener noreferrer">Watch on YouTube</a></p>
+	      <p><a href="https://youtu.be/{{ render.get('info').get('id') }}" target="_blank" rel="noopener noreferrer">▶️ Watch on YouTube</a></p>
 		</div>
 		<div class="col s12 m3">
-	      <p><a href="https://archive.ventilaar.net/videos/automatic/{{ render.get('info').get('channel_id') }}/{{ render.get('info').get('id') }}/">Source files</a></p>
+	      <p><a href="https://archive.ventilaar.net/videos/automatic/{{ render.get('info').get('channel_id') }}/{{ render.get('info').get('id') }}/">🗄️ Source files</a></p>
 		</div>
 		<div class="col s12 m3">
 	      <p>Sample text</p>
 		</div>
 		<div class="col s12 m3 input-field">
-		  <select id="report">
-            <option value="" disabled selected></option>
-            <option value="auto-video">Auto/Video Problems</option>
-            <option value="metadata">Incorrect metadata</option>
-            <option value="illegal">Illegal video</option>
-          </select>
-          <label for="report">Report a problem</label>
-		  <button for="report" class="btn" type="submit" name="action">Submit report</button>
+		  <form method="post">
+		    <select id="report" name="reason">
+              <option value="" disabled selected></option>
+              <option value="auto-video">Auto/Video Problems</option>
+              <option value="metadata">Incorrect metadata</option>
+              <option value="illegal">Illegal video</option>
+            </select>
+            <label for="report">Report a problem</label>
+		    <button for="report" class="btn" type="submit" name="action">Submit report</button>
+		  </form>
 		</div>
 	  </div>
 	</div>

requirements.txt

@@ -1,3 +1,5 @@
+# because we use basic functions and options the latest versions would work just fine
+
 flask
 flask-caching
 flask-login
@@ -8,3 +10,5 @@ pymongo
 yt-dlp
 argon2-cffi
 gunicorn
+celery
+sqlalchemy

run.py

@@ -1,4 +1,4 @@
 import ayta
 
 if __name__ == '__main__':
-    ayta.create_app().run(debug=True, host="0.0.0.0")
+    ayta.create_app().run(debug=True, host="0.0.0.0", port=5001)