Fix None iterable and add new background task

ayta/__init__.py

@@ -26,6 +26,7 @@ def create_app(test_config=None):
     config['CELERY']['beat_schedule'] = {}
     config['CELERY']['beat_schedule']['Renew WebSub endpoints'] = {'task': 'ayta.tasks.websub_renew_expiring', 'schedule': 4000}
     config['CELERY']['beat_schedule']['Process WebSub data'] = {'task': 'ayta.tasks.websub_process_data', 'schedule': 100}
+    config['CELERY']['beat_schedule']['Queue up new videos in static channel playlists'] = {'task': 'ayta.tasks.playlist_to_queue', 'schedule': 5000}
     
     app = Flask(__name__)
     app.config.from_mapping(config)

ayta/blueprints/admin.py

@@ -2,7 +2,7 @@ from flask import Blueprint, render_template, request, redirect, url_for, flash,
 from ..nosql import get_nosql
 from ..dlp import checkChannelId, getChannelInfo
 from ..decorators import login_required
-from ..tasks import test_sleep, websub_subscribe_callback, websub_unsubscribe_callback, video_download, video_queue
+from ..tasks import test_sleep, websub_subscribe_callback, websub_unsubscribe_callback, video_download, video_queue, playlist_to_queue
 from datetime import datetime
 from secrets import token_urlsafe
 
@@ -30,22 +30,29 @@ def channels():
     generic = {}
 
     if request.method == 'POST':
-        channelId = request.form.get('channel_id', None)
-        originalName = request.form.get('original_name', None)
-        addedDate = request.form.get('added_date', None)
+        task = request.form.get('task', None)
+        
+        if task == 'subscribe-websub':
+            channelId = request.form.get('channel_id', None)
+            originalName = request.form.get('original_name', None)
+            addedDate = request.form.get('added_date', None)
 
-        ### add some validation
-        
-        addedDate = datetime.strptime(addedDate, '%Y-%m-%d')
-        
-        if checkChannelId(channelId) is False:
-            channelId, originalName = getChannelInfo(channelId, ('channel_id', 'uploader'))
-        
-        if not get_nosql().insert_new_channel(channelId, originalName, addedDate):
-            flash('Error inserting new channel, you probably made a mistake somewhere')
-            return redirect(url_for('admin.channels'))
+            ### add some validation
+            
+            addedDate = datetime.strptime(addedDate, '%Y-%m-%d')
+            
+            if checkChannelId(channelId) is False:
+                channelId, originalName = getChannelInfo(channelId, ('channel_id', 'uploader'))
+            
+            if not get_nosql().insert_new_channel(channelId, originalName, addedDate):
+                flash('Error inserting new channel, you probably made a mistake somewhere')
+                return redirect(url_for('admin.channels'))
 
-        return redirect(url_for('admin.channel', channelId=channelId))
+            return redirect(url_for('admin.channel', channelId=channelId))
+            
+        elif task == 'playlist-queue':
+            task =  playlist_to_queue.delay()
+            flash(f'Task playlist-queue has been queued: {task.id}')
 
     generic['currentDate'] = datetime.utcnow()
     channelIds = get_nosql().list_all_channels()
@@ -210,8 +217,10 @@ def queue():
             flash(f'Queue has been emptied')
             
         elif task == 'queue-run-once':
-            task =  video_queue.delay()
-            flash(f'Task has been started on the oldest queued item: {task.id}')
+            value = int(value) if value.isdigit() else 1
+            for x in range(value):
+                task =  video_queue.delay()
+                flash(f'Task has been started on the oldest queued item: {task.id}')
         
         return redirect(url_for('admin.queue'))
         
@@ -249,7 +258,7 @@ def users():
     return render_template('admin/users.html', users=users)
     
 @bp.route('/workers', methods=['GET', 'POST'])
-#@login_required
+@login_required
 def workers():
     if request.method == 'POST':
         task = request.form.get('task', None)
@@ -259,4 +268,5 @@ def workers():
     celery = current_app.extensions.get('celery')
     
     tasks = celery.control.inspect().active()
-    return render_template('admin/workers.html', tasks=tasks)
+    reserved = celery.control.inspect().reserved()
+    return render_template('admin/workers.html', tasks=tasks, reserved=reserved)

ayta/oidc.py

@@ -4,6 +4,7 @@ class OIDC():
     Additionally this class provides the function to generate redirect url's and check bearer tokens on their validity as well as caching jwt signing keys.
     Fairly barebones and should be 100% secure. (famous last words)
     This is made for form posted JWT's. While not the most secure it is the most easy way to implement. Moving on to a code based solution might be preferred in the future.
+    The nonce and state store is in memory, so only one instance can be used at a time until central key caching is implemented.
     """
     def __init__(self, app=None):
         self.states = {}
@@ -151,7 +152,6 @@ class OIDC():
                                  
         # Any exception (invalid JWT, invalid formatting etc...) must return False
         except Exception as e:
-            print(e, flush=True)
             return False
 
         # Double check if given token is really requested by us by matching the nonce in the signed key

ayta/tasks.py

@@ -164,6 +164,19 @@ def websub_renew_expiring(hours=6):
         if count >= 256:
             break
 
+@shared_task()
+def playlist_to_queue(hours=6):
+    from .nosql import get_nosql
+    
+    channels = get_nosql().list_all_channels(active=True)
+    
+    for channel in channels:
+        info = get_nosql().get_channel_info()
+        
+        for item in info['playlist']['entries']:
+            videoId = item['id']
+            get_nosql().queue_insertQueue(videoId, 'Playlist mirroring')
+
 ##########################################
 #              TASK MODULES              #
 ##########################################

ayta/templates/admin/channels.html

@@ -15,6 +15,18 @@
   </div>
 </div>
 <div class="row">
+  <div class="col s12 l4 m-4">
+    <div class="card">
+      <div class="card-content">
+        <span class="card-title">Direct actions</span>
+        <form class="mt-4" method="post">
+          <button class="btn mb-2 green" type="submit" name="task" value="playlist-queue">Playlist to Queue</button>
+          <br>
+          <span class="supporting-text">Forcerun playlist to queue task</span>
+        </form>
+      </div>
+    </div>
+  </div>
   <div class="col s12 l4 m-4">
 	<div class="card">
       <div class="card-content">
@@ -38,7 +50,7 @@
               });
             </script>
 		  </div>
-          <button class="btn mt-4" type="submit" name="action" value="add_channel">Add</button>
+          <button class="btn mt-4" type="submit" name="task" value="add_channel">Add</button>
 	    </form>
       </div>
     </div>

ayta/templates/admin/queue.html

@@ -29,7 +29,8 @@
           <br>
           <span class="supporting-text">Prunes all deactivated endpoints, but keeps last 3 days</span>
         </form>
-        <form class="mt-4" method="post" onsubmit="return confirm('Are you sure?');">
+        <form class="mt-4 input-field" method="post" onsubmit="return confirm('Are you sure?');">
+          <input type="number" style="width: 80px" value="1" name="value" min="1" max="99">
           <button class="btn mb-2 green" type="submit" name="task" value="queue-run-once">Download oldest queued</button>
           <br>
           <span class="supporting-text">Will download the oldest queued video ID</span>
@@ -151,7 +152,7 @@
           <td>
             <form method="post">
               <input type="text" value="{{ id.get('id') }}" name="value" hidden>
-              <button class="btn-small waves-effect waves-light" type="submit" name="task" value="delete-queue" title="Delete from queue" {% if id.get('status') != 'queued' %}disabled{% endif %}>🗑️</button>
+              <button class="btn-small waves-effect waves-light" type="submit" name="task" value="delete-queue" title="Delete from queue" {% if id.get('status') == 'working' %}disabled{% endif %}>🗑️</button>
             </form>
             <form method="post">
               <input type="text" value="{{ id.get('id') }}" name="value" hidden>

ayta/templates/admin/workers.html

@@ -19,8 +19,40 @@
 </form>
 <div class="divider"></div>
 <div class="row">
-  <div class="col s12">
-    <h6>Current workers</h6>
+  <div class="col s12 m-4">
+    <h5>Reserved tasks per worker</h5>
+    <p>Usually 4 tasks per worker</p>
+	{% if reserved is none %}
+	<h6>No workers with reserved tasks, are there any workers with stuck tasks or are they even online?</h6>
+	{% else %}
+    {% for worker in reserved %}
+    <span>{{ worker }}</span>
+    <table class="striped highlight responsive-table" style=" border: 1px solid black;">
+      <thead>
+        <tr>
+          <th>ID</th>
+          <th>Task</th>
+          <th>Arguments</th>
+        </tr>
+      </thead>
+      <tbody>
+        {% for task in reserved[worker] %}
+        <tr>
+          <td>{{ task.get('id') }}</td>
+          <td>{{ task.get('name') }}</td>
+          <td>{{ task.get('args') }} {{ task.get('kwargs') }}</td>
+        </tr>
+        {% endfor %}
+      </tbody>
+    </table>
+    {% endfor %}
+	{% endif %}
+  </div>
+  <div class="col s12 m-4">
+    <h5>Current workers and processing tasks</h5>
+	{% if tasks is none %}
+	<h6>No workers with running tasks, are there any workers with stuck tasks or are they even online?</h6>
+	{% else %}
     {% for worker in tasks %}
     <span>{{ worker }}</span>
     <table class="striped highlight responsive-table" style=" border: 1px solid black;">
@@ -35,13 +67,14 @@
         {% for task in tasks[worker] %}
         <tr>
           <td>{{ task.get('id') }}</td>
-          <td>{{ task.get('type') }}</td>
+          <td>{{ task.get('name') }}</td>
           <td>{{ task.get('time_start')|epoch_time }}</td>
         </tr>
         {% endfor %}
       </tbody>
     </table>
     {% endfor %}
+	{% endif %}
   </div>
 </div>
 {% endblock %}

requirements.txt

@@ -3,6 +3,8 @@
 flask
 flask-caching
 flask-limiter
+flask-sqlalchemy
+flask-migrate
 pymongo
 yt-dlp
 gunicorn