Small concurrency and logging fix

ayta/blueprints/admin.py

@@ -208,6 +208,10 @@ def queue():
         elif task == 'empty-queue':
             get_nosql().queue_emptyQueue()
             flash(f'Queue has been emptied')
+            
+        elif task == 'queue-run-once':
+            task =  video_queue.delay()
+            flash(f'Task has been started on the oldest queued item: {task.id}')
         
         return redirect(url_for('admin.queue'))
         
@@ -251,8 +255,6 @@ def workers():
         task = request.form.get('task', None)
         if task == 'test-sleep':
             test_sleep.delay()
-        elif task == 'queue-single':
-            video_queue.delay()
 
     celery = current_app.extensions.get('celery')
     

ayta/nosql.py

@@ -411,13 +411,17 @@ class Mango:
     ##########################################
         
     def queue_insertQueue(self, videoId, endpointId=None):
-        # if no document exists
+        # if already queued
-        if not self.download_queue.count_documents({'id': videoId}) >= 1:
+        if self.download_queue.count_documents({'id': videoId}) >= 1:
-            self.download_queue.insert_one({'id': videoId, 'endpoint': endpointId, 'created_time': current_time(object=True), 'status': 'queued'}).inserted_id
+            return False
-            return True
+        
-            
+        # if already in archive
-        # key already in queue
+        if self.check_exists(videoId):
-        return False
+            return False
+        
+        # add to queue
+        self.download_queue.insert_one({'id': videoId, 'endpoint': endpointId, 'created_time': current_time(object=True), 'status': 'queued'}).inserted_id
+        return True
         
     def queue_deleteQueue(self, videoId):
         if self.download_queue.delete_one({'id': videoId}):
@@ -430,13 +434,19 @@ class Mango:
     def queue_emptyQueue(self):
         return self.download_queue.delete_many({})
         
-    def queue_getNext(self, newest=False):
+    def queue_setFailed(self, videoId):
-        sort = []
+        return self.download_queue.update_one({'id': videoId}, {'$set': {'status': 'failed'}})
         
-        if newest:
+    def queue_getNext(self):
-            sort = [( 'created_time', pymongo.DESCENDING )]
+        """ Returns a LIST of queue parameters. Function first checks if ID exists, if so deletes and then checks the next queued until queue is empty (None) or queued id does not exist yet."""
-            
+        while True:
-        return self.download_queue.find_one({}, sort=sort)
+            queueItem = self.download_queue.find_one({'status': 'queued'})
+            if not queueItem:
+                return None
+            elif self.check_exists(queueItem['id']):
+                self.queue_deleteQueue(queueItem['id'])
+            self.download_queue.update_one({'id': queueItem['id']}, {'$set': {'status': 'working'}})
+            return queueItem
         
 ##########################################
 #           HELPER FUNCTIONS             #

ayta/tasks.py

@@ -42,6 +42,7 @@ def video_queue():
         get_nosql().queue_deleteQueue(videoId)
         return True
     else:
+        get_nosql().queue_setFailed(videoId)
         return False
 
 @shared_task()

ayta/templates/admin/queue.html

@@ -11,67 +11,71 @@
 <div class="divider"></div>
 <div class="row">
   <div class="col s12">
-	<h5>Options</h5>
+    <h5>Options</h5>
   </div>
 </div>
 <div class="row">
   <div class="col s12 l4 m-4">
-	<div class="card">
+    <div class="card">
       <div class="card-content">
         <span class="card-title">Direct actions</span>
-		<form class="mt-4" method="post" onsubmit="return confirm('Are you sure?');">
+        <form class="mt-4" method="post" onsubmit="return confirm('Are you sure?');">
           <button class="btn mb-2 red" type="submit" name="task" value="empty-queue">Empty Queue</button>
-		  <br>
+          <br>
-		  <span class="supporting-text">Removes all queued ids</span>
+          <span class="supporting-text">Removes all queued ids</span>
-	    </form>
+        </form>
-		<form class="mt-4" method="post" onsubmit="return confirm('Are you sure?');">
+        <form class="mt-4" method="post" onsubmit="return confirm('Are you sure?');">
           <button class="btn mb-2" type="submit" name="task" value="clean-retired">Clean retired</button>
-		  <br>
+          <br>
-		  <span class="supporting-text">Prunes all deactivated endpoints, but keeps last 3 days</span>
+          <span class="supporting-text">Prunes all deactivated endpoints, but keeps last 3 days</span>
-	    </form>
+        </form>
-		
+        <form class="mt-4" method="post" onsubmit="return confirm('Are you sure?');">
+          <button class="btn mb-2 green" type="submit" name="task" value="queue-run-once">Download oldest queued</button>
+          <br>
+          <span class="supporting-text">Will download the oldest queued video ID</span>
+        </form>
       </div>
     </div>
   </div>
   <div class="col s12 l4 m-4">
-	<div class="card">
+    <div class="card">
       <div class="card-content">
         <span class="card-title">Create new endpoint</span>
-	    <form method="post">
+        <form method="post">
-		  <div class="row">
+          <div class="row">
             <div class="col s12 input-field">
               <input placeholder="Custom endpoint" name="value" type="text" class="validate" minlength="12">
-			  <span class="supporting-text">Leaving this empty will create a random secure string</span>
+              <span class="supporting-text">Leaving this empty will create a random secure string</span>
-	        </div>
+            </div>
-			<div class="col s12 input-field">
+            <div class="col s12 input-field">
               <input placeholder="Description" name="description" type="text" class="validate" minlength="8" maxlength="64" required>
-			  <span class="supporting-text">Description for the endpoint for better administration</span>
+              <span class="supporting-text">Description for the endpoint for better administration</span>
-	        </div>
+            </div>
           <button class="btn mt-4" type="submit" name="task" value="add-endpoint">Create</button>
-		  </div>
+          </div>
-	    </form>
+        </form>
       </div>
     </div>
   </div>
   <div class="col s12 l4 m-4">
-	<div class="card">
+    <div class="card">
       <div class="card-content">
         <span class="card-title">Queue manually</span>
-	    <form method="post">
+        <form method="post">
-		  <div class="row">
+          <div class="row">
             <div class="col s12 input-field">
               <input placeholder="Youtube video ID" name="value" type="text" class="validate" minlength="11" maxlength="11" required>
-			  <span class="supporting-text">Must be a valid Youtube video ID</span>
+              <span class="supporting-text">Must be a valid Youtube video ID</span>
-	        </div>
+            </div>
-			<div class="col s12 mt-5 input-field">
+            <div class="col s12 mt-5 input-field">
               <div class="switch">
-		        <label>Queue<input type="checkbox" value="direct" name="direct"><span class="lever"></span>Direct</label>
+                <label>Queue<input type="checkbox" value="direct" name="direct"><span class="lever"></span>Direct</label>
-				<span class="supporting-text">Queue up or start directly</span>
+                <span class="supporting-text">Queue up or start directly</span>
-		      </div>
+              </div>
-	        </div>
+            </div>
           <button class="btn mt-4" type="submit" name="task" value="manual-queue">Queue</button>
-		  </div>
+          </div>
-	    </form>
+        </form>
       </div>
     </div>
   </div>
@@ -79,10 +83,10 @@
 <div class="divider"></div>
 <div class="row">
   <div class="col s6 l9">
-	<h5>Registered endpoints</h5>
+    <h5>Registered endpoints</h5>
   </div>
   <div class="col s6 l3 m-4 input-field">
-	<input id="filter_query" type="text">
+    <input id="filter_query" type="text">
     <label for="filter_query">Filter results</label>
   </div>
 </div>
@@ -91,30 +95,30 @@
     <table class="striped highlight responsive-table">
       <thead>
         <tr>
-		  <th>Actions</th>
+          <th>Actions</th>
-		  <th>id</th>
+          <th>id</th>
-		  <th>description</th>
+          <th>description</th>
-		  <th>status</th>
+          <th>status</th>
-		  <th>created_time</th>
+          <th>created_time</th>
-		  <th>retired_time</th>
+          <th>retired_time</th>
         </tr>
       </thead>
       <tbody>
-	    {% for endpoint in endpoints %}
+        {% for endpoint in endpoints %}
         <tr class="filterable">
-		  <td>
+          <td>
-		    <form method="post">
+            <form method="post">
-			  <input type="text" value="{{ endpoint.get('id') }}" name="value" hidden>
+              <input type="text" value="{{ endpoint.get('id') }}" name="value" hidden>
-		      <button class="btn-small waves-effect waves-light" type="submit" name="task" value="retire" title="Retire endpoint" {% if endpoint.get('status') != 'active' %}disabled{% endif %}>🗑️</button>
+              <button class="btn-small waves-effect waves-light" type="submit" name="task" value="retire" title="Retire endpoint" {% if endpoint.get('status') != 'active' %}disabled{% endif %}>🗑️</button>
-			</form>
+            </form>
-		  </td>
+          </td>
-		  <td>{{ endpoint.get('id') }}</td>
+          <td>{{ endpoint.get('id') }}</td>
-		  <td>{{ endpoint.get('description') }}</td>
+          <td>{{ endpoint.get('description') }}</td>
-		  <td>{{ endpoint.get('status') }}</td>
+          <td>{{ endpoint.get('status') }}</td>
-		  <td>{{ endpoint.get('created_time') }}</td>
+          <td>{{ endpoint.get('created_time') }}</td>
-		  <td>{{ endpoint.get('retired_time') }}</td>
+          <td>{{ endpoint.get('retired_time') }}</td>
         </tr>
-		{% endfor %}
+        {% endfor %}
       </tbody>
     </table>
   </div>
@@ -122,10 +126,10 @@
 <div class="divider"></div>
 <div class="row">
   <div class="col s6 l9">
-	<h5>Queued ID's</h5>
+    <h5>Queued ID's</h5>
   </div>
   <div class="col s6 l3 m-4 input-field">
-	<input id="filter_query" type="text">
+    <input id="filter_query" type="text">
     <label for="filter_query">Filter results</label>
   </div>
 </div>
@@ -134,28 +138,33 @@
     <table class="striped highlight responsive-table">
       <thead>
         <tr>
-		  <th>Actions</th>
+          <th>Actions</th>
-		  <th>id</th>
+          <th>id</th>
-		  <th>endpoint</th>
+          <th>endpoint</th>
-		  <th>status</th>
+          <th>status</th>
-		  <th>created_time</th>
+          <th>created_time</th>
         </tr>
       </thead>
       <tbody>
-	    {% for id in queue %}
+        {% for id in queue %}
         <tr class="filterable">
-		  <td>
+          <td>
-		    <form method="post">
+            <form method="post">
-			  <input type="text" value="{{ id.get('id') }}" name="value" hidden>
+              <input type="text" value="{{ id.get('id') }}" name="value" hidden>
-		      <button class="btn-small waves-effect waves-light" type="submit" name="task" value="delete-queue" title="Delete from queue" {% if id.get('status') != 'queued' %}disabled{% endif %}>🗑️</button>
+              <button class="btn-small waves-effect waves-light" type="submit" name="task" value="delete-queue" title="Delete from queue" {% if id.get('status') != 'queued' %}disabled{% endif %}>🗑️</button>
-			</form>
+            </form>
-		  </td>
+            <form method="post">
-		  <td>{{ id.get('id') }}</td>
+              <input type="text" value="{{ id.get('id') }}" name="value" hidden>
-		  <td>{{ id.get('endpoint') }}</td>
+              <button class="btn-small waves-effect waves-light" type="submit" name="task" value="run-download" title="Run download task" disabled>⏩</button>
-		  <td>{{ id.get('status') }}</td>
+			  <!-- This function fill not work until the download queue and video download process is rewritten -->
-		  <td>{{ id.get('created_time') }}</td>
+            </form>
+          </td>
+          <td>{{ id.get('id') }}</td>
+          <td>{{ id.get('endpoint') }}</td>
+          <td>{{ id.get('status') }}</td>
+          <td>{{ id.get('created_time') }}</td>
         </tr>
-		{% endfor %}
+        {% endfor %}
       </tbody>
     </table>
   </div>

ayta/templates/admin/workers.html

@@ -16,7 +16,6 @@
 </div>
 <form method="POST">
       <input title="test-sleep" type="submit" value="test-sleep" name="task">
-	  <input title="test-sleep" type="submit" value="queue-single" name="task">
 </form>
 <div class="divider"></div>
 <div class="row">