Quick change to load slugified files

ayta/__init__.py

@@ -39,7 +39,7 @@ def create_app(test_config=None):
     from .blueprints import search
     from .blueprints import channel
     from .blueprints import auth
-    from .blueprints import websub
+    from .blueprints import api
 
     app.register_blueprint(watch.bp)
     app.register_blueprint(index.bp)
@@ -47,6 +47,6 @@ def create_app(test_config=None):
     app.register_blueprint(search.bp)
     app.register_blueprint(channel.bp)
     app.register_blueprint(auth.bp)
-    app.register_blueprint(websub.bp)
+    app.register_blueprint(api.bp)
 
     return app

ayta/blueprints/admin.py

@@ -5,7 +5,7 @@ from ..dlp import checkChannelId, getChannelInfo
 from ..decorators import login_required
 from ..tasks import subscribe_websub_callback, unsubscribe_websub_callback
 from datetime import datetime
-import requests
+from secrets import token_urlsafe
 
 bp = Blueprint('admin', __name__, url_prefix='/admin')
 
@@ -150,6 +150,50 @@ def reports():
    
     return render_template('admin/reports.html', reports=reports)
 
+@bp.route('/posters', methods=['GET', 'POST'])
+@login_required
+def posters():
+    if request.method == 'POST':
+        task = request.form.get('task', None)
+        value = request.form.get('value', None)
+        
+        if task == 'add-endpoint':
+            description = request.form.get('description', None)
+            if not description or len(description) <= 7:
+                flash('Description must be at least 8 characters long')
+            
+            if value and len(value) >= 12:
+                get_nosql().poster_newEndpoint(value, description)
+                flash(f'Created endpoint ID: {value}')
+            else:
+                value = token_urlsafe(16)
+                get_nosql().poster_newEndpoint(value, description)
+                flash(f'Created endpoint ID: {value}')
+        elif task == 'retire':
+            get_nosql().poster_retireEndpoint(value)
+            flash(f'Endpoint retired: {value}')
+            
+        elif task == 'clean-retired':
+            get_nosql().poster_cleanRetired()
+            flash(f'Cleaned retired endpoints')
+            
+        elif task == 'manual-queue':
+            get_nosql().poster_insertQueue('manual', value)
+            flash(f'Added to queue: {value}')
+            
+        elif task == 'delete-queue':
+            get_nosql().poster_deleteQueue(value)
+            flash(f'Deleted from queue: {value}')
+        
+        return redirect(url_for('admin.posters'))
+        
+    endpoints = get_nosql().poster_getEndpoints()
+    queue = get_nosql().poster_getQueue()
+   
+    return render_template('admin/posters.html', endpoints=endpoints, queue=queue)
+
+
+
 @bp.route('/files', methods=['GET', 'POST'])
 @login_required
 def files():

ayta/blueprints/api.py

@@ -2,12 +2,12 @@ from flask import Blueprint, render_template, request, redirect, url_for, flash,
 from ..nosql import get_nosql
 from ..extensions import caching, caching_unless
 
-bp = Blueprint('websub', __name__, url_prefix='/websub')
+import re
 
-@bp.route('/c/<cap>', methods=['GET', 'POST'])
-# Caching GET requests should be save since this endpoint is used as a capability URL
-@caching.cached(unless=caching_unless)
-def callback(cap):
+bp = Blueprint('api', __name__, url_prefix='/api')
+
+@bp.route('/websub/<cap>', methods=['GET', 'POST'])
+def websub(cap):
     if request.method == 'GET':
         topic = request.args.get('hub.topic')
         challenge = request.args.get('hub.challenge')
@@ -37,4 +37,30 @@ def callback(cap):
             return abort(500)
         return '', 202
     
-    return abort(404)
+    return abort(404)
+    
+@bp.route('/poster/<cap>', methods=['POST'])
+def poster(cap):
+    # if endpoint does not exist
+    if not get_nosql().poster_isActive(cap):
+        return abort(404)
+    
+    videoId = request.form.get('v')
+    
+    # if request is not valid
+    if not videoId:
+        return abort(400)
+        
+    # if requested string is not correct
+    if not re.match(r"^[a-zA-Z0-9_-]{11}$", videoId):
+        return abort(422)
+        
+    # if given string is already in the archive
+    if get_nosql().check_exists(videoId):
+        return abort(409)
+    
+    # try to insert
+    if get_nosql().poster_insertQueue(cap, videoId):
+        return '', 202
+    else:
+        return abort(409)

ayta/blueprints/channel.py

@@ -41,8 +41,10 @@ def channel(channelId):
 def orphaned():
     videoIds = get_nosql().get_orphaned_videos()
     
-    videos = {}
+    videos = []
     for videoId in videoIds:
-        videos[videoId] = get_nosql().get_video_info(videoId, limited=True)
+        videos.append(get_nosql().get_video_info(videoId, limited=True))
+        
+    videos = sorted(videos, key=lambda x: x.get('epoch', 0), reverse=True)
 
     return render_template('channel/orphaned.html', videos=videos)

ayta/nosql.py

@@ -51,12 +51,13 @@ class Mango:
         self.db = self.client['ayta']
         self.channels = self.db['channels']
         self.info_json = self.db['info_json']
-        self.download_queue = self.db['download_queue']
+        self.posters_queue = self.db['posters_queue']
         self.run_log = self.db['run_log']
         self.channel_log = self.db['channel_log']
         self.websub_callbacks = self.db['websub_callbacks']
         self.websub_data = self.db['websub_data']
         self.reports = self.db['reports']
+        self.posters_endpoints = self.db['posters_endpoints']
             
         self.ensure_indexes()
         #self.clean_info_json()
@@ -112,13 +113,9 @@ class Mango:
         
         stats['videos'] = self.info_json.count_documents({})
         stats['channels'] = self.channels.count_documents({})
-        stats['queue'] = self.download_queue.count_documents({})
+        stats['queue'] = self.posters_queue.count_documents({})
         
         return stats
-    
-    def insert_download_queue(self, video):
-        if not self.download_queue.count_documents({'id': video}) >= 1:
-            return self.download_queue.insert_one({'id': video}).inserted_id
             
     def search_videos(self, query):
         # search the index for the requested query. return limited keys
@@ -202,7 +199,7 @@ class Mango:
         
     def get_video_info(self, videoId, limited=False):
         if limited:
-            projection = {'_id': 1, 'id': 1, 'title': 1, 'upload_date': 1, 'description': 1, 'channel_id': 1}
+            projection = {'_id': 1, 'id': 1, 'title': 1, 'upload_date': 1, 'description': 1, 'channel_id': 1, 'epoch': 1, 'title_slug': 1}
         else:
             projection = {}
         
@@ -262,7 +259,7 @@ class Mango:
     ##########################################
     
     def websub_newCallback(self, channelId):
-        callbackId = secrets.token_hex(8)
+        callbackId = secrets.token_urlsafe(16)
         
         self.websub_callbacks.insert_one({'id': callbackId, 'channel': channelId, 'status': 'new', 'created_time': current_time(object=True)})
         
@@ -336,10 +333,60 @@ class Mango:
     def websub_cleanRetired(self, days=3):
         days = self.datetime.utcnow() - self.timedelta(days=days)
 
-        self.websub_callbacks.delete_many({'status': 'retired', 'retiring_time': {'$lt': days}})
+        self.websub_callbacks.delete_many({'status': 'retired', 'retired_time': {'$lt': days}})
         
         return True
 
+    ##########################################
+    #           POSTER FUNCTIONS             #
+    ##########################################
+    
+    def poster_newEndpoint(self, endpointId, description=''):
+        self.posters_endpoints.insert_one({'id': endpointId, 'description': description, 'status': 'active', 'created_time': current_time(object=True)})
+        return endpointId
+        
+    def poster_insertQueue(self, endpointId, videoId):
+        # if no document exists
+        if not self.posters_queue.count_documents({'id': videoId}) >= 1:
+            self.posters_queue.insert_one({'id': videoId, 'endpoint': endpointId, 'created_time': current_time(object=True), 'status': 'queued'}).inserted_id
+            return True
+            
+        # key already in queue
+        return False
+        
+    def poster_deleteQueue(self, videoId):
+        if self.posters_queue.delete_one({'id': videoId}):
+            return True
+        return False
+
+    def poster_retireEndpoint(self, endpointId):
+        return self.posters_endpoints.update_one({'id': endpointId}, {'$set': {'status': 'retired', 'retired_time': current_time(object=True)}})
+        
+    def poster_isActive(self, endpointId):
+        status = self.posters_endpoints.find_one({'id': endpointId}, {'status': 1})
+
+        if not status:
+            return False
+            
+        status = status.get('status')
+        
+        if status == 'active':
+            return True
+            
+        return False
+        
+    def poster_getEndpoints(self):
+        return self.posters_endpoints.find({})
+        
+    def poster_getQueue(self):
+        return self.posters_queue.find({})
+        
+    def poster_cleanRetired(self, days=3):
+        days = self.datetime.utcnow() - self.timedelta(days=days)
+
+        self.posters_endpoints.delete_many({'status': 'retired', 'retired_time': {'$lt': days}})
+        
+        return True
 
 ##########################################
 #           HELPER FUNCTIONS             #

ayta/tasks.py

@@ -10,7 +10,7 @@ def subscribe_websub_callback(channelId):
     
     url = 'https://pubsubhubbub.appspot.com/subscribe'
     data = {
-        'hub.callback': f'https://{current_app.config["DOMAIN"]}/websub/c/{callbackId}',
+        'hub.callback': f'https://{current_app.config["DOMAIN"]}/api/websub//{callbackId}',
         'hub.topic': f'https://www.youtube.com/xml/feeds/videos.xml?channel_id={channelId}',
         'hub.verify': 'async',
         'hub.mode': 'subscribe',
@@ -32,7 +32,7 @@ def unsubscribe_websub_callback(callbackId, channelId):
     from .nosql import get_nosql
     
     url = 'https://pubsubhubbub.appspot.com/subscribe'
-    data = {'hub.callback': f'https://{current_app.config["DOMAIN"]}/websub/c/{callbackId}',
+    data = {'hub.callback': f'https://{current_app.config["DOMAIN"]}/api/websub/{callbackId}',
             'hub.topic': f'https://www.youtube.com/xml/feeds/videos.xml?channel_id={channelId}',
             'hub.verify': 'async',
             'hub.mode': 'unsubscribe'

ayta/templates/admin/index.html

@@ -65,5 +65,15 @@
       </div>
 	</a>
   </div>
+  <div class="col s6 l4 m-4">
+	<a href="{{ url_for('admin.posters') }}">
+	  <div class="card black-text">
+        <div class="card-content">
+          <span class="card-title">Posters</span>
+		  <p class="grey-text">User extension posters</p>
+        </div>
+      </div>
+	</a>
+  </div>
 </div>
 {% endblock %}

ayta/templates/admin/posters.html

@@ -0,0 +1,150 @@
+{% extends 'material_base.html' %}
+{% block title %}Posters administration page{% endblock %}
+{% block description %}Posters administration page of the AYTA system{% endblock %}
+
+{% block content %}
+<div class="row">
+  <div class="col s12 l11">
+    <h4>Posters administration page</h4>
+  </div>
+  <div class="col s12 l1 m-5">
+    <form method="POST">
+      <input title="Prunes all deleted endpoints, but keeps last 3 days" type="submit" value="clean-retired" name="task">
+    </form>
+  </div>
+</div>
+<div class="divider"></div>
+<div class="row">
+  <div class="col s12">
+	<h5>Poster options</h5>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12 l4 m-4">
+	<div class="card">
+      <div class="card-content">
+        <span class="card-title">Create new endpoint</span>
+	    <form method="post">
+		  <div class="row">
+            <div class="col s12 input-field">
+              <input placeholder="Custom endpoint" name="value" type="text" class="validate" minlength="12">
+			  <span class="supporting-text">Leaving this empty will create a random secure string</span>
+	        </div>
+			<div class="col s12 input-field">
+              <input placeholder="Description" name="description" type="text" class="validate" minlength="8" maxlength="64" required>
+			  <span class="supporting-text">Description for the endpoint for better administration</span>
+	        </div>
+          <button class="btn mt-4" type="submit" name="task" value="add-endpoint">Create</button>
+		  </div>
+	    </form>
+      </div>
+    </div>
+  </div>
+  <div class="col s12 l4 m-4">
+	<div class="card">
+      <div class="card-content">
+        <span class="card-title">Queue manually</span>
+	    <form method="post">
+		  <div class="row">
+            <div class="col s12 input-field">
+              <input placeholder="Youtube video ID" name="value" type="text" class="validate" minlength="11" maxlength="11" required>
+			  <span class="supporting-text">Must be a valid Youtube video ID</span>
+	        </div>
+			<div class="col s12 mt-5 input-field">
+              <div class="switch">
+		        <label>Queue<input type="checkbox" value="direct" name="value" disabled><span class="lever"></span>Direct</label>
+				<span class="supporting-text">Queue up or start directly</span>
+		      </div>
+	        </div>
+          <button class="btn mt-4" type="submit" name="task" value="manual-queue">Queue</button>
+		  </div>
+	    </form>
+      </div>
+    </div>
+  </div>
+</div>
+<div class="divider"></div>
+<div class="row">
+  <div class="col s6 l9">
+	<h5>Registered endpoints</h5>
+  </div>
+  <div class="col s6 l3 m-4 input-field">
+	<input id="filter_query" type="text">
+    <label for="filter_query">Filter results</label>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12">
+    <table class="striped highlight responsive-table">
+      <thead>
+        <tr>
+		  <th>Actions</th>
+		  <th>id</th>
+		  <th>description</th>
+		  <th>status</th>
+		  <th>created_time</th>
+		  <th>retired_time</th>
+        </tr>
+      </thead>
+      <tbody>
+	    {% for endpoint in endpoints %}
+        <tr class="filterable">
+		  <td>
+		    <form method="post">
+			  <input type="text" value="{{ endpoint.get('id') }}" name="value" hidden>
+		      <button class="btn-small waves-effect waves-light" type="submit" name="task" value="retire" title="Retire endpoint" {% if endpoint.get('status') != 'active' %}disabled{% endif %}>🗑️</button>
+			</form>
+		  </td>
+		  <td>{{ endpoint.get('id') }}</td>
+		  <td>{{ endpoint.get('description') }}</td>
+		  <td>{{ endpoint.get('status') }}</td>
+		  <td>{{ endpoint.get('created_time') }}</td>
+		  <td>{{ endpoint.get('retired_time') }}</td>
+        </tr>
+		{% endfor %}
+      </tbody>
+    </table>
+  </div>
+</div>
+<div class="divider"></div>
+<div class="row">
+  <div class="col s6 l9">
+	<h5>Queued ID's</h5>
+  </div>
+  <div class="col s6 l3 m-4 input-field">
+	<input id="filter_query" type="text">
+    <label for="filter_query">Filter results</label>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12">
+    <table class="striped highlight responsive-table">
+      <thead>
+        <tr>
+		  <th>Actions</th>
+		  <th>id</th>
+		  <th>endpoint</th>
+		  <th>status</th>
+		  <th>created_time</th>
+        </tr>
+      </thead>
+      <tbody>
+	    {% for id in queue %}
+        <tr class="filterable">
+		  <td>
+		    <form method="post">
+			  <input type="text" value="{{ id.get('id') }}" name="value" hidden>
+		      <button class="btn-small waves-effect waves-light" type="submit" name="task" value="delete-queue" title="Delete from queue" {% if id.get('status') != 'queued' %}disabled{% endif %}>🗑️</button>
+			</form>
+		  </td>
+		  <td>{{ id.get('id') }}</td>
+		  <td>{{ id.get('endpoint') }}</td>
+		  <td>{{ id.get('status') }}</td>
+		  <td>{{ id.get('created_time') }}</td>
+        </tr>
+		{% endfor %}
+      </tbody>
+    </table>
+  </div>
+</div>
+{% endblock %}

ayta/templates/channel/channel.html

@@ -25,7 +25,7 @@
 	<div class="card medium black-text">
 	  <a href="{{ url_for('watch.base') }}?v={{ video.get('id') }}">
 	    <div class="card-image">
-		  <img loading="lazy" src="https://archive.ventilaar.net/videos/automatic/{{ video.get('channel_id') }}/{{ video.get('id') }}/{{ video.get('title') }}.jpg">
+		  <img loading="lazy" src="https://archive.ventilaar.net/videos/automatic/{{ video.get('channel_id') }}/{{ video.get('id') }}/{{ video.get('title_slug') }}.jpg">
         </div>
 	  </a>
       <div class="card-content activator">

ayta/templates/channel/orphaned.html

@@ -12,7 +12,7 @@
 <div class="row">
   <div class="col s6 l9">
 	<h5>Orphaned videos</h5>
-	<p>Videos in the archive which do not have a permanent channel linked. There is a high chance that the videos are manually downloaded.</p>
+	<p>Videos in the archive which do not have a permanent channel linked. There is a high chance that the videos are manually downloaded. Sorted by last added.</p>
   </div>
   <div class="col s6 l3 m-4 input-field">
 	<input id="filter_query" type="text">
@@ -23,18 +23,18 @@
   {% for video in videos %}
   <div class="col s6 l4 m-4 filterable">
 	<div class="card medium black-text">
-	  <a href="{{ url_for('watch.base') }}?v={{ video }}">
+	  <a href="{{ url_for('watch.base') }}?v={{ video.get('id') }}">
 	    <div class="card-image">
-		  <img loading="lazy" src="https://archive.ventilaar.net/videos/automatic/{{ videos[video].get('channel_id') }}/{{ videos[video].get('id') }}/{{ videos[video].get('title') }}.jpg">
+		  <img loading="lazy" src="https://archive.ventilaar.net/videos/automatic/{{ video.get('channel_id') }}/{{ video.get('id') }}/{{ video.get('title') }}.jpg">
         </div>
 	  </a>
       <div class="card-content activator">
-        <span class="card-title">{{ videos[video].get('title') }}</span>
-		<p class="grey-text">{{ videos[video].get('id') }} | {{ videos[video].get('upload_date')|pretty_time }}</p>
+        <span class="card-title">{{ video.get('title') }}</span>
+		<p class="grey-text">{{ video.get('id') }} | {{ video.get('upload_date')|pretty_time }}</p>
       </div>
       <div class="card-reveal">
-        <span class="card-title truncate">{{ videos[video].get('title') }}</span>
-        <p style="white-space: pre-wrap;">{{ videos[video].get('description') }}</p>
+        <span class="card-title truncate">{{ video.get('title') }}</span>
+        <p style="white-space: pre-wrap;">{{ video.get('description') }}</p>
       </div>
 	</div>
   </div>

ayta/templates/watch/index.html

@@ -31,8 +31,8 @@
 <div class="row">
   <div class="col s12 center-align">
     <video controls class="responsive-video">
-    <source src="https://archive.ventilaar.net/videos/automatic/{{ render.get('info').get('channel_id') }}/{{ render.get('info').get('id') }}/{{ render.get('info').get('title') }}.mp4">
-	<source src="https://archive.ventilaar.net/videos/automatic/{{ render.get('info').get('channel_id') }}/{{ render.get('info').get('id') }}/{{ render.get('info').get('title') }}.webm">
+    <source src="https://archive.ventilaar.net/videos/automatic/{{ render.get('info').get('channel_id') }}/{{ render.get('info').get('id') }}/{{ render.get('info').get('title_slug') }}.mp4">
+	<source src="https://archive.ventilaar.net/videos/automatic/{{ render.get('info').get('channel_id') }}/{{ render.get('info').get('id') }}/{{ render.get('info').get('title_slug') }}.webm">
     Your browser does not support the video tag.
     </video>
   </div>