From e532c45a9227975fd749662f39834e91f5003e2a Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Tue, 25 Feb 2025 16:23:31 -0500
Subject: [PATCH 01/64] Add video proxying support

Requires S3 setup
---
 app.py           | 171 ++++++++++++++++++++++++++++++++++++++++++++++-
 requirements.txt |   3 +
 2 files changed, 172 insertions(+), 2 deletions(-)

diff --git a/app.py b/app.py
index 4161969..3bf55c0 100644
--- a/app.py
+++ b/app.py
@@ -7,13 +7,25 @@ from flask import Flask, Response
 from diskcache import Cache
 import logging
 
+import threading
+import time
+import tempfile
+import nndownload
+import boto3
+from botocore.client import Config as BotoConfig
+
+
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
-
 app = Flask(__name__)
 
+S3_BUCKET_NAME = os.environ.get('NICONICOGAY_S3_BUCKET_NAME')
+S3_REGION = os.environ.get('NICONICOGAY_S3_REGION')
+CDN_BASE_URL = os.environ.get('NICONICOGAY_CDN_BASE_URL')
+MAX_CONCURRENT_DOWNLOADS = 3
 CACHE_EXPIRATION_SECONDS = 3600  # 1 hour
 CACHE_SIZE_LIMIT = 100 * 1024 * 1024  # 100 MB
+
 cache = None if os.environ.get('NICONICOGAY_DISABLE_CACHE', '') != '' else Cache("disk_cache", size_limit=CACHE_SIZE_LIMIT)
 
 cookie_jar = http.cookiejar.MozillaCookieJar('cookies.txt')
@@ -24,10 +36,147 @@ except FileNotFoundError:
 
 s = requests.Session()
 s.headers.update({
-    "User-Agent": "Twitterbot/1.0"
+    "User-Agent": os.environ.get('NICONICOGAY_USER_AGENT', 'Twitterbot/1.0')
 })
 s.cookies = cookie_jar  # type: ignore
 
+if all(key in os.environ for key in [
+    'NICONICOGAY_S3_ACCESS_KEY',
+    'NICONICOGAY_S3_SECRET_KEY',
+]):
+    s3_session = boto3.Session()
+    s3_client = s3_session.client(
+        's3',
+        aws_access_key_id=os.environ['NICONICOGAY_S3_ACCESS_KEY'],
+        aws_secret_access_key=os.environ['NICONICOGAY_S3_SECRET_KEY'],
+        region_name=S3_REGION,
+        endpoint_url=f"https://{S3_REGION}.digitaloceanspaces.com",
+        config=BotoConfig(s3={'addressing_style': 'virtual'}),
+    )
+else:
+    logger.warning("S3 credentials not provided, exiting")
+    exit(1)
+
+download_tracker = {
+    'active_downloads': 0,
+    'in_progress': set(),
+}
+download_lock = threading.Lock()
+download_queue = []
+
+def download_and_upload_video(video_id, url, video_quality):
+    try:
+        with download_lock:
+            download_tracker['active_downloads'] += 1
+            download_tracker['in_progress'].add(video_id)
+        
+        with tempfile.NamedTemporaryFile(suffix='.mp4', delete=True) as temp_file:
+            temp_path = temp_file.name
+        
+        try:
+            logger.info(f"Starting download for video ID: {video_id}")
+            nndownload.execute(
+                "--no-login",
+                "--user-agent", "Googlebot/2.1",
+                "--video-quality", video_quality,
+                "--output-path", temp_path,
+                url
+            )
+            
+            if os.path.exists(temp_path) and s3_client:
+                logger.info(f"Downloaded video {video_id}, uploading to CDN")
+                try:
+                    s3_key = f"niconico/{video_id}.mp4"
+                    s3_client.upload_file(
+                        temp_path, 
+                        S3_BUCKET_NAME, 
+                        s3_key,
+                        ExtraArgs={'ContentType': 'video/mp4', 'ACL': 'public-read'}
+                    )
+                    
+                    logger.info(f"Successfully uploaded video {video_id} to CDN")
+                    
+                    # Clear cache for this video to ensure next view gets updated HTML
+                    if cache:
+                        cache.delete(video_id)
+                        logger.info(f"Cleared cache for video ID: {video_id}")
+                    
+                    return True
+                except Exception as e:
+                    logger.error(f"Error uploading video {video_id} to CDN: {e}")
+                    return False
+            else:
+                logger.error(f"Failed to download video {video_id} or S3 client not configured")
+                return False
+        finally:
+            if os.path.exists(temp_path):
+                os.unlink(temp_path)
+                logger.info(f"Removed temporary file: {temp_path}")
+    except Exception as e:
+        logger.error(f"Error in download process for video {video_id}: {e}")
+        return False
+    finally:
+        with download_lock:
+            download_tracker['active_downloads'] -= 1
+            download_tracker['in_progress'].discard(video_id)
+
+def download_worker():
+    while True:
+        try:
+            with download_lock:
+                can_download = download_tracker['active_downloads'] < MAX_CONCURRENT_DOWNLOADS
+                queue_has_items = len(download_queue) > 0
+            
+            if queue_has_items and can_download:
+                with download_lock:
+                    # Get next video that is not already being downloaded
+                    for i, (video_id, _, _) in enumerate(download_queue):
+                        if video_id not in download_tracker['in_progress']:
+                            video_info = download_queue.pop(i)
+                            threading.Thread(target=download_and_upload_video, 
+                                            args=(video_info[0], video_info[1], video_info[2])).start()
+                            break
+            time.sleep(1)
+        except Exception as e:
+            logger.error(f"Error in download worker: {e}")
+            time.sleep(5)  # Back off in case of error
+
+worker_thread = threading.Thread(target=download_worker, daemon=True)
+worker_thread.start()
+
+def is_video_in_cdn(video_id):
+    """Check if video exists in CDN"""
+    if not s3_client:
+        return False
+        
+    try:
+        s3_client.head_object(Bucket=S3_BUCKET_NAME, Key=f"niconico/{video_id}.mp4")
+        return True
+    except Exception:
+        return False
+    
+def is_video_being_downloaded(video_id):
+    """Check if video is currently being downloaded"""
+    with download_lock:
+        return video_id in download_tracker['in_progress']
+
+def get_cdn_url(video_id):
+    """Get the CDN URL for a video"""
+    return f"{CDN_BASE_URL}/niconico/{video_id}.mp4"
+
+def allow_download(params):
+    if params['video']['duration'] > 60 * 15:
+        return False
+    return True
+
+def get_video_quality(params, quality_level_threshold=3):
+    """Get the code of the best video quality available (optionally below a certain threshold)"""
+    videos = params['media']['domand']['videos']
+    eligible_videos = [v for v in videos if v['qualityLevel'] < quality_level_threshold]
+    if not eligible_videos:
+        return None
+    return str(max(eligible_videos, key=lambda x: int(x['qualityLevel']))['id'])
+
 @app.route("/watch/<video_id>")
 def proxy(video_id):
     logger.info(f"Received request for video ID: {video_id}")
@@ -43,6 +192,7 @@ def proxy(video_id):
     try:
         logger.info(f"Fetching content from URL: {real_url}")
         r = s.get(real_url, timeout=10)
+        r.raise_for_status()
     except requests.RequestException as e:
         logger.error(f"Error fetching the page for video ID '{video_id}': {e}")
         return Response(status=500)
@@ -64,11 +214,28 @@ def proxy(video_id):
         logger.warning(f"Failed to extract thumbnail info for video ID '{video_id}': {e}")
         pass
 
+    download_allowed = allow_download(params) if params else False
+    video_quality = get_video_quality(params) if params else None
+    if download_allowed and video_quality is not None:
+        video_in_cdn = is_video_in_cdn(video_id)
+        video_in_progress = is_video_being_downloaded(video_id)
+        if not video_in_cdn and not video_in_progress and s3_client:
+            with download_lock:
+                # Add to queue if not already in it
+                queue_video_ids = [item[0] for item in download_queue]
+                if video_id not in queue_video_ids:
+                    download_queue.append((video_id, real_url, video_quality))
+                    logger.info(f"Queued video ID {video_id} for download")
+
+    cdn_video_url = get_cdn_url(video_id)
     og_tags = soup.find_all("meta", property=lambda x: x)  # type: ignore
     for tag in og_tags:
         # Fix thumbnail
         if tag.get("property") == "og:image" and thumbnail_url:
             tag["content"] = thumbnail_url
+        # Fix video URL
+        if tag.get("property") == "og:video:url" or tag.get("property") == "og:video:secure_url":
+            tag["content"] = cdn_video_url
 
     og_tags_str = "\n".join(str(tag) for tag in og_tags)
     html_response = f"""
diff --git a/requirements.txt b/requirements.txt
index 01f2ec1..25ba17a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,6 @@
 beautifulsoup4==4.12.3
 Flask==3.1.0
 Requests==2.32.3
+diskcache==5.6.3
+nndownload==1.19
+boto3
\ No newline at end of file

From add8f1bbde14ea2fb32132a4025ac68e9c01b6eb Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Tue, 25 Feb 2025 16:28:12 -0500
Subject: [PATCH 02/64] Load .env

---
 app.py           | 2 ++
 requirements.txt | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 3bf55c0..e486fd9 100644
--- a/app.py
+++ b/app.py
@@ -14,6 +14,8 @@ import nndownload
 import boto3
 from botocore.client import Config as BotoConfig
 
+from dotenv import load_dotenv
+load_dotenv()
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
diff --git a/requirements.txt b/requirements.txt
index 25ba17a..7bc5696 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,4 +3,5 @@ Flask==3.1.0
 Requests==2.32.3
 diskcache==5.6.3
 nndownload==1.19
-boto3
\ No newline at end of file
+boto3
+python-dotenv

From e9eb88c13fee213db00d96d1d0ec7c5781bc33d1 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Tue, 25 Feb 2025 17:23:50 -0500
Subject: [PATCH 03/64] Add more meta tags

---
 app.py | 36 +++++++++++++++++++++++++-----------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/app.py b/app.py
index e486fd9..f824be3 100644
--- a/app.py
+++ b/app.py
@@ -231,6 +231,8 @@ def proxy(video_id):
 
     cdn_video_url = get_cdn_url(video_id)
     og_tags = soup.find_all("meta", property=lambda x: x)  # type: ignore
+    og_video_width = None
+    og_video_height = None
     for tag in og_tags:
         # Fix thumbnail
         if tag.get("property") == "og:image" and thumbnail_url:
@@ -238,19 +240,31 @@ def proxy(video_id):
         # Fix video URL
         if tag.get("property") == "og:video:url" or tag.get("property") == "og:video:secure_url":
             tag["content"] = cdn_video_url
+        # Set vars
+        if tag.get("property") == "og:video:width":
+            og_video_width = tag.get("content")
+        if tag.get("property") == "og:video:height":
+            og_video_height = tag.get("content")
 
     og_tags_str = "\n".join(str(tag) for tag in og_tags)
-    html_response = f"""
-        <!DOCTYPE html>
-        <html lang="en">
-        <head>
-            <meta charset="UTF-8">
-            {og_tags_str}
-        </head>
-        <body>
-        </body>
-        </html>
-    """
+    og_tags_str += '\n<meta content="0" property="twitter:image"/>'
+    og_tags_str += '\n<meta content="player" property="twitter:card"/>'
+    og_tags_str += f'\n<meta content="{cdn_video_url}" property="twitter:player:stream"/>'
+    if og_video_width:
+        og_tags_str += f'\n<meta content="{og_video_width}" property="twitter:player:width"/>'
+    if og_video_height:
+        og_tags_str += f'\n<meta content="{og_video_height}" property="twitter:player:height"/>'
+    html_response = f"""<!DOCTYPE html>
+<!--
+niconico proxy - brought to you by https://mmaker.moe
+
+this service is intended to be used by social media open graph embed generators and discordbot.
+please do not abuse! the videos returned by the CDN are lower quality and intended to only be proxied by discord, not hotlinked.
+if you want to download videos, please consider using a tool like nndownload: https://github.com/AlexAplin/nndownload
+-->
+<html lang="en"><head><meta charset="UTF-8">
+{og_tags_str}
+</head><body></body></html>"""
 
     if cache:
         logging.info(f"Caching response for video ID: {video_id}")

From e48159ce14e6e89676484c2e15fb4c93e5dcc6a5 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Tue, 25 Feb 2025 17:56:06 -0500
Subject: [PATCH 04/64] owoembed

(oEmbed)
---
 app.py | 124 +++++++++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 99 insertions(+), 25 deletions(-)

diff --git a/app.py b/app.py
index f824be3..94e3045 100644
--- a/app.py
+++ b/app.py
@@ -3,7 +3,7 @@ import http.cookiejar
 import json
 import requests
 from bs4 import BeautifulSoup
-from flask import Flask, Response
+from flask import Flask, Response, request, jsonify
 from diskcache import Cache
 import logging
 
@@ -13,6 +13,7 @@ import tempfile
 import nndownload
 import boto3
 from botocore.client import Config as BotoConfig
+import urllib.parse
 
 from dotenv import load_dotenv
 load_dotenv()
@@ -21,6 +22,7 @@ logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 app = Flask(__name__)
 
+HOST = os.environ.get('NICONICOGAY_HOST', 'https://nicovideo.gay')
 S3_BUCKET_NAME = os.environ.get('NICONICOGAY_S3_BUCKET_NAME')
 S3_REGION = os.environ.get('NICONICOGAY_S3_REGION')
 CDN_BASE_URL = os.environ.get('NICONICOGAY_CDN_BASE_URL')
@@ -179,6 +181,58 @@ def get_video_quality(params, quality_level_threshold=3):
         return None
     return str(max(eligible_videos, key=lambda x: int(x['qualityLevel']))['id'])
 
+def get_data(video_id, real_url):
+    """Get the server response for a given video ID"""
+    try:
+        logger.info(f"Fetching content from URL: {real_url}")
+        r = s.get(real_url, timeout=10)
+        r.raise_for_status()
+    except requests.RequestException as e:
+        logger.error(f"Error fetching the page for video ID '{video_id}': {e}")
+        return None, None
+
+    soup = BeautifulSoup(r.text, "html.parser")
+    try:
+        server_response = soup.find("meta", {"name": "server-response"})
+        if server_response:
+            params = json.loads(server_response["content"])["data"]["response"]  # type: ignore
+            return params, soup
+    except (KeyError, json.JSONDecodeError) as e:
+        logger.warning(f"Failed to extract thumbnail info for video ID '{video_id}': {e}")
+        pass
+
+    return None, soup
+
+def get_oembed_url(params):
+    """Get the oEmbed (/owoembed) URL based on the given params (server response)"""
+    author_id = params.get('owner', {}).get('id')
+    author_name = params.get('owner', {}).get('nickname')
+    video_id = params.get('video', {}).get('id')
+
+    if not video_id:
+        return None
+
+    view_count = str(params.get('video', {}).get('count', {}).get('view')) or "n/a"
+    comment_count = str(params.get('video', {}).get('count', {}).get('comment')) or "n/a"
+    like_count = str(params.get('video', {}).get('count', {}).get('like')) or "n/a"
+    mylist_count = str(params.get('video', {}).get('count', {}).get('mylist')) or "n/a"
+    provder_stats = f"👁️ {view_count}   💬 {comment_count}   ❤️ {like_count}   📋 {mylist_count}"
+
+    author_name_encoded = urllib.parse.quote(author_name)
+    provider_stats_encoded = urllib.parse.quote(provder_stats)
+
+    oembed_url = (
+        f"{HOST}/owoembed?"
+        f"author_id={author_id}&"
+        f"author_name={author_name_encoded}&"
+        f"video_id={video_id}&"
+        f"provider={provider_stats_encoded}"
+    )
+
+    return oembed_url
+
+
+
 @app.route("/watch/<video_id>")
 def proxy(video_id):
     logger.info(f"Received request for video ID: {video_id}")
@@ -191,30 +245,14 @@ def proxy(video_id):
 
     # Not in cache or cache expired; fetch from nicovideo.jp
     real_url = f"https://www.nicovideo.jp/watch/{video_id}"
-    try:
-        logger.info(f"Fetching content from URL: {real_url}")
-        r = s.get(real_url, timeout=10)
-        r.raise_for_status()
-    except requests.RequestException as e:
-        logger.error(f"Error fetching the page for video ID '{video_id}': {e}")
-        return Response(status=500)
-
-    soup = BeautifulSoup(r.text, "html.parser")
-    thumbnail_url = None
-    try:
-        server_response = soup.find("meta", {"name": "server-response"})
-        if server_response:
-            params = json.loads(server_response["content"])["data"]["response"]  # type: ignore
-            thumbnail_url = (
-                params["video"]["thumbnail"].get("ogp") or
-                params["video"]["thumbnail"].get("player") or
-                params["video"]["thumbnail"].get("largeUrl") or
-                params["video"]["thumbnail"].get("middleUrl") or
-                params["video"]["thumbnail"].get("url")
-            )
-    except (KeyError, json.JSONDecodeError) as e:
-        logger.warning(f"Failed to extract thumbnail info for video ID '{video_id}': {e}")
-        pass
+    params, soup = get_data(video_id, real_url)
+    thumbnail_url = (
+        params["video"]["thumbnail"].get("ogp") or
+        params["video"]["thumbnail"].get("player") or
+        params["video"]["thumbnail"].get("largeUrl") or
+        params["video"]["thumbnail"].get("middleUrl") or
+        params["video"]["thumbnail"].get("url")
+    ) if params else None
 
     download_allowed = allow_download(params) if params else False
     video_quality = get_video_quality(params) if params else None
@@ -254,6 +292,7 @@ def proxy(video_id):
         og_tags_str += f'\n<meta content="{og_video_width}" property="twitter:player:width"/>'
     if og_video_height:
         og_tags_str += f'\n<meta content="{og_video_height}" property="twitter:player:height"/>'
+    og_tags_str += f'\n<link rel="alternate" href="{get_oembed_url(params)}" type="application/json+oembed" title="{video_id}"/>'
     html_response = f"""<!DOCTYPE html>
 <!--
 niconico proxy - brought to you by https://mmaker.moe
@@ -271,3 +310,38 @@ if you want to download videos, please consider using a tool like nndownload: ht
         cache.set(video_id, html_response, expire=CACHE_EXPIRATION_SECONDS)
 
     return Response(html_response, mimetype="text/html")
+
+@app.route("/owoembed")
+def owoembed():
+    """
+    Handles oEmbed requests with parameters in the URL
+    Returns JSON payload in oEmbed format
+    """
+    logger.info("Received request for oEmbed endpoint")
+    
+    # Get parameters from query string
+    author_id = request.args.get('author_id', '')
+    author_name = request.args.get('author_name', '')
+    video_id = request.args.get('video_id', '')
+    provider = request.args.get('provider', '')
+    
+    author_name_decoded = urllib.parse.unquote(author_name)
+    provider_decoded = urllib.parse.unquote(provider)
+    
+    # Create the author_url and provider_url
+    author_url = f"https://www.nicovideo.jp/user/{author_id}"
+    video_url = f"https://www.nicovideo.jp/watch/{video_id}"
+    
+    # Create oEmbed response
+    oembed_response = {
+        "author_name": author_name_decoded,
+        "author_url": author_url,
+        "provider_name": provider_decoded,
+        "provider_url": video_url,
+        "title": "Embed",
+        "type": "link",
+        "version": "1.0"
+    }
+    
+    logger.info(f"Returning oEmbed response for video ID: {video_id}")
+    return jsonify(oembed_response)

From be23df35916e74e389d61f24fe3ed87a86b4a4b6 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Tue, 25 Feb 2025 18:04:59 -0500
Subject: [PATCH 05/64] Meta tag cleanup

Maybe fixes site title not displaying?
---
 app.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/app.py b/app.py
index 94e3045..6be13c0 100644
--- a/app.py
+++ b/app.py
@@ -205,6 +205,9 @@ def get_data(video_id, real_url):
 
 def get_oembed_url(params):
     """Get the oEmbed (/owoembed) URL based on the given params (server response)"""
+    if not params:
+        return None
+
     author_id = params.get('owner', {}).get('id')
     author_name = params.get('owner', {}).get('nickname')
     video_id = params.get('video', {}).get('id')
@@ -272,6 +275,8 @@ def proxy(video_id):
     og_video_width = None
     og_video_height = None
     for tag in og_tags:
+        if 'data-server' in tag.attrs:
+            del tag.attrs['data-server']
         # Fix thumbnail
         if tag.get("property") == "og:image" and thumbnail_url:
             tag["content"] = thumbnail_url
@@ -287,6 +292,7 @@ def proxy(video_id):
     og_tags_str = "\n".join(str(tag) for tag in og_tags)
     og_tags_str += '\n<meta content="0" property="twitter:image"/>'
     og_tags_str += '\n<meta content="player" property="twitter:card"/>'
+    og_tags_str += '\n<meta content="video/mp4" property="twitter:player:stream:content_type"/>'
     og_tags_str += f'\n<meta content="{cdn_video_url}" property="twitter:player:stream"/>'
     if og_video_width:
         og_tags_str += f'\n<meta content="{og_video_width}" property="twitter:player:width"/>'

From 3201aea8561714ca35ad3a9b5df58538303bcdd6 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Tue, 25 Feb 2025 18:06:46 -0500
Subject: [PATCH 06/64] Change mylist emoji

---
 app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 6be13c0..c6b8f82 100644
--- a/app.py
+++ b/app.py
@@ -219,7 +219,7 @@ def get_oembed_url(params):
     comment_count = str(params.get('video', {}).get('count', {}).get('comment')) or "n/a"
     like_count = str(params.get('video', {}).get('count', {}).get('like')) or "n/a"
     mylist_count = str(params.get('video', {}).get('count', {}).get('mylist')) or "n/a"
-    provder_stats = f"👁️ {view_count}   💬 {comment_count}   ❤️ {like_count}   📋 {mylist_count}"
+    provder_stats = f"👁️ {view_count}   💬 {comment_count}   ❤️ {like_count}   📝 {mylist_count}"
 
     author_name_encoded = urllib.parse.quote(author_name)
     provider_stats_encoded = urllib.parse.quote(provder_stats)

From b3539d7a47ea15b9bc4a40a19decc9c99da94380 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Tue, 25 Feb 2025 18:08:15 -0500
Subject: [PATCH 07/64] Format numbers

---
 app.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/app.py b/app.py
index c6b8f82..bfc9637 100644
--- a/app.py
+++ b/app.py
@@ -203,6 +203,17 @@ def get_data(video_id, real_url):
 
     return None, soup
 
+def human_format(num):
+    """Format a number in a human-readable way (e.g., 1K, 2M, etc.)"""
+    if num is None:
+        return None
+    num = float('{:.3g}'.format(num))
+    magnitude = 0
+    while abs(num) >= 1000:
+        magnitude += 1
+        num /= 1000.0
+    return '{}{}'.format('{:f}'.format(num).rstrip('0').rstrip('.'), ['', 'K', 'M', 'B', 'T'][magnitude])
+
 def get_oembed_url(params):
     """Get the oEmbed (/owoembed) URL based on the given params (server response)"""
     if not params:
@@ -215,10 +226,10 @@ def get_oembed_url(params):
     if not video_id:
         return None
 
-    view_count = str(params.get('video', {}).get('count', {}).get('view')) or "n/a"
-    comment_count = str(params.get('video', {}).get('count', {}).get('comment')) or "n/a"
-    like_count = str(params.get('video', {}).get('count', {}).get('like')) or "n/a"
-    mylist_count = str(params.get('video', {}).get('count', {}).get('mylist')) or "n/a"
+    view_count = human_format(params.get('video', {}).get('count', {}).get('view')) or "n/a"
+    comment_count = human_format(params.get('video', {}).get('count', {}).get('comment')) or "n/a"
+    like_count = human_format(params.get('video', {}).get('count', {}).get('like')) or "n/a"
+    mylist_count = human_format(params.get('video', {}).get('count', {}).get('mylist')) or "n/a"
     provder_stats = f"👁️ {view_count}   💬 {comment_count}   ❤️ {like_count}   📝 {mylist_count}"
 
     author_name_encoded = urllib.parse.quote(author_name)

From 8ebacc84b01c9ae05ec89d546c248ef7b19be496 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Tue, 25 Feb 2025 18:10:12 -0500
Subject: [PATCH 08/64] Try to fix site name again

---
 app.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app.py b/app.py
index bfc9637..5f37a8c 100644
--- a/app.py
+++ b/app.py
@@ -300,7 +300,7 @@ def proxy(video_id):
         if tag.get("property") == "og:video:height":
             og_video_height = tag.get("content")
 
-    og_tags_str = "\n".join(str(tag) for tag in og_tags)
+    og_tags_str = "\n".join(str(tag) for tag in og_tags if tag.get("property") not in ["og:site_name"])
     og_tags_str += '\n<meta content="0" property="twitter:image"/>'
     og_tags_str += '\n<meta content="player" property="twitter:card"/>'
     og_tags_str += '\n<meta content="video/mp4" property="twitter:player:stream:content_type"/>'
@@ -309,6 +309,7 @@ def proxy(video_id):
         og_tags_str += f'\n<meta content="{og_video_width}" property="twitter:player:width"/>'
     if og_video_height:
         og_tags_str += f'\n<meta content="{og_video_height}" property="twitter:player:height"/>'
+    og_tags_str += '\n<meta content="ニコニコ動画" property="og:site_name"/>'
     og_tags_str += f'\n<link rel="alternate" href="{get_oembed_url(params)}" type="application/json+oembed" title="{video_id}"/>'
     html_response = f"""<!DOCTYPE html>
 <!--

From c456200ae00f27fdad110843800f20f3fcabef6b Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Tue, 25 Feb 2025 18:12:13 -0500
Subject: [PATCH 09/64] Handle unknown videos

---
 app.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 5f37a8c..b6bf674 100644
--- a/app.py
+++ b/app.py
@@ -260,6 +260,11 @@ def proxy(video_id):
     # Not in cache or cache expired; fetch from nicovideo.jp
     real_url = f"https://www.nicovideo.jp/watch/{video_id}"
     params, soup = get_data(video_id, real_url)
+
+    if not params and not soup:
+        logger.error(f"Failed to retrieve data for video ID '{video_id}'")
+        return Response("Video not found", status=404)
+
     thumbnail_url = (
         params["video"]["thumbnail"].get("ogp") or
         params["video"]["thumbnail"].get("player") or
@@ -282,7 +287,7 @@ def proxy(video_id):
                     logger.info(f"Queued video ID {video_id} for download")
 
     cdn_video_url = get_cdn_url(video_id)
-    og_tags = soup.find_all("meta", property=lambda x: x)  # type: ignore
+    og_tags = soup.find_all("meta", property=lambda x: x)
     og_video_width = None
     og_video_height = None
     for tag in og_tags:

From 77f9545db612f804427ed75bf6355fe1ea730f48 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Tue, 25 Feb 2025 18:21:29 -0500
Subject: [PATCH 10/64] Fixes

---
 app.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/app.py b/app.py
index b6bf674..d202673 100644
--- a/app.py
+++ b/app.py
@@ -219,8 +219,11 @@ def get_oembed_url(params):
     if not params:
         return None
 
-    author_id = params.get('owner', {}).get('id')
-    author_name = params.get('owner', {}).get('nickname')
+    author_id = None
+    author_name = None
+    if params.get('owner'):
+        author_id = params['owner'].get('id')
+        author_name = params['owner'].get('nickname')
     video_id = params.get('video', {}).get('id')
 
     if not video_id:
@@ -232,12 +235,12 @@ def get_oembed_url(params):
     mylist_count = human_format(params.get('video', {}).get('count', {}).get('mylist')) or "n/a"
     provder_stats = f"👁️ {view_count}   💬 {comment_count}   ❤️ {like_count}   📝 {mylist_count}"
 
-    author_name_encoded = urllib.parse.quote(author_name)
+    author_name_encoded = urllib.parse.quote(author_name) if author_name else ""
     provider_stats_encoded = urllib.parse.quote(provder_stats)
 
     oembed_url = (
         f"{HOST}/owoembed?"
-        f"author_id={author_id}&"
+        f"author_id={author_id if author_id else ''}&"
         f"author_name={author_name_encoded}&"
         f"video_id={video_id}&"
         f"provider={provider_stats_encoded}"

From 5a194507a0e777a33b98318d212d66bcddebc338 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Wed, 26 Feb 2025 00:31:51 -0500
Subject: [PATCH 11/64] Get correct video dimensions

---
 app.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/app.py b/app.py
index d202673..3d92c31 100644
--- a/app.py
+++ b/app.py
@@ -173,6 +173,12 @@ def allow_download(params):
         return False
     return True
 
+def get_video_resolution(params):
+    if not params:
+        return None, None
+    video = params['media']['domand']['videos'][0]
+    return video['width'], video['height']
+
 def get_video_quality(params, quality_level_threshold=3):
     """Get the code of the best video quality available (optionally below a certain threshold)"""
     videos = params['media']['domand']['videos']
@@ -275,6 +281,7 @@ def proxy(video_id):
         params["video"]["thumbnail"].get("middleUrl") or
         params["video"]["thumbnail"].get("url")
     ) if params else None
+    video_width, video_height = get_video_resolution(params) if params else (None, None)
 
     download_allowed = allow_download(params) if params else False
     video_quality = get_video_quality(params) if params else None
@@ -291,8 +298,6 @@ def proxy(video_id):
 
     cdn_video_url = get_cdn_url(video_id)
     og_tags = soup.find_all("meta", property=lambda x: x)
-    og_video_width = None
-    og_video_height = None
     for tag in og_tags:
         if 'data-server' in tag.attrs:
             del tag.attrs['data-server']
@@ -302,21 +307,16 @@ def proxy(video_id):
         # Fix video URL
         if tag.get("property") == "og:video:url" or tag.get("property") == "og:video:secure_url":
             tag["content"] = cdn_video_url
-        # Set vars
-        if tag.get("property") == "og:video:width":
-            og_video_width = tag.get("content")
-        if tag.get("property") == "og:video:height":
-            og_video_height = tag.get("content")
 
     og_tags_str = "\n".join(str(tag) for tag in og_tags if tag.get("property") not in ["og:site_name"])
     og_tags_str += '\n<meta content="0" property="twitter:image"/>'
     og_tags_str += '\n<meta content="player" property="twitter:card"/>'
     og_tags_str += '\n<meta content="video/mp4" property="twitter:player:stream:content_type"/>'
     og_tags_str += f'\n<meta content="{cdn_video_url}" property="twitter:player:stream"/>'
-    if og_video_width:
-        og_tags_str += f'\n<meta content="{og_video_width}" property="twitter:player:width"/>'
-    if og_video_height:
-        og_tags_str += f'\n<meta content="{og_video_height}" property="twitter:player:height"/>'
+    if video_width:
+        og_tags_str += f'\n<meta content="{video_width}" property="twitter:player:width"/>'
+    if video_height:
+        og_tags_str += f'\n<meta content="{video_height}" property="twitter:player:height"/>'
     og_tags_str += '\n<meta content="ニコニコ動画" property="og:site_name"/>'
     og_tags_str += f'\n<link rel="alternate" href="{get_oembed_url(params)}" type="application/json+oembed" title="{video_id}"/>'
     html_response = f"""<!DOCTYPE html>

From 71b7dac492e6836042bf1fd7d9dc50d0aecff613 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Wed, 26 Feb 2025 08:45:20 -0500
Subject: [PATCH 12/64] Remove tags that might be breaking Twitter

---
 app.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/app.py b/app.py
index 3d92c31..597b500 100644
--- a/app.py
+++ b/app.py
@@ -309,14 +309,14 @@ def proxy(video_id):
             tag["content"] = cdn_video_url
 
     og_tags_str = "\n".join(str(tag) for tag in og_tags if tag.get("property") not in ["og:site_name"])
-    og_tags_str += '\n<meta content="0" property="twitter:image"/>'
-    og_tags_str += '\n<meta content="player" property="twitter:card"/>'
-    og_tags_str += '\n<meta content="video/mp4" property="twitter:player:stream:content_type"/>'
-    og_tags_str += f'\n<meta content="{cdn_video_url}" property="twitter:player:stream"/>'
-    if video_width:
-        og_tags_str += f'\n<meta content="{video_width}" property="twitter:player:width"/>'
-    if video_height:
-        og_tags_str += f'\n<meta content="{video_height}" property="twitter:player:height"/>'
+    # og_tags_str += '\n<meta content="0" property="twitter:image"/>'
+    # og_tags_str += '\n<meta content="player" property="twitter:card"/>'
+    # og_tags_str += '\n<meta content="video/mp4" property="twitter:player:stream:content_type"/>'
+    # og_tags_str += f'\n<meta content="{cdn_video_url}" property="twitter:player:stream"/>'
+    # if video_width:
+    #     og_tags_str += f'\n<meta content="{video_width}" property="twitter:player:width"/>'
+    # if video_height:
+    #     og_tags_str += f'\n<meta content="{video_height}" property="twitter:player:height"/>'
     og_tags_str += '\n<meta content="ニコニコ動画" property="og:site_name"/>'
     og_tags_str += f'\n<link rel="alternate" href="{get_oembed_url(params)}" type="application/json+oembed" title="{video_id}"/>'
     html_response = f"""<!DOCTYPE html>

From 1802eeffe3c479f003b9ad6856f1307a67856009 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Wed, 26 Feb 2025 13:36:36 -0500
Subject: [PATCH 13/64] Add cleanup script

---
 .gitignore |  3 +-
 clean.py   | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+), 1 deletion(-)
 create mode 100644 clean.py

diff --git a/.gitignore b/.gitignore
index 8f5e87a..6e36383 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 venv
 .venv
 __pycache__
-cookies.txt
\ No newline at end of file
+cookies.txt
+.env
\ No newline at end of file
diff --git a/clean.py b/clean.py
new file mode 100644
index 0000000..e20faa0
--- /dev/null
+++ b/clean.py
@@ -0,0 +1,87 @@
+import os
+import sys
+import datetime
+import argparse
+import logging
+import boto3
+from botocore.client import Config as BotoConfig
+from dotenv import load_dotenv
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S'
+)
+logger = logging.getLogger(__name__)
+
+def cleanup_old_files(dry_run=False, days=7, directory_prefix="niconico"):
+    required_env_vars = [
+        'NICONICOGAY_S3_ACCESS_KEY',
+        'NICONICOGAY_S3_SECRET_KEY',
+        'NICONICOGAY_S3_BUCKET_NAME',
+        'NICONICOGAY_S3_REGION'
+    ]
+    missing_vars = [var for var in required_env_vars if not os.environ.get(var)]
+    if missing_vars:
+        logger.error(f"Missing required environment variables: {', '.join(missing_vars)}")
+        sys.exit(1)
+    
+    try:
+        s3_session = boto3.Session()
+        s3_client = s3_session.client(
+            's3',
+            aws_access_key_id=os.environ['NICONICOGAY_S3_ACCESS_KEY'],
+            aws_secret_access_key=os.environ['NICONICOGAY_S3_SECRET_KEY'],
+            region_name=os.environ['NICONICOGAY_S3_REGION'],
+            endpoint_url=f"https://{os.environ['NICONICOGAY_S3_REGION']}.digitaloceanspaces.com",
+            config=BotoConfig(s3={'addressing_style': 'virtual'}),
+        )
+        
+        bucket_name = os.environ['NICONICOGAY_S3_BUCKET_NAME']
+        cutoff_date = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=days)
+        paginator = s3_client.get_paginator('list_objects_v2')
+        page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=f"{directory_prefix}/")
+        
+        total_files = 0
+        objects_to_delete = []
+        
+        for page in page_iterator:
+            if 'Contents' not in page:
+                continue
+            
+            for obj in page['Contents']:
+                total_files += 1
+                if obj['LastModified'] < cutoff_date:  # type: ignore
+                    objects_to_delete.append({'Key': obj['Key']})  # type: ignore
+
+        if len(objects_to_delete) == 0:
+            logger.info("No files to delete")
+            return
+        
+        if dry_run:
+            logger.info(f"DRY RUN: Would delete {len(objects_to_delete)} out of {total_files} files")
+        else:
+            # Delete files in batches of 1000 (S3 limit?)
+            for i in range(0, len(objects_to_delete), 1000):
+                batch = objects_to_delete[i:i+1000]
+                s3_client.delete_objects(
+                    Bucket=bucket_name,
+                    Delete={'Objects': batch}
+                )
+            logger.info(f"Successfully deleted {len(objects_to_delete)} out of {total_files} files")
+            
+    except Exception as e:
+        logger.error(f"Error: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    load_dotenv()
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--dry-run", 
+        action="store_true", 
+        help="Show what would be deleted without actually deleting anything"
+    )
+    args = parser.parse_args()
+    
+    cleanup_old_files(dry_run=args.dry_run)

From 1ce10dfae4e63c2f5873ce088bc8b8ad44e76c8a Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 06:58:15 -0500
Subject: [PATCH 14/64] Remove raise for status check temporarily

---
 app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 597b500..704fda0 100644
--- a/app.py
+++ b/app.py
@@ -192,7 +192,7 @@ def get_data(video_id, real_url):
     try:
         logger.info(f"Fetching content from URL: {real_url}")
         r = s.get(real_url, timeout=10)
-        r.raise_for_status()
+        # r.raise_for_status()
     except requests.RequestException as e:
         logger.error(f"Error fetching the page for video ID '{video_id}': {e}")
         return None, None

From fe5c5470553baf8645f729daaa2a24c51598857f Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 07:43:10 -0500
Subject: [PATCH 15/64] Pass in user session for nndownload

---
 app.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/app.py b/app.py
index 704fda0..f4912ce 100644
--- a/app.py
+++ b/app.py
@@ -32,9 +32,11 @@ CACHE_SIZE_LIMIT = 100 * 1024 * 1024  # 100 MB
 
 cache = None if os.environ.get('NICONICOGAY_DISABLE_CACHE', '') != '' else Cache("disk_cache", size_limit=CACHE_SIZE_LIMIT)
 
+user_session = None
 cookie_jar = http.cookiejar.MozillaCookieJar('cookies.txt')
 try:
     cookie_jar.load(ignore_discard=True, ignore_expires=True)
+    user_session = next((cookie.value for cookie in cookie_jar if cookie.name == 'user_session'), None)
 except FileNotFoundError:
     logger.warning("cookies.txt not found, starting with empty cookie jar")
 
@@ -79,13 +81,17 @@ def download_and_upload_video(video_id, url, video_quality):
         
         try:
             logger.info(f"Starting download for video ID: {video_id}")
-            nndownload.execute(
+            nndownload_args = [
                 "--no-login",
                 "--user-agent", "Googlebot/2.1",
                 "--video-quality", video_quality,
                 "--output-path", temp_path,
                 url
-            )
+            ]
+            if user_session:
+                nndownload_args += ["--session-cookie", user_session]
+                nndownload_args = nndownload_args[1:]
+            nndownload.execute(*nndownload_args)
             
             if os.path.exists(temp_path) and s3_client:
                 logger.info(f"Downloaded video {video_id}, uploading to CDN")

From ac86c5f5ee326800bd47293381957cf3ca768ec0 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 10:29:36 -0500
Subject: [PATCH 16/64] Only download video if request from Discord

---
 app.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app.py b/app.py
index f4912ce..0aacf00 100644
--- a/app.py
+++ b/app.py
@@ -290,6 +290,10 @@ def proxy(video_id):
     video_width, video_height = get_video_resolution(params) if params else (None, None)
 
     download_allowed = allow_download(params) if params else False
+    request_user_agent = request.headers.get('User-Agent', '')
+    if download_allowed and 'discordbot' not in request_user_agent:
+        logger.info(f"Download ignored for video ID {video_id} due to user agent ({request_user_agent})")
+        download_allowed = False
     video_quality = get_video_quality(params) if params else None
     if download_allowed and video_quality is not None:
         video_in_cdn = is_video_in_cdn(video_id)

From aa836a4f55adec625c799f0c2378741293b7a403 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 10:48:46 -0500
Subject: [PATCH 17/64] oops

---
 app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 0aacf00..a2b3597 100644
--- a/app.py
+++ b/app.py
@@ -290,7 +290,7 @@ def proxy(video_id):
     video_width, video_height = get_video_resolution(params) if params else (None, None)
 
     download_allowed = allow_download(params) if params else False
-    request_user_agent = request.headers.get('User-Agent', '')
+    request_user_agent = request.headers.get('User-Agent', '').lower()
     if download_allowed and 'discordbot' not in request_user_agent:
         logger.info(f"Download ignored for video ID {video_id} due to user agent ({request_user_agent})")
         download_allowed = False

From ae803c0fe076c8e2deec2c5f13c75fbf44069b39 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 10:54:51 -0500
Subject: [PATCH 18/64] Make S3 optional

---
 app.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app.py b/app.py
index a2b3597..437fff7 100644
--- a/app.py
+++ b/app.py
@@ -46,6 +46,7 @@ s.headers.update({
 })
 s.cookies = cookie_jar  # type: ignore
 
+s3_client = None
 if all(key in os.environ for key in [
     'NICONICOGAY_S3_ACCESS_KEY',
     'NICONICOGAY_S3_SECRET_KEY',
@@ -60,8 +61,7 @@ if all(key in os.environ for key in [
         config=BotoConfig(s3={'addressing_style': 'virtual'}),
     )
 else:
-    logger.warning("S3 credentials not provided, exiting")
-    exit(1)
+    logger.warning("S3 credentials not provided. Videos will not be downloaded.")
 
 download_tracker = {
     'active_downloads': 0,

From 8f222ff95734d539fe28597ddd0f2a4b97868463 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 11:01:48 -0500
Subject: [PATCH 19/64] Cleaner log format

---
 app.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 437fff7..10f99c2 100644
--- a/app.py
+++ b/app.py
@@ -18,7 +18,11 @@ import urllib.parse
 from dotenv import load_dotenv
 load_dotenv()
 
-logging.basicConfig(level=logging.INFO)
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S'
+)
 logger = logging.getLogger(__name__)
 app = Flask(__name__)
 

From 1963ba53d96659c571ef2b04e1aac64149edc86b Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 11:12:42 -0500
Subject: [PATCH 20/64] Cache video CDN status

---
 app.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app.py b/app.py
index 10f99c2..d4625ce 100644
--- a/app.py
+++ b/app.py
@@ -109,6 +109,8 @@ def download_and_upload_video(video_id, url, video_quality):
                     )
                     
                     logger.info(f"Successfully uploaded video {video_id} to CDN")
+                    if cache:
+                        cache.set(f"{video_id}_uploaded", True, expire=CACHE_EXPIRATION_SECONDS)
                     
                     # Clear cache for this video to ensure next view gets updated HTML
                     if cache:
@@ -160,6 +162,9 @@ worker_thread.start()
 
 def is_video_in_cdn(video_id):
     """Check if video exists in CDN"""
+    if cache and cache.get(f"{video_id}_uploaded"):
+        return True
+
     if not s3_client:
         return False
         

From 6e95c1dd52f77c69485a3e7cb402f9c2879278ce Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 11:13:50 -0500
Subject: [PATCH 21/64] Add some logs

---
 app.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app.py b/app.py
index d4625ce..88a7387 100644
--- a/app.py
+++ b/app.py
@@ -163,9 +163,11 @@ worker_thread.start()
 def is_video_in_cdn(video_id):
     """Check if video exists in CDN"""
     if cache and cache.get(f"{video_id}_uploaded"):
+        logger.info(f"Video {video_id} is already uploaded to CDN (cached)")
         return True
 
     if not s3_client:
+        logger.warning("S3 client not configured. Cannot check if video exists in CDN.")
         return False
         
     try:

From 246de3e29d4c2cbcbbe64ce0de4ec716ac0839bc Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 11:16:14 -0500
Subject: [PATCH 22/64] Add diskcache logging

---
 app.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 88a7387..4312897 100644
--- a/app.py
+++ b/app.py
@@ -34,7 +34,12 @@ MAX_CONCURRENT_DOWNLOADS = 3
 CACHE_EXPIRATION_SECONDS = 3600  # 1 hour
 CACHE_SIZE_LIMIT = 100 * 1024 * 1024  # 100 MB
 
-cache = None if os.environ.get('NICONICOGAY_DISABLE_CACHE', '') != '' else Cache("disk_cache", size_limit=CACHE_SIZE_LIMIT)
+cache = None
+if os.environ.get('NICONICOGAY_DISABLE_CACHE', '') != '1':
+    Cache("disk_cache", size_limit=CACHE_SIZE_LIMIT)
+    logger.info("Using disk cache")
+else:
+    logger.info("Disk cache disabled")
 
 user_session = None
 cookie_jar = http.cookiejar.MozillaCookieJar('cookies.txt')

From e2d6cabed5c6d1e3be8c82cd05983f2ecf3f3419 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 11:18:24 -0500
Subject: [PATCH 23/64] More logging

---
 app.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app.py b/app.py
index 4312897..4807747 100644
--- a/app.py
+++ b/app.py
@@ -283,11 +283,14 @@ def proxy(video_id):
     logger.info(f"Received request for video ID: {video_id}")
 
     if cache:
+        logging.info(f"Checking cache for video ID: {video_id}")
         cached_html = cache.get(video_id)
         if cached_html is not None:
             logger.info(f"Using cached response for video ID: {video_id}")
             return Response(cached_html, mimetype="text/html")  # type: ignore
 
+    logging.info(f"Cache miss for video ID: {video_id} - fetching")
+
     # Not in cache or cache expired; fetch from nicovideo.jp
     real_url = f"https://www.nicovideo.jp/watch/{video_id}"
     params, soup = get_data(video_id, real_url)

From c3ceb007f3ca1d742eb942fb9ff7fcbc3352558d Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 11:19:38 -0500
Subject: [PATCH 24/64] Found the issue :^)

---
 app.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 4807747..bdadcd0 100644
--- a/app.py
+++ b/app.py
@@ -36,7 +36,7 @@ CACHE_SIZE_LIMIT = 100 * 1024 * 1024  # 100 MB
 
 cache = None
 if os.environ.get('NICONICOGAY_DISABLE_CACHE', '') != '1':
-    Cache("disk_cache", size_limit=CACHE_SIZE_LIMIT)
+    cache = Cache("disk_cache", size_limit=CACHE_SIZE_LIMIT)
     logger.info("Using disk cache")
 else:
     logger.info("Disk cache disabled")
@@ -364,6 +364,7 @@ if you want to download videos, please consider using a tool like nndownload: ht
         logging.info(f"Caching response for video ID: {video_id}")
         cache.set(video_id, html_response, expire=CACHE_EXPIRATION_SECONDS)
 
+    logging.info(f"Returning response for video ID: {video_id}")
     return Response(html_response, mimetype="text/html")
 
 @app.route("/owoembed")

From 4ac1fba2400232495066c5f996aa05966728f18d Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 11:24:39 -0500
Subject: [PATCH 25/64] Found the real cache issue :^)

---
 app.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/app.py b/app.py
index bdadcd0..67ffd30 100644
--- a/app.py
+++ b/app.py
@@ -114,12 +114,11 @@ def download_and_upload_video(video_id, url, video_quality):
                     )
                     
                     logger.info(f"Successfully uploaded video {video_id} to CDN")
-                    if cache:
-                        cache.set(f"{video_id}_uploaded", True, expire=CACHE_EXPIRATION_SECONDS)
                     
-                    # Clear cache for this video to ensure next view gets updated HTML
-                    if cache:
-                        cache.delete(video_id)
+                    if cache is not None:
+                        cache.set(f"{video_id}_uploaded", True, expire=CACHE_EXPIRATION_SECONDS)
+                        # Clear HTML cache for this video to ensure next view gets updated HTML
+                        cache.delete(f"{video_id}_html")
                         logger.info(f"Cleared cache for video ID: {video_id}")
                     
                     return True
@@ -167,7 +166,7 @@ worker_thread.start()
 
 def is_video_in_cdn(video_id):
     """Check if video exists in CDN"""
-    if cache and cache.get(f"{video_id}_uploaded"):
+    if cache is not None and cache.get(f"{video_id}_uploaded"):
         logger.info(f"Video {video_id} is already uploaded to CDN (cached)")
         return True
 
@@ -282,9 +281,9 @@ def get_oembed_url(params):
 def proxy(video_id):
     logger.info(f"Received request for video ID: {video_id}")
 
-    if cache:
+    if cache is not None:
         logging.info(f"Checking cache for video ID: {video_id}")
-        cached_html = cache.get(video_id)
+        cached_html = cache.get(f"{video_id}_html")
         if cached_html is not None:
             logger.info(f"Using cached response for video ID: {video_id}")
             return Response(cached_html, mimetype="text/html")  # type: ignore
@@ -311,7 +310,7 @@ def proxy(video_id):
     download_allowed = allow_download(params) if params else False
     request_user_agent = request.headers.get('User-Agent', '').lower()
     if download_allowed and 'discordbot' not in request_user_agent:
-        logger.info(f"Download ignored for video ID {video_id} due to user agent ({request_user_agent})")
+        logger.info(f"Video download ignored for {video_id} due to user agent ({request_user_agent})")
         download_allowed = False
     video_quality = get_video_quality(params) if params else None
     if download_allowed and video_quality is not None:
@@ -360,9 +359,9 @@ if you want to download videos, please consider using a tool like nndownload: ht
 {og_tags_str}
 </head><body></body></html>"""
 
-    if cache:
+    if cache is not None:
         logging.info(f"Caching response for video ID: {video_id}")
-        cache.set(video_id, html_response, expire=CACHE_EXPIRATION_SECONDS)
+        cache.set(f"{video_id}_html", html_response, expire=CACHE_EXPIRATION_SECONDS)
 
     logging.info(f"Returning response for video ID: {video_id}")
     return Response(html_response, mimetype="text/html")

From 419dd19faa8dd87f73ae78a9f31de26781431164 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 11:30:12 -0500
Subject: [PATCH 26/64] Clean up logging levels

---
 app.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/app.py b/app.py
index 67ffd30..6abbe59 100644
--- a/app.py
+++ b/app.py
@@ -37,7 +37,7 @@ CACHE_SIZE_LIMIT = 100 * 1024 * 1024  # 100 MB
 cache = None
 if os.environ.get('NICONICOGAY_DISABLE_CACHE', '') != '1':
     cache = Cache("disk_cache", size_limit=CACHE_SIZE_LIMIT)
-    logger.info("Using disk cache")
+    logger.debug("Using disk cache")
 else:
     logger.info("Disk cache disabled")
 
@@ -47,7 +47,7 @@ try:
     cookie_jar.load(ignore_discard=True, ignore_expires=True)
     user_session = next((cookie.value for cookie in cookie_jar if cookie.name == 'user_session'), None)
 except FileNotFoundError:
-    logger.warning("cookies.txt not found, starting with empty cookie jar")
+    logger.info("cookies.txt not found, starting with empty cookie jar")
 
 s = requests.Session()
 s.headers.update({
@@ -70,7 +70,7 @@ if all(key in os.environ for key in [
         config=BotoConfig(s3={'addressing_style': 'virtual'}),
     )
 else:
-    logger.warning("S3 credentials not provided. Videos will not be downloaded.")
+    logger.info("S3 credentials not provided. Videos will not be downloaded.")
 
 download_tracker = {
     'active_downloads': 0,
@@ -89,7 +89,7 @@ def download_and_upload_video(video_id, url, video_quality):
             temp_path = temp_file.name
         
         try:
-            logger.info(f"Starting download for video ID: {video_id}")
+            logger.info(f"Starting video download of {video_id}")
             nndownload_args = [
                 "--no-login",
                 "--user-agent", "Googlebot/2.1",
@@ -119,7 +119,7 @@ def download_and_upload_video(video_id, url, video_quality):
                         cache.set(f"{video_id}_uploaded", True, expire=CACHE_EXPIRATION_SECONDS)
                         # Clear HTML cache for this video to ensure next view gets updated HTML
                         cache.delete(f"{video_id}_html")
-                        logger.info(f"Cleared cache for video ID: {video_id}")
+                        logger.debug(f"Cleared cache for video ID: {video_id}")
                     
                     return True
                 except Exception as e:
@@ -131,7 +131,7 @@ def download_and_upload_video(video_id, url, video_quality):
         finally:
             if os.path.exists(temp_path):
                 os.unlink(temp_path)
-                logger.info(f"Removed temporary file: {temp_path}")
+                logger.debug(f"Removed temporary file: {temp_path}")
     except Exception as e:
         logger.error(f"Error in download process for video {video_id}: {e}")
         return False
@@ -167,7 +167,7 @@ worker_thread.start()
 def is_video_in_cdn(video_id):
     """Check if video exists in CDN"""
     if cache is not None and cache.get(f"{video_id}_uploaded"):
-        logger.info(f"Video {video_id} is already uploaded to CDN (cached)")
+        logger.debug(f"Video {video_id} is already uploaded to CDN (cached)")
         return True
 
     if not s3_client:
@@ -211,7 +211,7 @@ def get_video_quality(params, quality_level_threshold=3):
 def get_data(video_id, real_url):
     """Get the server response for a given video ID"""
     try:
-        logger.info(f"Fetching content from URL: {real_url}")
+        logger.debug(f"Fetching content from URL: {real_url}")
         r = s.get(real_url, timeout=10)
         # r.raise_for_status()
     except requests.RequestException as e:
@@ -282,13 +282,13 @@ def proxy(video_id):
     logger.info(f"Received request for video ID: {video_id}")
 
     if cache is not None:
-        logging.info(f"Checking cache for video ID: {video_id}")
+        logger.debug(f"Checking cache for video ID: {video_id}")
         cached_html = cache.get(f"{video_id}_html")
         if cached_html is not None:
-            logger.info(f"Using cached response for video ID: {video_id}")
+            logger.debug(f"Using cached response for video ID: {video_id}")
             return Response(cached_html, mimetype="text/html")  # type: ignore
 
-    logging.info(f"Cache miss for video ID: {video_id} - fetching")
+    logger.debug(f"Cache miss for video ID: {video_id} - fetching")
 
     # Not in cache or cache expired; fetch from nicovideo.jp
     real_url = f"https://www.nicovideo.jp/watch/{video_id}"
@@ -360,10 +360,10 @@ if you want to download videos, please consider using a tool like nndownload: ht
 </head><body></body></html>"""
 
     if cache is not None:
-        logging.info(f"Caching response for video ID: {video_id}")
+        logger.info(f"Caching response for video ID: {video_id}")
         cache.set(f"{video_id}_html", html_response, expire=CACHE_EXPIRATION_SECONDS)
 
-    logging.info(f"Returning response for video ID: {video_id}")
+    logger.debug(f"Returning response for video ID: {video_id}")
     return Response(html_response, mimetype="text/html")
 
 @app.route("/owoembed")
@@ -398,5 +398,5 @@ def owoembed():
         "version": "1.0"
     }
     
-    logger.info(f"Returning oEmbed response for video ID: {video_id}")
+    logger.debug(f"Returning oEmbed response for video ID: {video_id}")
     return jsonify(oembed_response)

From c120d9ba92ee17f476e9f71a062288642a4a62e2 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 11:30:58 -0500
Subject: [PATCH 27/64] Tweak response logging level

---
 app.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app.py b/app.py
index 6abbe59..0bed796 100644
--- a/app.py
+++ b/app.py
@@ -363,7 +363,7 @@ if you want to download videos, please consider using a tool like nndownload: ht
         logger.info(f"Caching response for video ID: {video_id}")
         cache.set(f"{video_id}_html", html_response, expire=CACHE_EXPIRATION_SECONDS)
 
-    logger.debug(f"Returning response for video ID: {video_id}")
+    logger.info(f"Returning response for video ID: {video_id}")
     return Response(html_response, mimetype="text/html")
 
 @app.route("/owoembed")
@@ -398,5 +398,5 @@ def owoembed():
         "version": "1.0"
     }
     
-    logger.debug(f"Returning oEmbed response for video ID: {video_id}")
+    logger.info(f"Returning oEmbed response for video ID: {video_id}")
     return jsonify(oembed_response)

From c7a2ae2b6e7442200a5142eab89385b54257de98 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 11:31:33 -0500
Subject: [PATCH 28/64] nit

---
 app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 0bed796..0d70d94 100644
--- a/app.py
+++ b/app.py
@@ -285,7 +285,7 @@ def proxy(video_id):
         logger.debug(f"Checking cache for video ID: {video_id}")
         cached_html = cache.get(f"{video_id}_html")
         if cached_html is not None:
-            logger.debug(f"Using cached response for video ID: {video_id}")
+            logger.info(f"Returning cached response for video ID: {video_id}")
             return Response(cached_html, mimetype="text/html")  # type: ignore
 
     logger.debug(f"Cache miss for video ID: {video_id} - fetching")

From cc21a2322ecd601bc6a73a544a64e2c86730d6e9 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 11:32:18 -0500
Subject: [PATCH 29/64] Granular time logging

---
 app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 0d70d94..e5e7684 100644
--- a/app.py
+++ b/app.py
@@ -20,7 +20,7 @@ load_dotenv()
 
 logging.basicConfig(
     level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s',
+    format='%(asctime)s.%(msecs)03d - %(levelname)s - %(message)s',
     datefmt='%Y-%m-%d %H:%M:%S'
 )
 logger = logging.getLogger(__name__)

From 2ca6d6aa735f327defcb60c217e846b0c28a7d8b Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 11:38:48 -0500
Subject: [PATCH 30/64] Better cache control, linter cleanup

---
 app.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/app.py b/app.py
index e5e7684..399dff0 100644
--- a/app.py
+++ b/app.py
@@ -31,7 +31,8 @@ S3_BUCKET_NAME = os.environ.get('NICONICOGAY_S3_BUCKET_NAME')
 S3_REGION = os.environ.get('NICONICOGAY_S3_REGION')
 CDN_BASE_URL = os.environ.get('NICONICOGAY_CDN_BASE_URL')
 MAX_CONCURRENT_DOWNLOADS = 3
-CACHE_EXPIRATION_SECONDS = 3600  # 1 hour
+CACHE_EXPIRATION_HTML = 60 * 60  # 1 hour
+CACHE_EXPIRATION_CDN = 60 * 60 * 24 * 7  # 1 week
 CACHE_SIZE_LIMIT = 100 * 1024 * 1024  # 100 MB
 
 cache = None
@@ -102,7 +103,7 @@ def download_and_upload_video(video_id, url, video_quality):
                 nndownload_args = nndownload_args[1:]
             nndownload.execute(*nndownload_args)
             
-            if os.path.exists(temp_path) and s3_client:
+            if os.path.exists(temp_path) and s3_client and S3_BUCKET_NAME:
                 logger.info(f"Downloaded video {video_id}, uploading to CDN")
                 try:
                     s3_key = f"niconico/{video_id}.mp4"
@@ -116,7 +117,7 @@ def download_and_upload_video(video_id, url, video_quality):
                     logger.info(f"Successfully uploaded video {video_id} to CDN")
                     
                     if cache is not None:
-                        cache.set(f"{video_id}_uploaded", True, expire=CACHE_EXPIRATION_SECONDS)
+                        cache.set(f"{video_id}_cdn", True, expire=CACHE_EXPIRATION_CDN)
                         # Clear HTML cache for this video to ensure next view gets updated HTML
                         cache.delete(f"{video_id}_html")
                         logger.debug(f"Cleared cache for video ID: {video_id}")
@@ -166,14 +167,14 @@ worker_thread.start()
 
 def is_video_in_cdn(video_id):
     """Check if video exists in CDN"""
-    if cache is not None and cache.get(f"{video_id}_uploaded"):
+    if cache is not None and cache.get(f"{video_id}_cdn"):
         logger.debug(f"Video {video_id} is already uploaded to CDN (cached)")
         return True
 
-    if not s3_client:
+    if not s3_client or not S3_BUCKET_NAME:
         logger.warning("S3 client not configured. Cannot check if video exists in CDN.")
         return False
-        
+
     try:
         s3_client.head_object(Bucket=S3_BUCKET_NAME, Key=f"niconico/{video_id}.mp4")
         return True
@@ -294,7 +295,7 @@ def proxy(video_id):
     real_url = f"https://www.nicovideo.jp/watch/{video_id}"
     params, soup = get_data(video_id, real_url)
 
-    if not params and not soup:
+    if not params or not soup:
         logger.error(f"Failed to retrieve data for video ID '{video_id}'")
         return Response("Video not found", status=404)
 
@@ -325,7 +326,7 @@ def proxy(video_id):
                     logger.info(f"Queued video ID {video_id} for download")
 
     cdn_video_url = get_cdn_url(video_id)
-    og_tags = soup.find_all("meta", property=lambda x: x)
+    og_tags = soup.find_all("meta", attrs={"property": True})
     for tag in og_tags:
         if 'data-server' in tag.attrs:
             del tag.attrs['data-server']
@@ -361,7 +362,7 @@ if you want to download videos, please consider using a tool like nndownload: ht
 
     if cache is not None:
         logger.info(f"Caching response for video ID: {video_id}")
-        cache.set(f"{video_id}_html", html_response, expire=CACHE_EXPIRATION_SECONDS)
+        cache.set(f"{video_id}_html", html_response, expire=CACHE_EXPIRATION_HTML)
 
     logger.info(f"Returning response for video ID: {video_id}")
     return Response(html_response, mimetype="text/html")

From e6d7278624f16ee5413d2f10d82f56482be2d77f Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 11:53:51 -0500
Subject: [PATCH 31/64] Ignore hidden videos

---
 app.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app.py b/app.py
index 399dff0..1577f6c 100644
--- a/app.py
+++ b/app.py
@@ -299,6 +299,11 @@ def proxy(video_id):
         logger.error(f"Failed to retrieve data for video ID '{video_id}'")
         return Response("Video not found", status=404)
 
+    reason_code = params.get('reasonCode', '').upper()
+    if reason_code in ['HIDDEN_VIDEO']:
+        logger.warning(f"Video ID '{video_id}' is hidden")
+        return Response("Video not found", status=404)
+
     thumbnail_url = (
         params["video"]["thumbnail"].get("ogp") or
         params["video"]["thumbnail"].get("player") or

From 0b8f0dc1b93fa08efb5766edfc9d39f4a8606373 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 12:03:17 -0500
Subject: [PATCH 32/64] Try to ignore connection pool warnings

---
 app.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app.py b/app.py
index 1577f6c..a8a737a 100644
--- a/app.py
+++ b/app.py
@@ -23,6 +23,7 @@ logging.basicConfig(
     format='%(asctime)s.%(msecs)03d - %(levelname)s - %(message)s',
     datefmt='%Y-%m-%d %H:%M:%S'
 )
+logging.getLogger("urllib3.connectionpool").setLevel(logging.DEBUG)
 logger = logging.getLogger(__name__)
 app = Flask(__name__)
 

From c6d53e0c1cc39124330f63fbe7b25bdb0fb6486b Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 12:08:35 -0500
Subject: [PATCH 33/64] Opposite

---
 app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app.py b/app.py
index a8a737a..a5ed9b0 100644
--- a/app.py
+++ b/app.py
@@ -23,7 +23,7 @@ logging.basicConfig(
     format='%(asctime)s.%(msecs)03d - %(levelname)s - %(message)s',
     datefmt='%Y-%m-%d %H:%M:%S'
 )
-logging.getLogger("urllib3.connectionpool").setLevel(logging.DEBUG)
+logging.getLogger("urllib3.connectionpool").setLevel(logging.ERROR)
 logger = logging.getLogger(__name__)
 app = Flask(__name__)
 

From 96326f543fe8e701dce010355828b187d4cd93d0 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 12:39:38 -0500
Subject: [PATCH 34/64] Better log phrasing

---
 app.py | 44 ++++++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/app.py b/app.py
index a5ed9b0..1075b55 100644
--- a/app.py
+++ b/app.py
@@ -91,7 +91,7 @@ def download_and_upload_video(video_id, url, video_quality):
             temp_path = temp_file.name
         
         try:
-            logger.info(f"Starting video download of {video_id}")
+            logger.info(f"{video_id}: Starting download")
             nndownload_args = [
                 "--no-login",
                 "--user-agent", "Googlebot/2.1",
@@ -105,7 +105,7 @@ def download_and_upload_video(video_id, url, video_quality):
             nndownload.execute(*nndownload_args)
             
             if os.path.exists(temp_path) and s3_client and S3_BUCKET_NAME:
-                logger.info(f"Downloaded video {video_id}, uploading to CDN")
+                logger.info(f"{video_id}: Downloaded, uploading to CDN")
                 try:
                     s3_key = f"niconico/{video_id}.mp4"
                     s3_client.upload_file(
@@ -115,27 +115,27 @@ def download_and_upload_video(video_id, url, video_quality):
                         ExtraArgs={'ContentType': 'video/mp4', 'ACL': 'public-read'}
                     )
                     
-                    logger.info(f"Successfully uploaded video {video_id} to CDN")
+                    logger.info(f"{video_id}: Upload successful to CDN")
                     
                     if cache is not None:
                         cache.set(f"{video_id}_cdn", True, expire=CACHE_EXPIRATION_CDN)
                         # Clear HTML cache for this video to ensure next view gets updated HTML
                         cache.delete(f"{video_id}_html")
-                        logger.debug(f"Cleared cache for video ID: {video_id}")
+                        logger.debug(f"{video_id}: Cleared HTML cache")
                     
                     return True
                 except Exception as e:
-                    logger.error(f"Error uploading video {video_id} to CDN: {e}")
+                    logger.error(f"{video_id}: Error uploading to CDN: {e}")
                     return False
             else:
-                logger.error(f"Failed to download video {video_id} or S3 client not configured")
+                logger.error(f"{video_id}: Failed to download or S3 client not configured")
                 return False
         finally:
             if os.path.exists(temp_path):
                 os.unlink(temp_path)
                 logger.debug(f"Removed temporary file: {temp_path}")
     except Exception as e:
-        logger.error(f"Error in download process for video {video_id}: {e}")
+        logger.error(f"{video_id}: Error in download process: {e}")
         return False
     finally:
         with download_lock:
@@ -169,7 +169,7 @@ worker_thread.start()
 def is_video_in_cdn(video_id):
     """Check if video exists in CDN"""
     if cache is not None and cache.get(f"{video_id}_cdn"):
-        logger.debug(f"Video {video_id} is already uploaded to CDN (cached)")
+        logger.debug(f"{video_id}: Already uploaded to CDN (cached)")
         return True
 
     if not s3_client or not S3_BUCKET_NAME:
@@ -213,11 +213,11 @@ def get_video_quality(params, quality_level_threshold=3):
 def get_data(video_id, real_url):
     """Get the server response for a given video ID"""
     try:
-        logger.debug(f"Fetching content from URL: {real_url}")
+        logger.debug(f"{video_id}: Fetching content from URL: {real_url}")
         r = s.get(real_url, timeout=10)
         # r.raise_for_status()
     except requests.RequestException as e:
-        logger.error(f"Error fetching the page for video ID '{video_id}': {e}")
+        logger.error(f"{video_id}: Error fetching the page ('{real_url}'): {e}")
         return None, None
 
     soup = BeautifulSoup(r.text, "html.parser")
@@ -227,7 +227,7 @@ def get_data(video_id, real_url):
             params = json.loads(server_response["content"])["data"]["response"]  # type: ignore
             return params, soup
     except (KeyError, json.JSONDecodeError) as e:
-        logger.warning(f"Failed to extract thumbnail info for video ID '{video_id}': {e}")
+        logger.warning(f"{video_id}: Failed to extract thumbnail info: {e}")
         pass
 
     return None, soup
@@ -281,28 +281,28 @@ def get_oembed_url(params):
 
 @app.route("/watch/<video_id>")
 def proxy(video_id):
-    logger.info(f"Received request for video ID: {video_id}")
+    logger.info(f"{video_id}: Received request")
 
     if cache is not None:
-        logger.debug(f"Checking cache for video ID: {video_id}")
+        logger.debug(f"{video_id}: Checking cache")
         cached_html = cache.get(f"{video_id}_html")
         if cached_html is not None:
-            logger.info(f"Returning cached response for video ID: {video_id}")
+            logger.info(f"{video_id}: Returning cached response")
             return Response(cached_html, mimetype="text/html")  # type: ignore
 
-    logger.debug(f"Cache miss for video ID: {video_id} - fetching")
+    logger.debug(f"{video_id}: Cache miss - fetching")
 
     # Not in cache or cache expired; fetch from nicovideo.jp
     real_url = f"https://www.nicovideo.jp/watch/{video_id}"
     params, soup = get_data(video_id, real_url)
 
     if not params or not soup:
-        logger.error(f"Failed to retrieve data for video ID '{video_id}'")
+        logger.error(f"{video_id}: Failed to fetch data")
         return Response("Video not found", status=404)
 
     reason_code = params.get('reasonCode', '').upper()
     if reason_code in ['HIDDEN_VIDEO']:
-        logger.warning(f"Video ID '{video_id}' is hidden")
+        logger.warning(f"{video_id}: Video is hidden - returning 404")
         return Response("Video not found", status=404)
 
     thumbnail_url = (
@@ -317,7 +317,7 @@ def proxy(video_id):
     download_allowed = allow_download(params) if params else False
     request_user_agent = request.headers.get('User-Agent', '').lower()
     if download_allowed and 'discordbot' not in request_user_agent:
-        logger.info(f"Video download ignored for {video_id} due to user agent ({request_user_agent})")
+        logger.info(f"{video_id}: Video download ignored for due to user agent ({request_user_agent})")
         download_allowed = False
     video_quality = get_video_quality(params) if params else None
     if download_allowed and video_quality is not None:
@@ -329,7 +329,7 @@ def proxy(video_id):
                 queue_video_ids = [item[0] for item in download_queue]
                 if video_id not in queue_video_ids:
                     download_queue.append((video_id, real_url, video_quality))
-                    logger.info(f"Queued video ID {video_id} for download")
+                    logger.info(f"{video_id}: Queued for download")
 
     cdn_video_url = get_cdn_url(video_id)
     og_tags = soup.find_all("meta", attrs={"property": True})
@@ -367,10 +367,10 @@ if you want to download videos, please consider using a tool like nndownload: ht
 </head><body></body></html>"""
 
     if cache is not None:
-        logger.info(f"Caching response for video ID: {video_id}")
+        logger.info(f"{video_id}: Caching HTML response")
         cache.set(f"{video_id}_html", html_response, expire=CACHE_EXPIRATION_HTML)
 
-    logger.info(f"Returning response for video ID: {video_id}")
+    logger.info(f"{video_id}: Returning response")
     return Response(html_response, mimetype="text/html")
 
 @app.route("/owoembed")
@@ -405,5 +405,5 @@ def owoembed():
         "version": "1.0"
     }
     
-    logger.info(f"Returning oEmbed response for video ID: {video_id}")
+    logger.info(f"{video_id}: Returning oEmbed response")
     return jsonify(oembed_response)

From 19befc9eb593aead6ff3e518a77eec4f01c5d1d6 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 12:52:37 -0500
Subject: [PATCH 35/64] Refactor out allow check

---
 app.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/app.py b/app.py
index 1075b55..a2c14dc 100644
--- a/app.py
+++ b/app.py
@@ -191,11 +191,6 @@ def get_cdn_url(video_id):
     """Get the CDN URL for a video"""
     return f"{CDN_BASE_URL}/niconico/{video_id}.mp4"
 
-def allow_download(params):
-    if params['video']['duration'] > 60 * 15:
-        return False
-    return True
-
 def get_video_resolution(params):
     if not params:
         return None, None
@@ -314,10 +309,13 @@ def proxy(video_id):
     ) if params else None
     video_width, video_height = get_video_resolution(params) if params else (None, None)
 
-    download_allowed = allow_download(params) if params else False
+    download_allowed = True
+    if params['video']['duration'] > 60 * 20:  # 20 minutes
+        logger.info(f"{video_id}: Video download ignored due to duration ({params['video']['duration']} seconds)")
+        download_allowed = False
     request_user_agent = request.headers.get('User-Agent', '').lower()
     if download_allowed and 'discordbot' not in request_user_agent:
-        logger.info(f"{video_id}: Video download ignored for due to user agent ({request_user_agent})")
+        logger.info(f"{video_id}: Video download ignored due to user agent ({request_user_agent})")
         download_allowed = False
     video_quality = get_video_quality(params) if params else None
     if download_allowed and video_quality is not None:
@@ -334,6 +332,7 @@ def proxy(video_id):
     cdn_video_url = get_cdn_url(video_id)
     og_tags = soup.find_all("meta", attrs={"property": True})
     for tag in og_tags:
+        # Remove attribute(s) added by niconico
         if 'data-server' in tag.attrs:
             del tag.attrs['data-server']
         # Fix thumbnail

From dd95661352f4af699232c41c18240e986eac8abf Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 13:14:09 -0500
Subject: [PATCH 36/64] Handle deleted videos

---
 app.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app.py b/app.py
index a2c14dc..c7bfa21 100644
--- a/app.py
+++ b/app.py
@@ -296,8 +296,8 @@ def proxy(video_id):
         return Response("Video not found", status=404)
 
     reason_code = params.get('reasonCode', '').upper()
-    if reason_code in ['HIDDEN_VIDEO']:
-        logger.warning(f"{video_id}: Video is hidden - returning 404")
+    if reason_code in ['HIDDEN_VIDEO', 'ADMINISTRATOR_DELETE_VIDEO']:
+        logger.warning(f"{video_id}: Video is hidden or deleted")
         return Response("Video not found", status=404)
 
     thumbnail_url = (

From d8ffe438570f095ee21c26114eee0d0b5e488f1e Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 13:18:30 -0500
Subject: [PATCH 37/64] Clearer log

---
 app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app.py b/app.py
index c7bfa21..80ce5e8 100644
--- a/app.py
+++ b/app.py
@@ -297,7 +297,7 @@ def proxy(video_id):
 
     reason_code = params.get('reasonCode', '').upper()
     if reason_code in ['HIDDEN_VIDEO', 'ADMINISTRATOR_DELETE_VIDEO']:
-        logger.warning(f"{video_id}: Video is hidden or deleted")
+        logger.warning(f"{video_id}: Video is hidden or deleted ({reason_code}) - returning 404")
         return Response("Video not found", status=404)
 
     thumbnail_url = (

From 5d5588f4f56c3065464a01b3fe206b8695aff266 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 15:15:11 -0500
Subject: [PATCH 38/64] Experimental placeholder video functionality

---
 app.py | 40 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 38 insertions(+), 2 deletions(-)

diff --git a/app.py b/app.py
index 80ce5e8..ac8f47e 100644
--- a/app.py
+++ b/app.py
@@ -3,8 +3,9 @@ import http.cookiejar
 import json
 import requests
 from bs4 import BeautifulSoup
-from flask import Flask, Response, request, jsonify
+from flask import Flask, Response, request, jsonify, send_file, make_response
 from diskcache import Cache
+from io import BytesIO
 import logging
 
 import threading
@@ -36,6 +37,17 @@ CACHE_EXPIRATION_HTML = 60 * 60  # 1 hour
 CACHE_EXPIRATION_CDN = 60 * 60 * 24 * 7  # 1 week
 CACHE_SIZE_LIMIT = 100 * 1024 * 1024  # 100 MB
 
+placeholder_video = None
+PLACEHOLDER_VIDEO_PATH = os.environ.get('NICONICOGAY_PLACEHOLDER_VIDEO', 'placeholder.mp4')
+try:
+    with open(PLACEHOLDER_VIDEO_PATH, 'rb') as f:
+        placeholder_video = BytesIO(f.read())
+    logger.debug("Loaded placeholder video")
+except FileNotFoundError:
+    logger.warning(f"Placeholder video file '{PLACEHOLDER_VIDEO_PATH}' not found")
+except Exception as e:
+    logger.error(f"Error loading placeholder video: {e}")
+
 cache = None
 if os.environ.get('NICONICOGAY_DISABLE_CACHE', '') != '1':
     cache = Cache("disk_cache", size_limit=CACHE_SIZE_LIMIT)
@@ -273,6 +285,29 @@ def get_oembed_url(params):
     return oembed_url
 
 
+@app.route("/cdn/<video_id>.mp4")
+def cdn_redirect(video_id):
+    """
+    Checks if a video exists in CDN and redirects accordingly.
+    Returns CDN URL if video exists, otherwise returns a placeholder video URL.
+    Only responds to requests from Discord bots.
+    """
+    logger.info(f"{video_id}: CDN redirect request received")
+    request_user_agent = request.headers.get('User-Agent', '').lower()
+    if 'discordbot' not in request_user_agent:
+        logger.info(f"{video_id}: Video CDN redirect ignored due to user agent ({request_user_agent})")
+        return Response("Video not found", status=404)
+
+    if placeholder_video is None or is_video_in_cdn(video_id):
+        cdn_url = get_cdn_url(video_id)
+        logger.info(f"{video_id}: Redirecting to CDN URL: {cdn_url}")
+        return Response("", status=302, headers={"Location": cdn_url})
+
+    logger.info(f"{video_id}: Video not found in CDN, returning placeholder")
+    response = make_response(send_file(placeholder_video, mimetype="video/mp4"))
+    response.headers['Content-Length'] = str(placeholder_video.getbuffer().nbytes)
+    return response
+
 
 @app.route("/watch/<video_id>")
 def proxy(video_id):
@@ -329,7 +364,7 @@ def proxy(video_id):
                     download_queue.append((video_id, real_url, video_quality))
                     logger.info(f"{video_id}: Queued for download")
 
-    cdn_video_url = get_cdn_url(video_id)
+    cdn_video_url = f"{HOST}/cdn/{video_id}.mp4" if placeholder_video else get_cdn_url(video_id)
     og_tags = soup.find_all("meta", attrs={"property": True})
     for tag in og_tags:
         # Remove attribute(s) added by niconico
@@ -372,6 +407,7 @@ if you want to download videos, please consider using a tool like nndownload: ht
     logger.info(f"{video_id}: Returning response")
     return Response(html_response, mimetype="text/html")
 
+
 @app.route("/owoembed")
 def owoembed():
     """

From 86b490bab13dae47f5dac54c7ed82f17f9cbf07d Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 15:33:07 -0500
Subject: [PATCH 39/64] fix

---
 app.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app.py b/app.py
index ac8f47e..055a97a 100644
--- a/app.py
+++ b/app.py
@@ -330,8 +330,8 @@ def proxy(video_id):
         logger.error(f"{video_id}: Failed to fetch data")
         return Response("Video not found", status=404)
 
-    reason_code = params.get('reasonCode', '').upper()
-    if reason_code in ['HIDDEN_VIDEO', 'ADMINISTRATOR_DELETE_VIDEO']:
+    reason_code = params.get('reasonCode')
+    if ('' or reason_code).upper() in ['HIDDEN_VIDEO', 'ADMINISTRATOR_DELETE_VIDEO']:
         logger.warning(f"{video_id}: Video is hidden or deleted ({reason_code}) - returning 404")
         return Response("Video not found", status=404)
 

From 2ade81b3beaa70773d40efabe8be6d7776fdd869 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 15:35:38 -0500
Subject: [PATCH 40/64] oops

---
 app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 055a97a..417a311 100644
--- a/app.py
+++ b/app.py
@@ -331,7 +331,7 @@ def proxy(video_id):
         return Response("Video not found", status=404)
 
     reason_code = params.get('reasonCode')
-    if ('' or reason_code).upper() in ['HIDDEN_VIDEO', 'ADMINISTRATOR_DELETE_VIDEO']:
+    if (reason_code or '').upper() in ['HIDDEN_VIDEO', 'ADMINISTRATOR_DELETE_VIDEO']:
         logger.warning(f"{video_id}: Video is hidden or deleted ({reason_code}) - returning 404")
         return Response("Video not found", status=404)
 

From 9905d914796b1853266665518e9eb0b7ac655b5d Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 15:40:34 -0500
Subject: [PATCH 41/64] Fix for Discord user agent for files

---
 app.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/app.py b/app.py
index 417a311..1e8d48e 100644
--- a/app.py
+++ b/app.py
@@ -293,8 +293,9 @@ def cdn_redirect(video_id):
     Only responds to requests from Discord bots.
     """
     logger.info(f"{video_id}: CDN redirect request received")
-    request_user_agent = request.headers.get('User-Agent', '').lower()
-    if 'discordbot' not in request_user_agent:
+    request_user_agent = request.headers.get('User-Agent', '')
+    # Discord uses a more realistic user agent for file requests
+    if 'Firefox/38.0' not in request_user_agent and 'Macintosh' not in request_user_agent:
         logger.info(f"{video_id}: Video CDN redirect ignored due to user agent ({request_user_agent})")
         return Response("Video not found", status=404)
 

From 7cbc5f84c9a6ac70b19409bc73c7d4a4753abd71 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 15:53:05 -0500
Subject: [PATCH 42/64] Fix placeholder video return

---
 app.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 1e8d48e..7c74ff0 100644
--- a/app.py
+++ b/app.py
@@ -295,7 +295,7 @@ def cdn_redirect(video_id):
     logger.info(f"{video_id}: CDN redirect request received")
     request_user_agent = request.headers.get('User-Agent', '')
     # Discord uses a more realistic user agent for file requests
-    if 'Firefox/38.0' not in request_user_agent and 'Macintosh' not in request_user_agent:
+    if 'Firefox/38.0' not in request_user_agent:
         logger.info(f"{video_id}: Video CDN redirect ignored due to user agent ({request_user_agent})")
         return Response("Video not found", status=404)
 
@@ -305,6 +305,7 @@ def cdn_redirect(video_id):
         return Response("", status=302, headers={"Location": cdn_url})
 
     logger.info(f"{video_id}: Video not found in CDN, returning placeholder")
+    placeholder_video.seek(0)
     response = make_response(send_file(placeholder_video, mimetype="video/mp4"))
     response.headers['Content-Length'] = str(placeholder_video.getbuffer().nbytes)
     return response

From bca73594f53c5a9cda238931db59b0b368e2a812 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 15:56:20 -0500
Subject: [PATCH 43/64] Fix again

---
 app.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/app.py b/app.py
index 7c74ff0..17de262 100644
--- a/app.py
+++ b/app.py
@@ -41,7 +41,7 @@ placeholder_video = None
 PLACEHOLDER_VIDEO_PATH = os.environ.get('NICONICOGAY_PLACEHOLDER_VIDEO', 'placeholder.mp4')
 try:
     with open(PLACEHOLDER_VIDEO_PATH, 'rb') as f:
-        placeholder_video = BytesIO(f.read())
+        placeholder_video = f.read()
     logger.debug("Loaded placeholder video")
 except FileNotFoundError:
     logger.warning(f"Placeholder video file '{PLACEHOLDER_VIDEO_PATH}' not found")
@@ -305,9 +305,8 @@ def cdn_redirect(video_id):
         return Response("", status=302, headers={"Location": cdn_url})
 
     logger.info(f"{video_id}: Video not found in CDN, returning placeholder")
-    placeholder_video.seek(0)
-    response = make_response(send_file(placeholder_video, mimetype="video/mp4"))
-    response.headers['Content-Length'] = str(placeholder_video.getbuffer().nbytes)
+    response = make_response(send_file(BytesIO(placeholder_video), mimetype="video/mp4"))
+    response.headers['Content-Length'] = str(len(placeholder_video))
     return response
 
 

From 753d4c691a40b4ecb2a3dc819b53134363b88068 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 16:05:39 -0500
Subject: [PATCH 44/64] Try to prevent caching of placeholder

---
 app.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app.py b/app.py
index 17de262..b52b221 100644
--- a/app.py
+++ b/app.py
@@ -307,6 +307,9 @@ def cdn_redirect(video_id):
     logger.info(f"{video_id}: Video not found in CDN, returning placeholder")
     response = make_response(send_file(BytesIO(placeholder_video), mimetype="video/mp4"))
     response.headers['Content-Length'] = str(len(placeholder_video))
+    response.headers['Cache-Control'] = 'no-cache'
+    response.headers['Pragma'] = 'no-cache'
+    response.headers['Expires'] = '0'
     return response
 
 

From eccea590703f9ff74e4953fcfb4bc22815ece679 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Thu, 27 Feb 2025 16:07:14 -0500
Subject: [PATCH 45/64] Revert experimental placeholder video

Discord seems to cache the video regardless of
what it's told.
---
 app.py | 48 ++++--------------------------------------------
 1 file changed, 4 insertions(+), 44 deletions(-)

diff --git a/app.py b/app.py
index b52b221..80ce5e8 100644
--- a/app.py
+++ b/app.py
@@ -3,9 +3,8 @@ import http.cookiejar
 import json
 import requests
 from bs4 import BeautifulSoup
-from flask import Flask, Response, request, jsonify, send_file, make_response
+from flask import Flask, Response, request, jsonify
 from diskcache import Cache
-from io import BytesIO
 import logging
 
 import threading
@@ -37,17 +36,6 @@ CACHE_EXPIRATION_HTML = 60 * 60  # 1 hour
 CACHE_EXPIRATION_CDN = 60 * 60 * 24 * 7  # 1 week
 CACHE_SIZE_LIMIT = 100 * 1024 * 1024  # 100 MB
 
-placeholder_video = None
-PLACEHOLDER_VIDEO_PATH = os.environ.get('NICONICOGAY_PLACEHOLDER_VIDEO', 'placeholder.mp4')
-try:
-    with open(PLACEHOLDER_VIDEO_PATH, 'rb') as f:
-        placeholder_video = f.read()
-    logger.debug("Loaded placeholder video")
-except FileNotFoundError:
-    logger.warning(f"Placeholder video file '{PLACEHOLDER_VIDEO_PATH}' not found")
-except Exception as e:
-    logger.error(f"Error loading placeholder video: {e}")
-
 cache = None
 if os.environ.get('NICONICOGAY_DISABLE_CACHE', '') != '1':
     cache = Cache("disk_cache", size_limit=CACHE_SIZE_LIMIT)
@@ -285,33 +273,6 @@ def get_oembed_url(params):
     return oembed_url
 
 
-@app.route("/cdn/<video_id>.mp4")
-def cdn_redirect(video_id):
-    """
-    Checks if a video exists in CDN and redirects accordingly.
-    Returns CDN URL if video exists, otherwise returns a placeholder video URL.
-    Only responds to requests from Discord bots.
-    """
-    logger.info(f"{video_id}: CDN redirect request received")
-    request_user_agent = request.headers.get('User-Agent', '')
-    # Discord uses a more realistic user agent for file requests
-    if 'Firefox/38.0' not in request_user_agent:
-        logger.info(f"{video_id}: Video CDN redirect ignored due to user agent ({request_user_agent})")
-        return Response("Video not found", status=404)
-
-    if placeholder_video is None or is_video_in_cdn(video_id):
-        cdn_url = get_cdn_url(video_id)
-        logger.info(f"{video_id}: Redirecting to CDN URL: {cdn_url}")
-        return Response("", status=302, headers={"Location": cdn_url})
-
-    logger.info(f"{video_id}: Video not found in CDN, returning placeholder")
-    response = make_response(send_file(BytesIO(placeholder_video), mimetype="video/mp4"))
-    response.headers['Content-Length'] = str(len(placeholder_video))
-    response.headers['Cache-Control'] = 'no-cache'
-    response.headers['Pragma'] = 'no-cache'
-    response.headers['Expires'] = '0'
-    return response
-
 
 @app.route("/watch/<video_id>")
 def proxy(video_id):
@@ -334,8 +295,8 @@ def proxy(video_id):
         logger.error(f"{video_id}: Failed to fetch data")
         return Response("Video not found", status=404)
 
-    reason_code = params.get('reasonCode')
-    if (reason_code or '').upper() in ['HIDDEN_VIDEO', 'ADMINISTRATOR_DELETE_VIDEO']:
+    reason_code = params.get('reasonCode', '').upper()
+    if reason_code in ['HIDDEN_VIDEO', 'ADMINISTRATOR_DELETE_VIDEO']:
         logger.warning(f"{video_id}: Video is hidden or deleted ({reason_code}) - returning 404")
         return Response("Video not found", status=404)
 
@@ -368,7 +329,7 @@ def proxy(video_id):
                     download_queue.append((video_id, real_url, video_quality))
                     logger.info(f"{video_id}: Queued for download")
 
-    cdn_video_url = f"{HOST}/cdn/{video_id}.mp4" if placeholder_video else get_cdn_url(video_id)
+    cdn_video_url = get_cdn_url(video_id)
     og_tags = soup.find_all("meta", attrs={"property": True})
     for tag in og_tags:
         # Remove attribute(s) added by niconico
@@ -411,7 +372,6 @@ if you want to download videos, please consider using a tool like nndownload: ht
     logger.info(f"{video_id}: Returning response")
     return Response(html_response, mimetype="text/html")
 
-
 @app.route("/owoembed")
 def owoembed():
     """

From 6e41e842fd0f1ebacf6001ef8125465708f080a2 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Mon, 3 Mar 2025 12:44:40 -0500
Subject: [PATCH 46/64] Attempt to fix Twitter card display

---
 app.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/app.py b/app.py
index 80ce5e8..adb2273 100644
--- a/app.py
+++ b/app.py
@@ -331,10 +331,18 @@ def proxy(video_id):
 
     cdn_video_url = get_cdn_url(video_id)
     og_tags = soup.find_all("meta", attrs={"property": True})
+    og_title = None
+    og_description = None
     for tag in og_tags:
         # Remove attribute(s) added by niconico
         if 'data-server' in tag.attrs:
             del tag.attrs['data-server']
+        # Set title
+        if tag.get("property") == "og:title":
+            og_title = tag["content"]
+        # Set description
+        if tag.get("property") == "og:description":
+            og_description = tag["content"]
         # Fix thumbnail
         if tag.get("property") == "og:image" and thumbnail_url:
             tag["content"] = thumbnail_url
@@ -343,8 +351,14 @@ def proxy(video_id):
             tag["content"] = cdn_video_url
 
     og_tags_str = "\n".join(str(tag) for tag in og_tags if tag.get("property") not in ["og:site_name"])
-    # og_tags_str += '\n<meta content="0" property="twitter:image"/>'
-    # og_tags_str += '\n<meta content="player" property="twitter:card"/>'
+    og_tags_str += f'\n<meta content="{thumbnail_url}" property="twitter:image"/>'
+    og_tags_str += '\n<meta content="summary_large_image" property="twitter:card"/>'
+    og_tags_str += '\n<meta content="www.nicovideo.gay" name="twitter:domain"/>'
+    og_tags_str += f'\n<meta content="{request.url}" name="twitter:url"/>'
+    if og_title:
+        og_tags_str += f'\n<meta content="{og_title}" name="twitter:title"/>'
+    if og_description:
+        og_tags_str += f'\n<meta content="{og_description}" name="twitter:description"/>'
     # og_tags_str += '\n<meta content="video/mp4" property="twitter:player:stream:content_type"/>'
     # og_tags_str += f'\n<meta content="{cdn_video_url}" property="twitter:player:stream"/>'
     # if video_width:

From a07da68e4f9f85e2220a7021f88608d26b886a5f Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Mon, 3 Mar 2025 12:46:25 -0500
Subject: [PATCH 47/64] Revert "Attempt to fix Twitter card display"

This reverts commit 6e41e842fd0f1ebacf6001ef8125465708f080a2.
---
 app.py | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/app.py b/app.py
index adb2273..80ce5e8 100644
--- a/app.py
+++ b/app.py
@@ -331,18 +331,10 @@ def proxy(video_id):
 
     cdn_video_url = get_cdn_url(video_id)
     og_tags = soup.find_all("meta", attrs={"property": True})
-    og_title = None
-    og_description = None
     for tag in og_tags:
         # Remove attribute(s) added by niconico
         if 'data-server' in tag.attrs:
             del tag.attrs['data-server']
-        # Set title
-        if tag.get("property") == "og:title":
-            og_title = tag["content"]
-        # Set description
-        if tag.get("property") == "og:description":
-            og_description = tag["content"]
         # Fix thumbnail
         if tag.get("property") == "og:image" and thumbnail_url:
             tag["content"] = thumbnail_url
@@ -351,14 +343,8 @@ def proxy(video_id):
             tag["content"] = cdn_video_url
 
     og_tags_str = "\n".join(str(tag) for tag in og_tags if tag.get("property") not in ["og:site_name"])
-    og_tags_str += f'\n<meta content="{thumbnail_url}" property="twitter:image"/>'
-    og_tags_str += '\n<meta content="summary_large_image" property="twitter:card"/>'
-    og_tags_str += '\n<meta content="www.nicovideo.gay" name="twitter:domain"/>'
-    og_tags_str += f'\n<meta content="{request.url}" name="twitter:url"/>'
-    if og_title:
-        og_tags_str += f'\n<meta content="{og_title}" name="twitter:title"/>'
-    if og_description:
-        og_tags_str += f'\n<meta content="{og_description}" name="twitter:description"/>'
+    # og_tags_str += '\n<meta content="0" property="twitter:image"/>'
+    # og_tags_str += '\n<meta content="player" property="twitter:card"/>'
     # og_tags_str += '\n<meta content="video/mp4" property="twitter:player:stream:content_type"/>'
     # og_tags_str += f'\n<meta content="{cdn_video_url}" property="twitter:player:stream"/>'
     # if video_width:

From 255a12fcc62b2ae0983ae2cb773c9dd97bb9df0d Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Mon, 3 Mar 2025 12:49:17 -0500
Subject: [PATCH 48/64] Attempt to fix Twitter card display v2

---
 app.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/app.py b/app.py
index 80ce5e8..1fbf4b2 100644
--- a/app.py
+++ b/app.py
@@ -331,10 +331,18 @@ def proxy(video_id):
 
     cdn_video_url = get_cdn_url(video_id)
     og_tags = soup.find_all("meta", attrs={"property": True})
+    og_title = None
+    og_description = None
     for tag in og_tags:
         # Remove attribute(s) added by niconico
         if 'data-server' in tag.attrs:
             del tag.attrs['data-server']
+        # Set title
+        if tag.get("property") == "og:title":
+            og_title = tag["content"]
+        # Set description
+        if tag.get("property") == "og:description":
+            og_description = tag["content"]
         # Fix thumbnail
         if tag.get("property") == "og:image" and thumbnail_url:
             tag["content"] = thumbnail_url
@@ -343,8 +351,15 @@ def proxy(video_id):
             tag["content"] = cdn_video_url
 
     og_tags_str = "\n".join(str(tag) for tag in og_tags if tag.get("property") not in ["og:site_name"])
-    # og_tags_str += '\n<meta content="0" property="twitter:image"/>'
-    # og_tags_str += '\n<meta content="player" property="twitter:card"/>'
+    if 'twitterbot' in request_user_agent:
+        og_tags_str += f'\n<meta content="{thumbnail_url}" property="twitter:image"/>'
+        og_tags_str += '\n<meta content="summary_large_image" property="twitter:card"/>'
+        og_tags_str += '\n<meta content="www.nicovideo.gay" name="twitter:domain"/>'
+        og_tags_str += f'\n<meta content="{request.url}" name="twitter:url"/>'
+        if og_title:
+            og_tags_str += f'\n<meta content="{og_title}" name="twitter:title"/>'
+        if og_description:
+            og_tags_str += f'\n<meta content="{og_description}" name="twitter:description"/>'
     # og_tags_str += '\n<meta content="video/mp4" property="twitter:player:stream:content_type"/>'
     # og_tags_str += f'\n<meta content="{cdn_video_url}" property="twitter:player:stream"/>'
     # if video_width:

From a075a5a7a2fcfa9231b1eceaaddd95a159bc7132 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Mon, 3 Mar 2025 13:01:26 -0500
Subject: [PATCH 49/64] Warning if no OG tags found

---
 app.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app.py b/app.py
index 1fbf4b2..e9d6295 100644
--- a/app.py
+++ b/app.py
@@ -331,6 +331,8 @@ def proxy(video_id):
 
     cdn_video_url = get_cdn_url(video_id)
     og_tags = soup.find_all("meta", attrs={"property": True})
+    if len(og_tags) == 0:
+        logger.warning(f"{video_id}: No Open Graph tags found")
     og_title = None
     og_description = None
     for tag in og_tags:

From aa755dc186bb79608d55fd9f0685d9433f0e34a7 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Mon, 3 Mar 2025 13:06:22 -0500
Subject: [PATCH 50/64] Try to fix oEmbed

---
 app.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/app.py b/app.py
index e9d6295..8956e84 100644
--- a/app.py
+++ b/app.py
@@ -353,7 +353,10 @@ def proxy(video_id):
             tag["content"] = cdn_video_url
 
     og_tags_str = "\n".join(str(tag) for tag in og_tags if tag.get("property") not in ["og:site_name"])
+    og_tags_str += '\n<meta content="ニコニコ動画" property="og:site_name"/>'
+    og_tags_str += f'\n<link rel="alternate" href="{get_oembed_url(params)}" type="application/json+oembed" title="{video_id}"/>'
     if 'twitterbot' in request_user_agent:
+        logger.info(f"{video_id}: Twitterbot detected - adding Twitter tags")
         og_tags_str += f'\n<meta content="{thumbnail_url}" property="twitter:image"/>'
         og_tags_str += '\n<meta content="summary_large_image" property="twitter:card"/>'
         og_tags_str += '\n<meta content="www.nicovideo.gay" name="twitter:domain"/>'
@@ -368,8 +371,6 @@ def proxy(video_id):
     #     og_tags_str += f'\n<meta content="{video_width}" property="twitter:player:width"/>'
     # if video_height:
     #     og_tags_str += f'\n<meta content="{video_height}" property="twitter:player:height"/>'
-    og_tags_str += '\n<meta content="ニコニコ動画" property="og:site_name"/>'
-    og_tags_str += f'\n<link rel="alternate" href="{get_oembed_url(params)}" type="application/json+oembed" title="{video_id}"/>'
     html_response = f"""<!DOCTYPE html>
 <!--
 niconico proxy - brought to you by https://mmaker.moe

From 6a78b81084618925e70cd28cde456fcc695e26ae Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Mon, 3 Mar 2025 13:14:12 -0500
Subject: [PATCH 51/64] Remove title and category from OG description

---
 app.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app.py b/app.py
index 8956e84..22e4bdd 100644
--- a/app.py
+++ b/app.py
@@ -1,6 +1,7 @@
 import os
 import http.cookiejar
 import json
+import re
 import requests
 from bs4 import BeautifulSoup
 from flask import Flask, Response, request, jsonify
@@ -364,6 +365,8 @@ def proxy(video_id):
         if og_title:
             og_tags_str += f'\n<meta content="{og_title}" name="twitter:title"/>'
         if og_description:
+            if og_title:
+                og_description = re.sub(rf"^{re.escape(og_title)}(\s+\[.*?\])?\s+", "", og_description)
             og_tags_str += f'\n<meta content="{og_description}" name="twitter:description"/>'
     # og_tags_str += '\n<meta content="video/mp4" property="twitter:player:stream:content_type"/>'
     # og_tags_str += f'\n<meta content="{cdn_video_url}" property="twitter:player:stream"/>'

From b0b552ee825c4925502e766dbe68ad1914e87ff0 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Mon, 3 Mar 2025 13:15:19 -0500
Subject: [PATCH 52/64] Log HTML response

---
 app.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app.py b/app.py
index 22e4bdd..0b052dd 100644
--- a/app.py
+++ b/app.py
@@ -391,6 +391,7 @@ if you want to download videos, please consider using a tool like nndownload: ht
         cache.set(f"{video_id}_html", html_response, expire=CACHE_EXPIRATION_HTML)
 
     logger.info(f"{video_id}: Returning response")
+    logger.debug(f"{video_id}: HTML response:\n----------\n{html_response}----------")
     return Response(html_response, mimetype="text/html")
 
 @app.route("/owoembed")

From d770c7df41f54c2cdcf9c7c6ce4c03e0a420437c Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Mon, 3 Mar 2025 13:16:34 -0500
Subject: [PATCH 53/64] Set log level via env var

---
 app.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app.py b/app.py
index 0b052dd..95fad21 100644
--- a/app.py
+++ b/app.py
@@ -26,6 +26,7 @@ logging.basicConfig(
 )
 logging.getLogger("urllib3.connectionpool").setLevel(logging.ERROR)
 logger = logging.getLogger(__name__)
+logger.setLevel(os.environ.get('NICONICOGAY_LOG', 'INFO').upper())
 app = Flask(__name__)
 
 HOST = os.environ.get('NICONICOGAY_HOST', 'https://nicovideo.gay')

From 3456e74afb04411372bc5e9ce9791e9af3ceef11 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Mon, 3 Mar 2025 13:17:32 -0500
Subject: [PATCH 54/64] Extra newline in log

---
 app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 95fad21..00cb7c6 100644
--- a/app.py
+++ b/app.py
@@ -392,7 +392,7 @@ if you want to download videos, please consider using a tool like nndownload: ht
         cache.set(f"{video_id}_html", html_response, expire=CACHE_EXPIRATION_HTML)
 
     logger.info(f"{video_id}: Returning response")
-    logger.debug(f"{video_id}: HTML response:\n----------\n{html_response}----------")
+    logger.debug(f"{video_id}: HTML response:\n----------\n{html_response}\n----------")
     return Response(html_response, mimetype="text/html")
 
 @app.route("/owoembed")

From 36c34bd4f7ad1bc2fe115ec59c9ce982a39ecbe8 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Mon, 3 Mar 2025 13:28:44 -0500
Subject: [PATCH 55/64] Better description modification

---
 app.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/app.py b/app.py
index 00cb7c6..445ae21 100644
--- a/app.py
+++ b/app.py
@@ -337,6 +337,7 @@ def proxy(video_id):
         logger.warning(f"{video_id}: No Open Graph tags found")
     og_title = None
     og_description = None
+    og_category = None
     for tag in og_tags:
         # Remove attribute(s) added by niconico
         if 'data-server' in tag.attrs:
@@ -347,6 +348,14 @@ def proxy(video_id):
         # Set description
         if tag.get("property") == "og:description":
             og_description = tag["content"]
+            if og_description and og_title:
+                # The description is formatted like "Title [Category] Description"
+                # Extract category (just incase this is useful later), and keep only the description part.
+                match = re.search(rf"^{re.escape(og_title)}(\s+\[(.*?)\])?\s+(.*)", og_description)
+                if match:
+                    og_category = match.group(2) if match.group(2) else None
+                    og_description = match.group(3)
+                    tag["content"] = og_description
         # Fix thumbnail
         if tag.get("property") == "og:image" and thumbnail_url:
             tag["content"] = thumbnail_url
@@ -366,8 +375,6 @@ def proxy(video_id):
         if og_title:
             og_tags_str += f'\n<meta content="{og_title}" name="twitter:title"/>'
         if og_description:
-            if og_title:
-                og_description = re.sub(rf"^{re.escape(og_title)}(\s+\[.*?\])?\s+", "", og_description)
             og_tags_str += f'\n<meta content="{og_description}" name="twitter:description"/>'
     # og_tags_str += '\n<meta content="video/mp4" property="twitter:player:stream:content_type"/>'
     # og_tags_str += f'\n<meta content="{cdn_video_url}" property="twitter:player:stream"/>'

From 070eed8f4103d2edf2ff2941d0cbf4a13a21ce7e Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Mon, 3 Mar 2025 13:38:29 -0500
Subject: [PATCH 56/64] Print oEmbed response in debug

---
 app.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app.py b/app.py
index 445ae21..8eb14b4 100644
--- a/app.py
+++ b/app.py
@@ -435,4 +435,5 @@ def owoembed():
     }
     
     logger.info(f"{video_id}: Returning oEmbed response")
+    logger.debug(f"{video_id}: oEmbed response:\n----------\n{json.dumps(oembed_response, indent=2)}\n----------")
     return jsonify(oembed_response)

From df8537e811e42ff7b09ac7b939e48cbb072b7887 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Mon, 3 Mar 2025 22:03:32 -0500
Subject: [PATCH 57/64] Remove play button for videos not downloaded

---
 app.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 8eb14b4..2d9d9e2 100644
--- a/app.py
+++ b/app.py
@@ -366,7 +366,11 @@ def proxy(video_id):
     og_tags_str = "\n".join(str(tag) for tag in og_tags if tag.get("property") not in ["og:site_name"])
     og_tags_str += '\n<meta content="ニコニコ動画" property="og:site_name"/>'
     og_tags_str += f'\n<link rel="alternate" href="{get_oembed_url(params)}" type="application/json+oembed" title="{video_id}"/>'
-    if 'twitterbot' in request_user_agent:
+
+    # Discord seems to ignore video URLs when Twitter meta tags are present,
+    # so in addition to including these when the User Agent is a Twitter bot,
+    # we also include them when the video is too long to download in order to remove the play button.
+    if 'twitterbot' in request_user_agent or not download_allowed:
         logger.info(f"{video_id}: Twitterbot detected - adding Twitter tags")
         og_tags_str += f'\n<meta content="{thumbnail_url}" property="twitter:image"/>'
         og_tags_str += '\n<meta content="summary_large_image" property="twitter:card"/>'

From a31d7d5a9074d15020f0f768c32f79c3a98aef51 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Mon, 3 Mar 2025 22:06:15 -0500
Subject: [PATCH 58/64] Add extra logging

---
 app.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 2d9d9e2..98c8899 100644
--- a/app.py
+++ b/app.py
@@ -371,7 +371,10 @@ def proxy(video_id):
     # so in addition to including these when the User Agent is a Twitter bot,
     # we also include them when the video is too long to download in order to remove the play button.
     if 'twitterbot' in request_user_agent or not download_allowed:
-        logger.info(f"{video_id}: Twitterbot detected - adding Twitter tags")
+        if 'twitterbot' in request_user_agent:
+            logger.info(f"{video_id}: Twitterbot detected - adding Twitter tags")
+        if not download_allowed:
+            logger.info(f"{video_id}: Video too long to download - will not show play button")
         og_tags_str += f'\n<meta content="{thumbnail_url}" property="twitter:image"/>'
         og_tags_str += '\n<meta content="summary_large_image" property="twitter:card"/>'
         og_tags_str += '\n<meta content="www.nicovideo.gay" name="twitter:domain"/>'

From d8da55520fd91d16a1c9ae3152ae82f4c283ae89 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Tue, 4 Mar 2025 10:22:54 -0500
Subject: [PATCH 59/64] Handle another reason code

---
 app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 98c8899..8e64f97 100644
--- a/app.py
+++ b/app.py
@@ -298,7 +298,7 @@ def proxy(video_id):
         return Response("Video not found", status=404)
 
     reason_code = params.get('reasonCode', '').upper()
-    if reason_code in ['HIDDEN_VIDEO', 'ADMINISTRATOR_DELETE_VIDEO']:
+    if reason_code in ['HIDDEN_VIDEO', 'ADMINISTRATOR_DELETE_VIDEO', 'RIGHT_HOLDER_DELETE_VIDEO']:
         logger.warning(f"{video_id}: Video is hidden or deleted ({reason_code}) - returning 404")
         return Response("Video not found", status=404)
 

From 8089130b7e53107a931b1a2d4f329051b4f56305 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Tue, 4 Mar 2025 10:23:38 -0500
Subject: [PATCH 60/64] Only log "too long" if not a Twitterbot

---
 app.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app.py b/app.py
index 8e64f97..a41cd68 100644
--- a/app.py
+++ b/app.py
@@ -368,12 +368,12 @@ def proxy(video_id):
     og_tags_str += f'\n<link rel="alternate" href="{get_oembed_url(params)}" type="application/json+oembed" title="{video_id}"/>'
 
     # Discord seems to ignore video URLs when Twitter meta tags are present,
-    # so in addition to including these when the User Agent is a Twitter bot,
+    # so in addition to including these when the User Agent is a Twitterbot,
     # we also include them when the video is too long to download in order to remove the play button.
     if 'twitterbot' in request_user_agent or not download_allowed:
         if 'twitterbot' in request_user_agent:
             logger.info(f"{video_id}: Twitterbot detected - adding Twitter tags")
-        if not download_allowed:
+        elif not download_allowed:
             logger.info(f"{video_id}: Video too long to download - will not show play button")
         og_tags_str += f'\n<meta content="{thumbnail_url}" property="twitter:image"/>'
         og_tags_str += '\n<meta content="summary_large_image" property="twitter:card"/>'

From abbe5c3bd91cfe2b15cebfb6529b149b7ae00d16 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Tue, 4 Mar 2025 10:24:26 -0500
Subject: [PATCH 61/64] Re-order download allowed logic

---
 app.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app.py b/app.py
index a41cd68..a17780a 100644
--- a/app.py
+++ b/app.py
@@ -312,13 +312,13 @@ def proxy(video_id):
     video_width, video_height = get_video_resolution(params) if params else (None, None)
 
     download_allowed = True
-    if params['video']['duration'] > 60 * 20:  # 20 minutes
-        logger.info(f"{video_id}: Video download ignored due to duration ({params['video']['duration']} seconds)")
-        download_allowed = False
     request_user_agent = request.headers.get('User-Agent', '').lower()
     if download_allowed and 'discordbot' not in request_user_agent:
         logger.info(f"{video_id}: Video download ignored due to user agent ({request_user_agent})")
         download_allowed = False
+    if params['video']['duration'] > 60 * 20:  # 20 minutes
+        logger.info(f"{video_id}: Video download ignored due to duration ({params['video']['duration']} seconds)")
+        download_allowed = False
     video_quality = get_video_quality(params) if params else None
     if download_allowed and video_quality is not None:
         video_in_cdn = is_video_in_cdn(video_id)

From 71fa5ad6b608358cb80eb2b37890bcd481b0fbb4 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Tue, 4 Mar 2025 10:34:30 -0500
Subject: [PATCH 62/64] Separate HTML caches

---
 app.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/app.py b/app.py
index a17780a..977473d 100644
--- a/app.py
+++ b/app.py
@@ -280,9 +280,16 @@ def get_oembed_url(params):
 def proxy(video_id):
     logger.info(f"{video_id}: Received request")
 
+    cache_html_suffix = "_html"
+    request_user_agent = request.headers.get('User-Agent', '').lower()
+    if 'twitterbot' in request_user_agent:
+        cache_html_suffix = "_html_twitterbot"
+    elif 'discordbot' in request_user_agent:
+        cache_html_suffix = "_html_discordbot"
+
     if cache is not None:
         logger.debug(f"{video_id}: Checking cache")
-        cached_html = cache.get(f"{video_id}_html")
+        cached_html = cache.get(f"{video_id}{cache_html_suffix}")
         if cached_html is not None:
             logger.info(f"{video_id}: Returning cached response")
             return Response(cached_html, mimetype="text/html")  # type: ignore
@@ -312,7 +319,6 @@ def proxy(video_id):
     video_width, video_height = get_video_resolution(params) if params else (None, None)
 
     download_allowed = True
-    request_user_agent = request.headers.get('User-Agent', '').lower()
     if download_allowed and 'discordbot' not in request_user_agent:
         logger.info(f"{video_id}: Video download ignored due to user agent ({request_user_agent})")
         download_allowed = False
@@ -403,7 +409,7 @@ if you want to download videos, please consider using a tool like nndownload: ht
 
     if cache is not None:
         logger.info(f"{video_id}: Caching HTML response")
-        cache.set(f"{video_id}_html", html_response, expire=CACHE_EXPIRATION_HTML)
+        cache.set(f"{video_id}{cache_html_suffix}", html_response, expire=CACHE_EXPIRATION_HTML)
 
     logger.info(f"{video_id}: Returning response")
     logger.debug(f"{video_id}: HTML response:\n----------\n{html_response}\n----------")

From e43a67b0d5eb664b7d6465d72429b7234b129029 Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Sun, 30 Mar 2025 00:46:24 -0400
Subject: [PATCH 63/64] Add another reason code

---
 app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app.py b/app.py
index 977473d..a9b1c1a 100644
--- a/app.py
+++ b/app.py
@@ -305,7 +305,7 @@ def proxy(video_id):
         return Response("Video not found", status=404)
 
     reason_code = params.get('reasonCode', '').upper()
-    if reason_code in ['HIDDEN_VIDEO', 'ADMINISTRATOR_DELETE_VIDEO', 'RIGHT_HOLDER_DELETE_VIDEO']:
+    if reason_code in ['HIDDEN_VIDEO', 'ADMINISTRATOR_DELETE_VIDEO', 'RIGHT_HOLDER_DELETE_VIDEO', 'DELETED_VIDEO']:
         logger.warning(f"{video_id}: Video is hidden or deleted ({reason_code}) - returning 404")
         return Response("Video not found", status=404)
 

From c18260ebcbe80993af6fd5e722507eae8428dd4b Mon Sep 17 00:00:00 2001
From: MMaker <mmaker@mmaker.moe>
Date: Mon, 18 Aug 2025 21:28:42 -0400
Subject: [PATCH 64/64] Add access tracker for cleaning CDN

---
 .gitignore        |  3 ++-
 access_tracker.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++
 app.py            |  8 ++++++
 clean.py          | 32 ++++++++++++++++++++---
 4 files changed, 104 insertions(+), 4 deletions(-)
 create mode 100644 access_tracker.py

diff --git a/.gitignore b/.gitignore
index 6e36383..8bd1691 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,5 @@ venv
 .venv
 __pycache__
 cookies.txt
-.env
\ No newline at end of file
+.env
+access_times.json
\ No newline at end of file
diff --git a/access_tracker.py b/access_tracker.py
new file mode 100644
index 0000000..22661ab
--- /dev/null
+++ b/access_tracker.py
@@ -0,0 +1,65 @@
+import json
+import os
+import threading
+import time
+from typing import Dict, Optional
+import logging
+
+logger = logging.getLogger(__name__)
+
+class AccessTracker:
+    """Tracks when video URLs are accessed, storing data in JSON file and keeping it in memory"""
+    
+    def __init__(self, json_file_path: str = "access_times.json"):
+        self.json_file_path = json_file_path
+        self.access_times: Dict[str, float] = {}
+        self.lock = threading.Lock()
+        self._load_from_file()
+    
+    def _load_from_file(self) -> None:
+        """Load access times from JSON file into memory"""
+        try:
+            if os.path.exists(self.json_file_path):
+                with open(self.json_file_path, 'r') as f:
+                    self.access_times = json.load(f)
+                logger.info(f"Loaded {len(self.access_times)} access times from {self.json_file_path}")
+            else:
+                logger.info(f"Access times file {self.json_file_path} does not exist, starting fresh")
+        except Exception as e:
+            logger.error(f"Error loading access times from {self.json_file_path}: {e}")
+            self.access_times = {}
+    
+    def _save_to_file(self) -> None:
+        """Save current access times from memory to JSON file"""
+        try:
+            with open(self.json_file_path, 'w') as f:
+                json.dump(self.access_times, f, indent=2)
+            logger.debug(f"Saved {len(self.access_times)} access times to {self.json_file_path}")
+        except Exception as e:
+            logger.error(f"Error saving access times to {self.json_file_path}: {e}")
+    
+    def record_access(self, video_id: str) -> None:
+        """Record that a video was accessed at the current time"""
+        current_time = time.time()
+        with self.lock:
+            self.access_times[video_id] = current_time
+            self._save_to_file()
+        logger.debug(f"Recorded access for {video_id} at {current_time}")
+    
+    def get_last_access(self, video_id: str) -> Optional[float]:
+        """Get the last access time for a video (returns None if never accessed)"""
+        with self.lock:
+            return self.access_times.get(video_id)
+    
+    def get_all_access_times(self) -> Dict[str, float]:
+        """Get a copy of all access times"""
+        with self.lock:
+            return self.access_times.copy()
+    
+    def remove_access_record(self, video_id: str) -> None:
+        """Remove access record for a video (e.g., when video is deleted)"""
+        with self.lock:
+            if video_id in self.access_times:
+                del self.access_times[video_id]
+                self._save_to_file()
+                logger.debug(f"Removed access record for {video_id}")
diff --git a/app.py b/app.py
index a9b1c1a..a5de1eb 100644
--- a/app.py
+++ b/app.py
@@ -17,6 +17,7 @@ from botocore.client import Config as BotoConfig
 import urllib.parse
 
 from dotenv import load_dotenv
+from access_tracker import AccessTracker
 load_dotenv()
 
 logging.basicConfig(
@@ -83,6 +84,8 @@ download_tracker = {
 download_lock = threading.Lock()
 download_queue = []
 
+access_tracker = AccessTracker()
+
 def download_and_upload_video(video_id, url, video_quality):
     try:
         with download_lock:
@@ -411,6 +414,11 @@ if you want to download videos, please consider using a tool like nndownload: ht
         logger.info(f"{video_id}: Caching HTML response")
         cache.set(f"{video_id}{cache_html_suffix}", html_response, expire=CACHE_EXPIRATION_HTML)
 
+    # Record access time for CDN cleanup purposes
+    if is_video_in_cdn(video_id):
+        access_tracker.record_access(video_id)
+        logger.debug(f"{video_id}: Recorded access time for CDN tracking")
+
     logger.info(f"{video_id}: Returning response")
     logger.debug(f"{video_id}: HTML response:\n----------\n{html_response}\n----------")
     return Response(html_response, mimetype="text/html")
diff --git a/clean.py b/clean.py
index e20faa0..cfbd7c4 100644
--- a/clean.py
+++ b/clean.py
@@ -6,6 +6,7 @@ import logging
 import boto3
 from botocore.client import Config as BotoConfig
 from dotenv import load_dotenv
+from access_tracker import AccessTracker
 
 logging.basicConfig(
     level=logging.INFO,
@@ -15,6 +16,7 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 def cleanup_old_files(dry_run=False, days=7, directory_prefix="niconico"):
+    access_tracker = AccessTracker()
     required_env_vars = [
         'NICONICOGAY_S3_ACCESS_KEY',
         'NICONICOGAY_S3_SECRET_KEY',
@@ -38,12 +40,13 @@ def cleanup_old_files(dry_run=False, days=7, directory_prefix="niconico"):
         )
         
         bucket_name = os.environ['NICONICOGAY_S3_BUCKET_NAME']
-        cutoff_date = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=days)
+        cutoff_timestamp = datetime.datetime.now(datetime.timezone.utc).timestamp() - (days * 24 * 60 * 60)
         paginator = s3_client.get_paginator('list_objects_v2')
         page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=f"{directory_prefix}/")
         
         total_files = 0
         objects_to_delete = []
+        access_times = access_tracker.get_all_access_times()
         
         for page in page_iterator:
             if 'Contents' not in page:
@@ -51,8 +54,31 @@ def cleanup_old_files(dry_run=False, days=7, directory_prefix="niconico"):
             
             for obj in page['Contents']:
                 total_files += 1
-                if obj['LastModified'] < cutoff_date:  # type: ignore
-                    objects_to_delete.append({'Key': obj['Key']})  # type: ignore
+                key = obj['Key']  # type: ignore
+                
+                # Extract video_id from S3 key (e.g., "niconico/sm12345.mp4" -> "sm12345")
+                if key.startswith(f"{directory_prefix}/") and key.endswith('.mp4'):
+                    video_id = key[len(f"{directory_prefix}/"):-4]  # Remove prefix and .mp4 extension
+                    
+                    last_access = access_times.get(video_id)
+                    should_delete = False
+                    
+                    if last_access is None:
+                        # No access record - delete files that haven't been accessed since tracking started
+                        # For safety, only delete files older than the cutoff date
+                        if obj['LastModified'].timestamp() < cutoff_timestamp:  # type: ignore
+                            should_delete = True
+                            logger.debug(f"Will delete {video_id}: no access record and file is old")
+                    elif last_access < cutoff_timestamp:
+                        # Has access record but last access was too long ago
+                        should_delete = True
+                        logger.debug(f"Will delete {video_id}: last accessed {(datetime.datetime.now().timestamp() - last_access) / (24*60*60):.1f} days ago")
+                    
+                    if should_delete:
+                        objects_to_delete.append({'Key': key})
+                        # Remove the access record since we're deleting the file
+                        if not dry_run:
+                            access_tracker.remove_access_record(video_id)
 
         if len(objects_to_delete) == 0:
             logger.info("No files to delete")