From 1fb35e8048eabc5a44c8e28a2037ec0f480e2dd3 Mon Sep 17 00:00:00 2001
From: Wesley Moore <wes@wezm.net>
Date: Mon, 6 May 2024 10:33:43 +1000
Subject: [PATCH] Add YouTube subscription to OPML tooling

---
 .gitignore         |  1 +
 Makefile           | 20 ++++++++++++++++++++
 README.md          | 39 +++++++++++++++++++++++++++++++++++++++
 generate-json-opml | 20 ++++++++++++++++++++
 generate-opml      | 41 +++++++++++++++++++++++++++++++++++++++++
 5 files changed, 121 insertions(+)
 create mode 100644 Makefile
 create mode 100644 README.md
 create mode 100755 generate-json-opml
 create mode 100755 generate-opml

diff --git a/.gitignore b/.gitignore
index 0694fa3..ad12bf9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 /html
 /json
+/subscriptions.*
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..7606f42
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,20 @@
+JAQ?=jaq
+
+subscriptions.txt: subscriptions.json
+	$(JAQ) --raw-output '.[]' subscriptions.json > $@
+
+subscriptions.curl: subscriptions.json
+	$(JAQ) --raw-output '.[] | (split("/") | last) as $$name | . | "url \(.)\noutput \($$name).html"' subscriptions.json > $@
+
+fetch: subscriptions.curl
+	curl --location --output-dir html --create-dirs --rate 1/s --config subscriptions.curl
+
+channel-json: subscriptions.txt
+	# nproc is not portable :-/
+	xargs -n1 --max-procs=$$(nproc) --arg-file subscriptions.txt --verbose ./generate-json-opml
+
+# turn all the channel json files into an OPML file
+subscriptions.opml:
+	./generate-opml > $@
+
+.PHONY: channel-json subscriptions.opml
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..99c4986
--- /dev/null
+++ b/README.md
@@ -0,0 +1,39 @@
+# YouTube Subscriptions to OPML
+
+This repo contains a small collection of scripts that I used to turn my YouTube subscriptions into an OPML file for import in to [Feedbin].
+
+## Dependencies
+
+The scripts have only been run on a Linux system using GNU coreutils. They will
+probably need some tweaking to run on other UNIX-like systems.
+
+- [Scraper](https://lib.rs/crates/scraper)
+- [jaq](https://github.com/01mf02/jaq)
+- curl
+- Python
+- awk
+- GNU make (I haven't tested non-GNU make)
+
+## Usage
+
+1. Visit your [subscriptions page](https://www.youtube.com/feed/channels)
+2. Repeatedly scroll to the end of the page to make them all load
+3. Run the following in the JavaScript console to copy the list of subscriptions to you clipboard as JSON array:
+
+    ```javascript
+    copy(JSON.stringify(Array.from(new Set(Array.prototype.map.call(document.querySelectorAll('a.channel-link'), (link) => link.href))).filter((x) => !x.includes('/channel/')), null, 2))
+    ```
+
+    **Note:** I only tested the above on Firefox.
+
+    Also why do this instead of processing the subscriptions.csv from Google Takeout?
+
+    1. Takeout generates multiple gigabytes of archives I have to download to get the CSV file.
+    2. It's slow to generate. This process can be done whenever you want.
+
+4. Paste the list of subscriptions into `subscriptions.json`.
+5. Run `make fetch` to fetch the channel pages of all the subscriptions. This only needs to be run once.
+6. Run `make channel-json` to extract info from each channel page.
+7. Run `make subscriptions.opml` to generate the OPML file.
+
+[Feedbin]: https://feedbin.com/
diff --git a/generate-json-opml b/generate-json-opml
new file mode 100755
index 0000000..fa8893c
--- /dev/null
+++ b/generate-json-opml
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -eu
+
+URL="$1"
+NAME=$(echo "$URL" | awk -F / '{ print $NF }')
+HTML="html/${NAME}.html"
+CHANNEL_ID=$(scraper -a content 'meta[property="og:url"]' < "$HTML" | awk -F / '{ print $NF }')
+TITLE=$(scraper -a content 'meta[property="og:title"]' < "$HTML")
+XML_URL="https://www.youtube.com/feeds/videos.xml?channel_id=${CHANNEL_ID}"
+
+json_escape() {
+  echo "$1" | jaq --raw-input .
+}
+
+JSON_TITLE=$(json_escape "$TITLE")
+JSON_XML_URL=$(json_escape "$XML_URL")
+JSON_URL=$(json_escape "$URL")
+
+printf '{"title": %s, "xmlUrl": %s, "htmlUrl": %s}\n' "$JSON_TITLE" "$JSON_XML_URL" "$JSON_URL" > json/"$NAME".json
diff --git a/generate-opml b/generate-opml
new file mode 100755
index 0000000..4cff1bf
--- /dev/null
+++ b/generate-opml
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+
+import email.utils
+import glob
+import json
+import xml.etree.ElementTree as ET
+
+# This is what we're aiming to generate:
+#
+# <?xml version="1.0" encoding="UTF-8"?>
+# <opml version="1.0">
+#   <head>
+#     <title>RSS subscriptions for wes@wezm.net</title>
+#     <dateCreated>Sun, 05 May 2024 02:54:31 +0000</dateCreated>
+#     <ownerEmail>wes@wezm.net</ownerEmail>
+#   </head>
+#   <body>
+#     <outline text="3D Printing" title="3D Printing">
+# <outline text="CadHub Blog" title="CadHub Blog" type="rss" xmlUrl="https://learn.cadhub.xyz/blog/rss.xml" htmlUrl="https://learn.cadhub.xyz/blog"/>
+#     </outline>
+#   </body>
+# </opml>
+
+opml = ET.Element("opml")
+
+head = ET.SubElement(opml, "head")
+title = ET.SubElement(head, "title")
+title.text = "YouTube Subscription"
+dateCreated = ET.SubElement(head, "dateCreated")
+dateCreated.text = email.utils.formatdate(timeval=None, localtime=True)
+
+body = ET.SubElement(opml, "body")
+youtube = ET.SubElement(body, "outline", {"title": "YouTube", "text": "YouTube"})
+
+for path in glob.glob("json/*.json"):
+    with open(path) as f:
+        info = json.load(f)
+        ET.SubElement(youtube, "outline", info, type="rss", text=info["title"])
+
+ET.indent(opml)
+print(ET.tostring(opml, encoding="unicode", xml_declaration=True))