WIP tweets/timeline parser

.github/workflows/build-docker.yml

@@ -1,4 +1,4 @@
-name: Docker
+name: CI/CD
 
 on:
   push:
@@ -8,56 +8,31 @@ on:
       - master
 
 jobs:
-  tests:
+  build-docker:
-    uses: ./.github/workflows/run-tests.yml
+    runs-on: ubuntu-latest
-    secrets: inherit
-  build-docker-amd64:
-    needs: [tests]
-    runs-on: buildjet-2vcpu-ubuntu-2204
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 0
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: all
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v1
         with:
           version: latest
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Build and push AMD64 Docker image
+      - name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v2
         with:
           context: .
           file: ./Dockerfile
           platforms: linux/amd64
           push: true
           tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}
-  build-docker-arm64:
-    needs: [tests]
-    runs-on: buildjet-2vcpu-ubuntu-2204-arm
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          version: latest
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Build and push ARM64 Docker image
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          file: ./Dockerfile.arm64
-          platforms: linux/arm64
-          push: true
-          tags: zedeus/nitter:latest-arm64,zedeus/nitter:${{ github.sha }}-arm64

.github/workflows/run-tests.yml

@@ -1,108 +0,0 @@
-name: Tests
-
-on:
-  push:
-    paths-ignore:
-      - "*.md"
-    branches-ignore:
-      - master
-  workflow_call:
-
-# Ensure that multiple runs on the same branch do not overlap.
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell: bash
-
-jobs:
-  build-test:
-    name: Build and test
-    runs-on: buildjet-2vcpu-ubuntu-2204
-    strategy:
-      matrix:
-        nim: ["1.6.x", "2.0.x", "2.2.x", "devel"]
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Cache Nimble Dependencies
-        id: cache-nimble
-        uses: buildjet/cache@v4
-        with:
-          path: ~/.nimble
-          key: ${{ matrix.nim }}-nimble-v2-${{ hashFiles('*.nimble') }}
-          restore-keys: |
-            ${{ matrix.nim }}-nimble-v2-
-
-      - name: Setup Nim
-        uses: jiro4989/setup-nim-action@v2
-        with:
-          nim-version: ${{ matrix.nim }}
-          use-nightlies: true
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build Project
-        run: nimble build -d:release -Y
-
-  integration-test:
-    needs: [build-test]
-    name: Integration test
-    runs-on: buildjet-2vcpu-ubuntu-2204
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Cache Nimble Dependencies
-        id: cache-nimble
-        uses: buildjet/cache@v4
-        with:
-          path: ~/.nimble
-          key: devel-nimble-v2-${{ hashFiles('*.nimble') }}
-          restore-keys: |
-            devel-nimble-v2-
-
-      - name: Setup Python (3.10) with pip cache
-        uses: buildjet/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: pip
-
-      - name: Setup Nim
-        uses: jiro4989/setup-nim-action@v2
-        with:
-          nim-version: devel
-          use-nightlies: true
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build Project
-        run: nimble build -d:release -Y
-
-      - name: Install SeleniumBase and Chromedriver
-        run: |
-          pip install seleniumbase
-          seleniumbase install chromedriver
-
-      - name: Start Redis Service
-        uses: supercharge/redis-github-action@1.5.0
-
-      - name: Prepare Nitter Environment
-        run: |
-          sudo apt-get update && sudo apt-get install -y libsass-dev
-          cp nitter.example.conf nitter.conf
-          sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf
-          nimble md
-          nimble scss
-          echo '${{ secrets.SESSIONS }}' | head -n1
-          echo '${{ secrets.SESSIONS }}' > ./sessions.jsonl
-
-      - name: Run Tests
-        run: |
-          ./nitter &
-          pytest -n1 tests

.gitignore

@@ -3,13 +3,9 @@ nitter
 *.db
 /tests/__pycache__
 /tests/geckodriver.log
-/tests/downloaded_files
+/tests/downloaded_files/*
-/tests/latest_logs
 /tools/gencss
 /tools/rendermd
 /public/css/style.css
 /public/md/*.html
 nitter.conf
-guest_accounts.json*
-sessions.json*
-dump.rdb

Dockerfile

@@ -1,5 +1,6 @@
-FROM nimlang/nim:2.2.0-alpine-regular as nim
+FROM nimlang/nim:1.6.2-alpine-regular as nim
 LABEL maintainer="setenforce@protonmail.com"
+EXPOSE 8080
 
 RUN apk --no-cache add libsass-dev pcre
 
@@ -9,17 +10,14 @@ COPY nitter.nimble .
 RUN nimble install -y --depsOnly
 
 COPY . .
-RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
+RUN nimble build -d:danger -d:lto -d:strip \
     && nimble scss \
     && nimble md
 
 FROM alpine:latest
 WORKDIR /src/
-RUN apk --no-cache add pcre ca-certificates
+RUN apk --no-cache add pcre
 COPY --from=nim /src/nitter/nitter ./
 COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
 COPY --from=nim /src/nitter/public ./public
-EXPOSE 8080
-RUN adduser -h /src/ -D -s /bin/sh nitter
-USER nitter
 CMD ./nitter

Dockerfile.arm64

@@ -1,25 +0,0 @@
-FROM alpine:3.20.6 as nim
-LABEL maintainer="setenforce@protonmail.com"
-
-RUN apk --no-cache add libsass-dev pcre gcc git libc-dev nim nimble
-
-WORKDIR /src/nitter
-
-COPY nitter.nimble .
-RUN nimble install -y --depsOnly
-
-COPY . .
-RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
-    && nimble scss \
-    && nimble md
-
-FROM alpine:3.20.6
-WORKDIR /src/
-RUN apk --no-cache add pcre ca-certificates openssl
-COPY --from=nim /src/nitter/nitter ./
-COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
-COPY --from=nim /src/nitter/public ./public
-EXPOSE 8080
-RUN adduser -h /src/ -D -s /bin/sh nitter
-USER nitter
-CMD ./nitter

README.md

@@ -1,38 +1,29 @@
 # Nitter
 
-[![Test Matrix](https://github.com/zedeus/nitter/workflows/Tests/badge.svg)](https://github.com/zedeus/nitter/actions/workflows/run-tests.yml)
+[![Test Matrix](https://github.com/zedeus/nitter/workflows/CI/CD/badge.svg)](https://github.com/zedeus/nitter/actions?query=workflow%3ACI/CD)
-[![Test Matrix](https://github.com/zedeus/nitter/workflows/Docker/badge.svg)](https://github.com/zedeus/nitter/actions/workflows/build-docker.yml)
 [![License](https://img.shields.io/github/license/zedeus/nitter?style=flat)](#license)
 
-> [!NOTE]
-> Running a Nitter instance now requires real accounts, since Twitter removed the previous methods. \
-> This does not affect users. \
-> For instructions on how to obtain session tokens, see [Creating session tokens](https://github.com/zedeus/nitter/wiki/Creating-session-tokens).
-
 A free and open source alternative Twitter front-end focused on privacy and
 performance. \
-Inspired by the [Invidious](https://github.com/iv-org/invidious) project.
+Inspired by the [Invidious](https://github.com/iv-org/invidious)
+project.
 
 - No JavaScript or ads
 - All requests go through the backend, client never talks to Twitter
 - Prevents Twitter from tracking your IP or JavaScript fingerprint
-- Uses Twitter's unofficial API (no developer account required)
+- Uses Twitter's unofficial API (no rate limits or developer account required)
 - Lightweight (for [@nim_lang](https://nitter.net/nim_lang), 60KB vs 784KB from twitter.com)
 - RSS feeds
 - Themes
 - Mobile support (responsive design)
 - AGPLv3 licensed, no proprietary instances permitted
 
-<details>
+Liberapay: https://liberapay.com/zedeus \
-<summary>Donations</summary>
+Patreon: https://patreon.com/nitter \
-Liberapay: https://liberapay.com/zedeus<br>
+BTC: bc1qp7q4qz0fgfvftm5hwz3vy284nue6jedt44kxya \
-Patreon: https://patreon.com/nitter<br>
+ETH: 0x66d84bc3fd031b62857ad18c62f1ba072b011925 \
-BTC: bc1qpqpzjkcpgluhzf7x9yqe7jfe8gpfm5v08mdr55<br>
+LTC: ltc1qhsz5nxw6jw9rdtw9qssjeq2h8hqk2f85rdgpkr \
-ETH: 0x24a0DB59A923B588c7A5EBd0dBDFDD1bCe9c4460<br>
+XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL
-XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL<br>
-SOL: ANsyGNXFo6osuFwr1YnUqif2RdoYRhc27WdyQNmmETSW<br>
-ZEC: u1vndfqtzyy6qkzhkapxelel7ams38wmfeccu3fdpy2wkuc4erxyjm8ncjhnyg747x6t0kf0faqhh2hxyplgaum08d2wnj4n7cyu9s6zhxkqw2aef4hgd4s6vh5hpqvfken98rg80kgtgn64ff70djy7s8f839z00hwhuzlcggvefhdlyszkvwy3c7yw623vw3rvar6q6evd3xcvveypt
-</details>
 
 ## Roadmap
 
@@ -43,20 +34,19 @@ ZEC: u1vndfqtzyy6qkzhkapxelel7ams38wmfeccu3fdpy2wkuc4erxyjm8ncjhnyg747x6t0kf0faq
 
 ## Resources
 
-The wiki contains
+The wiki contains 
 [a list of instances](https://github.com/zedeus/nitter/wiki/Instances) and
 [browser extensions](https://github.com/zedeus/nitter/wiki/Extensions)
 maintained by the community.
 
 ## Why?
 
-It's impossible to use Twitter without JavaScript enabled, and as of 2024 you
+It's impossible to use Twitter without JavaScript enabled. For privacy-minded
-need to sign up. For privacy-minded folks, preventing JavaScript analytics and
+folks, preventing JavaScript analytics and IP-based tracking is important, but
-IP-based tracking is important, but apart from using a VPN and uBlock/uMatrix,
+apart from using a VPN and uBlock/uMatrix, it's impossible. Despite being behind
-it's impossible. Despite being behind a VPN and using heavy-duty adblockers,
+a VPN and using heavy-duty adblockers, you can get accurately tracked with your
-you can get accurately tracked with your [browser's
+[browser's fingerprint](https://restoreprivacy.com/browser-fingerprinting/),
-fingerprint](https://restoreprivacy.com/browser-fingerprinting/), [no
+[no JavaScript required](https://noscriptfingerprint.com/). This all became
-JavaScript required](https://noscriptfingerprint.com/). This all became
 particularly important after Twitter [removed the
 ability](https://www.eff.org/deeplinks/2020/04/twitter-removes-privacy-option-and-shows-why-we-need-strong-privacy-laws)
 for users to control whether their data gets sent to advertisers.
@@ -77,24 +67,21 @@ Twitter account.
 ## Installation
 
 ### Dependencies
-
+* libpcre
-- libpcre
+* libsass
-- libsass
+* redis
-- redis/valkey
 
 To compile Nitter you need a Nim installation, see
-[nim-lang.org](https://nim-lang.org/install.html) for details. It is possible
+[nim-lang.org](https://nim-lang.org/install.html) for details. It is possible to
-to install it system-wide or in the user directory you create below.
+install it system-wide or in the user directory you create below.
 
 To compile the scss files, you need to install `libsass`. On Ubuntu and Debian,
 you can use `libsass-dev`.
 
-Redis is required for caching and in the future for account info. As of 2024
+Redis is required for caching and in the future for account info. It should be
-Redis is no longer open source, so using the fork Valkey is recommended. It
+available on most distros as `redis` or `redis-server` (Ubuntu/Debian).
-should be available on most distros as `redis` or `redis-server`
+Running it with the default config is fine, Nitter's default config is set to
-(Ubuntu/Debian), or `valkey`/`valkey-server`. Running it with the default
+use the default Redis port and localhost.
-config is fine, Nitter's default config is set to use the default port and
-localhost.
 
 Here's how to create a `nitter` user, clone the repo, and build the project
 along with the scss and md files.
@@ -104,7 +91,7 @@ along with the scss and md files.
 # su nitter
 $ git clone https://github.com/zedeus/nitter
 $ cd nitter
-$ nimble build -d:danger --mm:refc
+$ nimble build -d:release
 $ nimble scss
 $ nimble md
 $ cp nitter.example.conf nitter.conf
@@ -121,32 +108,25 @@ performance reasons.
 
 ### Docker
 
-Page for the Docker image: https://hub.docker.com/r/zedeus/nitter
+#### NOTE: For ARM64/ARM support, please use [unixfox's image](https://quay.io/repository/unixfox/nitter?tab=tags), more info [here](https://github.com/zedeus/nitter/issues/399#issuecomment-997263495)
-
-#### NOTE: For ARM64 support, please use the separate ARM64 docker image: [`zedeus/nitter:latest-arm64`](https://hub.docker.com/r/zedeus/nitter/tags).
 
 To run Nitter with Docker, you'll need to install and run Redis separately
 before you can run the container. See below for how to also run Redis using
 Docker.
 
 To build and run Nitter in Docker:
-
 ```bash
 docker build -t nitter:latest .
 docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest
 ```
 
-Note: For ARM64, use this Dockerfile: [`Dockerfile.arm64`](https://github.com/zedeus/nitter/blob/master/Dockerfile.arm64).
-
 A prebuilt Docker image is provided as well:
-
 ```bash
 docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host zedeus/nitter:latest
 ```
 
 Using docker-compose to run both Nitter and Redis as different containers:
 Change `redisHost` from `localhost` to `nitter-redis` in `nitter.conf`, then run:
-
 ```bash
 docker-compose up -d
 ```

config.nims

@@ -1,13 +1,17 @@
 --define:ssl
 --define:useStdLib
---threads:off
 
 # workaround httpbeast file upload bug
 --assertions:off
 
 # disable annoying warnings
 warning("GcUnsafe2", off)
-warning("HoleEnumConv", off)
 hint("XDeclaredButNotUsed", off)
 hint("XCannotRaiseY", off)
 hint("User", off)
+
+const
+  nimVersion = (major: NimMajor, minor: NimMinor, patch: NimPatch)
+
+when nimVersion >= (1, 6, 0):
+  warning("HoleEnumConv", off)

docker-compose.yml

@@ -8,22 +8,10 @@ services:
     ports:
       - "127.0.0.1:8080:8080" # Replace with "8080:8080" if you don't use a reverse proxy
     volumes:
-      - ./nitter.conf:/src/nitter.conf:Z,ro
+      - ./nitter.conf:/src/nitter.conf:ro
-      - ./sessions.jsonl:/src/sessions.jsonl:Z,ro # Run get_sessions.py to get the credentials
     depends_on:
       - nitter-redis
     restart: unless-stopped
-    healthcheck:
-      test: wget -nv --tries=1 --spider http://127.0.0.1:8080/Jack/status/20 || exit 1
-      interval: 30s
-      timeout: 5s
-      retries: 2
-    user: "998:998"
-    read_only: true
-    security_opt:
-      - no-new-privileges:true
-    cap_drop:
-      - ALL
 
   nitter-redis:
     image: redis:6-alpine
@@ -32,17 +20,6 @@ services:
     volumes:
       - nitter-redis:/data
     restart: unless-stopped
-    healthcheck:
-      test: redis-cli ping
-      interval: 30s
-      timeout: 5s
-      retries: 2
-    user: "999:1000"
-    read_only: true
-    security_opt:
-      - no-new-privileges:true
-    cap_drop:
-      - ALL
 
 volumes:
   nitter-redis:

nitter.example.conf

@@ -1,11 +1,11 @@
 [Server]
-hostname = "nitter.net"  # for generating links, change this to your own domain/ip
-title = "nitter"
 address = "0.0.0.0"
 port = 8080
 https = false  # disable to enable cookies when not using https
 httpMaxConnections = 100
 staticDir = "./public"
+title = "nitter"
+hostname = "nitter.net"
 
 [Cache]
 listMinutes = 240  # how long to cache list info (not the tweets, so keep it high)
@@ -13,9 +13,9 @@ rssMinutes = 10  # how long to cache rss queries
 redisHost = "localhost"  # Change to "nitter-redis" if using docker-compose
 redisPort = 6379
 redisPassword = ""
-redisConnections = 20  # minimum open connections in pool
+redisConnections = 20  # connection pool size
 redisMaxConnections = 30
-# new connections are opened when none are available, but if the pool size
+# max, new connections are opened when none are available, but if the pool size
 # goes above this, they're closed when released. don't worry about this unless
 # you receive tons of requests per second
 
@@ -23,16 +23,23 @@ redisMaxConnections = 30
 hmacKey = "secretkey"  # random key for cryptographic signing of video urls
 base64Media = false  # use base64 encoding for proxied media urls
 enableRSS = true  # set this to false to disable RSS feeds
-enableDebug = false  # enable request logs and debug endpoints (/.sessions)
+enableDebug = false  # enable request logs and debug endpoints
 proxy = ""  # http/https url, SOCKS proxies are not supported
 proxyAuth = ""
+tokenCount = 10
+# minimum amount of usable tokens. tokens are used to authorize API requests,
+# but they expire after ~1 hour, and have a limit of 187 requests.
+# the limit gets reset every 15 minutes, and the pool is filled up so there's
+# always at least $tokenCount usable tokens. again, only increase this if
+# you receive major bursts all the time
 
 # Change default preferences here, see src/prefs_impl.nim for a complete list
 [Preferences]
 theme = "Nitter"
 replaceTwitter = "nitter.net"
-replaceYouTube = "piped.video"
+replaceYouTube = "piped.kavin.rocks"
 replaceReddit = "teddit.net"
+replaceInstagram = ""
 proxyVideos = true
 hlsPlayback = false
 infiniteScroll = false

nitter.nimble

@@ -10,20 +10,20 @@ bin           = @["nitter"]
 
 # Dependencies
 
-requires "nim >= 1.6.10"
+requires "nim >= 1.4.8"
-requires "jester#baca3f"
+requires "jester >= 0.5.0"
-requires "karax#5cf360c"
+requires "karax#c71bc92"
-requires "sass#7dfdd03"
+requires "sass#e683aa1"
-requires "nimcrypto#a079df9"
+requires "nimcrypto#a5742a9"
-requires "markdown#158efe3"
+requires "markdown#abdbe5e"
-requires "packedjson#9e6fbb6"
+requires "packedjson#d11d167"
-requires "supersnappy#6c94198"
+requires "supersnappy#2.1.1"
 requires "redpool#8b7c1db"
 requires "https://github.com/zedeus/redis#d0a0e6f"
-requires "zippy#ca5989a"
+requires "zippy#0.7.3"
-requires "flatty#e668085"
+requires "flatty#0.2.3"
-requires "jsony#1de1f08"
+requires "jsony#d0e69bd"
-requires "oauth#b8c163b"
+
 
 # Tasks
 

public/css/themes/dracula.css

@@ -1,41 +0,0 @@
-body {
-    --bg_color: #282a36;
-    --fg_color: #f8f8f2;
-    --fg_faded: #818eb6;
-    --fg_dark: var(--fg_faded);
-    --fg_nav: var(--accent);
-
-    --bg_panel: #343746;
-    --bg_elements: #292b36;
-    --bg_overlays: #44475a;
-    --bg_hover: #2f323f;
-
-    --grey: var(--fg_faded);
-    --dark_grey: #44475a;
-    --darker_grey: #3d4051;
-    --darkest_grey: #363948;
-    --border_grey: #44475a;
-
-    --accent: #bd93f9;
-    --accent_light: #caa9fa;
-    --accent_dark: var(--accent);
-    --accent_border: #ff79c696;
-
-    --play_button: #ffb86c;
-    --play_button_hover: #ffc689;
-
-    --more_replies_dots: #bd93f9;
-    --error_red: #ff5555;
-
-    --verified_blue: var(--accent);
-    --icon_text: ##F8F8F2;
-
-    --tab: #6272a4;
-    --tab_selected: var(--accent);
-
-    --profile_stat: #919cbf;
-}
-
-.search-bar > form input::placeholder{
-    color: var(--fg_faded);
-}

public/js/infiniteScroll.js

@@ -1,13 +1,11 @@
 // @license http://www.gnu.org/licenses/agpl-3.0.html AGPL-3.0
 // SPDX-License-Identifier: AGPL-3.0-only
-const LOADING_TEXT = "Loading...";
-
 function insertBeforeLast(node, elem) {
     node.insertBefore(elem, node.childNodes[node.childNodes.length - 2]);
 }
 
 function getLoadMore(doc) {
-    return doc.querySelector(".show-more:not(.timeline-item)");
+    return doc.querySelector('.show-more:not(.timeline-item)');
 }
 
 function isDuplicate(item, itemClass) {
@@ -17,173 +15,52 @@ function isDuplicate(item, itemClass) {
     return document.querySelector(itemClass + " .tweet-link[href='" + href + "']") != null;
 }
 
-function addScrollToURL(href) {
+window.onload = function() {
-    const url = new URL(href);
-    url.searchParams.append("scroll", "true");
-    return url.toString();
-}
-
-function fetchAndParse(url) {
-    return fetch(url)
-        .then(function (response) {
-            return response.text();
-        })
-        .then(function (html) {
-            var parser = new DOMParser();
-            return parser.parseFromString(html, "text/html");
-        });
-}
-
-window.onload = function () {
     const url = window.location.pathname;
     const isTweet = url.indexOf("/status/") !== -1;
-    const isIncompleteThread =
-        isTweet && document.querySelector(".timeline-item.more-replies") != null;
-
     const containerClass = isTweet ? ".replies" : ".timeline";
-    const itemClass = containerClass + " > div:not(.top-ref)";
+    const itemClass = containerClass + ' > div:not(.top-ref)';
 
     var html = document.querySelector("html");
-    var mainContainer = document.querySelector(containerClass);
+    var container = document.querySelector(containerClass);
     var loading = false;
 
-    function catchErrors(err) {
+    window.addEventListener('scroll', function() {
-        console.warn("Something went wrong.", err);
+        if (loading) return;
-        loading = true;
+        if (html.scrollTop + html.clientHeight >= html.scrollHeight - 3000) {
-    }
+            loading = true;
+            var loadMore = getLoadMore(document);
+            if (loadMore == null) return;
 
-    function appendLoadedReplies(loadMore) {
+            loadMore.children[0].text = "Loading...";
-        return function (doc) {
-            loadMore.remove();
 
-            for (var item of doc.querySelectorAll(itemClass)) {
+            var url = new URL(loadMore.children[0].href);
-                if (item.className == "timeline-item show-more") continue;
+            url.searchParams.append('scroll', 'true');
-                if (isDuplicate(item, itemClass)) continue;
-                if (isTweet) mainContainer.appendChild(item);
-                else insertBeforeLast(mainContainer, item);
-            }
 
-            loading = false;
+            fetch(url.toString()).then(function (response) {
-            const newLoadMore = getLoadMore(doc);
+                return response.text();
-            if (newLoadMore == null) return;
+            }).then(function (html) {
-            if (isTweet) mainContainer.appendChild(newLoadMore);
+                var parser = new DOMParser();
-            else insertBeforeLast(mainContainer, newLoadMore);
+                var doc = parser.parseFromString(html, 'text/html');
-        };
-    }
-
-    var scrollListener = null;
-    if (!isIncompleteThread) {
-        scrollListener = (e) => {
-            if (loading) return;
-
-            if (html.scrollTop + html.clientHeight >= html.scrollHeight - 3000) {
-                loading = true;
-                var loadMore = getLoadMore(document);
-                if (loadMore == null) return;
-
-                loadMore.children[0].text = LOADING_TEXT;
-
-                const fetchUrl = addScrollToURL(loadMore.children[0].href);
-                fetchAndParse(fetchUrl)
-                    .then(appendLoadedReplies(loadMore))
-                    .catch(catchErrors);
-            }
-        };
-    } else {
-        function getEarlierReplies(doc) {
-            return doc.querySelector(".timeline-item.more-replies.earlier-replies");
-        }
-
-        function getLaterReplies(doc) {
-            return doc.querySelector(".after-tweet > .timeline-item.more-replies");
-        }
-
-        function prependLoadedThread(loadMore) {
-            return function (doc) {
                 loadMore.remove();
 
-                const targetSelector = ".before-tweet.thread-line";
+                for (var item of doc.querySelectorAll(itemClass)) {
-                const threadContainer = document.querySelector(targetSelector);
+                    if (item.className == "timeline-item show-more") continue;
-
+                    if (isDuplicate(item, itemClass)) continue;
-                const earlierReplies = doc.querySelector(targetSelector);
+                    if (isTweet) container.appendChild(item);
-                for (var i = earlierReplies.children.length - 1; i >= 0; i--) {
+                    else insertBeforeLast(container, item);
-                    threadContainer.insertBefore(
-                        earlierReplies.children[i],
-                        threadContainer.children[0]
-                    );
                 }
+
                 loading = false;
-            };
+                const newLoadMore = getLoadMore(doc);
-        }
+                if (newLoadMore == null) return;
-
+                if (isTweet) container.appendChild(newLoadMore);
-        function appendLoadedThread(loadMore) {
+                else insertBeforeLast(container, newLoadMore);
-            return function (doc) {
+            }).catch(function (err) {
-                const targetSelector = ".after-tweet.thread-line";
+                console.warn('Something went wrong.', err);
-                const threadContainer = document.querySelector(targetSelector);
-
-                const laterReplies = doc.querySelector(targetSelector);
-                while (laterReplies && laterReplies.firstChild) {
-                    threadContainer.appendChild(laterReplies.firstChild);
-                }
-
-                const finalReply = threadContainer.lastElementChild;
-                if (finalReply.classList.contains("thread-last")) {
-                    fetchAndParse(finalReply.children[0].href).then(function (lastDoc) {
-                        loadMore.remove();
-                        const anyResponses = lastDoc.querySelector(".replies");
-                        anyResponses &&
-                            insertBeforeLast(
-                                threadContainer.parentElement.parentElement,
-                                anyResponses
-                            );
-                        loading = false;
-                    });
-                } else {
-                    loadMore.remove();
-                    loading = false;
-                }
-            };
-        }
-
-        scrollListener = (e) => {
-            if (loading) return;
-
-            if (html.scrollTop <= html.clientHeight) {
-                var loadMore = getEarlierReplies(document);
-                if (loadMore == null) return;
                 loading = true;
-
+            });
-                loadMore.children[0].text = LOADING_TEXT;
+        }
-
+    });
-                fetchAndParse(loadMore.children[0].href)
-                    .then(prependLoadedThread(loadMore))
-                    .catch(catchErrors);
-            } else if (html.scrollTop + html.clientHeight >= html.scrollHeight - 3000) {
-                var loadMore = getLaterReplies(document);
-                if (loadMore != null) {
-                    loading = true;
-
-                    loadMore.children[0].text = LOADING_TEXT;
-
-                    fetchAndParse(loadMore.children[0].href)
-                        .then(appendLoadedThread(loadMore))
-                        .catch(catchErrors);
-                } else {
-                    loadMore = getLoadMore(document);
-                    if (loadMore == null) return;
-                    loading = true;
-
-                    loadMore.children[0].text = LOADING_TEXT;
-
-                    mainContainer = document.querySelector(containerClass);
-                    fetchAndParse(loadMore.children[0].href)
-                        .then(appendLoadedReplies(loadMore))
-                        .catch(catchErrors);
-                }
-            }
-        };
-    }
-
-    window.addEventListener("scroll", scrollListener);
 };
 // @license-end

public/md/about.md

@@ -4,15 +4,15 @@ Nitter is a free and open source alternative Twitter front-end focused on
 privacy and performance. The source is available on GitHub at
 <https://github.com/zedeus/nitter>
 
-- No JavaScript or ads
+* No JavaScript or ads
-- All requests go through the backend, client never talks to Twitter
+* All requests go through the backend, client never talks to Twitter
-- Prevents Twitter from tracking your IP or JavaScript fingerprint
+* Prevents Twitter from tracking your IP or JavaScript fingerprint
-- Uses Twitter's unofficial API (no developer account required)
+* Uses Twitter's unofficial API (no rate limits or developer account required)
-- Lightweight (for [@nim_lang](/nim_lang), 60KB vs 784KB from twitter.com)
+* Lightweight (for [@nim_lang](/nim_lang), 60KB vs 784KB from twitter.com)
-- RSS feeds
+* RSS feeds
-- Themes
+* Themes
-- Mobile support (responsive design)
+* Mobile support (responsive design)
-- AGPLv3 licensed, no proprietary instances permitted
+* AGPLv3 licensed, no proprietary instances permitted
 
 Nitter's GitHub wiki contains
 [instances](https://github.com/zedeus/nitter/wiki/Instances) and
@@ -21,13 +21,12 @@ maintained by the community.
 
 ## Why use Nitter?
 
-It's impossible to use Twitter without JavaScript enabled, and as of 2024 you
+It's impossible to use Twitter without JavaScript enabled. For privacy-minded
-need to sign up. For privacy-minded folks, preventing JavaScript analytics and
+folks, preventing JavaScript analytics and IP-based tracking is important, but
-IP-based tracking is important, but apart from using a VPN and uBlock/uMatrix,
+apart from using a VPN and uBlock/uMatrix, it's impossible. Despite being behind
-it's impossible. Despite being behind a VPN and using heavy-duty adblockers,
+a VPN and using heavy-duty adblockers, you can get accurately tracked with your
-you can get accurately tracked with your [browser's
+[browser's fingerprint](https://restoreprivacy.com/browser-fingerprinting/),
-fingerprint](https://restoreprivacy.com/browser-fingerprinting/), [no
+[no JavaScript required](https://noscriptfingerprint.com/). This all became
-JavaScript required](https://noscriptfingerprint.com/). This all became
 particularly important after Twitter [removed the
 ability](https://www.eff.org/deeplinks/2020/04/twitter-removes-privacy-option-and-shows-why-we-need-strong-privacy-laws)
 for users to control whether their data gets sent to advertisers.
@@ -43,13 +42,12 @@ Twitter account.
 
 ## Donating
 
-Liberapay: https://liberapay.com/zedeus \
+Liberapay: <https://liberapay.com/zedeus> \
-Patreon: https://patreon.com/nitter \
+Patreon: <https://patreon.com/nitter> \
-BTC: bc1qpqpzjkcpgluhzf7x9yqe7jfe8gpfm5v08mdr55 \
+BTC: bc1qp7q4qz0fgfvftm5hwz3vy284nue6jedt44kxya \
-ETH: 0x24a0DB59A923B588c7A5EBd0dBDFDD1bCe9c4460 \
+ETH: 0x66d84bc3fd031b62857ad18c62f1ba072b011925 \
-XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL \
+LTC: ltc1qhsz5nxw6jw9rdtw9qssjeq2h8hqk2f85rdgpkr \
-SOL: ANsyGNXFo6osuFwr1YnUqif2RdoYRhc27WdyQNmmETSW \
+XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL
-ZEC: u1vndfqtzyy6qkzhkapxelel7ams38wmfeccu3fdpy2wkuc4erxyjm8ncjhnyg747x6t0kf0faqhh2hxyplgaum08d2wnj4n7cyu9s6zhxkqw2aef4hgd4s6vh5hpqvfken98rg80kgtgn64ff70djy7s8f839z00hwhuzlcggvefhdlyszkvwy3c7yw623vw3rvar6q6evd3xcvveypt
 
 ## Contact
 

public/robots.txt

@@ -1,5 +0,0 @@
-User-agent: *
-Disallow: /
-Crawl-delay: 1
-User-agent: Twitterbot
-Disallow:

src/api.nim

@@ -4,155 +4,121 @@ import packedjson
 import types, query, formatters, consts, apiutils, parser
 import experimental/parser as newParser
 
-proc mediaUrl(id: string; cursor: string): SessionAwareUrl =
+proc getGraphUser*(id: string): Future[User] {.async.} =
-  let
-    cookieVariables = userMediaVariables % [id, cursor]
-    oauthVariables = userTweetsVariables % [id, cursor]
-  result = SessionAwareUrl(
-    cookieUrl: graphUserMedia ? {"variables": cookieVariables, "features": gqlFeatures},
-    oauthUrl: graphUserMediaV2 ? {"variables": oauthVariables, "features": gqlFeatures}
-  )
-
-proc getGraphUser*(username: string): Future[User] {.async.} =
-  if username.len == 0: return
-  let
-    variables = """{"screen_name": "$1"}""" % username
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetchRaw(graphUser ? params, Api.userScreenName)
-  result = parseGraphUser(js)
-
-proc getGraphUserById*(id: string): Future[User] {.async.} =
   if id.len == 0 or id.any(c => not c.isDigit): return
   let
-    variables = """{"rest_id": "$1"}""" % id
+    variables = %*{"userId": id, "withSuperFollowsUserFields": true}
-    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetchRaw(graphUser ? {"variables": $variables}, Api.userRestId)
-    js = await fetchRaw(graphUserById ? params, Api.userRestId)
   result = parseGraphUser(js)
 
-proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
-  if id.len == 0: return
-  let
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = userTweetsVariables % [id, cursor]
-    params = {"variables": variables, "features": gqlFeatures}
-    js = case kind
-      of TimelineKind.tweets:
-        await fetch(graphUserTweets ? params, Api.userTweets)
-      of TimelineKind.replies:
-        await fetch(graphUserTweetsAndReplies ? params, Api.userTweetsAndReplies)
-      of TimelineKind.media:
-        await fetch(mediaUrl(id, cursor), Api.userMedia)
-  result = parseGraphTimeline(js, after)
-
-proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
-  if id.len == 0: return
-  let
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = listTweetsVariables % [id, cursor]
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetch(graphListTweets ? params, Api.listTweets)
-  result = parseGraphTimeline(js, after).tweets
-
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
   let
-    variables = %*{"screenName": name, "listSlug": list}
+    variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false}
-    params = {"variables": $variables, "features": gqlFeatures}
+    url = graphListBySlug ? {"variables": $variables}
-    url = graphListBySlug ? params
   result = parseGraphList(await fetch(url, Api.listBySlug))
 
 proc getGraphList*(id: string): Future[List] {.async.} =
   let
-    variables = """{"listId": "$1"}""" % id
+    variables = %*{"listId": id, "withHighlightedLabel": false}
-    params = {"variables": variables, "features": gqlFeatures}
+    url = graphList ? {"variables": $variables}
-    url = graphListById ? params
   result = parseGraphList(await fetch(url, Api.list))
 
 proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
   if list.id.len == 0: return
-  var
+  let
     variables = %*{
       "listId": list.id,
+      "cursor": after,
+      "withSuperFollowsUserFields": false,
       "withBirdwatchPivots": false,
       "withDownvotePerspective": false,
       "withReactionsMetadata": false,
-      "withReactionsPerspective": false
+      "withReactionsPerspective": false,
+      "withSuperFollowsTweetFields": false
     }
-  if after.len > 0:
+    url = graphListMembers ? {"variables": $variables}
-    variables["cursor"] = % after
-  let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
   result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
 
-proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
+proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
   if id.len == 0: return
   let
-    variables = """{"rest_id": "$1"}""" % id
+    ps = genParams({"list_id": id, "ranking_mode": "reverse_chronological"}, after)
-    params = {"variables": variables, "features": gqlFeatures}
+    url = listTimeline ? ps
-    js = await fetch(graphTweetResult ? params, Api.tweetResult)
+  result = parseTimeline(await fetch(url, Api.timeline), after)
-  result = parseGraphTweetResult(js)
 
-proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
+proc getUser*(username: string): Future[User] {.async.} =
+  if username.len == 0: return
+  let
+    ps = genParams({"screen_name": username})
+    json = await fetchRaw(userShow ? ps, Api.userShow)
+  result = parseUser(json, username)
+
+proc getUserById*(userId: string): Future[User] {.async.} =
+  if userId.len == 0: return
+  let
+    ps = genParams({"user_id": userId})
+    json = await fetchRaw(userShow ? ps, Api.userShow)
+  result = parseUser(json)
+
+proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
   if id.len == 0: return
   let
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
-    variables = tweetVariables % [id, cursor]
+    url = timeline / (id & ".json") ? ps
-    params = {"variables": variables, "features": gqlFeatures}
+  result = parseTimeline(await fetch(url, Api.timeline), after)
-    js = await fetch(graphTweet ? params, Api.tweetDetail)
+
-  result = parseGraphConversation(js, id)
+proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} =
+  if id.len == 0: return
+  let url = mediaTimeline / (id & ".json") ? genParams(cursor=after)
+  result = parseTimeline(await fetch(url, Api.timeline), after)
+
+proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
+  if name.len == 0: return
+  let
+    ps = genParams({"screen_name": name, "trim_user": "true"},
+                   count="18", ext=false)
+    url = photoRail ? ps
+  result = parsePhotoRail(await fetch(url, Api.timeline))
+
+proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
+  when T is User:
+    const
+      searchMode = ("result_filter", "user")
+      parse = parseUsers
+      fetchFunc = fetchRaw
+  else:
+    const
+      searchMode = ("tweet_search_mode", "live")
+      parse = parseTweets
+      fetchFunc = fetchRaw
+
+  let q = genQueryParam(query)
+  if q.len == 0 or q == emptyQuery:
+    return Result[T](beginning: true, query: query)
+
+  let url = search ? genParams(searchParams & @[("q", q), searchMode], after)
+  try:
+    result = parse(await fetchFunc(url, Api.search), after)
+    result.query = query
+  except InternalError:
+    return Result[T](beginning: true, query: query)
+
+proc getTweetImpl(id: string; after=""): Future[Conversation] {.async.} =
+  let url = tweet / (id & ".json") ? genParams(cursor=after)
+  result = parseConversation(await fetch(url, Api.tweet), id)
 
 proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
-  result = (await getGraphTweet(id, after)).replies
+  result = (await getTweetImpl(id, after)).replies
   result.beginning = after.len == 0
 
 proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
-  result = await getGraphTweet(id)
+  result = await getTweetImpl(id)
   if after.len > 0:
     result.replies = await getReplies(id, after)
 
-proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
+proc getStatus*(id: string): Future[Tweet] {.async.} =
-  let q = genQueryParam(query)
+  let url = status / (id & ".json") ? genParams()
-  if q.len == 0 or q == emptyQuery:
+  result = parseStatus(await fetch(url, Api.status))
-    return Timeline(query: query, beginning: true)
-
-  var
-    variables = %*{
-      "rawQuery": q,
-      "count": 20,
-      "product": "Latest",
-      "withDownvotePerspective": false,
-      "withReactionsMetadata": false,
-      "withReactionsPerspective": false
-    }
-  if after.len > 0:
-    variables["cursor"] = % after
-  let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
-  result = parseGraphSearch[Tweets](await fetch(url, Api.search), after)
-  result.query = query
-
-proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} =
-  if query.text.len == 0:
-    return Result[User](query: query, beginning: true)
-
-  var
-    variables = %*{
-      "rawQuery": query.text,
-      "count": 20,
-      "product": "People",
-      "withDownvotePerspective": false,
-      "withReactionsMetadata": false,
-      "withReactionsPerspective": false
-    }
-  if after.len > 0:
-    variables["cursor"] = % after
-    result.beginning = false
-
-  let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
-  result = parseGraphSearch[User](await fetch(url, Api.search), after)
-  result.query = query
-
-proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
-  if id.len == 0: return
-  let js = await fetch(mediaUrl(id, ""), Api.userMedia)
-  result = parseGraphPhotoRail(js)
 
 proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
   let client = newAsyncHttpClient(maxRedirects=0)

src/apiutils.nim

@@ -1,177 +1,121 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
+import httpclient, asyncdispatch, options, strutils, uri
-import jsony, packedjson, zippy, oauth1
+import jsony, packedjson, zippy
-import types, auth, consts, parserutils, http_pool
+import types, tokens, consts, parserutils, http_pool
 import experimental/types/common
 
 const
   rlRemaining = "x-rate-limit-remaining"
   rlReset = "x-rate-limit-reset"
-  rlLimit = "x-rate-limit-limit"
-  errorsToSkip = {doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
 
 var pool: HttpPool
 
-proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
+proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
-  let
+                count="20"; ext=true): seq[(string, string)] =
-    encodedUrl = url.replace(",", "%2C").replace("+", "%20")
+  result = timelineParams
-    params = OAuth1Parameters(
+  for p in pars:
-      consumerKey: consumerKey,
+    result &= p
-      signatureMethod: "HMAC-SHA1",
+  if ext:
-      timestamp: $int(round(epochTime())),
+    result &= ("ext", "mediaStats")
-      nonce: "0",
+    result &= ("include_ext_alt_text", "true")
-      isIncludeVersionToHeader: true,
+    result &= ("include_ext_media_availability", "true")
-      token: oauthToken
+  if count.len > 0:
-    )
+    result &= ("count", count)
-    signature = getSignature(HttpGet, encodedUrl, "", params, consumerSecret, oauthTokenSecret)
+  if cursor.len > 0:
+    # The raw cursor often has plus signs, which sometimes get turned into spaces,
+    # so we need to them back into a plus
+    if " " in cursor:
+      result &= ("cursor", cursor.replace(" ", "+"))
+    else:
+      result &= ("cursor", cursor)
 
-  params.signature = percentEncode(signature)
+proc genHeaders*(token: Token = nil): HttpHeaders =
-
-  return getOauth1RequestHeader(params)["authorization"]
-
-proc getCookieHeader(authToken, ct0: string): string =
-  "auth_token=" & authToken & "; ct0=" & ct0
-
-proc genHeaders*(session: Session, url: string): HttpHeaders =
   result = newHttpHeaders({
     "connection": "keep-alive",
+    "authorization": auth,
     "content-type": "application/json",
+    "x-guest-token": if token == nil: "" else: token.tok,
     "x-twitter-active-user": "yes",
-    "x-twitter-client-language": "en",
+    "authority": "api.twitter.com",
-    "authority": "api.x.com",
     "accept-encoding": "gzip",
     "accept-language": "en-US,en;q=0.9",
     "accept": "*/*",
-    "DNT": "1",
+    "DNT": "1"
-    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
   })
 
-  case session.kind
+template updateToken() =
-  of SessionKind.oauth:
+  if api != Api.search and resp.headers.hasKey(rlRemaining):
-    result["authorization"] = getOauthHeader(url, session.oauthToken, session.oauthSecret)
+    let
-  of SessionKind.cookie:
+      remaining = parseInt(resp.headers[rlRemaining])
-    result["authorization"] = "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
+      reset = parseInt(resp.headers[rlReset])
-    result["x-twitter-auth-type"] = "OAuth2Session"
+    token.setRateLimit(api, remaining, reset)
-    result["x-csrf-token"] = session.ct0
-    result["cookie"] = getCookieHeader(session.authToken, session.ct0)
-
-proc getAndValidateSession*(api: Api): Future[Session] {.async.} =
-  result = await getSession(api)
-  case result.kind
-  of SessionKind.oauth:
-    if result.oauthToken.len == 0:
-      echo "[sessions] Empty oauth token, session: ", result.id
-      raise rateLimitError()
-  of SessionKind.cookie:
-    if result.authToken.len == 0 or result.ct0.len == 0:
-      echo "[sessions] Empty cookie credentials, session: ", result.id
-      raise rateLimitError()
 
 template fetchImpl(result, fetchBody) {.dirty.} =
   once:
     pool = HttpPool()
 
+  var token = await getToken(api)
+  if token.tok.len == 0:
+    raise rateLimitError()
+
   try:
     var resp: AsyncResponse
-    pool.use(genHeaders(session, $url)):
+    pool.use(genHeaders(token)):
-      template getContent =
+      resp = await c.get($url)
-        resp = await c.get($url)
+      result = await resp.body
-        result = await resp.body
-
-      getContent()
 
       if resp.status == $Http503:
         badClient = true
-        raise newException(BadClientError, "Bad client")
+        raise newException(InternalError, result)
-
-    if resp.headers.hasKey(rlRemaining):
-      let
-        remaining = parseInt(resp.headers[rlRemaining])
-        reset = parseInt(resp.headers[rlReset])
-        limit = parseInt(resp.headers[rlLimit])
-      session.setRateLimit(api, remaining, reset, limit)
 
     if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":
         result = uncompress(result, dfGzip)
-
+      else:
-      if result.startsWith("{\"errors"):
+        echo "non-gzip body, url: ", url, ", body: ", result
-        let errors = result.fromJson(Errors)
-        if errors notin errorsToSkip:
-          echo "Fetch error, API: ", api, ", errors: ", errors
-          if errors in {expiredToken, badToken, locked}:
-            invalidate(session)
-            raise rateLimitError()
-          elif errors in {rateLimited}:
-            # rate limit hit, resets after 24 hours
-            setLimited(session, api)
-            raise rateLimitError()
-      elif result.startsWith("429 Too Many Requests"):
-        echo "[sessions] 429 error, API: ", api, ", session: ", session.id
-        session.apis[api].remaining = 0
-        # rate limit hit, resets after the 15 minute window
-        raise rateLimitError()
 
     fetchBody
 
+    release(token, used=true)
+
     if resp.status == $Http400:
-      echo "ERROR 400, ", api, ": ", result
       raise newException(InternalError, $url)
   except InternalError as e:
     raise e
-  except BadClientError as e:
-    raise e
-  except OSError as e:
-    raise e
   except Exception as e:
-    let id = if session.isNil: "null" else: $session.id
+    echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url
-    echo "error: ", e.name, ", msg: ", e.msg, ", sessionId: ", id, ", url: ", url
+    if "length" notin e.msg and "descriptor" notin e.msg:
+      release(token, invalid=true)
     raise rateLimitError()
-  finally:
-    release(session)
 
-template retry(bod) =
+proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
-  try:
+  var body: string
-    bod
+  fetchImpl body:
-  except RateLimitError:
+    if body.startsWith('{') or body.startsWith('['):
-    echo "[sessions] Rate limited, retrying ", api, " request..."
+      result = parseJson(body)
-    bod
+    else:
+      echo resp.status, ": ", body, " --- url: ", url
+      result = newJNull()
 
-proc fetch*(url: Uri | SessionAwareUrl; api: Api): Future[JsonNode] {.async.} =
+    updateToken()
-  retry:
-    var 
-      body: string
-      session = await getAndValidateSession(api)
 
-    when url is SessionAwareUrl:
+    let error = result.getError
-      let url = case session.kind
+    if error in {invalidToken, forbidden, badToken}:
-        of SessionKind.oauth: url.oauthUrl
+      echo "fetch error: ", result.getError
-        of SessionKind.cookie: url.cookieUrl
+      release(token, invalid=true)
+      raise rateLimitError()
 
-    fetchImpl body:
+proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
-      if body.startsWith('{') or body.startsWith('['):
+  fetchImpl result:
-        result = parseJson(body)
+    if not (result.startsWith('{') or result.startsWith('[')):
-      else:
+      echo resp.status, ": ", result, " --- url: ", url
-        echo resp.status, ": ", body, " --- url: ", url
+      result.setLen(0)
-        result = newJNull()
 
-      let error = result.getError
+    updateToken()
-      if error != null and error notin errorsToSkip:
-        echo "Fetch error, API: ", api, ", error: ", error
-        if error in {expiredToken, badToken, locked}:
-          invalidate(session)
-          raise rateLimitError()
 
-proc fetchRaw*(url: Uri | SessionAwareUrl; api: Api): Future[string] {.async.} =
+    if result.startsWith("{\"errors"):
-  retry:
+      let errors = result.fromJson(Errors)
-    var session = await getAndValidateSession(api)
+      if errors in {invalidToken, forbidden, badToken}:
-
+        echo "fetch error: ", errors
-    when url is SessionAwareUrl:
+        release(token, invalid=true)
-      let url = case session.kind
+        raise rateLimitError()
-        of SessionKind.oauth: url.oauthUrl
-        of SessionKind.cookie: url.cookieUrl
-
-    fetchImpl result:
-      if not (result.startsWith('{') or result.startsWith('[')):
-        echo resp.status, ": ", result, " --- url: ", url
-        result.setLen(0)

src/auth.nim

@@ -1,203 +0,0 @@
-#SPDX-License-Identifier: AGPL-3.0-only
-import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os]
-import types
-import experimental/parser/session
-
-# max requests at a time per session to avoid race conditions
-const
-  maxConcurrentReqs = 2
-  hourInSeconds = 60 * 60
-  apiMaxReqs: Table[Api, int] = {
-    Api.search: 50,
-    Api.tweetDetail: 500,
-    Api.userTweets: 500,
-    Api.userTweetsAndReplies: 500,
-    Api.userMedia: 500,
-    Api.userRestId: 500,
-    Api.userScreenName: 500,
-    Api.tweetResult: 500,
-    Api.list: 500,
-    Api.listTweets: 500,
-    Api.listMembers: 500,
-    Api.listBySlug: 500
-  }.toTable
-
-var
-  sessionPool: seq[Session]
-  enableLogging = false
-
-template log(str: varargs[string, `$`]) =
-  echo "[sessions] ", str.join("")
-
-proc snowflakeToEpoch(flake: int64): int64 =
-  int64(((flake shr 22) + 1288834974657) div 1000)
-
-proc getSessionPoolHealth*(): JsonNode =
-  let now = epochTime().int
-
-  var
-    totalReqs = 0
-    limited: PackedSet[int64]
-    reqsPerApi: Table[string, int]
-    oldest = now.int64
-    newest = 0'i64
-    average = 0'i64
-
-  for session in sessionPool:
-    let created = snowflakeToEpoch(session.id)
-    if created > newest:
-      newest = created
-    if created < oldest:
-      oldest = created
-    average += created
-
-    if session.limited:
-      limited.incl session.id
-
-    for api in session.apis.keys:
-      let
-        apiStatus = session.apis[api]
-        limit = if apiStatus.limit > 0: apiStatus.limit else: apiMaxReqs.getOrDefault(api, 0)
-        reqs = limit - apiStatus.remaining
-
-      # no requests made with this session and endpoint since the limit reset
-      if apiStatus.reset < now:
-        continue
-
-      reqsPerApi.mgetOrPut($api, 0).inc reqs
-      totalReqs.inc reqs
-
-  if sessionPool.len > 0:
-    average = average div sessionPool.len
-  else:
-    oldest = 0
-    average = 0
-
-  return %*{
-    "sessions": %*{
-      "total": sessionPool.len,
-      "limited": limited.card,
-      "oldest": $fromUnix(oldest),
-      "newest": $fromUnix(newest),
-      "average": $fromUnix(average)
-    },
-    "requests": %*{
-      "total": totalReqs,
-      "apis": reqsPerApi
-    }
-  }
-
-proc getSessionPoolDebug*(): JsonNode =
-  let now = epochTime().int
-  var list = newJObject()
-
-  for session in sessionPool:
-    let sessionJson = %*{
-      "apis": newJObject(),
-      "pending": session.pending,
-    }
-
-    if session.limited:
-      sessionJson["limited"] = %true
-
-    for api in session.apis.keys:
-      let
-        apiStatus = session.apis[api]
-        obj = %*{}
-
-      if apiStatus.reset > now.int:
-        obj["remaining"] = %apiStatus.remaining
-        obj["reset"] = %apiStatus.reset
-
-      if "remaining" notin obj:
-        continue
-
-      sessionJson{"apis", $api} = obj
-      list[$session.id] = sessionJson
-
-  return %list
-
-proc rateLimitError*(): ref RateLimitError =
-  newException(RateLimitError, "rate limited")
-
-proc noSessionsError*(): ref NoSessionsError =
-  newException(NoSessionsError, "no sessions available")
-
-proc isLimited(session: Session; api: Api): bool =
-  if session.isNil:
-    return true
-
-  if session.limited and api != Api.userTweets:
-    if (epochTime().int - session.limitedAt) > hourInSeconds:
-      session.limited = false
-      log "resetting limit: ", session.id
-      return false
-    else:
-      return true
-
-  if api in session.apis:
-    let limit = session.apis[api]
-    return limit.remaining <= 10 and limit.reset > epochTime().int
-  else:
-    return false
-
-proc isReady(session: Session; api: Api): bool =
-  not (session.isNil or session.pending > maxConcurrentReqs or session.isLimited(api))
-
-proc invalidate*(session: var Session) =
-  if session.isNil: return
-  log "invalidating: ", session.id
-
-  # TODO: This isn't sufficient, but it works for now
-  let idx = sessionPool.find(session)
-  if idx > -1: sessionPool.delete(idx)
-  session = nil
-
-proc release*(session: Session) =
-  if session.isNil: return
-  dec session.pending
-
-proc getSession*(api: Api): Future[Session] {.async.} =
-  for i in 0 ..< sessionPool.len:
-    if result.isReady(api): break
-    result = sessionPool.sample()
-
-  if not result.isNil and result.isReady(api):
-    inc result.pending
-  else:
-    log "no sessions available for API: ", api
-    raise noSessionsError()
-
-proc setLimited*(session: Session; api: Api) =
-  session.limited = true
-  session.limitedAt = epochTime().int
-  log "rate limited by api: ", api, ", reqs left: ", session.apis[api].remaining, ", id: ", session.id
-
-proc setRateLimit*(session: Session; api: Api; remaining, reset, limit: int) =
-  # avoid undefined behavior in race conditions
-  if api in session.apis:
-    let rateLimit = session.apis[api]
-    if rateLimit.reset >= reset and rateLimit.remaining < remaining:
-      return
-    if rateLimit.reset == reset and rateLimit.remaining >= remaining:
-      session.apis[api].remaining = remaining
-      return
-
-  session.apis[api] = RateLimit(limit: limit, remaining: remaining, reset: reset)
-
-proc initSessionPool*(cfg: Config; path: string) =
-  enableLogging = cfg.enableDebug
-
-  if path.endsWith(".json"):
-    log "ERROR: .json is not supported, the file must be a valid JSONL file ending in .jsonl"
-    quit 1
-
-  if not fileExists(path):
-    log "ERROR: ", path, " not found. This file is required to authenticate API requests."
-    quit 1
-
-  log "parsing JSONL account sessions file: ", path
-  for line in path.lines:
-    sessionPool.add parseSession(line)
-
-  log "successfully added ", sessionPool.len, " valid account sessions"

src/consts.nim

@@ -1,128 +1,59 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import uri, strutils
+import uri, sequtils
 
 const
-  consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
+  auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
-  consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
 
-  gql = parseUri("https://api.x.com") / "graphql"
+  api = parseUri("https://api.twitter.com")
+  activate* = $(api / "1.1/guest/activate.json")
 
-  graphUser* = gql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery"
+  userShow* = api / "1.1/users/show.json"
-  graphUserById* = gql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery"
+  photoRail* = api / "1.1/statuses/media_timeline.json"
-  graphUserTweets* = gql / "JLApJKFY0MxGTzCoK6ps8Q/UserWithProfileTweetsQueryV2"
+  status* = api / "1.1/statuses/show"
-  graphUserTweetsAndReplies* = gql / "Y86LQY7KMvxn5tu3hFTyPg/UserWithProfileTweetsAndRepliesQueryV2"
+  search* = api / "2/search/adaptive.json"
-  graphUserMedia* = gql / "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
-  graphUserMediaV2* = gql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2"
-  graphTweet* = gql / "Vorskcd2tZ-tc4Gx3zbk4Q/ConversationTimelineV2"
-  graphTweetResult* = gql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery"
-  graphSearchTimeline* = gql / "KI9jCXUx3Ymt-hDKLOZb9Q/SearchTimeline"
-  graphListById* = gql / "oygmAig8kjn0pKsx_bUadQ/ListByRestId"
-  graphListBySlug* = gql / "88GTz-IPPWLn1EiU8XoNVg/ListBySlug"
-  graphListMembers* = gql / "kSmxeqEeelqdHSR7jMnb_w/ListMembers"
-  graphListTweets* = gql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline"
 
-  gqlFeatures* = """{
+  timelineApi = api / "2/timeline"
-  "android_graphql_skip_api_media_color_palette": false,
+  timeline* = timelineApi / "profile"
-  "blue_business_profile_image_shape_enabled": false,
+  mediaTimeline* = timelineApi / "media"
-  "creator_subscriptions_subscription_count_enabled": false,
+  listTimeline* = timelineApi / "list.json"
-  "creator_subscriptions_tweet_preview_api_enabled": true,
+  tweet* = timelineApi / "conversation"
-  "freedom_of_speech_not_reach_fetch_enabled": true,
-  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
-  "hidden_profile_likes_enabled": false,
-  "highlights_tweets_tab_ui_enabled": false,
-  "interactive_text_enabled": false,
-  "longform_notetweets_consumption_enabled": true,
-  "longform_notetweets_inline_media_enabled": true,
-  "longform_notetweets_richtext_consumption_enabled": true,
-  "longform_notetweets_rich_text_read_enabled": true,
-  "responsive_web_edit_tweet_api_enabled": true,
-  "responsive_web_enhance_cards_enabled": false,
-  "responsive_web_graphql_exclude_directive_enabled": true,
-  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
-  "responsive_web_graphql_timeline_navigation_enabled": true,
-  "responsive_web_media_download_video_enabled": false,
-  "responsive_web_text_conversations_enabled": false,
-  "responsive_web_twitter_article_tweet_consumption_enabled": true,
-  "responsive_web_twitter_blue_verified_badge_is_enabled": true,
-  "rweb_lists_timeline_redesign_enabled": true,
-  "spaces_2022_h2_clipping": true,
-  "spaces_2022_h2_spaces_communities": true,
-  "standardized_nudges_misinfo": true,
-  "subscriptions_verification_info_enabled": true,
-  "subscriptions_verification_info_reason_enabled": true,
-  "subscriptions_verification_info_verified_since_enabled": true,
-  "super_follow_badge_privacy_enabled": false,
-  "super_follow_exclusive_tweet_notifications_enabled": false,
-  "super_follow_tweet_api_enabled": false,
-  "super_follow_user_api_enabled": false,
-  "tweet_awards_web_tipping_enabled": false,
-  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
-  "tweetypie_unmention_optimization_enabled": false,
-  "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
-  "verified_phone_label_enabled": false,
-  "vibe_api_enabled": false,
-  "view_counts_everywhere_api_enabled": true,
-  "premium_content_api_read_enabled": false,
-  "communities_web_enable_tweet_community_results_fetch": true,
-  "responsive_web_jetfuel_frame": true,
-  "responsive_web_grok_analyze_button_fetch_trends_enabled": false,
-  "responsive_web_grok_image_annotation_enabled": true,
-  "responsive_web_grok_imagine_annotation_enabled": true,
-  "rweb_tipjar_consumption_enabled": true,
-  "profile_label_improvements_pcf_label_in_post_enabled": true,
-  "creator_subscriptions_quote_tweet_preview_enabled": false,
-  "c9s_tweet_anatomy_moderator_badge_enabled": true,
-  "responsive_web_grok_analyze_post_followups_enabled": true,
-  "rweb_video_timestamps_enabled": false,
-  "responsive_web_grok_share_attachment_enabled": true,
-  "articles_preview_enabled": true,
-  "immersive_video_status_linkable_timestamps": false,
-  "articles_api_enabled": false,
-  "responsive_web_grok_analysis_button_from_backend": true,
-  "rweb_video_screen_enabled": false,
-  "payments_enabled": false,
-  "responsive_web_profile_redirect_enabled": false,
-  "responsive_web_grok_show_grok_translated_post": false,
-  "responsive_web_grok_community_note_auto_translation_is_enabled": false
-}""".replace(" ", "").replace("\n", "")
 
-  tweetVariables* = """{
+  graphql = api / "graphql"
-  "focalTweetId": "$1",
+  graphUser* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
-  $2
+  graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
-  "includeHasBirdwatchNotes": false,
+  graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
-  "includePromotedContent": false,
+  graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
-  "withBirdwatchNotes": false,
-  "withVoice": false,
-  "withV2Timeline": true
-}""".replace(" ", "").replace("\n", "")
 
-#   oldUserTweetsVariables* = """{
+  timelineParams* = {
-#   "userId": "$1", $2
+    "include_profile_interstitial_type": "0",
-#   "count": 20,
+    "include_blocking": "0",
-#   "includePromotedContent": false,
+    "include_blocked_by": "0",
-#   "withDownvotePerspective": false,
+    "include_followed_by": "0",
-#   "withReactionsMetadata": false,
+    "include_want_retweets": "0",
-#   "withReactionsPerspective": false,
+    "include_mute_edge": "0",
-#   "withVoice": false,
+    "include_can_dm": "0",
-#   "withV2Timeline": true
+    "include_can_media_tag": "1",
-# }
+    "skip_status": "1",
-# """
+    "cards_platform": "Web-12",
+    "include_cards": "1",
+    "include_composer_source": "false",
+    "include_reply_count": "1",
+    "tweet_mode": "extended",
+    "include_entities": "true",
+    "include_user_entities": "true",
+    "include_ext_media_color": "false",
+    "send_error_codes": "true",
+    "simple_quoted_tweet": "true",
+    "include_quote_count": "true"
+  }.toSeq
 
-  userTweetsVariables* = """{
+  searchParams* = {
-  "rest_id": "$1", $2
+    "query_source": "typed_query",
-  "count": 20
+    "pc": "1",
-}"""
+    "spelling_corrections": "1"
-
+  }.toSeq
-  listTweetsVariables* = """{
+  ## top: nothing
-  "rest_id": "$1", $2
+  ## latest: "tweet_search_mode: live"
-  "count": 20
+  ## user:   "result_filter: user"
-}"""
+  ## photos: "result_filter: photos"
-
+  ## videos: "result_filter: videos"
-  userMediaVariables* = """{
-  "userId": "$1", $2
-  "count": 20,
-  "includePromotedContent": false,
-  "withClientEventToken": false,
-  "withBirdwatchNotes": false,
-  "withVoice": true
-}""".replace(" ", "").replace("\n", "")

src/experimental/parser.nim

@@ -1,2 +1,2 @@
-import parser/[user, graphql]
+import parser/[user, graphql, timeline]
-export user, graphql
+export user, graphql, timeline

src/experimental/parser/graphql.nim

@@ -1,21 +1,11 @@
-import options
 import jsony
 import user, ../types/[graphuser, graphlistmembers]
-from ../../types import User, VerifiedType, Result, Query, QueryKind
+from ../../types import User, Result, Query, QueryKind
 
 proc parseGraphUser*(json: string): User =
-  if json.len == 0 or json[0] != '{':
-    return
-
   let raw = json.fromJson(GraphUser)
-
+  result = toUser raw.data.user.result.legacy
-  if raw.data.userResult.result.unavailableReason.get("") == "Suspended":
+  result.id = raw.data.user.result.restId
-    return User(suspended: true)
-
-  result = raw.data.userResult.result.legacy
-  result.id = raw.data.userResult.result.restId
-  if result.verifiedType == VerifiedType.none and raw.data.userResult.result.isBlueVerified:
-    result.verifiedType = blue
 
 proc parseGraphListMembers*(json, cursor: string): Result[User] =
   result = Result[User](
@@ -31,7 +21,7 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] =
         of TimelineTimelineItem:
           let userResult = entry.content.itemContent.userResults.result
           if userResult.restId.len > 0:
-            result.content.add userResult.legacy
+            result.content.add toUser userResult.legacy
         of TimelineTimelineCursor:
           if entry.content.cursorType == "Bottom":
             result.bottom = entry.content.value

src/experimental/parser/media.nim

@@ -0,0 +1,44 @@
+import std/[json, strutils, times, math]
+import utils
+import ".."/types/[media, tweet]
+from ../../types import Poll, Gif, Video, VideoVariant, VideoType
+
+proc parseVideo*(entity: Entity): Video =
+  result = Video(
+    thumb: entity.mediaUrlHttps.getImageUrl,
+    views: entity.ext.mediaStats{"r", "ok", "viewCount"}.getStr,
+    available: entity.extMediaAvailability.status == "available",
+    title: entity.extAltText,
+    durationMs: entity.videoInfo.durationMillis,
+    description: entity.additionalMediaInfo.description,
+    variants: entity.videoInfo.variants
+    # playbackType: mp4
+  )
+
+  if entity.additionalMediaInfo.title.len > 0:
+    result.title = entity.additionalMediaInfo.title
+
+proc parseGif*(entity: Entity): Gif =
+  result = Gif(
+    url: entity.videoInfo.variants[0].url.getImageUrl,
+    thumb: entity.getImageUrl
+  )
+
+proc parsePoll*(card: Card): Poll =
+  let vals = card.bindingValues
+
+  # name format is pollNchoice_*
+  for i in '1' .. card.name[4]:
+    let choice = "choice" & i
+    result.values.add parseInt(vals{choice & "_count", "string_value"}.getStr("0"))
+    result.options.add vals{choice & "_label", "string_value"}.getStr
+
+  let time = vals{"end_datetime_utc", "string_value"}.getStr.parseIsoDate
+  if time > now():
+    let timeLeft = $(time - now())
+    result.status = timeLeft[0 ..< timeLeft.find(",")]
+  else:
+    result.status = "Final results"
+
+  result.leader = result.values.find(max(result.values))
+  result.votes = result.values.sum

src/experimental/parser/session.nim

@@ -1,28 +0,0 @@
-import std/strutils
-import jsony
-import ../types/session
-from ../../types import Session, SessionKind
-
-proc parseSession*(raw: string): Session =
-  let session = raw.fromJson(RawSession)
-  let kind = if session.kind == "": "oauth" else: session.kind
-
-  case kind
-  of "oauth":
-    let id = session.oauthToken[0 ..< session.oauthToken.find('-')]
-    result = Session(
-      kind: SessionKind.oauth,
-      id: parseBiggestInt(id),
-      oauthToken: session.oauthToken,
-      oauthSecret: session.oauthTokenSecret
-    )
-  of "cookie":
-    let id = if session.id.len > 0: parseBiggestInt(session.id) else: 0
-    result = Session(
-      kind: SessionKind.cookie,
-      id: id,
-      authToken: session.authToken,
-      ct0: session.ct0
-    )
-  else:
-    raise newException(ValueError, "Unknown session kind: " & kind)

src/experimental/parser/slices.nim

@@ -1,15 +1,14 @@
 import std/[macros, htmlgen, unicode]
-import ../types/common
 import ".."/../[formatters, utils]
 
 type
-  ReplaceSliceKind = enum
+  ReplaceSliceKind* = enum
     rkRemove, rkUrl, rkHashtag, rkMention
 
   ReplaceSlice* = object
-    slice: Slice[int]
+    slice*: Slice[int]
-    kind: ReplaceSliceKind
+    kind*: ReplaceSliceKind
-    url, display: string
+    url*, display*: string
 
 proc cmp*(x, y: ReplaceSlice): int = cmp(x.slice.a, y.slice.b)
 
@@ -27,11 +26,14 @@ proc dedupSlices*(s: var seq[ReplaceSlice]) =
         inc j
     inc i
 
-proc extractUrls*(result: var seq[ReplaceSlice]; url: Url;
+proc extractHashtags*(result: var seq[ReplaceSlice]; slice: Slice[int]) =
-                  textLen: int; hideTwitter = false) =
+  result.add ReplaceSlice(kind: rkHashtag, slice: slice)
+
+proc extractUrls*[T](result: var seq[ReplaceSlice]; entity: T;
+                     textLen: int; hideTwitter = false) =
   let
-    link = url.expandedUrl
+    link = entity.expandedUrl
-    slice = url.indices[0] ..< url.indices[1]
+    slice = entity.indices
 
   if hideTwitter and slice.b.succ >= textLen and link.isTwitterUrl:
     if slice.a < textLen:

src/experimental/parser/timeline.nim

@@ -0,0 +1,84 @@
+import std/[strutils, tables, options]
+import jsony
+import user, tweet, utils, ../types/timeline
+from ../../types import Result, User, Tweet
+
+proc parseHook(s: string; i: var int; v: var Slice[int]) =
+  var slice: array[2, int]
+  parseHook(s, i, slice)
+  v = slice[0] ..< slice[1]
+
+proc getId(id: string): string {.inline.} =
+  let start = id.rfind("-")
+  if start < 0: return id
+  id[start + 1 ..< id.len]
+
+proc processTweet(id: string; objects: GlobalObjects;
+                  userCache: var Table[string, User]): Tweet =
+  let raw = objects.tweets[id]
+  result = toTweet raw
+
+  let uid = result.user.id
+  if uid.len > 0 and uid in objects.users:
+    if uid notin userCache:
+      userCache[uid] = toUser objects.users[uid]
+    result.user = userCache[uid]
+
+  let rtId = raw.retweetedStatusIdStr
+  if rtId.len > 0:
+    if rtId in objects.tweets:
+      result.retweet = some processTweet(rtId, objects, userCache)
+    else:
+      result.retweet = some Tweet(id: rtId.toId)
+
+  let qId = raw.quotedStatusIdStr
+  if qId.len > 0:
+    if qId in objects.tweets:
+      result.quote = some processTweet(qId, objects, userCache)
+    else:
+      result.quote = some Tweet(id: qId.toId)
+
+proc parseCursor[T](e: Entry; result: var Result[T]) =
+  let cursor = e.content.operation.cursor
+  if cursor.cursorType == "Top":
+    result.top = cursor.value
+  elif cursor.cursorType == "Bottom":
+    result.bottom = cursor.value
+
+proc parseUsers*(json: string; after=""): Result[User] =
+  result = Result[User](beginning: after.len == 0)
+
+  let raw = json.fromJson(Search)
+  if raw.timeline.instructions.len == 0:
+    return
+
+  for e in raw.timeline.instructions[0].addEntries.entries:
+    let
+      eId = e.entryId
+      id = eId.getId
+
+    if eId.startsWith("user") or eId.startsWith("sq-U"):
+      if id in raw.globalObjects.users:
+        result.content.add toUser raw.globalObjects.users[id]
+    elif eId.startsWith("cursor") or eId.startsWith("sq-C"):
+      parseCursor(e, result)
+
+proc parseTweets*(json: string; after=""): Result[Tweet] =
+  result = Result[Tweet](beginning: after.len == 0)
+
+  let raw = json.fromJson(Search)
+  if raw.timeline.instructions.len == 0:
+    return
+
+  var userCache: Table[string, User]
+
+  for e in raw.timeline.instructions[0].addEntries.entries:
+    let
+      eId = e.entryId
+      id = eId.getId
+
+    if eId.startsWith("tweet") or eId.startsWith("sq-I-t"):
+      if id in raw.globalObjects.tweets:
+        result.content.add processTweet(id, raw.globalObjects, userCache)
+    elif eId.startsWith("cursor") or eId.startsWith("sq-C"):
+      parseCursor(e, result)

src/experimental/parser/tweet.nim

@@ -0,0 +1,97 @@
+import std/[strutils, options, algorithm, json]
+import std/unicode except strip
+import utils, slices, media, user
+import ../types/tweet
+from ../types/media as mediaTypes import MediaType
+from ../../types import Tweet, User, TweetStats
+
+proc expandTweetEntities(tweet: var Tweet; raw: RawTweet) =
+  let
+    orig = raw.fullText.toRunes
+    textRange = raw.displayTextRange
+    textSlice = textRange[0] .. textRange[1]
+    hasCard = raw.card.isSome
+
+  var replyTo = ""
+  if tweet.replyId > 0:
+    tweet.reply.add raw.inReplyToScreenName
+    replyTo = raw.inReplyToScreenName
+
+  var replacements = newSeq[ReplaceSlice]()
+
+  for u in raw.entities.urls:
+    if u.url.len == 0 or u.url notin raw.fullText:
+      continue
+
+    replacements.extractUrls(u, textSlice.b, hideTwitter=raw.isQuoteStatus)
+    # if hasCard and u.url == get(tweet.card).url:
+    #   get(tweet.card).url = u.expandedUrl
+
+  for m in raw.entities.media:
+    replacements.extractUrls(m, textSlice.b, hideTwitter=true)
+
+  for hashtag in raw.entities.hashtags:
+    replacements.extractHashtags(hashtag.indices)
+
+  for symbol in raw.entities.symbols:
+    replacements.extractHashtags(symbol.indices)
+
+  for mention in raw.entities.userMentions:
+    let
+      name = mention.screenName
+      idx = tweet.reply.find(name)
+
+    if mention.indices.a >= textSlice.a:
+      replacements.add ReplaceSlice(kind: rkMention, slice: mention.indices,
+        url: "/" & name, display: mention.name)
+      if idx > -1 and name != replyTo:
+        tweet.reply.delete idx
+    elif idx == -1 and tweet.replyId != 0:
+      tweet.reply.add name
+
+  replacements.dedupSlices
+  replacements.sort(cmp)
+
+  tweet.text = orig.replacedWith(replacements, textSlice)
+                   .strip(leading=false)
+
+proc toTweet*(raw: RawTweet): Tweet =
+  result = Tweet(
+    id: raw.idStr.toId,
+    threadId: raw.conversationIdStr.toId,
+    replyId: raw.inReplyToStatusIdStr.toId,
+    time: parseTwitterDate(raw.createdAt),
+    hasThread: raw.selfThread.idStr.len > 0,
+    available: true,
+    user: User(id: raw.userIdStr),
+    stats: TweetStats(
+      replies: raw.replyCount,
+      retweets: raw.retweetCount,
+      likes: raw.favoriteCount,
+      quotes: raw.quoteCount
+    )
+  )
+
+  result.expandTweetEntities(raw)
+
+  if raw.card.isSome:
+    let card = raw.card.get
+    if "poll" in card.name:
+      result.poll = some parsePoll(card)
+      if "image" in card.name:
+        result.photos.add card.bindingValues{"image_large", "image_value", "url"}
+                                            .getStr.getImageUrl
+    # elif card.name == "amplify":
+    #   discard
+    #   # result.video = some(parsePromoVideo(jsCard{"binding_values"}))
+    # else:
+    #   result.card = some parseCard(card, raw.entities.urls)
+
+  for m in raw.extendedEntities.media:
+    case m.kind
+    of photo: result.photos.add m.getImageUrl
+    of video:
+      result.video = some parseVideo(m)
+      if m.additionalMediaInfo.sourceUser.isSome:
+        result.attribution = some toUser get(m.additionalMediaInfo.sourceUser)
+    of animatedGif: result.gif = some parseGif(m)

src/experimental/parser/unifiedcard.nim

@@ -1,11 +1,8 @@
-import std/[options, tables, strutils, strformat, sugar]
+import std/[options, tables, strformat]
 import jsony
-import user, ../types/unifiedcard
+import utils
+import ".."/types/[unifiedcard, media]
 from ../../types import Card, CardKind, Video
-from ../../utils import twimg, https
-
-proc getImageUrl(entity: MediaEntity): string =
-  entity.mediaUrlHttps.dup(removePrefix(twimg), removePrefix(https))
 
 proc parseDestination(id: string; card: UnifiedCard; result: var Card) =
   let destination = card.destinationObjects[id].data
@@ -27,14 +24,6 @@ proc parseMediaDetails(data: ComponentData; card: UnifiedCard; result: var Card)
   result.text = data.topicDetail.title
   result.dest = "Topic"
 
-proc parseJobDetails(data: ComponentData; card: UnifiedCard; result: var Card) =
-  data.destination.parseDestination(card, result)
-
-  result.kind = CardKind.jobDetails
-  result.title = data.title
-  result.text = data.shortDescriptionText
-  result.dest = &"@{data.profileUser.username} · {data.location}"
-
 proc parseAppDetails(data: ComponentData; card: UnifiedCard; result: var Card) =
   let app = card.appStoreData[data.appId][0]
 
@@ -74,8 +63,7 @@ proc parseMedia(component: Component; card: UnifiedCard; result: var Card) =
       durationMs: videoInfo.durationMillis,
       variants: videoInfo.variants
     )
-  of model3d:
+  of animatedGif: discard
-    result.title = "Unsupported 3D model ad"
 
 proc parseUnifiedCard*(json: string): Card =
   let card = json.fromJson(UnifiedCard)
@@ -88,16 +76,10 @@ proc parseUnifiedCard*(json: string): Card =
       component.data.parseAppDetails(card, result)
     of mediaWithDetailsHorizontal:
       component.data.parseMediaDetails(card, result)
-    of media, swipeableMedia:
+    of ComponentType.media, swipeableMedia:
       component.parseMedia(card, result)
     of buttonGroup:
       discard
-    of ComponentType.jobDetails:
-      component.data.parseJobDetails(card, result)
-    of ComponentType.hidden:
-      result.kind = CardKind.hidden
-    of ComponentType.unknown:
-      echo "ERROR: Unknown component type: ", json
 
     case component.kind
     of twitterListDetails:

src/experimental/parser/user.nim

@@ -1,14 +1,14 @@
-import std/[algorithm, unicode, re, strutils, strformat, options, nre]
+import std/[algorithm, unicode, re, strutils, strformat, options]
 import jsony
 import utils, slices
 import ../types/user as userType
-from ../../types import Result, User, Error
+from ../../types import User, Error
 
 let
-  unRegex = re.re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
+  unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
   unReplace = "$1<a href=\"/$2\">@$2</a>"
 
-  htRegex = nre.re"""(*U)(^|[^\w-_.?])([#＃$])([\w_]*+)(?!</a>|">|#)"""
+  htRegex = re"(^|[^\w-_./?])([#＃$])([\w_]+)"
   htReplace = "$1<a href=\"/search?q=%23$3\">$2$3</a>"
 
 proc expandUserEntities(user: var User; raw: RawUser) =
@@ -29,7 +29,7 @@ proc expandUserEntities(user: var User; raw: RawUser) =
 
   user.bio = orig.replacedWith(replacements, 0 .. orig.len)
                  .replacef(unRegex, unReplace)
-                 .replace(htRegex, htReplace)
+                 .replacef(htRegex, htReplace)
 
 proc getBanner(user: RawUser): string =
   if user.profileBannerUrl.len > 0:
@@ -56,7 +56,7 @@ proc toUser*(raw: RawUser): User =
     tweets: raw.statusesCount,
     likes: raw.favouritesCount,
     media: raw.mediaCount,
-    verifiedType: raw.verifiedType,
+    verified: raw.verified,
     protected: raw.protected,
     joinDate: parseTwitterDate(raw.createdAt),
     banner: getBanner(raw),
@@ -68,11 +68,6 @@ proc toUser*(raw: RawUser): User =
 
   result.expandUserEntities(raw)
 
-proc parseHook*(s: string; i: var int; v: var User) =
-  var u: RawUser
-  parseHook(s, i, u)
-  v = toUser u
-
 proc parseUser*(json: string; username=""): User =
   handleErrors:
     case error.code
@@ -80,13 +75,4 @@ proc parseUser*(json: string; username=""): User =
     of userNotFound: return
     else: echo "[error - parseUser]: ", error
 
-  result = json.fromJson(User)
+  result = toUser json.fromJson(RawUser)
-
-proc parseUsers*(json: string; after=""): Result[User] =
-  result = Result[User](beginning: after.len == 0)
-
-  # starting with '{' means it's an error
-  if json[0] == '[':
-    let raw = json.fromJson(seq[RawUser])
-    for user in raw:
-      result.content.add user.toUser

src/experimental/parser/utils.nim

@@ -1,12 +1,16 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import std/[sugar, strutils, times]
-import ../types/common
+import ".."/types/[common, media, tweet]
 import ../../utils as uutils
 
 template parseTime(time: string; f: static string; flen: int): DateTime =
   if time.len != flen: return
   parse(time, f, utc())
 
+proc toId*(id: string): int64 =
+  if id.len == 0: 0'i64
+  else: parseBiggestInt(id)
+
 proc parseIsoDate*(date: string): DateTime =
   date.parseTime("yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", 20)
 
@@ -16,6 +20,9 @@ proc parseTwitterDate*(date: string): DateTime =
 proc getImageUrl*(url: string): string =
   url.dup(removePrefix(twimg), removePrefix(https))
 
+proc getImageUrl*(entity: MediaEntity | Entity): string =
+  entity.mediaUrlHttps.getImageUrl
+
 template handleErrors*(body) =
   if json.startsWith("{\"errors"):
     for error {.inject.} in json.fromJson(Errors).errors:

src/experimental/types/common.nim

@@ -1,3 +1,4 @@
+import jsony
 from ../../types import Error
 
 type
@@ -5,7 +6,7 @@ type
     url*: string
     expandedUrl*: string
     displayUrl*: string
-    indices*: array[2, int]
+    indices*: Slice[int]
 
   ErrorObj* = object
     code*: Error
@@ -18,3 +19,8 @@ proc contains*(codes: set[Error]; errors: Errors): bool =
   for e in errors.errors:
     if e.code in codes:
       return true
+
+proc parseHook*(s: string; i: var int; v: var Slice[int]) =
+  var slice: array[2, int]
+  parseHook(s, i, slice)
+  v = slice[0] ..< slice[1]

src/experimental/types/graphuser.nim

@@ -1,15 +1,12 @@
-import options
+import user
-from ../../types import User
 
 type
   GraphUser* = object
-    data*: tuple[userResult: UserData]
+    data*: tuple[user: UserData]
 
   UserData* = object
     result*: UserResult
 
   UserResult = object
-    legacy*: User
+    legacy*: RawUser
     restId*: string
-    isBlueVerified*: bool
-    unavailableReason*: Option[string]

src/experimental/types/media.nim

@@ -0,0 +1,15 @@
+import options
+from ../../types import VideoType, VideoVariant
+
+type
+  MediaType* = enum
+    photo, video, animatedGif
+
+  MediaEntity* = object
+    kind*: MediaType
+    mediaUrlHttps*: string
+    videoInfo*: Option[VideoInfo]
+
+  VideoInfo* = object
+    durationMillis*: int
+    variants*: seq[VideoVariant]

src/experimental/types/session.nim

@@ -1,9 +0,0 @@
-type
-  RawSession* = object
-    kind*: string
-    username*: string
-    id*: string
-    oauthToken*: string
-    oauthTokenSecret*: string
-    authToken*: string
-    ct0*: string

src/experimental/types/timeline.nim

@@ -1,13 +1,14 @@
 import std/tables
-from ../../types import User
+import user, tweet
 
 type
   Search* = object
     globalObjects*: GlobalObjects
     timeline*: Timeline
 
-  GlobalObjects = object
+  GlobalObjects* = object
-    users*: Table[string, User]
+    users*: Table[string, RawUser]
+    tweets*: Table[string, RawTweet]
 
   Timeline = object
     instructions*: seq[Instructions]
@@ -15,9 +16,13 @@ type
   Instructions = object
     addEntries*: tuple[entries: seq[Entry]]
 
-  Entry = object
+  Entry* = object
     entryId*: string
     content*: tuple[operation: Operation]
 
   Operation = object
     cursor*: tuple[value, cursorType: string]
+
+proc renameHook*(v: var Entity; fieldName: var string) =
+  if fieldName == "type":
+    fieldName = "kind"

src/experimental/types/tweet.nim

@@ -0,0 +1,85 @@
+import options
+import jsony
+from json import JsonNode
+import user, media, common
+
+type
+  RawTweet* = object
+    createdAt*: string
+    idStr*: string
+    fullText*: string
+    displayTextRange*: array[2, int]
+    entities*: Entities
+    extendedEntities*: ExtendedEntities
+    inReplyToStatusIdStr*: string
+    inReplyToScreenName*: string
+    userIdStr*: string
+    isQuoteStatus*: bool
+    replyCount*: int
+    retweetCount*: int
+    favoriteCount*: int
+    quoteCount*: int
+    conversationIdStr*: string
+    favorited*: bool
+    retweeted*: bool
+    selfThread*: tuple[idStr: string]
+    card*: Option[Card]
+    quotedStatusIdStr*: string
+    retweetedStatusIdStr*: string
+
+  Card* = object
+    name*: string
+    url*: string
+    bindingValues*: JsonNode
+
+  Entities* = object
+    hashtags*: seq[Hashtag]
+    symbols*: seq[Hashtag]
+    userMentions*: seq[UserMention]
+    urls*: seq[Url]
+    media*: seq[Entity]
+
+  Hashtag* = object
+    indices*: Slice[int]
+
+  UserMention* = object
+    screenName*: string
+    name*: string
+    indices*: Slice[int]
+
+  ExtendedEntities* = object
+    media*: seq[Entity]
+
+  Entity* = object
+    kind*: MediaType
+    indices*: Slice[int]
+    mediaUrlHttps*: string
+    url*: string
+    expandedUrl*: string
+    videoInfo*: VideoInfo
+    ext*: Ext
+    extMediaAvailability*: tuple[status: string]
+    extAltText*: string
+    additionalMediaInfo*: AdditionalMediaInfo
+    sourceStatusIdStr*: string
+    sourceUserIdStr*: string
+
+  AdditionalMediaInfo* = object
+    sourceUser*: Option[RawUser]
+    title*: string
+    description*: string
+
+  Ext* = object
+    mediaStats*: JsonNode
+
+  MediaStats* = object
+    ok*: tuple[viewCount: string]
+
+proc renameHook*(v: var Entity; fieldName: var string) =
+  if fieldName == "type":
+    fieldName = "kind"
+
+proc parseHook*(s: string; i: var int; v: var Slice[int]) =
+  var slice: array[2, int]
+  parseHook(s, i, slice)
+  v = slice[0] ..< slice[1]

src/experimental/types/unifiedcard.nim

@@ -1,10 +1,7 @@
-import std/[options, tables, times]
+import options, tables
-import jsony
+import media as mediaTypes
-from ../../types import VideoType, VideoVariant, User
 
 type
-  Text* = distinct string
-
   UnifiedCard* = object
     componentObjects*: Table[string, Component]
     destinationObjects*: Table[string, Destination]
@@ -16,13 +13,10 @@ type
     media
     swipeableMedia
     buttonGroup
-    jobDetails
     appStoreDetails
     twitterListDetails
     communityDetails
     mediaWithDetailsHorizontal
-    hidden
-    unknown
 
   Component* = object
     kind*: ComponentType
@@ -33,39 +27,24 @@ type
     appId*: string
     mediaId*: string
     destination*: string
-    location*: string
     title*: Text
     subtitle*: Text
     name*: Text
     memberCount*: int
     mediaList*: seq[MediaItem]
     topicDetail*: tuple[title: Text]
-    profileUser*: User
-    shortDescriptionText*: string
 
   MediaItem* = object
     id*: string
     destination*: string
 
-  Destination* = object
-    kind*: string
-    data*: tuple[urlData: UrlData]
-
   UrlData* = object
     url*: string
     vanity*: string
 
-  MediaType* = enum
+  Destination* = object
-    photo, video, model3d
+    kind*: string
-
+    data*: tuple[urlData: UrlData]
-  MediaEntity* = object
-    kind*: MediaType
-    mediaUrlHttps*: string
-    videoInfo*: Option[VideoInfo]
-
-  VideoInfo* = object
-    durationMillis*: int
-    variants*: seq[VideoVariant]
 
   AppType* = enum
     androidApp, iPhoneApp, iPadApp
@@ -76,53 +55,13 @@ type
     title*: Text
     category*: Text
 
-  TypeField = Component | Destination | MediaEntity | AppStoreData
+  Text = object
+    content: string
 
-converter fromText*(text: Text): string = string(text)
+  HasTypeField = Component | Destination | MediaEntity | AppStoreData
 
-proc renameHook*(v: var TypeField; fieldName: var string) =
+converter fromText*(text: Text): string = text.content
+
+proc renameHook*(v: var HasTypeField; fieldName: var string) =
   if fieldName == "type":
     fieldName = "kind"
-
-proc enumHook*(s: string; v: var ComponentType) =
-  v = case s
-      of "details": details
-      of "media": media
-      of "swipeable_media": swipeableMedia
-      of "button_group": buttonGroup
-      of "job_details": jobDetails
-      of "app_store_details": appStoreDetails
-      of "twitter_list_details": twitterListDetails
-      of "community_details": communityDetails
-      of "media_with_details_horizontal": mediaWithDetailsHorizontal
-      of "commerce_drop_details": hidden
-      else: echo "ERROR: Unknown enum value (ComponentType): ", s; unknown
-
-proc enumHook*(s: string; v: var AppType) =
-  v = case s
-      of "android_app": androidApp
-      of "iphone_app": iPhoneApp
-      of "ipad_app": iPadApp
-      else: echo "ERROR: Unknown enum value (AppType): ", s; androidApp
-
-proc enumHook*(s: string; v: var MediaType) =
-  v = case s
-      of "video": video
-      of "photo": photo
-      of "model3d": model3d
-      else: echo "ERROR: Unknown enum value (MediaType): ", s; photo
-
-proc parseHook*(s: string; i: var int; v: var DateTime) =
-  var str: string
-  parseHook(s, i, str)
-  v = parse(str, "yyyy-MM-dd hh:mm:ss")
-
-proc parseHook*(s: string; i: var int; v: var Text) =
-  if s[i] == '"':
-    var str: string
-    parseHook(s, i, str)
-    v = Text(str)
-  else:
-    var t: tuple[content: string]
-    parseHook(s, i, t)
-    v = Text(t.content)

src/experimental/types/user.nim

@@ -1,6 +1,6 @@
 import options
+import jsony
 import common
-from ../../types import VerifiedType
 
 type
   RawUser* = object
@@ -16,7 +16,7 @@ type
     favouritesCount*: int
     statusesCount*: int
     mediaCount*: int
-    verifiedType*: VerifiedType
+    verified*: bool
     protected*: bool
     profileLinkColor*: string
     profileBannerUrl*: string
@@ -42,3 +42,8 @@ type
 
   Color* = object
     red*, green*, blue*: int
+
+proc parseHook*(s: string; i: var int; v: var Slice[int]) =
+  var slice: array[2, int]
+  parseHook(s, i, slice)
+  v = slice[0] ..< slice[1]

src/formatters.nim

@@ -11,10 +11,9 @@ const
 let
   twRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?twitter\.com"
   twLinkRegex = re"""<a href="https:\/\/twitter.com([^"]+)">twitter\.com(\S+)</a>"""
-  xRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?x\.com"
-  xLinkRegex = re"""<a href="https:\/\/x.com([^"]+)">x\.com(\S+)</a>"""
 
-  ytRegex = re(r"([A-z.]+\.)?youtu(be\.com|\.be)", {reStudy, reIgnoreCase})
+  ytRegex = re"([A-z.]+\.)?youtu(be\.com|\.be)"
+  igRegex = re"(www\.)?instagram\.com"
 
   rdRegex = re"(?<![.b])((www|np|new|amp|old)\.)?reddit.com"
   rdShortRegex = re"(?<![.b])redd\.it\/"
@@ -57,19 +56,15 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
 
   if prefs.replaceYouTube.len > 0 and "youtu" in result:
     result = result.replace(ytRegex, prefs.replaceYouTube)
+    if prefs.replaceYouTube in result:
+      result = result.replace("/c/", "/")
 
-  if prefs.replaceTwitter.len > 0:
+  if prefs.replaceTwitter.len > 0 and ("twitter.com" in body or tco in body):
-    if tco in result:
+    result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
-      result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
+    result = result.replace(cards, prefs.replaceTwitter & "/cards")
-    if "x.com" in result:
+    result = result.replace(twRegex, prefs.replaceTwitter)
-      result = result.replace(xRegex, prefs.replaceTwitter)
+    result = result.replacef(twLinkRegex, a(
-      result = result.replacef(xLinkRegex, a(
+      prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
-        prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
-    if "twitter.com" in result:
-      result = result.replace(cards, prefs.replaceTwitter & "/cards")
-      result = result.replace(twRegex, prefs.replaceTwitter)
-      result = result.replacef(twLinkRegex, a(
-        prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
 
   if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result):
     result = result.replace(rdShortRegex, prefs.replaceReddit & "/comments/")
@@ -77,8 +72,11 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
     if prefs.replaceReddit in result and "/gallery/" in result:
       result = result.replace("/gallery/", "/comments/")
 
+  if prefs.replaceInstagram.len > 0 and "instagram.com" in result:
+    result = result.replace(igRegex, prefs.replaceInstagram)
+
   if absolute.len > 0 and "href" in result:
-    result = result.replace("href=\"/", &"href=\"{absolute}/")
+    result = result.replace("href=\"/", "href=\"" & absolute & "/")
 
 proc getM3u8Url*(content: string): string =
   var matches: array[1, string]
@@ -90,8 +88,6 @@ proc proxifyVideo*(manifest: string; proxy: bool): string =
   for line in manifest.splitLines:
     let url =
       if line.startsWith("#EXT-X-MAP:URI"): line[16 .. ^2]
-      elif line.startsWith("#EXT-X-MEDIA") and "URI=" in line:
-        line[line.find("URI=") + 5 .. -1 + line.find("\"", start= 5 + line.find("URI="))]
       else: line
     if url.startsWith('/'):
       let path = "https://video.twimg.com" & url

src/http_pool.nim

@@ -39,11 +39,8 @@ template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped =
 
   try:
     body
-  except BadClientError, ProtocolError:
+  except ProtocolError:
-    # Twitter returned 503 or closed the connection, we need a new client
+    # Twitter closed the connection, retry
-    pool.release(c, true)
-    badClient = false
-    c = pool.acquire(heads)
     body
   finally:
     pool.release(c, badClient)

src/nitter.nim

@@ -6,7 +6,7 @@ from os import getEnv
 
 import jester
 
-import types, config, prefs, formatters, redis_cache, http_pool, auth
+import types, config, prefs, formatters, redis_cache, http_pool, tokens
 import views/[general, about]
 import routes/[
   preferences, timeline, status, media, search, rss, list, debug,
@@ -15,13 +15,8 @@ import routes/[
 const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
 const issuesUrl = "https://github.com/zedeus/nitter/issues"
 
-let
+let configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
-  configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
+let (cfg, fullCfg) = getConfig(configPath)
-  (cfg, fullCfg) = getConfig(configPath)
-
-  sessionsPath = getEnv("NITTER_SESSIONS_FILE", "./sessions.jsonl")
-
-initSessionPool(cfg, sessionsPath)
 
 if not cfg.enableDebug:
   # Silence Jester's query warning
@@ -43,6 +38,8 @@ waitFor initRedisPool(cfg)
 stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n"
 stdout.flushFile
 
+asyncCheck initTokenPool(cfg)
+
 createUnsupportedRouter(cfg)
 createResolverRouter(cfg)
 createPrefRouter(cfg)
@@ -59,7 +56,6 @@ settings:
   port = Port(cfg.port)
   staticDir = cfg.staticDir
   bindAddr = cfg.address
-  reusePort = true
 
 routes:
   get "/":
@@ -88,28 +84,20 @@ routes:
     resp Http500, showError(
       &"An error occurred, please {link} with the URL you tried to visit.", cfg)
 
-  error BadClientError:
-    echo error.exc.name, ": ", error.exc.msg
-    resp Http500, showError("Network error occurred, please try again.", cfg)
-
   error RateLimitError:
+    echo error.exc.name, ": ", error.exc.msg
     const link = a("another instance", href = instancesUrl)
     resp Http429, showError(
       &"Instance has been rate limited.<br>Use {link} or try again later.", cfg)
 
-  error NoSessionsError:
+  extend unsupported, ""
-    const link = a("another instance", href = instancesUrl)
-    resp Http429, showError(
-      &"Instance has no auth tokens, or is fully rate limited.<br>Use {link} or try again later.", cfg)
-
-  extend rss, ""
-  extend status, ""
-  extend search, ""
-  extend timeline, ""
-  extend media, ""
-  extend list, ""
   extend preferences, ""
   extend resolver, ""
+  extend rss, ""
+  extend search, ""
+  extend timeline, ""
+  extend list, ""
+  extend status, ""
+  extend media, ""
   extend embed, ""
   extend debug, ""
-  extend unsupported, ""

src/parser.nim

@@ -1,11 +1,9 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, options, times, math
+import strutils, options, tables, times, math
 import packedjson, packedjson/deserialiser
 import types, parserutils, utils
 import experimental/parser/unifiedcard
 
-proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet
-
 proc parseUser(js: JsonNode; id=""): User =
   if js.isNull: return
   result = User(
@@ -21,38 +19,13 @@ proc parseUser(js: JsonNode; id=""): User =
     tweets: js{"statuses_count"}.getInt,
     likes: js{"favourites_count"}.getInt,
     media: js{"media_count"}.getInt,
-    verifiedType: parseEnum[VerifiedType](js{"verified_type"}.getStr("None")),
+    verified: js{"verified"}.getBool,
     protected: js{"protected"}.getBool,
     joinDate: js{"created_at"}.getTime
   )
 
   result.expandUserEntities(js)
 
-proc parseGraphUser(js: JsonNode): User =
-  var user = js{"user_result", "result"}
-  if user.isNull:
-    user = ? js{"user_results", "result"}
-
-  if user.isNull:
-    if js{"core"}.notNull and js{"legacy"}.notNull:
-      user = js
-    else:
-      return
-
-  result = parseUser(user{"legacy"}, user{"rest_id"}.getStr)
-
-  # fallback to support UserMedia/recent GraphQL updates
-  if result.username.len == 0 and user{"core", "screen_name"}.notNull:
-    result.username = user{"core", "screen_name"}.getStr
-    result.fullname = user{"core", "name"}.getStr
-    result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
-
-    if user{"is_blue_verified"}.getBool(false):
-      result.verifiedType = blue
-    elif user{"verification", "verified_type"}.notNull:
-      let verifiedType = user{"verification", "verified_type"}.getStr("None")
-      result.verifiedType = parseEnum[VerifiedType](verifiedType)
-
 proc parseGraphList*(js: JsonNode): List =
   if js.isNull: return
 
@@ -65,13 +38,14 @@ proc parseGraphList*(js: JsonNode): List =
   result = List(
     id: list{"id_str"}.getStr,
     name: list{"name"}.getStr,
-    username: list{"user_results", "result", "legacy", "screen_name"}.getStr,
+    username: list{"user", "legacy", "screen_name"}.getStr,
-    userId: list{"user_results", "result", "rest_id"}.getStr,
+    userId: list{"user", "rest_id"}.getStr,
     description: list{"description"}.getStr,
     members: list{"member_count"}.getInt,
-    banner: list{"custom_banner_media", "media_info", "original_img_url"}.getImageStr
+    banner: list{"custom_banner_media", "media_info", "url"}.getImageStr
   )
 
+
 proc parsePoll(js: JsonNode): Poll =
   let vals = js{"binding_values"}
   # name format is pollNchoice_*
@@ -99,17 +73,13 @@ proc parseGif(js: JsonNode): Gif =
 proc parseVideo(js: JsonNode): Video =
   result = Video(
     thumb: js{"media_url_https"}.getImageStr,
-    views: getVideoViewCount(js),
+    views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr,
-    available: true,
+    available: js{"ext_media_availability", "status"}.getStr == "available",
     title: js{"ext_alt_text"}.getStr,
     durationMs: js{"video_info", "duration_millis"}.getInt
     # playbackType: mp4
   )
 
-  with status, js{"ext_media_availability", "status"}:
-    if status.getStr.len > 0 and status.getStr.toLowerAscii != "available":
-      result.available = false
-
   with title, js{"additional_media_info", "title"}:
     result.title = title.getStr
 
@@ -117,15 +87,10 @@ proc parseVideo(js: JsonNode): Video =
     result.description = description.getStr
 
   for v in js{"video_info", "variants"}:
-    let
-      contentType = parseEnum[VideoType](v{"content_type"}.getStr("summary"))
-      url = v{"url"}.getStr
-
     result.variants.add VideoVariant(
-      contentType: contentType,
+      contentType: parseEnum[VideoType](v{"content_type"}.getStr("summary")),
       bitrate: v{"bitrate"}.getInt,
-      url: url,
+      url: v{"url"}.getStr
-      resolution: if contentType == mp4: getMp4Resolution(url) else: 0
     )
 
 proc parsePromoVideo(js: JsonNode): Video =
@@ -216,7 +181,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
      result.url.len == 0 or result.url.startsWith("card://"):
     result.url = getPicUrl(result.image)
 
-proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
+proc parseTweet(js: JsonNode): Tweet =
   if js.isNull: return
   result = Tweet(
     id: js{"id_str"}.getId,
@@ -235,28 +200,16 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
     )
   )
 
-  # fix for pinned threads
+  result.expandTweetEntities(js)
-  if result.hasThread and result.threadId == 0:
-    result.threadId = js{"self_thread", "id_str"}.getId
 
-  if "retweeted_status" in js:
+  if js{"is_quote_status"}.getBool:
-    result.retweet = some Tweet()
-  elif js{"is_quote_status"}.getBool:
     result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId)
 
-  # legacy
   with rt, js{"retweeted_status_id_str"}:
     result.retweet = some Tweet(id: rt.getId)
     return
 
-  # graphql
+  with jsCard, js{"card"}:
-  with rt, js{"retweeted_status_result", "result"}:
-    # needed due to weird edgecase where the actual tweet data isn't included
-    if "legacy" in rt:
-      result.retweet = some parseGraphTweet(rt)
-      return
-
-  if jsCard.kind != JNull:
     let name = jsCard{"name"}.getStr
     if "poll" in name:
       if "image" in name:
@@ -268,8 +221,6 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
     else:
       result.card = some parseCard(jsCard, js{"entities", "urls"})
 
-  result.expandTweetEntities(js)
-
   with jsMedia, js{"extended_entities", "media"}:
     for m in jsMedia:
       case m{"type"}.getStr
@@ -278,19 +229,11 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
       of "video":
         result.video = some(parseVideo(m))
         with user, m{"additional_media_info", "source_user"}:
-          if user{"id"}.getInt > 0:
+          result.attribution = some(parseUser(user))
-            result.attribution = some(parseUser(user))
-          else:
-            result.attribution = some(parseGraphUser(user))
       of "animated_gif":
         result.gif = some(parseGif(m))
       else: discard
 
-      with url, m{"url"}:
-        if result.text.endsWith(url.getStr):
-          result.text.removeSuffix(url.getStr)
-          result.text = result.text.strip()
-
   with jsWithheld, js{"withheld_in_countries"}:
     let withheldInCountries: seq[string] =
       if jsWithheld.kind != JArray: @[]
@@ -305,250 +248,159 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
       result.text.removeSuffix(" Learn more.")
       result.available = false
 
-proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
+proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
-  if js.kind == JNull:
+  let intId = if id.len > 0: parseBiggestInt(id) else: 0
-    return Tweet()
+  result = global.tweets.getOrDefault(id, Tweet(id: intId))
-
-  case js{"__typename"}.getStr
-  of "TweetUnavailable":
-    return Tweet()
-  of "TweetTombstone":
-    with text, js{"tombstone", "richText"}:
-      return Tweet(text: text.getTombstone)
-    with text, js{"tombstone", "text"}:
-      return Tweet(text: text.getTombstone)
-    return Tweet()
-  of "TweetPreviewDisplay":
-    return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.")
-  of "TweetWithVisibilityResults":
-    return parseGraphTweet(js{"tweet"}, isLegacy)
-  else:
-    discard
-
-  if not js.hasKey("legacy"):
-    return Tweet()
-
-  var jsCard = copy(js{if isLegacy: "card" else: "tweet_card", "legacy"})
-  if jsCard.kind != JNull:
-    var values = newJObject()
-    for val in jsCard["binding_values"]:
-      values[val["key"].getStr] = val["value"]
-    jsCard["binding_values"] = values
-
-  result = parseTweet(js{"legacy"}, jsCard)
-  result.id = js{"rest_id"}.getId
-  result.user = parseGraphUser(js{"core"})
-
-  with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
-    result.expandNoteTweetEntities(noteTweet)
 
   if result.quote.isSome:
-    result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}, isLegacy))
+    let quote = get(result.quote).id
+    if $quote in global.tweets:
+      result.quote = some global.tweets[$quote]
+    else:
+      result.quote = some Tweet()
 
-proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
+  if result.retweet.isSome:
-  for t in js{"content", "items"}:
+    let rt = get(result.retweet).id
-    let entryId = t{"entryId"}.getStr
+    if $rt in global.tweets:
-    if "cursor-showmore" in entryId:
+      result.retweet = some finalizeTweet(global, $rt)
-      let cursor = t{"item", "content", "value"}
+    else:
-      result.thread.cursor = cursor.getStr
+      result.retweet = some Tweet()
-      result.thread.hasMore = true
-    elif "tweet" in entryId and "promoted" notin entryId:
-      let
-        isLegacy = t{"item"}.hasKey("itemContent")
-        (contentKey, resultKey) = if isLegacy: ("itemContent", "tweet_results")
-                                  else: ("content", "tweetResult")
 
-      with content, t{"item", contentKey}:
+proc parsePin(js: JsonNode; global: GlobalObjects): Tweet =
-        result.thread.content.add parseGraphTweet(content{resultKey, "result"}, isLegacy)
+  let pin = js{"pinEntry", "entry", "entryId"}.getStr
+  if pin.len == 0: return
 
-        if content{"tweetDisplayType"}.getStr == "SelfThread":
+  let id = pin.getId
-          result.self = true
+  if id notin global.tweets: return
 
-proc parseGraphTweetResult*(js: JsonNode): Tweet =
+  global.tweets[id].pinned = true
-  with tweet, js{"data", "tweet_result", "result"}:
+  return finalizeTweet(global, id)
-    result = parseGraphTweet(tweet, false)
-
-proc parseGraphConversation*(js: JsonNode; tweetId: string; v2=true): Conversation =
-  result = Conversation(replies: Result[Chain](beginning: true))
 
+proc parseGlobalObjects(js: JsonNode): GlobalObjects =
+  result = GlobalObjects()
   let
-    rootKey = if v2: "timeline_response" else: "threaded_conversation_with_injections_v2"
+    tweets = ? js{"globalObjects", "tweets"}
-    contentKey = if v2: "content" else: "itemContent"
+    users = ? js{"globalObjects", "users"}
-    resultKey = if v2: "tweetResult" else: "tweet_results"
 
-  let instructions = ? js{"data", rootKey, "instructions"}
+  for k, v in users:
-  if instructions.len == 0:
+    result.users[k] = parseUser(v, k)
-    return
 
-  for i in instructions:
+  for k, v in tweets:
-    if i{"__typename"}.getStr == "TimelineAddEntries":
+    var tweet = parseTweet(v)
-      for e in i{"entries"}:
+    if tweet.user.id in result.users:
-        let entryId = e{"entryId"}.getStr
+      tweet.user = result.users[tweet.user.id]
-        if entryId.startsWith("tweet"):
+    result.tweets[k] = tweet
-          with tweetResult, e{"content", contentKey, resultKey, "result"}:
-            let tweet = parseGraphTweet(tweetResult, not v2)
 
-            if not tweet.available:
+proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] =
-              tweet.id = parseBiggestInt(entryId.getId())
+  result.thread = Chain()
 
-            if $tweet.id == tweetId:
+  let thread = js{"content", "item", "content", "conversationThread"}
-              result.tweet = tweet
+  with cursor, thread{"showMoreCursor"}:
-            else:
+    result.thread.cursor = cursor{"value"}.getStr
-              result.before.content.add tweet
+    result.thread.hasMore = true
-        elif entryId.startsWith("conversationthread"):
-          let (thread, self) = parseGraphThread(e)
-          if self:
-            result.after = thread
-          elif thread.content.len > 0:
-            result.replies.content.add thread
-        elif entryId.startsWith("tombstone"):
-          let id = entryId.getId()
-          let tweet = Tweet(
-            id: parseBiggestInt(id),
-            available: false,
-            text: e{"content", contentKey, "tombstoneInfo", "richText"}.getTombstone
-          )
 
-          if id == tweetId:
+  for t in thread{"conversationComponents"}:
-            result.tweet = tweet
+    let content = t{"conversationTweetComponent", "tweet"}
-          else:
-            result.before.content.add tweet
-        elif entryId.startsWith("cursor-bottom"):
-          result.replies.bottom = e{"content", contentKey, "value"}.getStr
 
-proc extractTweetsFromEntry*(e: JsonNode; entryId: string): seq[Tweet] =
+    if content{"displayType"}.getStr == "SelfThread":
-  if e{"content", "items"}.notNull:
+      result.self = true
-    for item in e{"content", "items"}:
-      with tweetResult, item{"item", "itemContent", "tweet_results", "result"}:
-        var tweet = parseGraphTweet(tweetResult, false)
-        if not tweet.available:
-          tweet.id = parseBiggestInt(item{"entryId"}.getStr.getId())
-        result.add tweet
-    return
 
-  with tweetResult, e{"content", "content", "tweetResult", "result"}:
+    var tweet = finalizeTweet(global, content{"id"}.getStr)
-    var tweet = parseGraphTweet(tweetResult, false)
     if not tweet.available:
-      tweet.id = parseBiggestInt(entryId.getId())
+      tweet.tombstone = getTombstone(content{"tombstone"})
-    result.add tweet
+    result.thread.content.add tweet
 
-proc parseGraphTimeline*(js: JsonNode; after=""): Profile =
+proc parseConversation*(js: JsonNode; tweetId: string): Conversation =
-  result = Profile(tweets: Timeline(beginning: after.len == 0))
+  result = Conversation(replies: Result[Chain](beginning: true))
-
+  let global = parseGlobalObjects(? js)
-  let instructions =
-    if js{"data", "list"}.notNull:
-      ? js{"data", "list", "timeline_response", "timeline", "instructions"}
-    elif js{"data", "user"}.notNull:
-      ? js{"data", "user", "result", "timeline", "timeline", "instructions"}
-    else:
-      ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
 
+  let instructions = ? js{"timeline", "instructions"}
   if instructions.len == 0:
     return
 
-  for i in instructions:
+  for e in instructions[0]{"addEntries", "entries"}:
-    # TimelineAddToModule instruction is used by UserMedia
+    let entry = e{"entryId"}.getStr
-    if i{"moduleItems"}.notNull:
+    if "tweet" in entry or "tombstone" in entry:
-      for item in i{"moduleItems"}:
+      let tweet = finalizeTweet(global, e.getEntryId)
-        with tweetResult, item{"item", "itemContent", "tweet_results", "result"}:
+      if $tweet.id != tweetId:
-          let tweet = parseGraphTweet(tweetResult, false)
+        result.before.content.add tweet
-          if not tweet.available:
+      else:
-            tweet.id = parseBiggestInt(item{"entryId"}.getStr.getId())
+        result.tweet = tweet
-          result.tweets.content.add tweet
+    elif "conversationThread" in entry:
-      continue
+      let (thread, self) = parseThread(e, global)
+      if thread.content.len > 0:
+        if self:
+          result.after = thread
+        else:
+          result.replies.content.add thread
+    elif "cursor-showMore" in entry:
+      result.replies.bottom = e.getCursor
+    elif "cursor-bottom" in entry:
+      result.replies.bottom = e.getCursor
 
-    if i{"entries"}.notNull:
+proc parseStatus*(js: JsonNode): Tweet =
-      for e in i{"entries"}:
+  with e, js{"errors"}:
-        let entryId = e{"entryId"}.getStr
+    if e.getError == tweetNotFound:
-        if entryId.startsWith("tweet") or entryId.startsWith("profile-grid"):
+      return
-          for tweet in extractTweetsFromEntry(e, entryId):
-            result.tweets.content.add tweet
-        elif "-conversation-" in entryId or entryId.startsWith("homeConversation"):
-          let (thread, self) = parseGraphThread(e)
-          result.tweets.content.add thread.content
-        elif entryId.startsWith("cursor-bottom"):
-          result.tweets.bottom = e{"content", "value"}.getStr
 
-    if after.len == 0 and i{"__typename"}.getStr == "TimelinePinEntry":
+  result = parseTweet(js)
-      with tweetResult, i{"entry", "content", "content", "tweetResult", "result"}:
+  if not result.isNil:
-        let tweet = parseGraphTweet(tweetResult, false)
+    result.user = parseUser(js{"user"})
-        tweet.pinned = true
-        if not tweet.available and tweet.tombstone.len == 0:
-          let entryId = i{"entry", "entryId"}.getEntryId
-          if entryId.len > 0:
-            tweet.id = parseBiggestInt(entryId)
-        result.pinned = some tweet
 
-proc parseGraphPhotoRail*(js: JsonNode): PhotoRail =
+    with quote, js{"quoted_status"}:
-  result = @[]
+      result.quote = some parseStatus(js{"quoted_status"})
 
-  let instructions =
+proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNode) =
-    if js{"data", "user"}.notNull:
+  if js.kind != JArray or js.len == 0:
-      ? js{"data", "user", "result", "timeline", "timeline", "instructions"}
-    else:
-      ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
-
-  if instructions.len == 0:
     return
 
-  for i in instructions:
+  for i in js:
-    # TimelineAddToModule instruction is used by MediaTimelineV2
+    when T is Tweet:
-    if i{"moduleItems"}.notNull:
+      if res.beginning and i{"pinEntry"}.notNull:
-      for item in i{"moduleItems"}:
+        with pin, parsePin(i, global):
-        with tweetResult, item{"item", "itemContent", "tweet_results", "result"}:
+          res.content.add pin
-          let t = parseGraphTweet(tweetResult, false)
-          if not t.available:
-            t.id = parseBiggestInt(item{"entryId"}.getStr.getId())
 
-          let photo = extractGalleryPhoto(t)
+    with r, i{"replaceEntry", "entry"}:
-          if photo.url.len > 0:
+      if "top" in r{"entryId"}.getStr:
-            result.add photo
+        res.top = r.getCursor
+      elif "bottom" in r{"entryId"}.getStr:
+        res.bottom = r.getCursor
 
-          if result.len == 16:
+proc parseTimeline*(js: JsonNode; after=""): Timeline =
-            return
+  result = Timeline(beginning: after.len == 0)
-      continue
+  let global = parseGlobalObjects(? js)
 
-    let instrType = i{"type"}.getStr(i{"__typename"}.getStr)
+  let instructions = ? js{"timeline", "instructions"}
-    if instrType != "TimelineAddEntries":
+  if instructions.len == 0: return
-      continue
 
-    for e in i{"entries"}:
+  result.parseInstructions(global, instructions)
-      let entryId = e{"entryId"}.getStr
-      if entryId.startsWith("tweet") or entryId.startsWith("profile-grid"):
-        for t in extractTweetsFromEntry(e, entryId):
-          let photo = extractGalleryPhoto(t)
-          if photo.url.len > 0:
-            result.add photo
 
-          if result.len == 16:
+  for e in instructions[0]{"addEntries", "entries"}:
-            return
+    let entry = e{"entryId"}.getStr
+    if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry:
+      let tweet = finalizeTweet(global, e.getEntryId)
+      if not tweet.available: continue
+      result.content.add tweet
+    elif "cursor-top" in entry:
+      result.top = e.getCursor
+    elif "cursor-bottom" in entry:
+      result.bottom = e.getCursor
+    elif entry.startsWith("sq-C"):
+      with cursor, e{"content", "operation", "cursor"}:
+        if cursor{"cursorType"}.getStr == "Bottom":
+          result.bottom = cursor{"value"}.getStr
+        else:
+          result.top = cursor{"value"}.getStr
 
-proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] =
+proc parsePhotoRail*(js: JsonNode): PhotoRail =
-  result = Result[T](beginning: after.len == 0)
+  for tweet in js:
+    let
+      t = parseTweet(tweet)
+      url = if t.photos.len > 0: t.photos[0]
+            elif t.video.isSome: get(t.video).thumb
+            elif t.gif.isSome: get(t.gif).thumb
+            elif t.card.isSome: get(t.card).image
+            else: ""
 
-  let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
+    if url.len == 0: continue
-  if instructions.len == 0:
+    result.add GalleryPhoto(url: url, tweetId: $t.id)
-    return
-
-  for instruction in instructions:
-    let typ = instruction{"type"}.getStr
-    if typ == "TimelineAddEntries":
-      for e in instruction{"entries"}:
-        let entryId = e{"entryId"}.getStr
-        when T is Tweets:
-          if entryId.startsWith("tweet"):
-            with tweetRes, e{"content", "itemContent", "tweet_results", "result"}:
-              let tweet = parseGraphTweet(tweetRes)
-              if not tweet.available:
-                tweet.id = parseBiggestInt(entryId.getId())
-              result.content.add tweet
-        elif T is User:
-          if entryId.startsWith("user"):
-            with userRes, e{"content", "itemContent"}:
-              result.content.add parseGraphUser(userRes)
-
-        if entryId.startsWith("cursor-bottom"):
-          result.bottom = e{"content", "value"}.getStr
-    elif typ == "TimelineReplaceEntry":
-      if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"):
-        result.bottom = instruction{"entry", "content", "value"}.getStr

src/parserutils.nim

@@ -1,17 +1,9 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import std/[times, macros, htmlgen, options, algorithm, re]
+import std/[strutils, times, macros, htmlgen, options, algorithm, re]
-import std/strutils except escape
 import std/unicode except strip
-from xmltree import escape
 import packedjson
 import types, utils, formatters
 
-const
-  unicodeOpen = "\uFFFA"
-  unicodeClose = "\uFFFB"
-  xmlOpen = escape("<")
-  xmlClose = escape(">")
-
 let
   unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
   unReplace = "$1<a href=\"/$2\">@$2</a>"
@@ -36,16 +28,15 @@ template `?`*(js: JsonNode): untyped =
   if j.isNull: return
   j
 
-template with*(ident, value, body): untyped =
+template `with`*(ident, value, body): untyped =
-  if true:
+  block:
     let ident {.inject.} = value
     if ident != nil: body
 
-template with*(ident; value: JsonNode; body): untyped =
+template `with`*(ident; value: JsonNode; body): untyped =
-  if true:
+  block:
     let ident {.inject.} = value
-    # value.notNull causes a compilation error for versions < 1.6.14
+    if value.notNull: body
-    if notNull(value): body
 
 template getCursor*(js: JsonNode): string =
   js{"content", "operation", "cursor", "value"}.getStr
@@ -139,30 +130,9 @@ proc getBanner*(js: JsonNode): string =
       return
 
 proc getTombstone*(js: JsonNode): string =
-  result = js{"text"}.getStr
+  result = js{"tombstoneInfo", "richText", "text"}.getStr
   result.removeSuffix(" Learn more")
 
-proc getMp4Resolution*(url: string): int =
-  # parses the height out of a URL like this one:
-  # https://video.twimg.com/ext_tw_video/<tweet-id>/pu/vid/720x1280/<random>.mp4
-  const vidSep = "/vid/"
-  let
-    vidIdx = url.find(vidSep) + vidSep.len
-    resIdx = url.find('x', vidIdx) + 1
-    res = url[resIdx ..< url.find("/", resIdx)]
-
-  try:
-    return parseInt(res)
-  except ValueError:
-    # cannot determine resolution (e.g. m3u8/non-mp4 video)
-    return 0
-
-proc getVideoViewCount*(js: JsonNode): string =
-  with stats, js{"ext_media_stats"}:
-    return stats{"view_count"}.getStr($stats{"viewCount"}.getInt)
-
-  return $js{"mediaStats", "viewCount"}.getInt(0)
-
 proc extractSlice(js: JsonNode): Slice[int] =
   result = js["indices"][0].getInt ..< js["indices"][1].getInt
 
@@ -245,37 +215,47 @@ proc expandUserEntities*(user: var User; js: JsonNode) =
   user.bio = user.bio.replacef(unRegex, unReplace)
                      .replacef(htRegex, htReplace)
 
-proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlice: Slice[int];
+proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
-                        replyTo=""; hasRedundantLink=false) =
+  let
-  let hasCard = tweet.card.isSome
+    orig = tweet.text.toRunes
+    textRange = js{"display_text_range"}
+    textSlice = textRange{0}.getInt .. textRange{1}.getInt
+    hasQuote = js{"is_quote_status"}.getBool
+    hasCard = tweet.card.isSome
+
+  var replyTo = ""
+  if tweet.replyId != 0:
+    with reply, js{"in_reply_to_screen_name"}:
+      tweet.reply.add reply.getStr
+      replyTo = reply.getStr
+
+  let ent = ? js{"entities"}
 
   var replacements = newSeq[ReplaceSlice]()
 
-  with urls, entities{"urls"}:
+  with urls, ent{"urls"}:
     for u in urls:
       let urlStr = u["url"].getStr
-      if urlStr.len == 0 or urlStr notin text:
+      if urlStr.len == 0 or urlStr notin tweet.text:
         continue
-
+      replacements.extractUrls(u, textSlice.b, hideTwitter = hasQuote)
-      replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink)
-
       if hasCard and u{"url"}.getStr == get(tweet.card).url:
         get(tweet.card).url = u{"expanded_url"}.getStr
 
-  with media, entities{"media"}:
+  with media, ent{"media"}:
     for m in media:
       replacements.extractUrls(m, textSlice.b, hideTwitter = true)
 
-  if "hashtags" in entities:
+  if "hashtags" in ent:
-    for hashtag in entities["hashtags"]:
+    for hashtag in ent["hashtags"]:
       replacements.extractHashtags(hashtag)
 
-  if "symbols" in entities:
+  if "symbols" in ent:
-    for symbol in entities["symbols"]:
+    for symbol in ent["symbols"]:
       replacements.extractHashtags(symbol)
 
-  if "user_mentions" in entities:
+  if "user_mentions" in ent:
-    for mention in entities["user_mentions"]:
+    for mention in ent["user_mentions"]:
       let
         name = mention{"screen_name"}.getStr
         slice = mention.extractSlice
@@ -292,40 +272,5 @@ proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlic
   replacements.deduplicate
   replacements.sort(cmp)
 
-  tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false)
+  tweet.text = orig.replacedWith(replacements, textSlice)
-
+                   .strip(leading=false)
-proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
-  let
-    entities = ? js{"entities"}
-    textRange = js{"display_text_range"}
-    textSlice = textRange{0}.getInt .. textRange{1}.getInt
-    hasQuote = js{"is_quote_status"}.getBool
-    hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails
-
-  var replyTo = ""
-  if tweet.replyId != 0:
-    with reply, js{"in_reply_to_screen_name"}:
-      replyTo = reply.getStr
-      tweet.reply.add replyTo
-
-  tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote or hasJobCard)
-
-proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) =
-  let
-    entities = ? js{"entity_set"}
-    text = js{"text"}.getStr.multiReplace(("<", unicodeOpen), (">", unicodeClose))
-    textSlice = 0..text.runeLen
-
-  tweet.expandTextEntities(entities, text, textSlice)
-
-  tweet.text = tweet.text.multiReplace((unicodeOpen, xmlOpen), (unicodeClose, xmlClose))
-
-proc extractGalleryPhoto*(t: Tweet): GalleryPhoto =
-  let url =
-    if t.photos.len > 0: t.photos[0]
-    elif t.video.isSome: get(t.video).thumb
-    elif t.gif.isSome: get(t.gif).thumb
-    elif t.card.isSome: get(t.card).image
-    else: ""
-
-  result = GalleryPhoto(url: url, tweetId: $t.id)

src/prefs_impl.nim

@@ -83,7 +83,7 @@ genPrefs:
       "Enable mp4 video playback (only for gifs)"
 
     hlsPlayback(checkbox, false):
-      "Enable HLS video streaming (requires JavaScript)"
+      "Enable hls video streaming (requires JavaScript)"
 
     proxyVideos(checkbox, true):
       "Proxy video streaming through the server (might be slow)"
@@ -107,6 +107,10 @@ genPrefs:
       "Reddit -> Teddit/Libreddit"
       placeholder: "Teddit hostname"
 
+    replaceInstagram(input, ""):
+      "Instagram -> Bibliogram"
+      placeholder: "Bibliogram hostname"
+
 iterator allPrefs*(): Pref =
   for k, v in prefList:
     for pref in v:

src/query.nim

@@ -60,7 +60,7 @@ proc genQueryParam*(query: Query): string =
       param &= "OR "
 
   if query.fromUser.len > 0 and query.kind in {posts, media}:
-    param &= "filter:self_threads OR -filter:replies "
+    param &= "filter:self_threads OR-filter:replies "
 
   if "nativeretweets" notin query.excludes:
     param &= "include:nativeretweets "
@@ -93,11 +93,11 @@ proc genQueryUrl*(query: Query): string =
   if query.text.len > 0:
     params.add "q=" & encodeUrl(query.text)
   for f in query.filters:
-    params.add &"f-{f}=on"
+    params.add "f-" & f & "=on"
   for e in query.excludes:
-    params.add &"e-{e}=on"
+    params.add "e-" & e & "=on"
   for i in query.includes.filterIt(it != "nativeretweets"):
-    params.add &"i-{i}=on"
+    params.add "i-" & i & "=on"
 
   if query.since.len > 0:
     params.add "since=" & query.since

src/redis_cache.nim

@@ -52,7 +52,6 @@ proc initRedisPool*(cfg: Config) {.async.} =
     await migrate("profileDates", "p:*")
     await migrate("profileStats", "p:*")
     await migrate("userType", "p:*")
-    await migrate("verifiedType", "p:*")
 
     pool.withAcquire(r):
       # optimize memory usage for user ID buckets
@@ -86,7 +85,7 @@ proc cache*(data: List) {.async.} =
   await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
 
 proc cache*(data: PhotoRail; name: string) {.async.} =
-  await setEx("pr2:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data)))
+  await setEx("pr:" & toLower(name), baseCacheTime, compress(toFlatty(data)))
 
 proc cache*(data: User) {.async.} =
   if data.username.len == 0: return
@@ -119,11 +118,11 @@ proc getUserId*(username: string): Future[string] {.async.} =
   pool.withAcquire(r):
     result = await r.hGet(name.uidKey, name)
     if result == redisNil:
-      let user = await getGraphUser(username)
+      let user = await getUser(username)
       if user.suspended:
         return "suspended"
       else:
-        await all(cacheUserId(name, user.id), cache(user))
+        await cacheUserId(name, user.id)
         return user.id
 
 proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
@@ -131,7 +130,8 @@ proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
   if prof != redisNil:
     prof.deserialize(User)
   elif fetch:
-    result = await getGraphUser(username)
+    let userId = await getUserId(username)
+    result = await getGraphUser(userId)
     await cache(result)
 
 proc getCachedUsername*(userId: string): Future[string] {.async.} =
@@ -142,30 +142,28 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} =
   if username != redisNil:
     result = username
   else:
-    let user = await getGraphUserById(userId)
+    let user = await getUserById(userId)
     result = user.username
     await setEx(key, baseCacheTime, result)
-    if result.len > 0 and user.id.len > 0:
-      await all(cacheUserId(result, user.id), cache(user))
 
-# proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
+proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
-#   if id == 0: return
+  if id == 0: return
-#   let tweet = await get(id.tweetKey)
+  let tweet = await get(id.tweetKey)
-#   if tweet != redisNil:
+  if tweet != redisNil:
-#     tweet.deserialize(Tweet)
+    tweet.deserialize(Tweet)
-#   else:
+  else:
-#     result = await getGraphTweetResult($id)
+    result = await getStatus($id)
-#     if not result.isNil:
+    if result.isNil:
-#       await cache(result)
+      await cache(result)
 
-proc getCachedPhotoRail*(id: string): Future[PhotoRail] {.async.} =
+proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
-  if id.len == 0: return
+  if name.len == 0: return
-  let rail = await get("pr2:" & toLower(id))
+  let rail = await get("pr:" & toLower(name))
   if rail != redisNil:
     rail.deserialize(PhotoRail)
   else:
-    result = await getPhotoRail(id)
+    result = await getPhotoRail(name)
-    await cache(result, id)
+    await cache(result, name)
 
 proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} =
   let list = if id.len == 0: redisNil

src/routes/debug.nim

@@ -1,13 +1,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import jester
 import router_utils
-import ".."/[auth, types]
+import ".."/[tokens, types]
 
 proc createDebugRouter*(cfg: Config) =
   router debug:
-    get "/.health":
+    get "/.tokens":
-      respJson getSessionPoolHealth()
-
-    get "/.sessions":
       cond cfg.enableDebug
-      respJson getSessionPoolDebug()
+      respJson getPoolJson()

src/routes/embed.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, strformat, options
+import asyncdispatch, strutils, options
 import jester, karax/vdom
 import ".."/[types, api]
 import ../views/[embed, tweet, general]
@@ -10,27 +10,27 @@ export api, embed, vdom, tweet, general, router_utils
 proc createEmbedRouter*(cfg: Config) =
   router embed:
     get "/i/videos/tweet/@id":
-      let tweet = await getGraphTweetResult(@"id")
+      let convo = await getTweet(@"id")
-      if tweet == nil or tweet.video.isNone:
+      if convo == nil or convo.tweet == nil or convo.tweet.video.isNone:
         resp Http404
 
-      resp renderVideoEmbed(tweet, cfg, request)
+      resp renderVideoEmbed(convo.tweet, cfg, request)
 
     get "/@user/status/@id/embed":
       let
-        tweet = await getGraphTweetResult(@"id")
+        convo = await getTweet(@"id")
         prefs = cookiePrefs()
         path = getPath()
 
-      if tweet == nil:
+      if convo == nil or convo.tweet == nil:
         resp Http404
 
-      resp renderTweetEmbed(tweet, path, prefs, cfg, request)
+      resp $renderTweetEmbed(convo.tweet, path, prefs, cfg, request)
 
     get "/embed/Tweet.html":
       let id = @"id"
 
       if id.len > 0:
-        redirect(&"/i/status/{id}/embed")
+        redirect("/i/status/" & id & "/embed")
       else:
         resp Http404

src/routes/list.nim

@@ -1,25 +1,23 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, strformat, uri
+import strutils, uri
 
 import jester
 
 import router_utils
 import ".."/[types, redis_cache, api]
 import ../views/[general, timeline, list]
+export getListTimeline, getGraphList
 
 template respList*(list, timeline, title, vnode: typed) =
   if list.id.len == 0 or list.name.len == 0:
-    resp Http404, showError(&"""List "{@"id"}" not found""", cfg)
+    resp Http404, showError("List " & @"id" & " not found", cfg)
 
   let
     html = renderList(vnode, timeline.query, list)
-    rss = &"""/i/lists/{@"id"}/rss"""
+    rss = "/i/lists/$1/rss" % [@"id"]
 
   resp renderMain(html, request, cfg, prefs, titleText=title, rss=rss, banner=list.banner)
 
-proc title*(list: List): string =
-  &"@{list.username}/{list.name}"
-
 proc createListRouter*(cfg: Config) =
   router list:
     get "/@name/lists/@slug/?":
@@ -30,22 +28,24 @@ proc createListRouter*(cfg: Config) =
         slug = decodeUrl(@"slug")
         list = await getCachedList(@"name", slug)
       if list.id.len == 0:
-        resp Http404, showError(&"""List "{@"slug"}" not found""", cfg)
+        resp Http404, showError("List \"" & @"slug" & "\" not found", cfg)
-      redirect(&"/i/lists/{list.id}")
+      redirect("/i/lists/" & list.id)
 
     get "/i/lists/@id/?":
       cond '.' notin @"id"
       let
         prefs = cookiePrefs()
         list = await getCachedList(id=(@"id"))
-        timeline = await getGraphListTweets(list.id, getCursor())
+        title = "@" & list.username & "/" & list.name
+        timeline = await getListTimeline(list.id, getCursor())
         vnode = renderTimelineTweets(timeline, prefs, request.path)
-      respList(list, timeline, list.title, vnode)
+      respList(list, timeline, title, vnode)
 
     get "/i/lists/@id/members":
       cond '.' notin @"id"
       let
         prefs = cookiePrefs()
         list = await getCachedList(id=(@"id"))
+        title = "@" & list.username & "/" & list.name
         members = await getGraphListMembers(list, getCursor())
-      respList(list, members, list.title, renderTimelineUsers(members, prefs, request.path))
+      respList(list, members, title, renderTimelineUsers(members, prefs, request.path))

src/routes/media.nim

@@ -37,8 +37,6 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
   try:
     let res = await client.get(url)
     if res.status != "200 OK":
-      if res.status != "404 Not Found":
-        echo "[media] Proxying failed, status: $1, url: $2" % [res.status, url]
       return Http404
 
     let hashed = $hash(url)
@@ -67,7 +65,6 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
         await request.client.send(data)
     data.setLen 0
   except HttpRequestError, ProtocolError, OSError:
-    echo "[media] Proxying exception, error: $1, url: $2" % [getCurrentExceptionMsg(), url]
     result = Http404
   finally:
     client.close()
@@ -91,20 +88,6 @@ proc createMediaRouter*(cfg: Config) =
     get "/pic/?":
       resp Http404
 
-    get re"^\/pic\/orig\/(enc)?\/?(.+)":
-      var url = decoded(request, 1)
-      if "twimg.com" notin url:
-        url.insert(twimg)
-      if not url.startsWith(https):
-        url.insert(https)
-      url.add("?name=orig")
-
-      let uri = parseUri(url)
-      cond isTwitterUrl(uri) == true
-
-      let code = await proxyMedia(request, url)
-      check code
-
     get re"^\/pic\/(enc)?\/?(.+)":
       var url = decoded(request, 1)
       if "twimg.com" notin url:
@@ -123,7 +106,7 @@ proc createMediaRouter*(cfg: Config) =
       cond "http" in url
 
       if getHmac(url) != request.matches[1]:
-        resp Http403, showError("Failed to verify signature", cfg)
+        resp showError("Failed to verify signature", cfg)
 
       if ".mp4" in url or ".ts" in url or ".m4s" in url:
         let code = await proxyMedia(request, url)

src/routes/rss.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, tables, times, hashes, uri
+import asyncdispatch, strutils, tables, times, hashes, uri
 
 import jester
 
@@ -10,11 +10,6 @@ include "../views/rss.nimf"
 
 export times, hashes
 
-proc redisKey*(page, name, cursor: string): string =
-  result = page & ":" & name
-  if cursor.len > 0:
-    result &= ":" & cursor
-
 proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} =
   var profile: Profile
   let
@@ -23,16 +18,18 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
     names = getNames(name)
 
   if names.len == 1:
-    profile = await fetchProfile(after, query, skipRail=true)
+    profile = await fetchProfile(after, query, skipRail=true, skipPinned=true)
   else:
     var q = query
     q.fromUser = names
-    profile.tweets = await getGraphTweetSearch(q, after)
+    profile = Profile(
-    # this is kinda dumb
+      tweets: await getSearch[Tweet](q, after),
-    profile.user = User(
+      # this is kinda dumb
-      username: name,
+      user: User(
-      fullname: names.join(" | "),
+        username: name,
-      userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png"
+        fullname: names.join(" | "),
+        userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png"
+      )
     )
 
   if profile.user.suspended:
@@ -45,8 +42,8 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
 template respRss*(rss, page) =
   if rss.cursor.len == 0:
     let info = case page
-               of "User": " \"" & @"name" & "\" "
+               of "User": " \"$1\" " % @"name"
-               of "List": " \"" & @"id" & "\" "
+               of "List": " $1 " % @"id"
                else: " "
 
     resp Http404, showError(page & info & "not found", cfg)
@@ -70,13 +67,13 @@ proc createRssRouter*(cfg: Config) =
 
       let
         cursor = getCursor()
-        key = redisKey("search", $hash(genQueryUrl(query)), cursor)
+        key = "search:" & $hash(genQueryUrl(query)) & ":" & cursor
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
         respRss(rss, "Search")
 
-      let tweets = await getGraphTweetSearch(query, cursor)
+      let tweets = await getSearch[Tweet](query, cursor)
       rss.cursor = tweets.bottom
       rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg)
 
@@ -87,8 +84,9 @@ proc createRssRouter*(cfg: Config) =
       cond cfg.enableRss
       cond '.' notin @"name"
       let
+        cursor = getCursor()
         name = @"name"
-        key = redisKey("twitter", name, getCursor())
+        key = "twitter:" & name & ":" & cursor
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
@@ -103,20 +101,18 @@ proc createRssRouter*(cfg: Config) =
       cond cfg.enableRss
       cond '.' notin @"name"
       cond @"tab" in ["with_replies", "media", "search"]
-      let
+      let name = @"name"
-        name = @"name"
+      let query =
-        tab = @"tab"
+        case @"tab"
-        query =
+        of "with_replies": getReplyQuery(name)
-          case tab
+        of "media": getMediaQuery(name)
-          of "with_replies": getReplyQuery(name)
+        of "search": initQuery(params(request), name=name)
-          of "media": getMediaQuery(name)
+        else: Query(fromUser: @[name])
-          of "search": initQuery(params(request), name=name)
-          else: Query(fromUser: @[name])
 
-      let searchKey = if tab != "search": ""
+      var key = @"tab" & ":" & @"name" & ":"
-                      else: ":" & $hash(genQueryUrl(query))
+      if @"tab" == "search":
-
+        key &= $hash(genQueryUrl(query)) & ":"
-      let key = redisKey(tab, name & searchKey, getCursor())
+      key &= getCursor()
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
@@ -147,17 +143,18 @@ proc createRssRouter*(cfg: Config) =
     get "/i/lists/@id/rss":
       cond cfg.enableRss
       let
-        id = @"id"
         cursor = getCursor()
-        key = redisKey("lists", id, cursor)
+        key =
+          if cursor.len == 0: "lists:" & @"id"
+          else: "lists:" & @"id" & ":" & cursor
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
         respRss(rss, "List")
 
       let
-        list = await getCachedList(id=id)
+        list = await getCachedList(id=(@"id"))
-        timeline = await getGraphListTweets(list.id, cursor)
+        timeline = await getListTimeline(list.id, cursor)
       rss.cursor = timeline.bottom
       rss.feed = renderListRss(timeline.content, list, cfg)
 

src/routes/search.nim

@@ -14,31 +14,25 @@ export search
 proc createSearchRouter*(cfg: Config) =
   router search:
     get "/search/?":
-      let q = @"q"
+      if @"q".len > 500:
-      if q.len > 500:
         resp Http400, showError("Search input too long.", cfg)
 
       let
         prefs = cookiePrefs()
         query = initQuery(params(request))
-        title = "Search" & (if q.len > 0: " (" & q & ")" else: "")
 
       case query.kind
       of users:
-        if "," in q:
+        if "," in @"q":
-          redirect("/" & q)
+          redirect("/" & @"q")
-        var users: Result[User]
+        let users = await getSearch[User](query, getCursor())
-        try:
+        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs)
-          users = await getGraphUserSearch(query, getCursor())
-        except InternalError:
-          users = Result[User](beginning: true, query: query)
-        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title)
       of tweets:
         let
-          tweets = await getGraphTweetSearch(query, getCursor())
+          tweets = await getSearch[Tweet](query, getCursor())
           rss = "/search/rss?" & genQueryUrl(query)
         resp renderMain(renderTweetSearch(tweets, prefs, getPath()),
-                        request, cfg, prefs, title, rss=rss)
+                        request, cfg, prefs, rss=rss)
       else:
         resp Http404, showError("Invalid search", cfg)
 
@@ -48,4 +42,4 @@ proc createSearchRouter*(cfg: Config) =
     get "/opensearch":
       let url = getUrlPrefix(cfg) & "/search?q="
       resp Http200, {"Content-Type": "application/opensearchdescription+xml"},
-                     generateOpenSearchXML(cfg.title, cfg.hostname, url)
+                    generateOpenSearchXML(cfg.title, cfg.hostname, url)

src/routes/status.nim

@@ -16,21 +16,17 @@ proc createStatusRouter*(cfg: Config) =
   router status:
     get "/@name/status/@id/?":
       cond '.' notin @"name"
-      let id = @"id"
+      cond not @"id".any(c => not c.isDigit)
-
-      if id.len > 19 or id.any(c => not c.isDigit):
-        resp Http404, showError("Invalid tweet ID", cfg)
-
       let prefs = cookiePrefs()
 
       # used for the infinite scroll feature
       if @"scroll".len > 0:
-        let replies = await getReplies(id, getCursor())
+        let replies = await getReplies(@"id", getCursor())
         if replies.content.len == 0:
           resp Http404, ""
         resp $renderReplies(replies, prefs, getPath())
 
-      let conv = await getTweet(id, getCursor())
+      let conv = await getTweet(@"id", getCursor())
       if conv == nil:
         echo "nil conv"
 
@@ -76,6 +72,3 @@ proc createStatusRouter*(cfg: Config) =
 
     get "/i/web/status/@id":
       redirect("/i/status/" & @"id")
-      
-    get "/@name/thread/@id/?":
-      redirect("/$1/status/$2" % [@"name", @"id"])

src/routes/timeline.nim

@@ -27,7 +27,8 @@ template skipIf[T](cond: bool; default; body: Future[T]): Future[T] =
   else:
     body
 
-proc fetchProfile*(after: string; query: Query; skipRail=false): Future[Profile] {.async.} =
+proc fetchProfile*(after: string; query: Query; skipRail=false;
+                   skipPinned=false): Future[Profile] {.async.} =
   let
     name = query.fromUser[0]
     userId = await getUserId(name)
@@ -44,21 +45,36 @@ proc fetchProfile*(after: string; query: Query; skipRail=false): Future[Profile]
     after.setLen 0
 
   let
+    timeline =
+      case query.kind
+      of posts: getTimeline(userId, after)
+      of replies: getTimeline(userId, after, replies=true)
+      of media: getMediaTimeline(userId, after)
+      else: getSearch[Tweet](query, after)
+
     rail =
       skipIf(skipRail or query.kind == media, @[]):
-        getCachedPhotoRail(userId)
+        getCachedPhotoRail(name)
 
-    user = getCachedUser(name)
+    user = await getCachedUser(name)
 
-  result =
+  var pinned: Option[Tweet]
-    case query.kind
+  if not skipPinned and user.pinnedTweet > 0 and
-    of posts: await getGraphUserTweets(userId, TimelineKind.tweets, after)
+     after.len == 0 and query.kind in {posts, replies}:
-    of replies: await getGraphUserTweets(userId, TimelineKind.replies, after)
+    let tweet = await getCachedTweet(user.pinnedTweet)
-    of media: await getGraphUserTweets(userId, TimelineKind.media, after)
+    if not tweet.isNil:
-    else: Profile(tweets: await getGraphTweetSearch(query, after))
+      tweet.pinned = true
+      pinned = some tweet
 
-  result.user = await user
+  result = Profile(
-  result.photoRail = await rail
+    user: user,
+    pinned: pinned,
+    tweets: await timeline,
+    photoRail: await rail
+  )
+
+  if result.user.protected or result.user.suspended:
+    return
 
   result.tweets.query = query
 
@@ -66,11 +82,11 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
                    rss, after: string): Future[string] {.async.} =
   if query.fromUser.len != 1:
     let
-      timeline = await getGraphTweetSearch(query, after)
+      timeline = await getSearch[Tweet](query, after)
       html = renderTweetSearch(timeline, prefs, getPath())
     return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
 
-  var profile = await fetchProfile(after, query)
+  var profile = await fetchProfile(after, query, skipPinned=prefs.hidePins)
   template u: untyped = profile.user
 
   if u.suspended:
@@ -107,7 +123,7 @@ proc createTimelineRouter*(cfg: Config) =
 
     get "/@name/?@tab?/?":
       cond '.' notin @"name"
-      cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]
+      cond @"name" notin ["pic", "gif", "video"]
       cond @"tab" in ["with_replies", "media", "search", ""]
       let
         prefs = cookiePrefs()
@@ -121,7 +137,7 @@ proc createTimelineRouter*(cfg: Config) =
       # used for the infinite scroll feature
       if @"scroll".len > 0:
         if query.fromUser.len != 1:
-          var timeline = await getGraphTweetSearch(query, after)
+          var timeline = await getSearch[Tweet](query, after)
           if timeline.content.len == 0: resp Http404
           timeline.beginning = true
           resp $renderTweetSearch(timeline, prefs, getPath())

src/sass/include/_variables.scss

@@ -28,8 +28,6 @@ $more_replies_dots: #AD433B;
 $error_red: #420A05;
 
 $verified_blue: #1DA1F2;
-$verified_business: #FAC82B;
-$verified_government: #C1B6A4;
 $icon_text: $fg_color;
 
 $tab: $fg_color;
@@ -39,5 +37,8 @@ $shadow: rgba(0,0,0,.6);
 $shadow_dark: rgba(0,0,0,.2);
 
 //fonts
-$font_stack: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;
+$font_0: Helvetica Neue;
-$font_icon: fontello;
+$font_1: Helvetica;
+$font_2: Arial;
+$font_3: sans-serif;
+$font_4: fontello;

src/sass/index.scss

@@ -39,8 +39,6 @@ body {
     --error_red: #{$error_red};
 
     --verified_blue: #{$verified_blue};
-    --verified_business: #{$verified_business};
-    --verified_government: #{$verified_government};
     --icon_text: #{$icon_text};
 
     --tab: #{$fg_color};
@@ -50,7 +48,7 @@ body {
 
     background-color: var(--bg_color);
     color: var(--fg_color);
-    font-family: $font_stack;
+    font-family: $font_0, $font_1, $font_2, $font_3;
     font-size: 14px;
     line-height: 1.3;
     margin: 0;
@@ -143,30 +141,17 @@ ul {
 
 .verified-icon {
     color: var(--icon_text);
+    background-color: var(--verified_blue);
     border-radius: 50%;
     flex-shrink: 0;
     margin: 2px 0 3px 3px;
-    padding-top: 3px;
+    padding-top: 2px;
-    height: 11px;
+    height: 12px;
     width: 14px;
     font-size: 8px;
     display: inline-block;
     text-align: center;
     vertical-align: middle;
-
-    &.blue {
-        background-color: var(--verified_blue);
-    }
-
-    &.business {
-        color: var(--bg_panel);
-        background-color: var(--verified_business);
-    }
-
-    &.government {
-        color: var(--bg_panel);
-        background-color: var(--verified_government);
-    }
 }
 
 @media(max-width: 600px) {

src/sass/inputs.scss

@@ -136,7 +136,7 @@ input::-webkit-datetime-edit-year-field:focus {
         left: 2px;
         bottom: 0;
         font-size: 13px;
-        font-family: $font_icon;
+        font-family: $font_4;
         content: '\e803';
     }
 }

src/sass/navbar.scss

@@ -70,9 +70,8 @@ nav {
 
 .lp {
     height: 14px;
-    display: inline-block;
+    margin-top: 2px;
-    position: relative;
+    display: block;
-    top: 2px;
     fill: var(--fg_nav);
 
     &:hover {

src/sass/profile/_base.scss

@@ -15,7 +15,7 @@
 }
 
 .profile-banner {
-    margin: 4px 0 4px 0;
+    margin-bottom: 4px;
     background-color: var(--bg_panel);
 
     a {

src/sass/profile/card.scss

@@ -73,9 +73,9 @@
         }
     }
 
-    .profile-joindate, .profile-location, .profile-website {
+    .profile-joindate, .profile-location, profile-website {
         color: var(--fg_faded);
-        margin: 1px 0;
+        margin: 2px 0;
         width: 100%;
     }
 }
@@ -115,7 +115,7 @@
     }
 
     .profile-card-tabs-name {
-        flex-shrink: 100;
+        @include breakable;
     }
 
     .profile-card-avatar {

src/sass/search.scss

@@ -14,8 +14,6 @@
     button {
         margin: 0 2px 0 0;
         height: 23px;
-        display: flex;
-        align-items: center;
     }
 
     .pref-input {

src/sass/tweet/_base.scss

@@ -17,7 +17,7 @@
 }
 
 .tweet-content {
-    font-family: $font_stack;
+    font-family: $font_3;
     line-height: 1.3em;
     pointer-events: all;
     display: inline;
@@ -98,9 +98,10 @@
 }
 
 .avatar {
+    position: absolute;
+
     &.round {
         border-radius: 50%;
-        -webkit-user-select: none;
     }
     
     &.mini {
@@ -120,22 +121,14 @@
     background-color: var(--bg_panel);
 
     .tweet-content { 
-        font-size: 18px;
+        font-size: 18px 
     }
     
-    .tweet-body {
+    .tweet-body { 
         display: flex;
         flex-direction: column;
         max-height: calc(100vh - 0.75em * 2);
     }
-
-    .card-image img {
-        height: auto;
-    }
-    
-    .avatar {
-        position: absolute;
-    }
 }
 
 .attribution {
@@ -200,11 +193,6 @@
 
 .tweet-stats {
     margin-bottom: -3px;
-    -webkit-user-select: none;
-
-    a {
-        pointer-events: all;
-    }
 }
 
 .tweet-stat {
@@ -236,7 +224,6 @@
     left: 0;
     top: 0;
     position: absolute;
-    -webkit-user-select: none;
 
     &:hover {
         background-color: var(--bg_hover);

src/sass/tweet/thread.scss

@@ -23,6 +23,7 @@
     font-size: 18px;
 }
 
+
 @media(max-width: 600px) {
     .main-tweet .tweet-content {
         font-size: 16px;
@@ -110,29 +111,3 @@
     margin-left: 58px;
     padding: 7px 0;
 }
-
-.timeline-item.thread.more-replies-thread {
-    padding: 0 0.75em;
-
-    &::before {
-        top: 40px;
-        margin-bottom: 31px;
-    }
-
-    .more-replies {
-        display: flex;
-        padding-top: unset !important;
-        margin-top: 8px;
-
-        &::before {
-            display: inline-block;
-            position: relative;
-            top: -1px;
-            line-height: 0.4em;
-        }
-
-        .more-replies-text {
-            display: inline;
-        }
-    }
-}

src/sass/tweet/video.scss

@@ -3,7 +3,7 @@
 
 video {
     max-height: 100%;
-    width: 100%;
+    max-width: 100%;
 }
 
 .gallery-video {
@@ -16,8 +16,6 @@ video {
 }
 
 .video-container {
-    min-height: 80px;
-    min-width: 200px;
     max-height: 530px;
     margin: 0;
     display: flex;

src/tokens.nim

@@ -0,0 +1,154 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import asyncdispatch, httpclient, times, sequtils, json, random
+import strutils, tables
+import zippy
+import types, consts, http_pool
+
+const
+  maxConcurrentReqs = 5  # max requests at a time per token, to avoid race conditions
+  maxLastUse = 1.hours   # if a token is unused for 60 minutes, it expires
+  maxAge = 2.hours + 55.minutes  # tokens expire after 3 hours
+  failDelay = initDuration(minutes=30)
+
+var
+  clientPool: HttpPool
+  tokenPool: seq[Token]
+  lastFailed: Time
+
+proc getPoolJson*(): JsonNode =
+  var
+    list = newJObject()
+    totalReqs = 0
+    totalPending = 0
+    reqsPerApi: Table[string, int]
+
+  for token in tokenPool:
+    totalPending.inc(token.pending)
+    list[token.tok] = %*{
+      "apis": newJObject(),
+      "pending": token.pending,
+      "init": $token.init,
+      "lastUse": $token.lastUse
+    }
+
+    for api in token.apis.keys:
+      list[token.tok]["apis"][$api] = %token.apis[api]
+
+      let
+        maxReqs =
+          case api
+          of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId: 500
+          of Api.timeline: 187
+          else: 180
+        reqs = maxReqs - token.apis[api].remaining
+
+      reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
+      totalReqs.inc(reqs)
+
+  return %*{
+    "amount": tokenPool.len,
+    "requests": totalReqs,
+    "pending": totalPending,
+    "apis": reqsPerApi,
+    "tokens": list
+  }
+
+proc rateLimitError*(): ref RateLimitError =
+  newException(RateLimitError, "rate limited")
+
+proc fetchToken(): Future[Token] {.async.} =
+  if getTime() - lastFailed < failDelay:
+    raise rateLimitError()
+
+  let headers = newHttpHeaders({
+    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+    "accept-encoding": "gzip",
+    "accept-language": "en-US,en;q=0.5",
+    "connection": "keep-alive",
+    "authorization": auth
+  })
+
+  try:
+    let
+      resp = clientPool.use(headers): await c.postContent(activate)
+      tokNode = parseJson(uncompress(resp))["guest_token"]
+      tok = tokNode.getStr($(tokNode.getInt))
+      time = getTime()
+
+    return Token(tok: tok, init: time, lastUse: time)
+  except Exception as e:
+    lastFailed = getTime()
+    echo "fetching token failed: ", e.msg
+
+proc expired(token: Token): bool =
+  let time = getTime()
+  token.init < time - maxAge or token.lastUse < time - maxLastUse
+
+proc isLimited(token: Token; api: Api): bool =
+  if token.isNil or token.expired:
+    return true
+
+  if api in token.apis:
+    let limit = token.apis[api]
+    return (limit.remaining <= 10 and limit.reset > epochTime().int)
+  else:
+    return false
+
+proc isReady(token: Token; api: Api): bool =
+  not (token.isNil or token.pending > maxConcurrentReqs or token.isLimited(api))
+
+proc release*(token: Token; used=false; invalid=false) =
+  if token.isNil: return
+  if invalid or token.expired:
+    let idx = tokenPool.find(token)
+    if idx > -1: tokenPool.delete(idx)
+  elif used:
+    dec token.pending
+    token.lastUse = getTime()
+
+proc getToken*(api: Api): Future[Token] {.async.} =
+  for i in 0 ..< tokenPool.len:
+    if result.isReady(api): break
+    release(result)
+    result = tokenPool.sample()
+
+  if not result.isReady(api):
+    release(result)
+    result = await fetchToken()
+    tokenPool.add result
+
+  if not result.isNil:
+    inc result.pending
+  else:
+    raise rateLimitError()
+
+proc setRateLimit*(token: Token; api: Api; remaining, reset: int) =
+  # avoid undefined behavior in race conditions
+  if api in token.apis:
+    let limit = token.apis[api]
+    if limit.reset >= reset and limit.remaining < remaining:
+      return
+
+  token.apis[api] = RateLimit(remaining: remaining, reset: reset)
+
+proc poolTokens*(amount: int) {.async.} =
+  var futs: seq[Future[Token]]
+  for i in 0 ..< amount:
+    futs.add fetchToken()
+
+  for token in futs:
+    var newToken: Token
+
+    try: newToken = await token
+    except: discard
+
+    if not newToken.isNil:
+      tokenPool.add newToken
+
+proc initTokenPool*(cfg: Config) {.async.} =
+  clientPool = HttpPool()
+
+  while true:
+    if tokenPool.countIt(not it.isLimited(Api.timeline)) < cfg.minTokens:
+      await poolTokens(min(4, cfg.minTokens - tokenPool.len))
+    await sleepAsync(2000)

src/types.nim

@@ -1,89 +1,50 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import times, sequtils, options, tables, uri
+import times, sequtils, options, tables
 import prefs_impl
 
 genPrefsType()
 
 type
   RateLimitError* = object of CatchableError
-  NoSessionsError* = object of CatchableError
   InternalError* = object of CatchableError
-  BadClientError* = object of CatchableError
-
-  TimelineKind* {.pure.} = enum
-    tweets, replies, media
 
   Api* {.pure.} = enum
-    tweetDetail
+    userShow
-    tweetResult
+    timeline
     search
+    tweet
     list
     listBySlug
     listMembers
-    listTweets
     userRestId
-    userScreenName
+    status
-    userTweets
-    userTweetsAndReplies
-    userMedia
 
   RateLimit* = object
-    limit*: int
     remaining*: int
     reset*: int
 
-  SessionKind* = enum
+  Token* = ref object
-    oauth
+    tok*: string
-    cookie
+    init*: Time
-
+    lastUse*: Time
-  Session* = ref object
-    id*: int64
     pending*: int
-    limited*: bool
-    limitedAt*: int
     apis*: Table[Api, RateLimit]
-    case kind*: SessionKind
-    of oauth:
-      oauthToken*: string
-      oauthSecret*: string
-    of cookie:
-      authToken*: string
-      ct0*: string
-
-  SessionAwareUrl* = object
-    oauthUrl*: Uri
-    cookieUrl*: Uri
 
   Error* = enum
     null = 0
     noUserMatches = 17
     protectedUser = 22
-    missingParams = 25
-    timeout = 29
     couldntAuth = 32
     doesntExist = 34
-    unauthorized = 37
-    invalidParam = 47
     userNotFound = 50
     suspended = 63
     rateLimited = 88
-    expiredToken = 89
+    invalidToken = 89
     listIdOrSlug = 112
     tweetNotFound = 144
-    tweetNotAuthorized = 179
     forbidden = 200
-    badRequest = 214
     badToken = 239
-    locked = 326
     noCsrf = 353
-    tweetUnavailable = 421
-    tweetCensored = 422
-
-  VerifiedType* = enum
-    none = "None"
-    blue = "Blue"
-    business = "Business"
-    government = "Government"
 
   User* = object
     id*: string
@@ -100,7 +61,7 @@ type
     tweets*: int
     likes*: int
     media*: int
-    verifiedType*: VerifiedType
+    verified*: bool
     protected*: bool
     suspended*: bool
     joinDate*: DateTime
@@ -114,7 +75,6 @@ type
     contentType*: VideoType
     url*: string
     bitrate*: int
-    resolution*: int
 
   Video* = object
     durationMs*: int
@@ -184,10 +144,8 @@ type
     imageDirectMessage = "image_direct_message"
     audiospace = "audiospace"
     newsletterPublication = "newsletter_publication"
-    jobDetails = "job_details"
-    hidden
     unknown
-
+    
   Card* = object
     kind*: CardKind
     url*: string
@@ -216,8 +174,6 @@ type
     available*: bool
     tombstone*: string
     location*: string
-    # Unused, needed for backwards compat
-    source*: string
     stats*: TweetStats
     retweet*: Option[Tweet]
     attribution*: Option[User]
@@ -229,8 +185,6 @@ type
     video*: Option[Video]
     photos*: seq[string]
 
-  Tweets* = seq[Tweet]
-
   Result*[T] = object
     content*: seq[T]
     top*, bottom*: string
@@ -238,7 +192,7 @@ type
     query*: Query
 
   Chain* = object
-    content*: Tweets
+    content*: seq[Tweet]
     hasMore*: bool
     cursor*: string
 
@@ -248,7 +202,7 @@ type
     after*: Chain
     replies*: Result[Chain]
 
-  Timeline* = Result[Tweets]
+  Timeline* = Result[Tweet]
 
   Profile* = object
     user*: User
@@ -300,6 +254,3 @@ type
 
 proc contains*(thread: Chain; tweet: Tweet): bool =
   thread.content.anyIt(it.id == tweet.id)
-
-proc add*(timeline: var seq[Tweets]; tweet: Tweet) =
-  timeline.add @[tweet]

src/utils.nim

@@ -16,8 +16,7 @@ const
     "twimg.com",
     "abs.twimg.com",
     "pbs.twimg.com",
-    "video.twimg.com",
+    "video.twimg.com"
-    "x.com"
   ]
 
 proc setHmacKey*(key: string) =
@@ -43,12 +42,6 @@ proc getPicUrl*(link: string): string =
   else:
     &"/pic/{encodeUrl(link)}"
 
-proc getOrigPicUrl*(link: string): string =
-  if base64Media:
-    &"/pic/orig/enc/{encode(link, safe=true)}"
-  else:
-    &"/pic/orig/{encodeUrl(link)}"
-
 proc filterParams*(params: Table): seq[(string, string)] =
   for p in params.pairs():
     if p[1].len > 0 and p[0] notin nitterParams:
@@ -58,4 +51,4 @@ proc isTwitterUrl*(uri: Uri): bool =
   uri.hostname in twitterDomains
 
 proc isTwitterUrl*(url: string): bool =
-  isTwitterUrl(parseUri(url))
+  parseUri(url).hostname in twitterDomains

src/views/embed.nim

@@ -11,12 +11,11 @@ const doctype = "<!DOCTYPE html>\n"
 proc renderVideoEmbed*(tweet: Tweet; cfg: Config; req: Request): string =
   let thumb = get(tweet.video).thumb
   let vidUrl = getVideoEmbed(cfg, tweet.id)
-  let prefs = Prefs(hlsPlayback: true, mp4Playback: true)
+  let prefs = Prefs(hlsPlayback: true)
   let node = buildHtml(html(lang="en")):
     renderHead(prefs, cfg, req, video=vidUrl, images=(@[thumb]))
 
-    body:
+    tdiv(class="embed-video"):
-      tdiv(class="embed-video"):
+      renderVideo(get(tweet.video), prefs, "")
-        renderVideo(get(tweet.video), prefs, "")
 
   result = doctype & $node

src/views/general.nim

@@ -52,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
   let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
 
   buildHtml(head):
-    link(rel="stylesheet", type="text/css", href="/css/style.css?v=19")
+    link(rel="stylesheet", type="text/css", href="/css/style.css?v=16")
     link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
 
     if theme.len > 0:
@@ -73,7 +73,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
       link(rel="alternate", type="application/rss+xml", href=rss, title="RSS feed")
 
     if prefs.hlsPlayback:
-      script(src="/js/hls.min.js", `defer`="")
+      script(src="/js/hls.light.min.js", `defer`="")
       script(src="/js/hlsPlayback.js", `defer`="")
 
     if prefs.infiniteScroll:
@@ -93,13 +93,14 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
     meta(property="og:site_name", content="Nitter")
     meta(property="og:locale", content="en_US")
 
-    if banner.len > 0 and not banner.startsWith('#'):
+    if banner.len > 0:
       let bannerUrl = getPicUrl(banner)
       link(rel="preload", type="image/png", href=bannerUrl, `as`="image")
 
     for url in images:
-      let preloadUrl = if "400x400" in url: getPicUrl(url)
+      let suffix = if "400x400" in url or url.endsWith("placeholder.png"): ""
-                       else: getSmallPic(url)
+                   else: "?name=small"
+      let preloadUrl = getPicUrl(url & suffix)
       link(rel="preload", type="image/png", href=preloadUrl, `as`="image")
 
       let image = getUrlPrefix(cfg) & getPicUrl(url)

src/views/profile.nim

@@ -50,7 +50,7 @@ proc renderUserCard*(user: User; prefs: Prefs): VNode =
           span:
             let url = replaceUrls(user.website, prefs)
             icon "link"
-            a(href=url): text url.shortLink
+            a(href=url): text shortLink(url)
 
       tdiv(class="profile-joindate"):
         span(title=getJoinDateFull(user)):
@@ -78,11 +78,8 @@ proc renderPhotoRail(profile: Profile): VNode =
     tdiv(class="photo-rail-grid"):
       for i, photo in profile.photoRail:
         if i == 16: break
-        let photoSuffix =
-          if "format" in photo.url or "placeholder" in photo.url: ""
-          else: ":thumb"
         a(href=(&"/{profile.user.username}/status/{photo.tweetId}#m")):
-          genImg(photo.url & photoSuffix)
+          genImg(photo.url & (if "format" in photo.url: "" else: ":thumb"))
 
 proc renderBanner(banner: string): VNode =
   buildHtml():
@@ -108,7 +105,7 @@ proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
         renderBanner(profile.user.banner)
 
     let sticky = if prefs.stickyProfile: " sticky" else: ""
-    tdiv(class=("profile-tab" & sticky)):
+    tdiv(class=(&"profile-tab{sticky}")):
       renderUserCard(profile.user, prefs)
       if profile.photoRail.len > 0:
         renderPhotoRail(profile)

src/views/renderutils.nim

@@ -1,19 +1,11 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, strformat
+import strutils
 import karax/[karaxdsl, vdom, vstyles]
 import ".."/[types, utils]
 
-const smallWebp* = "?name=small&format=webp"
-
-proc getSmallPic*(url: string): string =
-  result = url
-  if "?" notin url and not url.endsWith("placeholder.png"):
-    result &= smallWebp
-  result = getPicUrl(result)
-
 proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
   var c = "icon-" & icon
-  if class.len > 0: c = &"{c} {class}"
+  if class.len > 0: c = c & " " & class
   buildHtml(tdiv(class="icon-container")):
     if href.len > 0:
       a(class=c, title=title, href=href)
@@ -23,13 +15,6 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
     if text.len > 0:
       text " " & text
 
-template verifiedIcon*(user: User): untyped {.dirty.} =
-  if user.verifiedType != VerifiedType.none:
-    let lower = ($user.verifiedType).toLowerAscii()
-    icon "ok", class=(&"verified-icon {lower}"), title=(&"Verified {lower} account")
-  else:
-    text ""
-
 proc linkUser*(user: User, class=""): VNode =
   let
     isName = "username" notin class
@@ -39,11 +24,11 @@ proc linkUser*(user: User, class=""): VNode =
 
   buildHtml(a(href=href, class=class, title=nameText)):
     text nameText
-    if isName:
+    if isName and user.verified:
-      verifiedIcon(user)
+      icon "ok", class="verified-icon", title="Verified account"
-      if user.protected:
+    if isName and user.protected:
-        text " "
+      text " "
-        icon "lock", title="Protected account"
+      icon "lock", title="Protected account"
 
 proc linkText*(text: string; class=""): VNode =
   let url = if "http" notin text: https & text else: text
@@ -66,23 +51,29 @@ proc buttonReferer*(action, text, path: string; class=""; `method`="post"): VNod
 proc genCheckbox*(pref, label: string; state: bool): VNode =
   buildHtml(label(class="pref-group checkbox-container")):
     text label
-    input(name=pref, `type`="checkbox", checked=state)
+    if state: input(name=pref, `type`="checkbox", checked="")
+    else: input(name=pref, `type`="checkbox")
     span(class="checkbox")
 
-proc genInput*(pref, label, state, placeholder: string; class=""; autofocus=true): VNode =
+proc genInput*(pref, label, state, placeholder: string; class=""): VNode =
   let p = placeholder
   buildHtml(tdiv(class=("pref-group pref-input " & class))):
     if label.len > 0:
       label(`for`=pref): text label
-    input(name=pref, `type`="text", placeholder=p, value=state, autofocus=(autofocus and state.len == 0))
+    if state.len == 0:
+      input(name=pref, `type`="text", placeholder=p, value=state, autofocus="")
+    else:
+      input(name=pref, `type`="text", placeholder=p, value=state)
 
 proc genSelect*(pref, label, state: string; options: seq[string]): VNode =
   buildHtml(tdiv(class="pref-group pref-input")):
     label(`for`=pref): text label
     select(name=pref):
       for opt in options:
-        option(value=opt, selected=(opt == state)):
+        if opt == state:
-          text opt
+          option(value=opt, selected=""): text opt
+        else:
+          option(value=opt): text opt
 
 proc genDate*(pref, state: string): VNode =
   buildHtml(span(class="date-input")):
@@ -91,12 +82,15 @@ proc genDate*(pref, state: string): VNode =
 
 proc genImg*(url: string; class=""): VNode =
   buildHtml():
-    img(src=getPicUrl(url), class=class, alt="", loading="lazy")
+    img(src=getPicUrl(url), class=class, alt="")
 
 proc getTabClass*(query: Query; tab: QueryKind): string =
-  if query.kind == tab: "tab-item active"
+  result = "tab-item"
-  else: "tab-item"
+  if query.kind == tab:
+    result &= " active"
 
 proc getAvatarClass*(prefs: Prefs): string =
-  if prefs.squareAvatars: "avatar"
+  if prefs.squareAvatars:
-  else: "avatar round"
+    "avatar"
+  else:
+    "avatar round"

src/views/rss.nimf

@@ -28,29 +28,15 @@
 Twitter feed for: ${desc}. Generated by ${cfg.hostname}
 #end proc
 #
-#proc getTweetsWithPinned(profile: Profile): seq[Tweets] =
-#result = profile.tweets.content
-#if profile.pinned.isSome and result.len > 0:
-#  let pinnedTweet = profile.pinned.get
-#  var inserted = false
-#  for threadIdx in 0 ..< result.len:
-#    if not inserted:
-#      for tweetIdx in 0 ..< result[threadIdx].len:
-#        if result[threadIdx][tweetIdx].id < pinnedTweet.id:
-#          result[threadIdx].insert(pinnedTweet, tweetIdx)
-#          inserted = true
-#        end if
-#      end for
-#    end if
-#  end for
-#end if
-#end proc
-#
 #proc renderRssTweet(tweet: Tweet; cfg: Config): string =
 #let tweet = tweet.retweet.get(tweet)
 #let urlPrefix = getUrlPrefix(cfg)
 #let text = replaceUrls(tweet.text, defaultPrefs, absolute=urlPrefix)
 <p>${text.replace("\n", "<br>\n")}</p>
+#if tweet.quote.isSome and get(tweet.quote).available:
+#  let quoteLink = getLink(get(tweet.quote))
+<p><a href="${urlPrefix}${quoteLink}">${cfg.hostname}${quoteLink}</a></p>
+#end if
 #if tweet.photos.len > 0:
 #  for photo in tweet.photos:
 <img src="${urlPrefix}${getPicUrl(photo)}" style="max-width:250px;" />
@@ -61,51 +47,33 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
 #  let thumb = &"{urlPrefix}{getPicUrl(get(tweet.gif).thumb)}"
 #  let url = &"{urlPrefix}{getPicUrl(get(tweet.gif).url)}"
 <video poster="${thumb}" autoplay muted loop style="max-width:250px;">
-  <source src="${url}" type="video/mp4"></video>
+  <source src="${url}" type="video/mp4"</source></video>
 #elif tweet.card.isSome:
 #  let card = tweet.card.get()
 #  if card.image.len > 0:
 <img src="${urlPrefix}${getPicUrl(card.image)}" style="max-width:250px;" />
 #  end if
 #end if
-#if tweet.quote.isSome and get(tweet.quote).available:
-#  let quoteLink = getLink(get(tweet.quote))
-<blockquote>
-  <p>
-    ${renderRssTweet(get(tweet.quote), cfg)}
-  </p>
-  <footer>
-    — <cite><a href="${urlPrefix}${quoteLink}">${cfg.hostname}${quoteLink}</a></cite>
-  </footer>
-</blockquote>
-#end if
 #end proc
 #
-#proc renderRssTweets(tweets: seq[Tweets]; cfg: Config; userId=""): string =
+#proc renderRssTweets(tweets: seq[Tweet]; cfg: Config): string =
 #let urlPrefix = getUrlPrefix(cfg)
 #var links: seq[string]
-#for thread in tweets:
+#for t in tweets:
-#  for tweet in thread:
+#  let retweet = if t.retweet.isSome: t.user.username else: ""
-#    if userId.len > 0 and tweet.user.id != userId: continue
+#  let tweet = if retweet.len > 0: t.retweet.get else: t
-#    end if
+#  let link = getLink(tweet)
-#
+#  if link in links: continue
-#    let retweet = if tweet.retweet.isSome: tweet.user.username else: ""
+#  end if
-#    let tweet = if retweet.len > 0: tweet.retweet.get else: tweet
+#  links.add link
-#    if not tweet.available: continue
+    <item>
-#    end if
+      <title>${getTitle(tweet, retweet)}</title>
-#    let link = getLink(tweet)
+      <dc:creator>@${tweet.user.username}</dc:creator>
-#    if link in links: continue
+      <description><![CDATA[${renderRssTweet(tweet, cfg).strip(chars={'\n'})}]]></description>
-#    end if
+      <pubDate>${getRfc822Time(tweet)}</pubDate>
-#    links.add link
+      <guid>${urlPrefix & link}</guid>
-      <item>
+      <link>${urlPrefix & link}</link>
-        <title>${getTitle(tweet, retweet)}</title>
+    </item>
-        <dc:creator>@${tweet.user.username}</dc:creator>
-        <description><![CDATA[${renderRssTweet(tweet, cfg).strip(chars={'\n'})}]]></description>
-        <pubDate>${getRfc822Time(tweet)}</pubDate>
-        <guid>${urlPrefix & link}</guid>
-        <link>${urlPrefix & link}</link>
-      </item>
-#  end for
 #end for
 #end proc
 #
@@ -133,15 +101,14 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
       <width>128</width>
       <height>128</height>
     </image>
-#let tweetsList = getTweetsWithPinned(profile)
+#if profile.tweets.content.len > 0:
-#if tweetsList.len > 0:
+${renderRssTweets(profile.tweets.content, cfg)}
-${renderRssTweets(tweetsList, cfg, userId=profile.user.id)}
 #end if
   </channel>
 </rss>
 #end proc
 #
-#proc renderListRss*(tweets: seq[Tweets]; list: List; cfg: Config): string =
+#proc renderListRss*(tweets: seq[Tweet]; list: List; cfg: Config): string =
 #let link = &"{getUrlPrefix(cfg)}/i/lists/{list.id}"
 #result = ""
 <?xml version="1.0" encoding="UTF-8"?>
@@ -150,7 +117,7 @@ ${renderRssTweets(tweetsList, cfg, userId=profile.user.id)}
     <atom:link href="${link}" rel="self" type="application/rss+xml" />
     <title>${xmltree.escape(list.name)} / @${list.username}</title>
     <link>${link}</link>
-    <description>${getDescription(&"{list.name} by @{list.username}", cfg)}</description>
+    <description>${getDescription(list.name & " by @" & list.username, cfg)}</description>
     <language>en-us</language>
     <ttl>40</ttl>
 ${renderRssTweets(tweets, cfg)}
@@ -158,7 +125,7 @@ ${renderRssTweets(tweets, cfg)}
 </rss>
 #end proc
 #
-#proc renderSearchRss*(tweets: seq[Tweets]; name, param: string; cfg: Config): string =
+#proc renderSearchRss*(tweets: seq[Tweet]; name, param: string; cfg: Config): string =
 #let link = &"{getUrlPrefix(cfg)}/search"
 #let escName = xmltree.escape(name)
 #result = ""
@@ -168,7 +135,7 @@ ${renderRssTweets(tweets, cfg)}
     <atom:link href="${link}" rel="self" type="application/rss+xml" />
     <title>Search results for "${escName}"</title>
     <link>${link}</link>
-    <description>${getDescription(&"Search \"{escName}\"", cfg)}</description>
+    <description>${getDescription("Search \"" & escName & "\"", cfg)}</description>
     <language>en-us</language>
     <ttl>40</ttl>
 ${renderRssTweets(tweets, cfg)}

src/views/search.nim

@@ -24,9 +24,9 @@ proc renderSearch*(): VNode =
   buildHtml(tdiv(class="panel-container")):
     tdiv(class="search-bar"):
       form(`method`="get", action="/search", autocomplete="off"):
-        hiddenField("f", "tweets")
+        hiddenField("f", "users")
         input(`type`="text", name="q", autofocus="",
-              placeholder="Search...", dir="auto")
+              placeholder="Enter username...", dir="auto")
         button(`type`="submit"): icon "search"
 
 proc renderProfileTabs*(query: Query; username: string): VNode =
@@ -63,10 +63,12 @@ proc renderSearchPanel*(query: Query): VNode =
     hiddenField("f", "tweets")
     genInput("q", "", query.text, "Enter search...", class="pref-inline")
     button(`type`="submit"): icon "search"
-
+    if isPanelOpen(query):
-    input(id="search-panel-toggle", `type`="checkbox", checked=isPanelOpen(query))
+      input(id="search-panel-toggle", `type`="checkbox", checked="")
-    label(`for`="search-panel-toggle"): icon "down"
+    else:
-
+      input(id="search-panel-toggle", `type`="checkbox")
+    label(`for`="search-panel-toggle"):
+      icon "down"
     tdiv(class="search-panel"):
       for f in @["filter", "exclude"]:
         span(class="search-title"): text capitalize(f)
@@ -86,9 +88,9 @@ proc renderSearchPanel*(query: Query): VNode =
             genDate("until", query.until)
         tdiv:
           span(class="search-title"): text "Near"
-          genInput("near", "", query.near, "Location...", autofocus=false)
+          genInput("near", "", query.near, placeholder="Location...")
 
-proc renderTweetSearch*(results: Timeline; prefs: Prefs; path: string;
+proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string;
                         pinned=none(Tweet)): VNode =
   let query = results.query
   buildHtml(tdiv(class="timeline-container")):

src/views/timeline.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, strformat, algorithm, uri, options
+import strutils, strformat, sequtils, algorithm, uri, options
 import karax/[karaxdsl, vdom]
 
 import ".."/[types, query, formatters]
@@ -39,22 +39,24 @@ proc renderNoneFound(): VNode =
     h2(class="timeline-none"):
       text "No items found"
 
-proc renderThread(thread: Tweets; prefs: Prefs; path: string): VNode =
+proc renderThread(thread: seq[Tweet]; prefs: Prefs; path: string): VNode =
   buildHtml(tdiv(class="thread-line")):
     let sortedThread = thread.sortedByIt(it.id)
     for i, tweet in sortedThread:
-      # thread has a gap, display "more replies" link
-      if i > 0 and tweet.replyId != sortedThread[i - 1].id:
-        tdiv(class="timeline-item thread more-replies-thread"):
-          tdiv(class="more-replies"):
-            a(class="more-replies-text", href=getLink(tweet)):
-              text "more replies"
-
       let show = i == thread.high and sortedThread[0].id != tweet.threadId
       let header = if tweet.pinned or tweet.retweet.isSome: "with-header " else: ""
       renderTweet(tweet, prefs, path, class=(header & "thread"),
                   index=i, last=(i == thread.high), showThread=show)
 
+proc threadFilter(tweets: openArray[Tweet]; threads: openArray[int64]; it: Tweet): seq[Tweet] =
+  result = @[it]
+  if it.retweet.isSome or it.replyId in threads: return
+  for t in tweets:
+    if t.id == result[0].replyId:
+      result.insert t
+    elif t.replyId == result[0].id:
+      result.add t
+
 proc renderUser(user: User; prefs: Prefs): VNode =
   buildHtml(tdiv(class="timeline-item")):
     a(class="tweet-link", href=("/" & user.username))
@@ -87,7 +89,7 @@ proc renderTimelineUsers*(results: Result[User]; prefs: Prefs; path=""): VNode =
     else:
       renderNoMore()
 
-proc renderTimelineTweets*(results: Timeline; prefs: Prefs; path: string;
+proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string;
                            pinned=none(Tweet)): VNode =
   buildHtml(tdiv(class="timeline")):
     if not results.beginning:
@@ -103,26 +105,26 @@ proc renderTimelineTweets*(results: Timeline; prefs: Prefs; path: string;
       else:
         renderNoneFound()
     else:
-      var retweets: seq[int64]
+      var
+        threads: seq[int64]
+        retweets: seq[int64]
 
-      for thread in results.content:
+      for tweet in results.content:
-        if thread.len == 1:
+        let rt = if tweet.retweet.isSome: get(tweet.retweet).id else: 0
-          let
-            tweet = thread[0]
-            retweetId = if tweet.retweet.isSome: get(tweet.retweet).id else: 0
 
-          if retweetId in retweets or tweet.id in retweets or
+        if tweet.id in threads or rt in retweets or tweet.id in retweets or
-             tweet.pinned and prefs.hidePins:
+           tweet.pinned and prefs.hidePins: continue
-            continue
 
+        let thread = results.content.threadFilter(threads, tweet)
+        if thread.len < 2:
           var hasThread = tweet.hasThread
-          if retweetId != 0 and tweet.retweet.isSome:
+          if rt != 0:
-            retweets &= retweetId
+            retweets &= rt
             hasThread = get(tweet.retweet).hasThread
           renderTweet(tweet, prefs, path, showThread=hasThread)
         else:
           renderThread(thread, prefs, path)
+          threads &= thread.mapIt(it.id)
 
-      if results.bottom.len > 0:
+      renderMore(results.query, results.bottom)
-        renderMore(results.query, results.bottom)
       renderToTop()

src/views/tweet.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, sequtils, strformat, options, algorithm, uri
+import strutils, sequtils, strformat, options
 import karax/[karaxdsl, vdom, vstyles]
 from jester import Request
 
@@ -7,20 +7,27 @@ import renderutils
 import ".."/[types, utils, formatters]
 import general
 
-const doctype = "<!DOCTYPE html>\n"
+proc getSmallPic(url: string): string =
+  result = url
+  if "?" notin url and not url.endsWith("placeholder.png"):
+    result &= ":small"
+  result = getPicUrl(result)
 
 proc renderMiniAvatar(user: User; prefs: Prefs): VNode =
-  genImg(user.getUserPic("_mini"), class=(prefs.getAvatarClass & " mini"))
+  let url = getPicUrl(user.getUserPic("_mini"))
+  buildHtml():
+    img(class=(prefs.getAvatarClass & " mini"), src=url)
 
-proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs): VNode =
+proc renderHeader(tweet: Tweet; retweet: string; prefs: Prefs): VNode =
   buildHtml(tdiv):
-    if pinned:
+    if retweet.len > 0:
-      tdiv(class="pinned"):
-        span: icon "pin", "Pinned Tweet"
-    elif retweet.len > 0:
       tdiv(class="retweet-header"):
         span: icon "retweet", retweet & " retweeted"
 
+    if tweet.pinned:
+      tdiv(class="pinned"):
+        span: icon "pin", "Pinned Tweet"
+
     tdiv(class="tweet-header"):
       a(class="tweet-avatar", href=("/" & tweet.user.username)):
         var size = "_bigger"
@@ -50,21 +57,19 @@ proc renderAlbum(tweet: Tweet): VNode =
           tdiv(class="attachment image"):
             let
               named = "name=" in photo
-              small = if named: photo else: photo & smallWebp
+              orig = if named: photo else: photo & "?name=orig"
-            a(href=getOrigPicUrl(photo), class="still-image", target="_blank"):
+              small = if named: photo else: photo & "?name=small"
+            a(href=getPicUrl(orig), class="still-image", target="_blank"):
               genImg(small)
 
-proc isPlaybackEnabled(prefs: Prefs; playbackType: VideoType): bool =
+proc isPlaybackEnabled(prefs: Prefs; video: Video): bool =
-  case playbackType
+  case video.playbackType
   of mp4: prefs.mp4Playback
   of m3u8, vmap: prefs.hlsPlayback
 
-proc hasMp4Url(video: Video): bool =
+proc renderVideoDisabled(video: Video; path: string): VNode =
-  video.variants.anyIt(it.contentType == mp4)
-
-proc renderVideoDisabled(playbackType: VideoType; path: string): VNode =
   buildHtml(tdiv(class="video-overlay")):
-    case playbackType
+    case video.playbackType
     of mp4:
       p: text "mp4 playback disabled in preferences"
     of m3u8, vmap:
@@ -79,34 +84,33 @@ proc renderVideoUnavailable(video: Video): VNode =
       p: text "This media is unavailable"
 
 proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode =
-  let
+  let container =
-    container = if video.description.len == 0 and video.title.len == 0: ""
+    if video.description.len > 0 or video.title.len > 0: " card-container"
-                else: " card-container"
+    else: ""
-    playbackType = if not prefs.proxyVideos and video.hasMp4Url: mp4
-                   else: video.playbackType
 
   buildHtml(tdiv(class="attachments card")):
     tdiv(class="gallery-video" & container):
       tdiv(class="attachment video-container"):
         let thumb = getSmallPic(video.thumb)
         if not video.available:
-          img(src=thumb, loading="lazy")
+          img(src=thumb)
           renderVideoUnavailable(video)
-        elif not prefs.isPlaybackEnabled(playbackType):
+        elif not prefs.isPlaybackEnabled(video):
-          img(src=thumb, loading="lazy")
+          img(src=thumb)
-          renderVideoDisabled(playbackType, path)
+          renderVideoDisabled(video, path)
         else:
-          let
+          let vid = video.variants.filterIt(it.contentType == video.playbackType)
-            vars = video.variants.filterIt(it.contentType == playbackType)
+          let source = getVidUrl(vid[0].url)
-            vidUrl = vars.sortedByIt(it.resolution)[^1].url
+          case video.playbackType
-            source = if prefs.proxyVideos: getVidUrl(vidUrl)
-                     else: vidUrl
-          case playbackType
           of mp4:
-            video(poster=thumb, controls="", muted=prefs.muteVideos):
+            if prefs.muteVideos:
-              source(src=source, `type`="video/mp4")
+              video(poster=thumb, controls="", muted=""):
+                source(src=source, `type`="video/mp4")
+            else:
+              video(poster=thumb, controls=""):
+                source(src=source, `type`="video/mp4")
           of m3u8, vmap:
-            video(poster=thumb, data-url=source, data-autoload="false", muted=prefs.muteVideos)
+            video(poster=thumb, data-url=source, data-autoload="false")
             verbatim "<div class=\"video-overlay\" onclick=\"playVideo(this)\">"
             tdiv(class="overlay-circle"): span(class="overlay-triangle")
             verbatim "</div>"
@@ -120,9 +124,14 @@ proc renderGif(gif: Gif; prefs: Prefs): VNode =
   buildHtml(tdiv(class="attachments media-gif")):
     tdiv(class="gallery-gif", style={maxHeight: "unset"}):
       tdiv(class="attachment"):
-        video(class="gif", poster=getSmallPic(gif.thumb), autoplay=prefs.autoplayGifs,
+        let thumb = getSmallPic(gif.thumb)
-              controls="", muted="", loop=""):
+        let url = getPicUrl(gif.url)
-          source(src=getPicUrl(gif.url), `type`="video/mp4")
+        if prefs.autoplayGifs:
+          video(class="gif", poster=thumb, controls="", autoplay="", muted="", loop=""):
+            source(src=url, `type`="video/mp4")
+        else:
+          video(class="gif", poster=thumb, controls="", muted="", loop=""):
+            source(src=url, `type`="video/mp4")
 
 proc renderPoll(poll: Poll): VNode =
   buildHtml(tdiv(class="poll")):
@@ -137,12 +146,12 @@ proc renderPoll(poll: Poll): VNode =
         span(class="poll-choice-value"): text percStr
         span(class="poll-choice-option"): text poll.options[i]
     span(class="poll-info"):
-      text &"{insertSep($poll.votes, ',')} votes • {poll.status}"
+      text insertSep($poll.votes, ',') & " votes • " & poll.status
 
 proc renderCardImage(card: Card): VNode =
   buildHtml(tdiv(class="card-image-container")):
     tdiv(class="card-image"):
-      genImg(card.image)
+      img(src=getPicUrl(card.image), alt="")
       if card.kind == player:
         tdiv(class="card-overlay"):
           tdiv(class="overlay-circle"):
@@ -178,11 +187,11 @@ func formatStat(stat: int): string =
   if stat > 0: insertSep($stat, ',')
   else: ""
 
-proc renderStats(tweet_id: int64; stats: TweetStats; views: string): VNode =
+proc renderStats(stats: TweetStats; views: string): VNode =
   buildHtml(tdiv(class="tweet-stats")):
     span(class="tweet-stat"): icon "comment", formatStat(stats.replies)
     span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets)
-    a(class="tweet-stat", href=("/search?q=" & encodeUrl(&"-from:quotedreplies url:{tweet_id}") & "&e-nativeretweets=on")): icon "quote", formatStat(stats.quotes)
+    span(class="tweet-stat"): icon "quote", formatStat(stats.quotes)
     span(class="tweet-stat"): icon "heart", formatStat(stats.likes)
     if views.len > 0:
       span(class="tweet-stat"): icon "play", insertSep(views, ',')
@@ -198,7 +207,8 @@ proc renderAttribution(user: User; prefs: Prefs): VNode =
   buildHtml(a(class="attribution", href=("/" & user.username))):
     renderMiniAvatar(user, prefs)
     strong: text user.fullname
-    verifiedIcon(user)
+    if user.verified:
+      icon "ok", class="verified-icon", title="Verified account"
 
 proc renderMediaTags(tags: seq[User]): VNode =
   buildHtml(tdiv(class="media-tag-block")):
@@ -286,10 +296,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       if tweet.quote.isSome:
         renderQuote(tweet.quote.get(), prefs, path)
 
-  let
+  let fullTweet = tweet
-    fullTweet = tweet
-    pinned = tweet.pinned
-
   var retweet: string
   var tweet = fullTweet
   if tweet.retweet.isSome:
@@ -302,7 +309,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
 
     tdiv(class="tweet-body"):
       var views = ""
-      renderHeader(tweet, retweet, pinned, prefs)
+      renderHeader(tweet, retweet, prefs)
 
       if not afterTweet and index == 0 and tweet.reply.len > 0 and
          (tweet.reply.len > 1 or tweet.reply[0] != tweet.user.username):
@@ -318,7 +325,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       if tweet.attribution.isSome:
         renderAttribution(tweet.attribution.get(), prefs)
 
-      if tweet.card.isSome and tweet.card.get().kind != hidden:
+      if tweet.card.isSome:
         renderCard(tweet.card.get(), prefs, path)
 
       if tweet.photos.len > 0:
@@ -337,24 +344,19 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         renderQuote(tweet.quote.get(), prefs, path)
 
       if mainTweet:
-        p(class="tweet-published"): text &"{getTime(tweet)}"
+        p(class="tweet-published"): text getTime(tweet)
 
       if tweet.mediaTags.len > 0:
         renderMediaTags(tweet.mediaTags)
 
       if not prefs.hideTweetStats:
-        renderStats(tweet.id, tweet.stats, views)
+        renderStats(tweet.stats, views)
 
       if showThread:
         a(class="show-thread", href=("/i/status/" & $tweet.threadId)):
           text "Show this thread"
 
-proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): string =
+proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): VNode =
-  let node = buildHtml(html(lang="en")):
+  buildHtml(tdiv(class="tweet-embed")):
     renderHead(prefs, cfg, req)
-
+    renderTweet(tweet, prefs, path, mainTweet=true)
-    body:
-      tdiv(class="tweet-embed"):
-        renderTweet(tweet, prefs, path, mainTweet=true)
-
-  result = doctype & $node

tests/requirements.txt

@@ -1 +0,0 @@
-seleniumbase

tests/test_card.py

@@ -3,6 +3,11 @@ from parameterized import parameterized
 
 
 card = [
+    ['Thom_Wolf/status/1122466524860702729',
+     'pytorch/fairseq',
+     'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - pytorch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',
+     'github.com', True],
+
     ['nim_lang/status/1136652293510717440',
      'Version 0.20.0 released',
      'We are very proud to announce Nim version 0.20. This is a massive release, both literally and figuratively. It contains more than 1,000 commits and it marks our release candidate for version 1.0!',
@@ -13,32 +18,37 @@ card = [
      'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim',
      'gist.github.com', True],
 
-    ['nim_lang/status/1082989146040340480',
+    ['FluentAI/status/1116417904831029248',
-     'Nim in 2018: A short recap',
+     'Amazon’s Alexa isn’t just AI — thousands of humans are listening',
-     'There were several big news in the Nim world in 2018 – two new major releases, partnership with Status, and much more. But let us go chronologically.',
+     'One of the only ways to improve Alexa is to have human beings check it for errors',
-     'nim-lang.org', True]
+     'theverge.com', True]
 ]
 
 no_thumb = [
-    ['FluentAI/status/1116417904831029248',
+    ['Bountysource/status/1141879700639215617',
-     'LinkedIn',
+     'Post a bounty on kivy/plyer!',
-     'This link will take you to a page that’s not on LinkedIn',
+     'Automation and Screen Reader Support',
-     'lnkd.in'],
+     'bountysource.com'],
-
-    ['Thom_Wolf/status/1122466524860702729',
-     'GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in',
-     '',
-     'github.com'],
 
     ['brent_p/status/1088857328680488961',
-     'GitHub - brentp/hts-nim: nim wrapper for htslib for parsing genomics data files',
+     'Hts Nim Sugar',
-     '',
+     'hts-nim is a library that allows one to use htslib via the nim programming language. Nim is a garbage-collected language that compiles to C and often has similar performance. I have become very...',
-     'github.com'],
+     'brentp.github.io'],
 
     ['voidtarget/status/1133028231672582145',
      'sinkingsugar/nimqt-example',
      'A sample of a Qt app written using mostly nim. Contribute to sinkingsugar/nimqt-example development by creating an account on GitHub.',
-     'github.com']
+     'github.com'],
+
+    ['mobile_test/status/490378953744318464',
+     'Nantasket Beach',
+     'Explore this photo titled Nantasket Beach by Ben Sandofsky (@sandofsky) on 500px',
+     '500px.com'],
+
+    ['nim_lang/status/1082989146040340480',
+     'Nim in 2018: A short recap',
+     'Posted in r/programming by u/miran1',
+     'reddit.com']
 ]
 
 playable = [
@@ -53,6 +63,17 @@ playable = [
      'youtube.com']
 ]
 
+# promo = [
+    # ['BangOlufsen/status/1145698701517754368',
+    #  'Upgrade your journey', '',
+    #  'www.bang-olufsen.com'],
+
+    # ['BangOlufsen/status/1154934429900406784',
+    #  'Learn more about Beosound Shape', '',
+    #  'www.bang-olufsen.com']
+# ]
+
+
 class CardTest(BaseTestCase):
     @parameterized.expand(card)
     def test_card(self, tweet, title, description, destination, large):
@@ -60,7 +81,7 @@ class CardTest(BaseTestCase):
         c = Card(Conversation.main + " ")
         self.assert_text(title, c.title)
         self.assert_text(destination, c.destination)
-        self.assertIn('/pic/', self.get_image_url(c.image + ' img'))
+        self.assertIn('_img', self.get_image_url(c.image + ' img'))
         if len(description) > 0:
             self.assert_text(description, c.description)
         if large:
@@ -83,7 +104,17 @@ class CardTest(BaseTestCase):
         c = Card(Conversation.main + " ")
         self.assert_text(title, c.title)
         self.assert_text(destination, c.destination)
-        self.assertIn('/pic/', self.get_image_url(c.image + ' img'))
+        self.assertIn('_img', self.get_image_url(c.image + ' img'))
         self.assert_element_visible('.card-overlay')
         if len(description) > 0:
             self.assert_text(description, c.description)
+
+    # @parameterized.expand(promo)
+    # def test_card_promo(self, tweet, title, description, destination):
+    #     self.open_nitter(tweet)
+    #     c = Card(Conversation.main + " ")
+    #     self.assert_text(title, c.title)
+    #     self.assert_text(destination, c.destination)
+    #     self.assert_element_visible('.video-overlay')
+    #     if len(description) > 0:
+    #         self.assert_text(description, c.description)

tests/test_profile.py

@@ -4,7 +4,7 @@ from parameterized import parameterized
 profiles = [
         ['mobile_test', 'Test account',
          'Test Account. test test Testing username with @mobile_test_2 and a #hashtag',
-         'San Francisco, CA', 'example.com/foobar', 'Joined October 2009', '97'],
+         'San Francisco, CA', 'example.com/foobar', 'Joined October 2009', '100'],
         ['mobile_test_2', 'mobile test 2', '', '', '', 'Joined January 2011', '13']
 ]
 
@@ -17,6 +17,11 @@ protected = [
 
 invalid = [['thisprofiledoesntexist'], ['%']]
 
+banner_color = [
+    ['nim_lang', '22, 25, 32'],
+    ['rustlang', '35, 31, 32']
+]
+
 banner_image = [
     ['mobile_test', 'profile_banners%2F82135242%2F1384108037%2F1500x500']
 ]
@@ -66,8 +71,14 @@ class ProfileTest(BaseTestCase):
         self.assert_text(f'User "{username}" not found')
 
     def test_suspended(self):
-        self.open_nitter('suspendme')
+        self.open_nitter('user')
-        self.assert_text('User "suspendme" has been suspended')
+        self.assert_text('User "user" has been suspended')
+
+    @parameterized.expand(banner_color)
+    def test_banner_color(self, username, color):
+        self.open_nitter(username)
+        banner = self.find_element(Profile.banner + ' a')
+        self.assertIn(color, banner.value_of_css_property('background-color'))
 
     @parameterized.expand(banner_image)
     def test_banner_image(self, username, url):

tests/test_quote.py

@@ -2,13 +2,15 @@ from base import BaseTestCase, Quote, Conversation
 from parameterized import parameterized
 
 text = [
-    ['nim_lang/status/1491461266849808397#m',
+    ['elonmusk/status/1138136540096319488',
-     'Nim', '@nim_lang',
+     'Tesla Owners Online', '@Model3Owners',
-     """What's better than Nim 1.6.0?
+     """As of March 58.4% of new car sales in Norway are electric. 
 
-Nim 1.6.2 :)
+What are we doing wrong? reuters.com/article/us-norwa…"""],
 
-nim-lang.org/blog/2021/12/17…"""]
+    ['nim_lang/status/924694255364341760',
+     'Hacker News', '@newsycombinator',
+     'Why Rust fails hard at scientific computing andre-ratsimbazafy.com/why-r…']
 ]
 
 image = [

tests/test_search.py

@@ -2,8 +2,8 @@ from base import BaseTestCase
 from parameterized import parameterized
 
 
-#class SearchTest(BaseTestCase):
+class SearchTest(BaseTestCase):
-    #@parameterized.expand([['@mobile_test'], ['@mobile_test_2']])
+    @parameterized.expand([['@mobile_test'], ['@mobile_test_2']])
-    #def test_username_search(self, username):
+    def test_username_search(self, username):
-        #self.search_username(username)
+        self.search_username(username)
-        #self.assert_text(f'{username}')
+        self.assert_text(f'{username}')

tests/test_timeline.py

@@ -1,18 +1,23 @@
 from base import BaseTestCase, Timeline
 from parameterized import parameterized
 
-normal = [['jack'], ['elonmusk']]
+normal = [['mobile_test'], ['mobile_test_2']]
 
-after = [['jack', '1681686036294803456'],
+after = [['mobile_test', 'HBaAgJPsqtGNhA0AAA%3D%3D'],
-         ['elonmusk', '1681686036294803456']]
+         ['mobile_test_2', 'HBaAgJPsqtGNhA0AAA%3D%3D']]
 
-no_more = [['mobile_test_8?cursor=DAABCgABF4YVAqN___kKAAICNn_4msIQAAgAAwAAAAIAAA']]
+no_more = [['mobile_test_8?cursor=HBaAwJCsk%2F6%2FtgQAAA%3D%3D']]
 
 empty = [['emptyuser'], ['mobile_test_10']]
 
 protected = [['mobile_test_7'], ['Empty_user']]
 
-photo_rail = [['mobile_test', ['Bo0nDsYIYAIjqVn', 'BoQbwJAIUAA0QCY', 'BoQbRQxIIAA3FWD', 'Bn8Qh8iIIAABXrG']]]
+photo_rail = [['mobile_test', [
+    'BzUnaDFCUAAmrjs', 'Bo0nDsYIYAIjqVn', 'Bos--KNIQAAA7Li', 'Boq1sDJIYAAxaoi',
+    'BonISmPIEAAhP3G', 'BoQbwJAIUAA0QCY', 'BoQbRQxIIAA3FWD', 'Bn8Qh8iIIAABXrG',
+    'Bn8QIG3IYAA0IGT', 'Bn8O3QeIUAAONai', 'Bn8NGViIAAATNG4', 'BkKovdrCUAAEz79',
+    'BkKoe_oCIAASAqr', 'BkKoRLNCAAAYfDf', 'BkKndxoCQAE1vFt', 'BPEmIbYCMAE44dl'
+]]]
 
 
 class TweetTest(BaseTestCase):
@@ -55,10 +60,10 @@ class TweetTest(BaseTestCase):
         self.assert_element_absent(Timeline.older)
         self.assert_element_absent(Timeline.end)
 
-    #@parameterized.expand(photo_rail)
+    @parameterized.expand(photo_rail)
-    #def test_photo_rail(self, username, images):
+    def test_photo_rail(self, username, images):
-        #self.open_nitter(username)
+        self.open_nitter(username)
-        #self.assert_element_visible(Timeline.photo_rail)
+        self.assert_element_visible(Timeline.photo_rail)
-        #for i, url in enumerate(images):
+        for i, url in enumerate(images):
-            #img = self.get_attribute(Timeline.photo_rail + f' a:nth-child({i + 1}) img', 'src')
+            img = self.get_attribute(Timeline.photo_rail + f' a:nth-child({i + 1}) img', 'src')
-            #self.assertIn(url, img)
+            self.assertIn(url, img)

tests/test_tweet.py

@@ -1,4 +1,4 @@
-from base import BaseTestCase, Tweet, Conversation, get_timeline_tweet
+from base import BaseTestCase, Tweet, get_timeline_tweet
 from parameterized import parameterized
 
 # image = tweet + 'div.attachments.media-body > div > div > a > div > img'
@@ -16,7 +16,7 @@ timeline = [
 ]
 
 status = [
-    [20, 'jack', 'jack', '21 Mar 2006', 'just setting up my twttr'],
+    [20, 'jack⚡️', 'jack', '21 Mar 2006', 'just setting up my twttr'],
     [134849778302464000, 'The Twoffice', 'TheTwoffice', '11 Nov 2011', 'test'],
     [105685475985080322, 'The Twoffice', 'TheTwoffice', '22 Aug 2011', 'regular tweet'],
     [572593440719912960, 'Test account', 'mobile_test', '3 Mar 2015', 'testing test']
@@ -28,15 +28,14 @@ invalid = [
 ]
 
 multiline = [
-    [1718660434457239868, 'WebDesignMuseum',
+    [400897186990284800, 'mobile_test_3',
      """
-Happy 32nd Birthday HTML tags! 
+♔
-
+  KEEP
-On October 29, 1991, the internet pioneer, Tim Berners-Lee, published a document entitled HTML Tags. 
+ CALM
-
+   AND
-The document contained a description of the first 18 HTML tags: <title>, <nextid>, <a>, <isindex>, <plaintext>, <listing>, <p>, <h1>…<h6>, <address>, <hp1>, <hp2>…, <dl>, <dt>, <dd>, <ul>, <li>,<menu> and <dir>. The design of the first version of HTML language was influenced by the SGML universal markup language.
+CLICHÉ
-
+    ON"""]
-#WebDesignHistory"""]
 ]
 
 link = [
@@ -72,7 +71,11 @@ emoji = [
 
 retweet = [
     [7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test', '1234'],
-    [3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr']
+    [3, 'mobile_test_8', 'mobile test 8', 'jack⚡️', '@jack', 'twttr']
+]
+
+reply = [
+    ['mobile_test/with_replies', 15]
 ]
 
 
@@ -100,18 +103,18 @@ class TweetTest(BaseTestCase):
     @parameterized.expand(multiline)
     def test_multiline_formatting(self, tid, username, text):
         self.open_nitter(f'{username}/status/{tid}')
-        self.assert_text(text.strip('\n'), Conversation.main)
+        self.assert_text(text.strip('\n'), '.main-tweet')
 
     @parameterized.expand(emoji)
     def test_emoji(self, tweet, text):
         self.open_nitter(tweet)
-        self.assert_text(text, Conversation.main)
+        self.assert_text(text, '.main-tweet')
 
     @parameterized.expand(link)
     def test_link(self, tweet, links):
         self.open_nitter(tweet)
         for link in links:
-            self.assert_text(link, Conversation.main)
+            self.assert_text(link, '.main-tweet')
 
     @parameterized.expand(username)
     def test_username(self, tweet, usernames):
@@ -134,8 +137,8 @@ class TweetTest(BaseTestCase):
         self.open_nitter(tweet)
         self.assert_text('Tweet not found', '.error-panel')
 
-    #@parameterized.expand(reply)
+    @parameterized.expand(reply)
-    #def test_thread(self, tweet, num):
+    def test_thread(self, tweet, num):
-        #self.open_nitter(tweet)
+        self.open_nitter(tweet)
-        #thread = self.find_element(f'.timeline > div:nth-child({num})')
+        thread = self.find_element(f'.timeline > div:nth-child({num})')
-        #self.assertIn(thread.get_attribute('class'), 'thread-line')
+        self.assertIn(thread.get_attribute('class'), 'thread-line')

tests/test_tweet_media.py

@@ -14,7 +14,7 @@ poll = [
 
 image = [
     ['mobile_test/status/519364660823207936', 'BzUnaDFCUAAmrjs'],
-    #['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj']
+    ['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj']
 ]
 
 gif = [
@@ -28,14 +28,14 @@ video_m3u8 = [
 ]
 
 gallery = [
-    # ['mobile_test/status/451108446603980803', [
+    ['mobile_test/status/451108446603980803', [
-    #     ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
+        ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
-    # ]],
+    ]],
 
-    # ['mobile_test/status/471539824713691137', [
+    ['mobile_test/status/471539824713691137', [
-    #     ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
+        ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
-    #     ['Bos--IqIQAAav23']
+        ['Bos--IqIQAAav23']
-    # ]],
+    ]],
 
     ['mobile_test/status/469530783384743936', [
         ['BoQbwJAIUAA0QCY', 'BoQbwN1IMAAuTiP'],

tools/get_session.py

@@ -1,162 +0,0 @@
-#!/usr/bin/env python3
-import requests
-import json
-import sys
-import pyotp
-import cloudscraper
-
-# NOTE: pyotp, requests and cloudscraper are dependencies
-# > pip install pyotp requests cloudscraper
-
-TW_CONSUMER_KEY = '3nVuSoBZnx6U4vzUxf5w'
-TW_CONSUMER_SECRET = 'Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys'
-
-def auth(username, password, otp_secret):
-    bearer_token_req = requests.post("https://api.twitter.com/oauth2/token",
-        auth=(TW_CONSUMER_KEY, TW_CONSUMER_SECRET),
-        headers={"Content-Type": "application/x-www-form-urlencoded"},
-        data='grant_type=client_credentials'
-    ).json()
-    bearer_token = ' '.join(str(x) for x in bearer_token_req.values())
-
-    guest_token = requests.post(
-        "https://api.twitter.com/1.1/guest/activate.json",
-        headers={'Authorization': bearer_token}
-    ).json().get('guest_token')
-
-    if not guest_token:
-        print("Failed to obtain guest token.")
-        sys.exit(1)
-
-    twitter_header = {
-        'Authorization': bearer_token,
-        "Content-Type": "application/json",
-        "User-Agent": "TwitterAndroid/10.21.0-release.0 (310210000-r-0) ONEPLUS+A3010/9 (OnePlus;ONEPLUS+A3010;OnePlus;OnePlus3;0;;1;2016)",
-        "X-Twitter-API-Version": '5',
-        "X-Twitter-Client": "TwitterAndroid",
-        "X-Twitter-Client-Version": "10.21.0-release.0",
-        "OS-Version": "28",
-        "System-User-Agent": "Dalvik/2.1.0 (Linux; U; Android 9; ONEPLUS A3010 Build/PKQ1.181203.001)",
-        "X-Twitter-Active-User": "yes",
-        "X-Guest-Token": guest_token,
-        "X-Twitter-Client-DeviceID": ""
-    }
-
-    scraper = cloudscraper.create_scraper()
-    scraper.headers = twitter_header
-
-    task1 = scraper.post(
-        'https://api.twitter.com/1.1/onboarding/task.json',
-        params={
-            'flow_name': 'login',
-            'api_version': '1',
-            'known_device_token': '',
-            'sim_country_code': 'us'
-        },
-        json={
-            "flow_token": None,
-            "input_flow_data": {
-                "country_code": None,
-                "flow_context": {
-                    "referrer_context": {
-                        "referral_details": "utm_source=google-play&utm_medium=organic",
-                        "referrer_url": ""
-                    },
-                    "start_location": {
-                        "location": "deeplink"
-                    }
-                },
-                "requested_variant": None,
-                "target_user_id": 0
-            }
-        }
-    )
-
-    scraper.headers['att'] = task1.headers.get('att')
-
-    task2 = scraper.post(
-        'https://api.twitter.com/1.1/onboarding/task.json',
-        json={
-            "flow_token": task1.json().get('flow_token'),
-            "subtask_inputs": [{
-                "enter_text": {
-                    "suggestion_id": None,
-                    "text": username,
-                    "link": "next_link"
-                },
-                "subtask_id": "LoginEnterUserIdentifier"
-            }]
-        }
-    )
-
-    task3 = scraper.post(
-        'https://api.twitter.com/1.1/onboarding/task.json',
-        json={
-            "flow_token": task2.json().get('flow_token'),
-            "subtask_inputs": [{
-                "enter_password": {
-                    "password": password,
-                    "link": "next_link"
-                },
-                "subtask_id": "LoginEnterPassword"
-            }],
-        }
-    )
-
-    for t3_subtask in task3.json().get('subtasks', []):
-        if "open_account" in t3_subtask:
-            return t3_subtask["open_account"]
-        elif "enter_text" in t3_subtask:
-            response_text = t3_subtask["enter_text"]["hint_text"]
-            totp = pyotp.TOTP(otp_secret)
-            generated_code = totp.now()
-            task4resp = scraper.post(
-                "https://api.twitter.com/1.1/onboarding/task.json",
-                json={
-                    "flow_token": task3.json().get("flow_token"),
-                    "subtask_inputs": [
-                        {
-                            "enter_text": {
-                                "suggestion_id": None,
-                                "text": generated_code,
-                                "link": "next_link",
-                            },
-                            "subtask_id": "LoginTwoFactorAuthChallenge",
-                        }
-                    ],
-                }
-            )
-            task4 = task4resp.json()
-            for t4_subtask in task4.get("subtasks", []):
-                if "open_account" in t4_subtask:
-                    return t4_subtask["open_account"]
-
-    return None
-
-if __name__ == "__main__":
-    if len(sys.argv) != 5:
-        print("Usage: python3 get_session.py <username> <password> <2fa secret> <path>")
-        sys.exit(1)
-
-    username = sys.argv[1]
-    password = sys.argv[2]
-    otp_secret = sys.argv[3]
-    path = sys.argv[4]
-
-    result = auth(username, password, otp_secret)
-    if result is None:
-        print("Authentication failed.")
-        sys.exit(1)
-
-    session_entry = {
-        "oauth_token": result.get("oauth_token"),
-        "oauth_token_secret": result.get("oauth_token_secret")
-    }
-
-    try:
-        with open(path, "a") as f:
-            f.write(json.dumps(session_entry) + "\n")
-        print("Authentication successful. Session appended to", path)
-    except Exception as e:
-        print(f"Failed to write session information: {e}")
-        sys.exit(1)

tools/get_web_session.py

@@ -1,158 +0,0 @@
-#!/usr/bin/env python3
-"""
-Authenticates with X.com/Twitter and extracts session cookies for use with Nitter.
-Handles 2FA, extracts user info, and outputs clean JSON for sessions.jsonl.
-
-Requirements:
-  pip install -r tools/requirements.txt
-
-Usage:
-  python3 tools/get_web_session.py <username> <password> [totp_seed] [--append sessions.jsonl] [--headless]
-
-Examples:
-  # Output to terminal
-  python3 tools/get_web_session.py myusername mypassword TOTP_BASE32_SECRET
-
-  # Append to sessions.jsonl
-  python3 tools/get_web_session.py myusername mypassword TOTP_SECRET --append sessions.jsonl
-
-  # Headless mode (may increase detection risk)
-  python3 tools/get_web_session.py myusername mypassword TOTP_SECRET --headless
-
-Output:
-  {"kind": "cookie", "username": "...", "id": "...", "auth_token": "...", "ct0": "..."}
-"""
-
-import sys
-import json
-import asyncio
-import pyotp
-import nodriver as uc
-import os
-
-
-async def login_and_get_cookies(username, password, totp_seed=None, headless=False):
-    """Authenticate with X.com and extract session cookies"""
-    # Note: headless mode may increase detection risk from bot-detection systems
-    browser = await uc.start(headless=headless)
-    tab = await browser.get('https://x.com/i/flow/login')
-
-    try:
-        # Enter username
-        print('[*] Entering username...', file=sys.stderr)
-        username_input = await tab.find('input[autocomplete="username"]', timeout=10)
-        await username_input.send_keys(username + '\n')
-        await asyncio.sleep(1)
-
-        # Enter password
-        print('[*] Entering password...', file=sys.stderr)
-        password_input = await tab.find('input[autocomplete="current-password"]', timeout=15)
-        await password_input.send_keys(password + '\n')
-        await asyncio.sleep(2)
-
-        # Handle 2FA if needed
-        page_content = await tab.get_content()
-        if 'verification code' in page_content or 'Enter code' in page_content:
-            if not totp_seed:
-                raise Exception('2FA required but no TOTP seed provided')
-
-            print('[*] 2FA detected, entering code...', file=sys.stderr)
-            totp_code = pyotp.TOTP(totp_seed).now()
-            code_input = await tab.select('input[type="text"]')
-            await code_input.send_keys(totp_code + '\n')
-            await asyncio.sleep(3)
-
-        # Get cookies
-        print('[*] Retrieving cookies...', file=sys.stderr)
-        for _ in range(20):  # 20 second timeout
-            cookies = await browser.cookies.get_all()
-            cookies_dict = {cookie.name: cookie.value for cookie in cookies}
-
-            if 'auth_token' in cookies_dict and 'ct0' in cookies_dict:
-                print('[*] Found both cookies', file=sys.stderr)
-
-                # Extract ID from twid cookie (may be URL-encoded)
-                user_id = None
-                if 'twid' in cookies_dict:
-                    twid = cookies_dict['twid']
-                    # Try to extract the ID from twid (format: u%3D<id> or u=<id>)
-                    if 'u%3D' in twid:
-                        user_id = twid.split('u%3D')[1].split('&')[0].strip('"')
-                    elif 'u=' in twid:
-                        user_id = twid.split('u=')[1].split('&')[0].strip('"')
-
-                cookies_dict['username'] = username
-                if user_id:
-                    cookies_dict['id'] = user_id
-
-                return cookies_dict
-
-            await asyncio.sleep(1)
-
-        raise Exception('Timeout waiting for cookies')
-
-    finally:
-        browser.stop()
-
-
-async def main():
-    if len(sys.argv) < 3:
-        print('Usage: python3 twitter-auth.py username password [totp_seed] [--append sessions.jsonl] [--headless]')
-        sys.exit(1)
-
-    username = sys.argv[1]
-    password = sys.argv[2]
-    totp_seed = None
-    append_file = None
-    headless = False
-
-    # Parse optional arguments
-    i = 3
-    while i < len(sys.argv):
-        arg = sys.argv[i]
-        if arg == '--append':
-            if i + 1 < len(sys.argv):
-                append_file = sys.argv[i + 1]
-                i += 2  # Skip '--append' and filename
-            else:
-                print('[!] Error: --append requires a filename', file=sys.stderr)
-                sys.exit(1)
-        elif arg == '--headless':
-            headless = True
-            i += 1
-        elif not arg.startswith('--'):
-            if totp_seed is None: 
-                totp_seed = arg
-            i += 1
-        else:
-            # Unkown args
-            print(f'[!] Warning: Unknown argument: {arg}', file=sys.stderr)
-            i += 1
-
-    try:
-        cookies = await login_and_get_cookies(username, password, totp_seed, headless)
-        session = {
-            'kind': 'cookie',
-            'username': cookies['username'],
-            'id': cookies.get('id'),
-            'auth_token': cookies['auth_token'],
-            'ct0': cookies['ct0']
-        }
-        output = json.dumps(session)
-
-        if append_file:
-            with open(append_file, 'a') as f:
-                f.write(output + '\n')
-            print(f'✓ Session appended to {append_file}', file=sys.stderr)
-        else:
-            print(output)
-
-        os._exit(0)
-
-    except Exception as error:
-        print(f'[!] Error: {error}', file=sys.stderr)
-        sys.exit(1)
-
-
-if __name__ == '__main__':
-    asyncio.run(main())

tools/requirements.txt

@@ -1,2 +0,0 @@
-nodriver>=0.48.0
-pyotp