Merge remote-tracking branch 'upstream/master'

* Add experimental x-client-transaction-id support

* Remove broken test

.github/FUNDING.yml

@@ -1,3 +0,0 @@
-github: zedeus
-liberapay: zedeus
-patreon: nitter

.github/workflows/build-docker.yml

@@ -1,63 +0,0 @@
-name: Docker
-
-on:
-  push:
-    paths-ignore:
-      - "README.md"
-    branches:
-      - master
-
-jobs:
-  tests:
-    uses: ./.github/workflows/run-tests.yml
-    secrets: inherit
-  build-docker-amd64:
-    needs: [tests]
-    runs-on: buildjet-2vcpu-ubuntu-2204
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          version: latest
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Build and push AMD64 Docker image
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          file: ./Dockerfile
-          platforms: linux/amd64
-          push: true
-          tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}
-  build-docker-arm64:
-    needs: [tests]
-    runs-on: buildjet-2vcpu-ubuntu-2204-arm
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          version: latest
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Build and push ARM64 Docker image
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          file: ./Dockerfile.arm64
-          platforms: linux/arm64
-          push: true
-          tags: zedeus/nitter:latest-arm64,zedeus/nitter:${{ github.sha }}-arm64

.github/workflows/run-tests.yml

@@ -1,108 +0,0 @@
-name: Tests
-
-on:
-  push:
-    paths-ignore:
-      - "*.md"
-    branches-ignore:
-      - master
-  workflow_call:
-
-# Ensure that multiple runs on the same branch do not overlap.
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell: bash
-
-jobs:
-  build-test:
-    name: Build and test
-    runs-on: buildjet-2vcpu-ubuntu-2204
-    strategy:
-      matrix:
-        nim: ["1.6.x", "2.0.x", "2.2.x", "devel"]
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Cache Nimble Dependencies
-        id: cache-nimble
-        uses: buildjet/cache@v4
-        with:
-          path: ~/.nimble
-          key: ${{ matrix.nim }}-nimble-v2-${{ hashFiles('*.nimble') }}
-          restore-keys: |
-            ${{ matrix.nim }}-nimble-v2-
-
-      - name: Setup Nim
-        uses: jiro4989/setup-nim-action@v2
-        with:
-          nim-version: ${{ matrix.nim }}
-          use-nightlies: true
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build Project
-        run: nimble build -d:release -Y
-
-  integration-test:
-    needs: [build-test]
-    name: Integration test
-    runs-on: buildjet-2vcpu-ubuntu-2204
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Cache Nimble Dependencies
-        id: cache-nimble
-        uses: buildjet/cache@v4
-        with:
-          path: ~/.nimble
-          key: devel-nimble-v2-${{ hashFiles('*.nimble') }}
-          restore-keys: |
-            devel-nimble-v2-
-
-      - name: Setup Python (3.10) with pip cache
-        uses: buildjet/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: pip
-
-      - name: Setup Nim
-        uses: jiro4989/setup-nim-action@v2
-        with:
-          nim-version: devel
-          use-nightlies: true
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build Project
-        run: nimble build -d:release -Y
-
-      - name: Install SeleniumBase and Chromedriver
-        run: |
-          pip install seleniumbase
-          seleniumbase install chromedriver
-
-      - name: Start Redis Service
-        uses: supercharge/redis-github-action@1.5.0
-
-      - name: Prepare Nitter Environment
-        run: |
-          sudo apt-get update && sudo apt-get install -y libsass-dev
-          cp nitter.example.conf nitter.conf
-          sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf
-          nimble md
-          nimble scss
-          echo '${{ secrets.SESSIONS }}' | head -n1
-          echo '${{ secrets.SESSIONS }}' > ./sessions.jsonl
-
-      - name: Run Tests
-        run: |
-          ./nitter &
-          pytest -n1 tests

.travis.yml

@@ -1,51 +0,0 @@
-jobs:
-  include:
-    - stage: test
-      if: (NOT type IN (pull_request)) AND (branch = master)
-      dist: bionic
-      language: python
-      python:
-        - 3.6
-      services:
-        - docker
-        - xvfb
-      script:
-        - sudo apt update
-        - sudo apt install --force-yes chromium-chromedriver
-        - wget https://www.dropbox.com/s/ckuoaubd1crrj2k/Linux_x64_737173_chrome-linux.zip -O chrome.zip
-        - unzip chrome.zip
-        - cd chrome-linux
-        - sudo rm /usr/bin/google-chrome
-        - sudo ln -s chrome /usr/bin/google-chrome
-        - cd ..
-        - pip3 install --upgrade pip
-        - pip3 install -U seleniumbase pytest
-        - docker run -d -p 127.0.0.1:8080:8080/tcp $IMAGE_NAME:$TRAVIS_COMMIT
-        - sleep 10
-        - cd tests
-        - pytest --headless -n 8 --reruns 10 --reruns-delay 2
-    - stage: pr
-      if: type IN (pull_request)
-      dist: bionic
-      language: python
-      python:
-        - 3.6
-      services:
-        - docker
-        - xvfb
-      script:
-        - sudo apt update
-        - sudo apt install --force-yes chromium-chromedriver
-        - wget https://www.dropbox.com/s/ckuoaubd1crrj2k/Linux_x64_737173_chrome-linux.zip -O chrome.zip
-        - unzip chrome.zip
-        - cd chrome-linux
-        - sudo rm /usr/bin/google-chrome
-        - sudo ln -s chrome /usr/bin/google-chrome
-        - cd ..
-        - pip3 install --upgrade pip
-        - pip3 install -U seleniumbase pytest
-        - docker build -t $IMAGE_NAME:$TRAVIS_COMMIT .
-        - docker run -d -p 127.0.0.1:8080:8080/tcp $IMAGE_NAME:$TRAVIS_COMMIT
-        - sleep 10
-        - cd tests
-        - pytest --headless -n 8 --reruns 3 --reruns-delay 2

Dockerfile.arm64

@@ -1,25 +0,0 @@
-FROM alpine:3.20.6 as nim
-LABEL maintainer="setenforce@protonmail.com"
-
-RUN apk --no-cache add libsass-dev pcre gcc git libc-dev nim nimble
-
-WORKDIR /src/nitter
-
-COPY nitter.nimble .
-RUN nimble install -y --depsOnly
-
-COPY . .
-RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
-    && nimble scss \
-    && nimble md
-
-FROM alpine:3.20.6
-WORKDIR /src/
-RUN apk --no-cache add pcre ca-certificates openssl
-COPY --from=nim /src/nitter/nitter ./
-COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
-COPY --from=nim /src/nitter/public ./public
-EXPOSE 8080
-RUN adduser -h /src/ -D -s /bin/sh nitter
-USER nitter
-CMD ./nitter

nitter.example.conf

@@ -24,15 +24,22 @@ hmacKey = "secretkey"  # random key for cryptographic signing of video urls
 base64Media = false  # use base64 encoding for proxied media urls
 enableRSS = true  # set this to false to disable RSS feeds
 enableDebug = false  # enable request logs and debug endpoints (/.sessions)
+logLevel = "info"  # log level (debug, info, warn, error, fatal)
 proxy = ""  # http/https url, SOCKS proxies are not supported
 proxyAuth = ""
+defaultFollowedAccounts = "eff,fsf" # default accounts to show when user follows none
+disableTid = false # enable this if cookie-based auth is failing
 
 # Change default preferences here, see src/prefs_impl.nim for a complete list
 [Preferences]
-theme = "Nitter"
+theme = "Kuuro"
-replaceTwitter = "nitter.net"
+replaceTwitter = "nt.kuuro.net"
-replaceYouTube = "piped.video"
+replaceYouTube = "inv.nadeko.net"
-replaceReddit = "teddit.net"
+replaceReddit = "rd.kuuro.net"
+replaceImgur = "rd.kuuro.net"
+replaceFandom = "ph.kuuro.net"
+replaceSoundCloud = "sc.kuuro.net"
 proxyVideos = true
 hlsPlayback = false
 infiniteScroll = false
+hideNsfw = true

nitter.nimble

@@ -10,7 +10,7 @@ bin           = @["nitter"]
 
 # Dependencies
 
-requires "nim >= 1.6.10"
+requires "nim >= 2.0.0"
 requires "jester#baca3f"
 requires "karax#5cf360c"
 requires "sass#7dfdd03"

public/css/fontello.css

@@ -1,16 +1,15 @@
 @font-face {
   font-family: 'fontello';
-  src: url('/fonts/fontello.eot?21002321');
+  src: url('/fonts/fontello.eot?61663884');
-  src: url('/fonts/fontello.eot?21002321#iefix') format('embedded-opentype'),
+  src: url('/fonts/fontello.eot?61663884#iefix') format('embedded-opentype'),
-       url('/fonts/fontello.woff2?21002321') format('woff2'),
+       url('/fonts/fontello.woff2?61663884') format('woff2'),
-       url('/fonts/fontello.woff?21002321') format('woff'),
+       url('/fonts/fontello.woff?61663884') format('woff'),
-       url('/fonts/fontello.ttf?21002321') format('truetype'),
+       url('/fonts/fontello.ttf?61663884') format('truetype'),
-       url('/fonts/fontello.svg?21002321#fontello') format('svg');
+       url('/fonts/fontello.svg?61663884#fontello') format('svg');
   font-weight: normal;
   font-style: normal;
 }
-
+[class^="icon-"]:before, [class*=" icon-"]:before {
- [class^="icon-"]:before, [class*=" icon-"]:before {
   font-family: "fontello";
   font-style: normal;
   font-weight: normal;
@@ -32,22 +31,23 @@
   -webkit-font-smoothing: antialiased;
   -moz-osx-font-smoothing: grayscale;
 }
- 
+
-.icon-heart:before { content: '\2665'; } /* '♥' */
+.icon-views:before { content: '\e800'; } /* '' */
-.icon-quote:before { content: '\275e'; } /* '❞' */
+.icon-heart:before { content: '\e801'; } /* '' */
-.icon-comment:before { content: '\e802'; } /* '' */
+.icon-quote:before { content: '\e802'; } /* '' */
-.icon-ok:before { content: '\e803'; } /* '' */
+.icon-comment:before { content: '\e803'; } /* '' */
-.icon-play:before { content: '\e804'; } /* '' */
+.icon-ok:before { content: '\e804'; } /* '' */
-.icon-link:before { content: '\e805'; } /* '' */
+.icon-play:before { content: '\e805'; } /* '' */
-.icon-calendar:before { content: '\e806'; } /* '' */
+.icon-link:before { content: '\e806'; } /* '' */
-.icon-location:before { content: '\e807'; } /* '' */
+.icon-calendar:before { content: '\e807'; } /* '' */
+.icon-location:before { content: '\e808'; } /* '' */
 .icon-picture:before { content: '\e809'; } /* '' */
 .icon-lock:before { content: '\e80a'; } /* '' */
 .icon-down:before { content: '\e80b'; } /* '' */
-.icon-retweet:before { content: '\e80d'; } /* '' */
+.icon-retweet:before { content: '\e80c'; } /* '' */
-.icon-search:before { content: '\e80e'; } /* '' */
+.icon-search:before { content: '\e80d'; } /* '' */
-.icon-pin:before { content: '\e80f'; } /* '' */
+.icon-pin:before { content: '\e80e'; } /* '' */
-.icon-cog:before { content: '\e812'; } /* '' */
+.icon-cog:before { content: '\e80f'; } /* '' */
-.icon-rss-feed:before { content: '\e813'; } /* '' */
+.icon-rss:before { content: '\e810'; } /* '' */
 .icon-info:before { content: '\f128'; } /* '' */
 .icon-bird:before { content: '\f309'; } /* '' */

public/css/themes/black.css

@@ -1,30 +1,30 @@
 body {
-    --bg_color: #000000;
+    --bg_color: #000;
-    --fg_color: #FFFFFF;
+    --fg_color: #EEE;
-    --fg_faded: #FFFFFFD4;
+    --fg_faded: #EEEEEED4;
     --fg_dark: var(--accent);
     --fg_nav: var(--accent);
 
     --bg_panel: #0C0C0C;
     --bg_elements: #000000;
     --bg_overlays: var(--bg_panel);
-    --bg_hover: #131313;
+    --bg_hover: #0F0F0F;
 
-    --grey: #E2E2E2;
+    --grey: #939393;
-    --dark_grey: #4E4E4E;
+    --dark_grey: #404040;
-    --darker_grey: #272727;
+    --darker_grey: #1F1F1F;
-    --darkest_grey: #212121;
+    --darkest_grey: #151515;
-    --border_grey: #7D7D7D;
+    --border_grey: #1c1c1c;
 
     --accent: #FF6C60;
     --accent_light: #FFACA0;
-    --accent_dark: #909090;
+    --accent_dark: #707070;
-    --accent_border: #FF6C6091;
+    --accent_border: #FF6C6060;
 
     --play_button: var(--accent);
     --play_button_hover: var(--accent_light);
 
-    --more_replies_dots: #A7A7A7;
+    --more_replies_dots: #808080;
     --error_red: #420A05;
 
     --verified_blue: #1DA1F2;

public/css/themes/kuuro.css

@@ -0,0 +1,59 @@
+body {
+    --bg_color: #050505;
+    --fg_color: #d2d2d2;
+    --fg_faded: #949494;
+    --fg_dark: var(--accent);
+    --fg_nav: var(--accent);
+
+    --bg_panel: #0a0a0a;
+    --bg_elements: #0f0f0f;
+    --bg_overlays: var(--bg_panel);
+    --bg_hover: #151515;
+
+    --grey: #949494;
+    --dark_grey: #444444;
+    --darker_grey: #333333;
+    --darkest_grey: #111111;
+    --border_grey: rgba(255, 255, 255, 0.06);
+
+    --accent: #f3c2e6;
+    --accent_light: #ffd6f2;
+    --accent_dark: #a6859d;
+    --accent_border: #f3c2e660;
+
+    --play_button: var(--accent);
+    --play_button_hover: var(--accent_light);
+
+    --more_replies_dots: #5f6364;
+    --error_red: #ec5f67;
+
+    --verified_blue: #6699cc;
+    --icon_text: var(--fg_color);
+
+    --tab: var(--fg_color);
+    --tab_selected: var(--accent);
+
+    --profile_stat: var(--fg_color);
+}
+
+*:not(.avatar) {
+	border-radius: 0 !important;
+}
+
+input, textarea, select, button,
+.card-container,
+.profile-card,
+.photo-rail-card,
+.timeline-item,
+.search-field,
+nav,
+.timeline-footer,
+.show-more,
+.unavailable-box,
+.tweet-embed,
+.overlay-panel,
+.timeline-header,
+.tab {
+		outline: 1px solid var(--border_grey) !important;
+		outline-offset: -1px !important;
+}

public/fonts/LICENSE.txt

@@ -1,6 +1,15 @@
 Font license info
 
 
+## Modern Pictograms
+
+   Copyright (c) 2012 by John Caserta. All rights reserved.
+
+   Author:    John Caserta
+   License:   SIL (http://scripts.sil.org/OFL)
+   Homepage:  http://thedesignoffice.org/project/modern-pictograms/
+
+
 ## Entypo
 
    Copyright (C) 2012 by Daniel Bruce
@@ -37,12 +46,3 @@ Font license info
    Homepage:  http://aristeides.com/
 
 
-## Modern Pictograms
-
-   Copyright (c) 2012 by John Caserta. All rights reserved.
-
-   Author:    John Caserta
-   License:   SIL (http://scripts.sil.org/OFL)
-   Homepage:  http://thedesignoffice.org/project/modern-pictograms/
-
-

public/fonts/fontello.svg

@@ -1,26 +1,28 @@
 <?xml version="1.0" standalone="no"?>
 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
 <svg xmlns="http://www.w3.org/2000/svg">
-<metadata>Copyright (C) 2020 by original authors @ fontello.com</metadata>
+<metadata>Copyright (C) 2025 by original authors @ fontello.com</metadata>
 <defs>
 <font id="fontello" horiz-adv-x="1000" >
 <font-face font-family="fontello" font-weight="400" font-stretch="normal" units-per-em="1000" ascent="850" descent="-150" />
 <missing-glyph horiz-adv-x="1000" />
-<glyph glyph-name="heart" unicode="&#x2665;" d="M790 644q70-64 70-156t-70-158l-360-330-360 330q-70 66-70 158t70 156q62 58 151 58t153-58l56-52 58 52q62 58 150 58t152-58z" horiz-adv-x="860" />
+<glyph glyph-name="views" unicode="&#xe800;" d="M180 516l0-538-180 0 0 538 180 0z m250-138l0-400-180 0 0 400 180 0z m250 344l0-744-180 0 0 744 180 0z" horiz-adv-x="680" />
 
-<glyph glyph-name="quote" unicode="&#x275e;" d="M18 685l335 0 0-334q0-140-98-238t-237-97l0 111q92 0 158 65t65 159l-223 0 0 334z m558 0l335 0 0-334q0-140-98-238t-237-97l0 111q92 0 158 65t65 159l-223 0 0 334z" horiz-adv-x="928" />
+<glyph glyph-name="heart" unicode="&#xe801;" d="M790 644q70-64 70-156t-70-158l-360-330-360 330q-70 66-70 158t70 156q62 58 151 58t153-58l56-52 58 52q62 58 150 58t152-58z" horiz-adv-x="860" />
 
-<glyph glyph-name="comment" unicode="&#xe802;" d="M1000 350q0-97-67-179t-182-130-251-48q-39 0-81 4-110-97-257-135-27-8-63-12-10-1-17 5t-10 16v1q-2 2 0 6t1 6 2 5l4 5t4 5 4 5q4 5 17 19t20 22 17 22 18 28 15 33 15 42q-88 50-138 123t-51 157q0 73 40 139t106 114 160 76 194 28q136 0 251-48t182-130 67-179z" horiz-adv-x="1000" />
+<glyph glyph-name="quote" unicode="&#xe802;" d="M18 685l335 0 0-334q0-140-98-238t-237-97l0 111q92 0 158 65t65 159l-223 0 0 334z m558 0l335 0 0-334q0-140-98-238t-237-97l0 111q92 0 158 65t65 159l-223 0 0 334z" horiz-adv-x="928" />
 
-<glyph glyph-name="ok" unicode="&#xe803;" d="M0 260l162 162 166-164 508 510 164-164-510-510-162-162-162 164z" horiz-adv-x="1000" />
+<glyph glyph-name="comment" unicode="&#xe803;" d="M1000 350q0-97-67-179t-182-130-251-48q-39 0-81 4-110-97-257-135-27-8-63-12-10-1-17 5t-10 16v1q-2 2 0 6t1 6 2 5l4 5t4 5 4 5q4 5 17 19t20 22 17 22 18 28 15 33 15 42q-88 50-138 123t-51 157q0 73 40 139t106 114 160 76 194 28q136 0 251-48t182-130 67-179z" horiz-adv-x="1000" />
 
-<glyph glyph-name="play" unicode="&#xe804;" d="M772 333l-741-412q-13-7-22-2t-9 20v822q0 14 9 20t22-2l741-412q13-7 13-17t-13-17z" horiz-adv-x="785.7" />
+<glyph glyph-name="ok" unicode="&#xe804;" d="M0 260l162 162 166-164 508 510 164-164-510-510-162-162-162 164z" horiz-adv-x="1000" />
 
-<glyph glyph-name="link" unicode="&#xe805;" d="M294 116q14 14 34 14t36-14q32-34 0-70l-42-40q-56-56-132-56-78 0-134 56t-56 132q0 78 56 134l148 148q70 68 144 77t128-43q16-16 16-36t-16-36q-36-32-70 0-50 48-132-34l-148-146q-26-26-26-64t26-62q26-26 63-26t63 26z m450 574q56-56 56-132 0-78-56-134l-158-158q-74-72-150-72-62 0-112 50-14 14-14 34t14 36q14 14 35 14t35-14q50-48 122 24l158 156q28 28 28 64 0 38-28 62-24 26-56 31t-60-21l-50-50q-16-14-36-14t-34 14q-34 34 0 70l50 50q54 54 127 51t129-61z" horiz-adv-x="800" />
+<glyph glyph-name="play" unicode="&#xe805;" d="M772 333l-741-412q-13-7-22-2t-9 20v822q0 14 9 20t22-2l741-412q13-7 13-17t-13-17z" horiz-adv-x="785.7" />
 
-<glyph glyph-name="calendar" unicode="&#xe806;" d="M800 700q42 0 71-29t29-71l0-600q0-40-29-70t-71-30l-700 0q-40 0-70 30t-30 70l0 600q0 42 30 71t70 29l46 0 0-100 160 0 0 100 290 0 0-100 160 0 0 100 44 0z m0-700l0 400-700 0 0-400 700 0z m-540 800l0-170-70 0 0 170 70 0z m450 0l0-170-70 0 0 170 70 0z" horiz-adv-x="900" />
+<glyph glyph-name="link" unicode="&#xe806;" d="M294 116q14 14 34 14t36-14q32-34 0-70l-42-40q-56-56-132-56-78 0-134 56t-56 132q0 78 56 134l148 148q70 68 144 77t128-43q16-16 16-36t-16-36q-36-32-70 0-50 48-132-34l-148-146q-26-26-26-64t26-62q26-26 63-26t63 26z m450 574q56-56 56-132 0-78-56-134l-158-158q-74-72-150-72-62 0-112 50-14 14-14 34t14 36q14 14 35 14t35-14q50-48 122 24l158 156q28 28 28 64 0 38-28 62-24 26-56 31t-60-21l-50-50q-16-14-36-14t-34 14q-34 34 0 70l50 50q54 54 127 51t129-61z" horiz-adv-x="800" />
 
-<glyph glyph-name="location" unicode="&#xe807;" d="M250 750q104 0 177-73t73-177q0-106-62-243t-126-223l-62-84q-10 12-27 35t-60 89-76 130-60 147-27 149q0 104 73 177t177 73z m0-388q56 0 96 40t40 96-40 95-96 39-95-39-39-95 39-96 95-40z" horiz-adv-x="500" />
+<glyph glyph-name="calendar" unicode="&#xe807;" d="M800 700q42 0 71-29t29-71l0-600q0-40-29-70t-71-30l-700 0q-40 0-70 30t-30 70l0 600q0 42 30 71t70 29l46 0 0-100 160 0 0 100 290 0 0-100 160 0 0 100 44 0z m0-700l0 400-700 0 0-400 700 0z m-540 800l0-170-70 0 0 170 70 0z m450 0l0-170-70 0 0 170 70 0z" horiz-adv-x="900" />
+
+<glyph glyph-name="location" unicode="&#xe808;" d="M250 750q104 0 177-73t73-177q0-106-62-243t-126-223l-62-84q-10 12-27 35t-60 89-76 130-60 147-27 149q0 104 73 177t177 73z m0-388q56 0 96 40t40 96-40 95-96 39-95-39-39-95 39-96 95-40z" horiz-adv-x="500" />
 
 <glyph glyph-name="picture" unicode="&#xe809;" d="M357 529q0-45-31-76t-76-32-76 32-31 76 31 76 76 31 76-31 31-76z m572-215v-250h-786v107l178 179 90-89 285 285z m53 393h-893q-7 0-12-5t-6-13v-678q0-7 6-13t12-5h893q7 0 13 5t5 13v678q0 8-5 13t-13 5z m89-18v-678q0-37-26-63t-63-27h-893q-36 0-63 27t-26 63v678q0 37 26 63t63 27h893q37 0 63-27t26-63z" horiz-adv-x="1071.4" />
 
@@ -28,19 +30,19 @@
 
 <glyph glyph-name="down" unicode="&#xe80b;" d="M939 399l-414-413q-10-11-25-11t-25 11l-414 413q-11 11-11 26t11 25l93 92q10 11 25 11t25-11l296-296 296 296q11 11 25 11t26-11l92-92q11-11 11-25t-11-26z" horiz-adv-x="1000" />
 
-<glyph glyph-name="retweet" unicode="&#xe80d;" d="M714 11q0-7-5-13t-13-5h-535q-5 0-8 1t-5 4-3 4-2 7 0 6v335h-107q-15 0-25 11t-11 25q0 13 8 23l179 214q11 12 27 12t28-12l178-214q9-10 9-23 0-15-11-25t-25-11h-107v-214h321q9 0 14-6l89-108q4-5 4-11z m357 232q0-13-8-23l-178-214q-12-13-28-13t-27 13l-179 214q-8 10-8 23 0 14 11 25t25 11h107v214h-322q-9 0-14 7l-89 107q-4 5-4 11 0 7 5 12t13 6h536q4 0 7-1t5-4 3-5 2-6 1-7v-334h107q14 0 25-11t10-25z" horiz-adv-x="1071.4" />
+<glyph glyph-name="retweet" unicode="&#xe80c;" d="M714 11q0-7-5-13t-13-5h-535q-5 0-8 1t-5 4-3 4-2 7 0 6v335h-107q-15 0-25 11t-11 25q0 13 8 23l179 214q11 12 27 12t28-12l178-214q9-10 9-23 0-15-11-25t-25-11h-107v-214h321q9 0 14-6l89-108q4-5 4-11z m357 232q0-13-8-23l-178-214q-12-13-28-13t-27 13l-179 214q-8 10-8 23 0 14 11 25t25 11h107v214h-322q-9 0-14 7l-89 107q-4 5-4 11 0 7 5 12t13 6h536q4 0 7-1t5-4 3-5 2-6 1-7v-334h107q14 0 25-11t10-25z" horiz-adv-x="1071.4" />
 
-<glyph glyph-name="search" unicode="&#xe80e;" d="M772 78q30-34 6-62l-46-46q-36-32-68 0l-190 190q-74-42-156-42-128 0-223 95t-95 223 90 219 218 91 224-95 96-223q0-88-46-162z m-678 358q0-88 68-156t156-68 151 63 63 153q0 88-68 155t-156 67-151-63-63-151z" horiz-adv-x="789" />
+<glyph glyph-name="search" unicode="&#xe80d;" d="M772 78q30-34 6-62l-46-46q-36-32-68 0l-190 190q-74-42-156-42-128 0-223 95t-95 223 90 219 218 91 224-95 96-223q0-88-46-162z m-678 358q0-88 68-156t156-68 151 63 63 153q0 88-68 155t-156 67-151-63-63-151z" horiz-adv-x="789" />
 
-<glyph glyph-name="pin" unicode="&#xe80f;" d="M268 368v250q0 8-5 13t-13 5-13-5-5-13v-250q0-8 5-13t13-5 13 5 5 13z m375-197q0-14-11-25t-25-10h-239l-29-270q-1-7-6-11t-11-5h-1q-15 0-17 15l-43 271h-225q-15 0-25 10t-11 25q0 69 44 124t99 55v286q-29 0-50 21t-22 50 22 50 50 22h357q29 0 50-22t21-50-21-50-50-21v-286q55 0 99-55t44-124z" horiz-adv-x="642.9" />
+<glyph glyph-name="pin" unicode="&#xe80e;" d="M268 368v250q0 8-5 13t-13 5-13-5-5-13v-250q0-8 5-13t13-5 13 5 5 13z m375-197q0-14-11-25t-25-10h-239l-29-270q-1-7-6-11t-11-5h-1q-15 0-17 15l-43 271h-225q-15 0-25 10t-11 25q0 69 44 124t99 55v286q-29 0-50 21t-22 50 22 50 50 22h357q29 0 50-22t21-50-21-50-50-21v-286q55 0 99-55t44-124z" horiz-adv-x="642.9" />
 
-<glyph glyph-name="cog" unicode="&#xe812;" d="M911 295l-133-56q-8-22-12-31l55-133-79-79-135 53q-9-4-31-12l-55-134-112 0-56 133q-11 4-33 13l-132-55-78 79 53 134q-1 3-4 9t-6 12-4 11l-131 55 0 112 131 56 14 33-54 132 78 79 133-54q22 9 33 13l55 132 112 0 56-132q14-5 31-13l133 55 80-79-54-135q6-12 12-30l133-56 0-112z m-447-111q69 0 118 48t49 118-49 119-118 50-119-50-49-119 49-118 119-48z" horiz-adv-x="928" />
+<glyph glyph-name="cog" unicode="&#xe80f;" d="M911 295l-133-56q-8-22-12-31l55-133-79-79-135 53q-9-4-31-12l-55-134-112 0-56 133q-11 4-33 13l-132-55-78 79 53 134q-1 3-4 9t-6 12-4 11l-131 55 0 112 131 56 14 33-54 132 78 79 133-54q22 9 33 13l55 132 112 0 56-132q14-5 31-13l133 55 80-79-54-135q6-12 12-30l133-56 0-112z m-447-111q69 0 118 48t49 118-49 119-118 50-119-50-49-119 49-118 119-48z" horiz-adv-x="928" />
 
-<glyph glyph-name="rss-feed" unicode="&#xe813;" d="M184 93c0-51-43-91-93-91s-91 40-91 91c0 50 41 91 91 91s93-41 93-91z m261-85l-125 0c0 174-140 323-315 323l0 118c231 0 440-163 440-441z m259 0l-136 0c0 300-262 561-563 561l0 129c370 0 699-281 699-690z" horiz-adv-x="704" />
+<glyph glyph-name="rss" unicode="&#xe810;" d="M184 93c0-51-43-91-93-91s-91 40-91 91c0 50 41 91 91 91s93-41 93-91z m261-85l-125 0c0 174-140 323-315 323l0 118c231 0 440-163 440-441z m259 0l-136 0c0 300-262 561-563 561l0 129c370 0 699-281 699-690z" horiz-adv-x="704" />
 
 <glyph glyph-name="info" unicode="&#xf128;" d="M393 149v-134q0-9-7-15t-15-7h-134q-9 0-16 7t-7 15v134q0 9 7 16t16 6h134q9 0 15-6t7-16z m176 335q0-30-8-56t-20-43-31-33-32-25-34-19q-23-13-38-37t-15-37q0-10-7-18t-16-9h-134q-8 0-14 11t-6 20v26q0 46 37 87t79 60q33 16 47 32t14 42q0 24-26 41t-60 18q-36 0-60-16-20-14-60-64-7-9-17-9-7 0-14 4l-91 70q-8 6-9 14t3 16q89 148 259 148 45 0 90-17t81-46 59-72 23-88z" horiz-adv-x="571.4" />
 
 <glyph glyph-name="bird" unicode="&#xf309;" d="M920 636q-36-54-94-98l0-24q0-130-60-250t-186-203-290-83q-160 0-290 84 14-2 46-2 132 0 234 80-62 2-110 38t-66 94q10-4 34-4 26 0 50 6-66 14-108 66t-42 120l0 2q36-20 84-24-84 58-84 158 0 48 26 94 154-188 390-196-6 18-6 42 0 78 55 133t135 55q82 0 136-58 60 12 120 44-20-66-82-104 56 8 108 30z" horiz-adv-x="920" />
 </font>
 </defs>
-</svg>
+</svg>

public/js/infiniteScroll.js

@@ -37,6 +37,10 @@ function fetchAndParse(url) {
 window.onload = function () {
     const url = window.location.pathname;
     const isTweet = url.indexOf("/status/") !== -1;
+    const isHomepage = url === "/" || url === "";
+    
+    if (isHomepage) return;
+    
     const isIncompleteThread =
         isTweet && document.querySelector(".timeline-item.more-replies") != null;
 

public/md/about.md

@@ -1,5 +1,7 @@
 # About
 
+This Nitter instance is hosted by [Kuuro.net](https://kuuro.net/services)
+
 Nitter is a free and open source alternative Twitter front-end focused on
 privacy and performance. The source is available on GitHub at
 <https://github.com/zedeus/nitter>
@@ -43,6 +45,28 @@ Twitter account.
 
 ## Donating
 
+You can either donate to me for hosting this instance, or to the Nitter project for development.
+
+Donating to me helps keep this Nitter instance running for everyone. Donating to the Nitter project helps the development of the software itself. Both are appreciated!
+
+### Donating to me
+<br>
+
+#### Credit/debit card and bank transfer
+
+Buy Me A Coffee: <https://buymeacoffee.com/kuu7o>
+
+#### Cryptocurrency
+
+Monero: 44AAFK5nJEM8CMFGHh1DnPDVLnsAXu9y7VcKTAar5F1AGwQcoCi5hwC22efxGWsp4HfwRRwRRMj2X5ypTo3vkjpJPMxwynZ \
+Bitcoin: bc1q8uzvmm3y23kfmvmvva2n2uptqty5hzmlg2vjyt \
+Ethereum: 0xe7Bf8f78Ffc92F4b50f879AEc7Fd41A463DC9c28 \
+Litecoin: LaqWXCsNBinE5ho2cUJiDZ4MFmPH6QYZ4s \
+Wownero: Wo4NPaJdM3yETa2JXfjJbgKbD4BHXSEsk4Vj6F5gbcb99yaG7fAgZQGLp2EwvNDqE4QKWm63mtWfvfkbbX8dipeR31zPxDbsd \
+Tron: TMbMarrcsiVVgB5CRMqhShcGHMXYozpocg
+
+### Donating to the Nitter project
+
 Liberapay: https://liberapay.com/zedeus \
 Patreon: https://patreon.com/nitter \
 BTC: bc1qpqpzjkcpgluhzf7x9yqe7jfe8gpfm5v08mdr55 \
@@ -53,4 +77,4 @@ ZEC: u1vndfqtzyy6qkzhkapxelel7ams38wmfeccu3fdpy2wkuc4erxyjm8ncjhnyg747x6t0kf0faq
 
 ## Contact
 
-Feel free to join our [Matrix channel](https://matrix.to/#/#nitter:matrix.org).
+Feel free to join Nitter [Matrix channel](https://matrix.to/#/#nitter:matrix.org).

src/api.nim

@@ -1,71 +1,101 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, httpclient, uri, strutils, sequtils, sugar
+import asyncdispatch, httpclient, strutils, sequtils, sugar
 import packedjson
 import types, query, formatters, consts, apiutils, parser
 import experimental/parser as newParser
 
-proc mediaUrl(id: string; cursor: string): SessionAwareUrl =
+# Helper to generate params object for GraphQL requests
-  let
+proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
-    cookieVariables = userMediaVariables % [id, cursor]
+  result.add ("variables", variables)
-    oauthVariables = userTweetsVariables % [id, cursor]
+  result.add ("features", gqlFeatures)
-  result = SessionAwareUrl(
+  if fieldToggles.len > 0:
-    cookieUrl: graphUserMedia ? {"variables": cookieVariables, "features": gqlFeatures},
+    result.add ("fieldToggles", fieldToggles)
-    oauthUrl: graphUserMediaV2 ? {"variables": oauthVariables, "features": gqlFeatures}
+
+proc apiUrl(endpoint, variables: string; fieldToggles = ""): ApiUrl =
+  return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles))
+
+proc apiReq(endpoint, variables: string; fieldToggles = ""): ApiReq =
+  let url = apiUrl(endpoint, variables, fieldToggles)
+  return ApiReq(cookie: url, oauth: url)
+
+proc mediaUrl(id: string; cursor: string): ApiReq =
+  result = ApiReq(
+    cookie: apiUrl(graphUserMedia, userMediaVars % [id, cursor]),
+    oauth: apiUrl(graphUserMediaV2, restIdVars % [id, cursor])
+  )
+
+proc userTweetsUrl(id: string; cursor: string): ApiReq =
+  result = ApiReq(
+    # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
+    oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor])
+  )
+  # might change this in the future pending testing
+  result.cookie = result.oauth
+
+proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq =
+  let cookieVars = userTweetsAndRepliesVars % [id, cursor]
+  result = ApiReq(
+    cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
+    oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor])
+  )
+
+proc tweetDetailUrl(id: string; cursor: string): ApiReq =
+  let cookieVars = tweetDetailVars % [id, cursor]
+  result = ApiReq(
+    cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
+    oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
+  )
+
+proc userUrl(username: string): ApiReq =
+  let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
+  result = ApiReq(
+    cookie: apiUrl(graphUser, cookieVars, tweetDetailFieldToggles),
+    oauth: apiUrl(graphUserV2, """{"screen_name": "$1"}""" % username)
   )
 
 proc getGraphUser*(username: string): Future[User] {.async.} =
   if username.len == 0: return
-  let
+  let js = await fetchRaw(userUrl(username))
-    variables = """{"screen_name": "$1"}""" % username
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetchRaw(graphUser ? params, Api.userScreenName)
   result = parseGraphUser(js)
 
 proc getGraphUserById*(id: string): Future[User] {.async.} =
   if id.len == 0 or id.any(c => not c.isDigit): return
   let
-    variables = """{"rest_id": "$1"}""" % id
+    url = apiReq(graphUserById, """{"rest_id": "$1"}""" % id)
-    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetchRaw(url)
-    js = await fetchRaw(graphUserById ? params, Api.userRestId)
   result = parseGraphUser(js)
 
 proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
   if id.len == 0: return
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = userTweetsVariables % [id, cursor]
+    url = case kind
-    params = {"variables": variables, "features": gqlFeatures}
+      of TimelineKind.tweets: userTweetsUrl(id, cursor)
-    js = case kind
+      of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor)
-      of TimelineKind.tweets:
+      of TimelineKind.media: mediaUrl(id, cursor)
-        await fetch(graphUserTweets ? params, Api.userTweets)
+    js = await fetch(url)
-      of TimelineKind.replies:
-        await fetch(graphUserTweetsAndReplies ? params, Api.userTweetsAndReplies)
-      of TimelineKind.media:
-        await fetch(mediaUrl(id, cursor), Api.userMedia)
   result = parseGraphTimeline(js, after)
 
 proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
   if id.len == 0: return
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = listTweetsVariables % [id, cursor]
+    url = apiReq(graphListTweets, restIdVars % [id, cursor])
-    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(url)
-    js = await fetch(graphListTweets ? params, Api.listTweets)
   result = parseGraphTimeline(js, after).tweets
 
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
   let
     variables = %*{"screenName": name, "listSlug": list}
-    params = {"variables": $variables, "features": gqlFeatures}
+    url = apiReq(graphListBySlug, $variables)
-    url = graphListBySlug ? params
+    js = await fetch(url)
-  result = parseGraphList(await fetch(url, Api.listBySlug))
+  result = parseGraphList(js)
 
 proc getGraphList*(id: string): Future[List] {.async.} =
-  let
+  let 
-    variables = """{"listId": "$1"}""" % id
+    url = apiReq(graphListById, """{"listId": "$1"}""" % id)
-    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(url)
-    url = graphListById ? params
+  result = parseGraphList(js)
-  result = parseGraphList(await fetch(url, Api.list))
 
 proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
   if list.id.len == 0: return
@@ -79,24 +109,23 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
     }
   if after.len > 0:
     variables["cursor"] = % after
-  let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
+  let 
-  result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
+    url = apiReq(graphListMembers, $variables)
+    js = await fetchRaw(url)
+  result = parseGraphListMembers(js, after)
 
 proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
   if id.len == 0: return
   let
-    variables = """{"rest_id": "$1"}""" % id
+    url = apiReq(graphTweetResult, """{"rest_id": "$1"}""" % id)
-    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(url)
-    js = await fetch(graphTweetResult ? params, Api.tweetResult)
   result = parseGraphTweetResult(js)
 
 proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
   if id.len == 0: return
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = tweetVariables % [id, cursor]
+    js = await fetch(tweetDetailUrl(id, cursor))
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetch(graphTweet ? params, Api.tweetDetail)
   result = parseGraphConversation(js, id)
 
 proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
@@ -116,6 +145,7 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
   var
     variables = %*{
       "rawQuery": q,
+      "query_source": "typedQuery",
       "count": 20,
       "product": "Latest",
       "withDownvotePerspective": false,
@@ -124,8 +154,10 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
     }
   if after.len > 0:
     variables["cursor"] = % after
-  let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
+  let 
-  result = parseGraphSearch[Tweets](await fetch(url, Api.search), after)
+    url = apiReq(graphSearchTimeline, $variables)
+    js = await fetch(url)
+  result = parseGraphSearch[Tweets](js, after)
   result.query = query
 
 proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} =
@@ -135,6 +167,7 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
   var
     variables = %*{
       "rawQuery": query.text,
+      "query_source": "typedQuery",
       "count": 20,
       "product": "People",
       "withDownvotePerspective": false,
@@ -145,13 +178,15 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
     variables["cursor"] = % after
     result.beginning = false
 
-  let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
+  let 
-  result = parseGraphSearch[User](await fetch(url, Api.search), after)
+    url = apiReq(graphSearchTimeline, $variables)
+    js = await fetch(url)
+  result = parseGraphSearch[User](js, after)
   result.query = query
 
 proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
   if id.len == 0: return
-  let js = await fetch(mediaUrl(id, ""), Api.userMedia)
+  let js = await fetch(mediaUrl(id, ""))
   result = parseGraphPhotoRail(js)
 
 proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =

src/apiutils.nim

@@ -1,16 +1,30 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
+import httpclient, asyncdispatch, options, strutils, uri, times, math, tables, logging
 import jsony, packedjson, zippy, oauth1
-import types, auth, consts, parserutils, http_pool
+import types, auth, consts, parserutils, http_pool, tid
 import experimental/types/common
 
 const
   rlRemaining = "x-rate-limit-remaining"
   rlReset = "x-rate-limit-reset"
   rlLimit = "x-rate-limit-limit"
-  errorsToSkip = {doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
+  errorsToSkip = {null, doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
 
-var pool: HttpPool
+var 
+  pool: HttpPool
+  disableTid: bool
+
+proc setDisableTid*(disable: bool) =
+  disableTid = disable
+
+proc toUrl(req: ApiReq; sessionKind: SessionKind): Uri =
+  case sessionKind
+  of oauth:  
+    let o = req.oauth
+    parseUri("https://api.x.com/graphql")   / o.endpoint ? o.params
+  of cookie: 
+    let c = req.cookie
+    parseUri("https://x.com/i/api/graphql") / c.endpoint ? c.params
 
 proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
   let
@@ -32,15 +46,15 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
 proc getCookieHeader(authToken, ct0: string): string =
   "auth_token=" & authToken & "; ct0=" & ct0
 
-proc genHeaders*(session: Session, url: string): HttpHeaders =
+proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
   result = newHttpHeaders({
     "connection": "keep-alive",
     "content-type": "application/json",
     "x-twitter-active-user": "yes",
     "x-twitter-client-language": "en",
-    "authority": "api.x.com",
+    "origin": "https://x.com",
     "accept-encoding": "gzip",
-    "accept-language": "en-US,en;q=0.9",
+    "accept-language": "en-US,en;q=0.5",
     "accept": "*/*",
     "DNT": "1",
     "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
@@ -48,23 +62,28 @@ proc genHeaders*(session: Session, url: string): HttpHeaders =
 
   case session.kind
   of SessionKind.oauth:
-    result["authorization"] = getOauthHeader(url, session.oauthToken, session.oauthSecret)
+    result["authority"] = "api.x.com"
+    result["authorization"] = getOauthHeader($url, session.oauthToken, session.oauthSecret)
   of SessionKind.cookie:
-    result["authorization"] = "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
     result["x-twitter-auth-type"] = "OAuth2Session"
     result["x-csrf-token"] = session.ct0
     result["cookie"] = getCookieHeader(session.authToken, session.ct0)
+    if disableTid:
+      result["authorization"] = bearerToken2
+    else:
+      result["authorization"] = bearerToken
+      result["x-client-transaction-id"] = await genTid(url.path)
 
-proc getAndValidateSession*(api: Api): Future[Session] {.async.} =
+proc getAndValidateSession*(req: ApiReq): Future[Session] {.async.} =
-  result = await getSession(api)
+  result = await getSession(req)
   case result.kind
   of SessionKind.oauth:
     if result.oauthToken.len == 0:
-      echo "[sessions] Empty oauth token, session: ", result.id
+      warn "[sessions] Empty oauth token, session: ", result.pretty
       raise rateLimitError()
   of SessionKind.cookie:
     if result.authToken.len == 0 or result.ct0.len == 0:
-      echo "[sessions] Empty cookie credentials, session: ", result.id
+      warn "[sessions] Empty cookie credentials, session: ", result.pretty
       raise rateLimitError()
 
 template fetchImpl(result, fetchBody) {.dirty.} =
@@ -73,7 +92,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
 
   try:
     var resp: AsyncResponse
-    pool.use(genHeaders(session, $url)):
+    pool.use(await genHeaders(session, url)):
       template getContent =
         resp = await c.get($url)
         result = await resp.body
@@ -89,7 +108,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
         remaining = parseInt(resp.headers[rlRemaining])
         reset = parseInt(resp.headers[rlReset])
         limit = parseInt(resp.headers[rlLimit])
-      session.setRateLimit(api, remaining, reset, limit)
+      session.setRateLimit(req, remaining, reset, limit)
 
     if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":
@@ -98,24 +117,22 @@ template fetchImpl(result, fetchBody) {.dirty.} =
       if result.startsWith("{\"errors"):
         let errors = result.fromJson(Errors)
         if errors notin errorsToSkip:
-          echo "Fetch error, API: ", api, ", errors: ", errors
+          error "Fetch error, API: ", url.path, ", errors: ", errors
           if errors in {expiredToken, badToken, locked}:
             invalidate(session)
             raise rateLimitError()
           elif errors in {rateLimited}:
             # rate limit hit, resets after 24 hours
-            setLimited(session, api)
+            setLimited(session, req)
             raise rateLimitError()
       elif result.startsWith("429 Too Many Requests"):
-        echo "[sessions] 429 error, API: ", api, ", session: ", session.id
+        warn "[sessions] 429 error, API: ", url.path, ", session: ", session.pretty
-        session.apis[api].remaining = 0
-        # rate limit hit, resets after the 15 minute window
         raise rateLimitError()
 
     fetchBody
 
     if resp.status == $Http400:
-      echo "ERROR 400, ", api, ": ", result
+      error "ERROR 400, ", url.path, ": ", result
       raise newException(InternalError, $url)
   except InternalError as e:
     raise e
@@ -124,8 +141,11 @@ template fetchImpl(result, fetchBody) {.dirty.} =
   except OSError as e:
     raise e
   except Exception as e:
-    let id = if session.isNil: "null" else: $session.id
+    let s = session.pretty
-    echo "error: ", e.name, ", msg: ", e.msg, ", sessionId: ", id, ", url: ", url
+    var safeUrl = $url
+    if safeUrl.len > 100:
+      safeUrl = safeUrl[0 .. 100] & "..."
+    error "error: ", e.name, ", msg: ", e.msg, ", session: ", s, ", url: ", safeUrl
     raise rateLimitError()
   finally:
     release(session)
@@ -134,44 +154,37 @@ template retry(bod) =
   try:
     bod
   except RateLimitError:
-    echo "[sessions] Rate limited, retrying ", api, " request..."
+    info "[sessions] Rate limited, retrying ", req.cookie.endpoint, " request..."
     bod
 
-proc fetch*(url: Uri | SessionAwareUrl; api: Api): Future[JsonNode] {.async.} =
+proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
   retry:
     var 
       body: string
-      session = await getAndValidateSession(api)
+      session = await getAndValidateSession(req)
 
-    when url is SessionAwareUrl:
+    let url = req.toUrl(session.kind)
-      let url = case session.kind
-        of SessionKind.oauth: url.oauthUrl
-        of SessionKind.cookie: url.cookieUrl
 
     fetchImpl body:
       if body.startsWith('{') or body.startsWith('['):
         result = parseJson(body)
       else:
-        echo resp.status, ": ", body, " --- url: ", url
+        warn resp.status, ": ", body, " --- url: ", url
         result = newJNull()
 
       let error = result.getError
       if error != null and error notin errorsToSkip:
-        echo "Fetch error, API: ", api, ", error: ", error
+        error "Fetch error, API: ", url.path, ", error: ", error
         if error in {expiredToken, badToken, locked}:
           invalidate(session)
           raise rateLimitError()
 
-proc fetchRaw*(url: Uri | SessionAwareUrl; api: Api): Future[string] {.async.} =
+proc fetchRaw*(req: ApiReq): Future[string] {.async.} =
   retry:
-    var session = await getAndValidateSession(api)
+    var session = await getAndValidateSession(req)
-
+    let url = req.toUrl(session.kind)
-    when url is SessionAwareUrl:
-      let url = case session.kind
-        of SessionKind.oauth: url.oauthUrl
-        of SessionKind.cookie: url.cookieUrl
 
     fetchImpl result:
       if not (result.startsWith('{') or result.startsWith('[')):
-        echo resp.status, ": ", result, " --- url: ", url
+        warn resp.status, ": ", result, " --- url: ", url
         result.setLen(0)

src/auth.nim

@@ -1,33 +1,41 @@
 #SPDX-License-Identifier: AGPL-3.0-only
-import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os]
+import std/[asyncdispatch, times, json, random, strutils, tables, packedsets, os, logging]
-import types
+import types, consts
 import experimental/parser/session
 
 # max requests at a time per session to avoid race conditions
 const
   maxConcurrentReqs = 2
   hourInSeconds = 60 * 60
-  apiMaxReqs: Table[Api, int] = {
-    Api.search: 50,
-    Api.tweetDetail: 500,
-    Api.userTweets: 500,
-    Api.userTweetsAndReplies: 500,
-    Api.userMedia: 500,
-    Api.userRestId: 500,
-    Api.userScreenName: 500,
-    Api.tweetResult: 500,
-    Api.list: 500,
-    Api.listTweets: 500,
-    Api.listMembers: 500,
-    Api.listBySlug: 500
-  }.toTable
 
 var
   sessionPool: seq[Session]
   enableLogging = false
 
-template log(str: varargs[string, `$`]) =
+proc logSession(args: varargs[string, `$`]) =
-  echo "[sessions] ", str.join("")
+  var s = "[sessions] "
+  for arg in args:
+    s.add arg
+  info s
+
+proc endpoint(req: ApiReq; session: Session): string =
+  case session.kind
+  of oauth: req.oauth.endpoint
+  of cookie: req.cookie.endpoint
+
+proc pretty*(session: Session): string =
+  if session.isNil:
+    return "<null>"
+
+  if session.id > 0 and session.username.len > 0:
+    result = $session.id & " (" & session.username & ")"
+  elif session.username.len > 0:
+    result = session.username
+  elif session.id > 0:
+    result = $session.id
+  else:
+    result = "<unknown>"
+  result = $session.kind & " " & result
 
 proc snowflakeToEpoch(flake: int64): int64 =
   int64(((flake shr 22) + 1288834974657) div 1000)
@@ -57,8 +65,7 @@ proc getSessionPoolHealth*(): JsonNode =
     for api in session.apis.keys:
       let
         apiStatus = session.apis[api]
-        limit = if apiStatus.limit > 0: apiStatus.limit else: apiMaxReqs.getOrDefault(api, 0)
+        reqs = apiStatus.limit - apiStatus.remaining
-        reqs = limit - apiStatus.remaining
 
       # no requests made with this session and endpoint since the limit reset
       if apiStatus.reset < now:
@@ -123,14 +130,15 @@ proc rateLimitError*(): ref RateLimitError =
 proc noSessionsError*(): ref NoSessionsError =
   newException(NoSessionsError, "no sessions available")
 
-proc isLimited(session: Session; api: Api): bool =
+proc isLimited(session: Session; req: ApiReq): bool =
   if session.isNil:
     return true
 
-  if session.limited and api != Api.userTweets:
+  let api = req.endpoint(session)
+  if session.limited and api != graphUserTweetsV2:
     if (epochTime().int - session.limitedAt) > hourInSeconds:
       session.limited = false
-      log "resetting limit: ", session.id
+      logSession "resetting limit: ", session.pretty
       return false
     else:
       return true
@@ -141,12 +149,12 @@ proc isLimited(session: Session; api: Api): bool =
   else:
     return false
 
-proc isReady(session: Session; api: Api): bool =
+proc isReady(session: Session; req: ApiReq): bool =
-  not (session.isNil or session.pending > maxConcurrentReqs or session.isLimited(api))
+  not (session.isNil or session.pending > maxConcurrentReqs or session.isLimited(req))
 
 proc invalidate*(session: var Session) =
   if session.isNil: return
-  log "invalidating: ", session.id
+  logSession "invalidating: ", session.pretty
 
   # TODO: This isn't sufficient, but it works for now
   let idx = sessionPool.find(session)
@@ -157,24 +165,26 @@ proc release*(session: Session) =
   if session.isNil: return
   dec session.pending
 
-proc getSession*(api: Api): Future[Session] {.async.} =
+proc getSession*(req: ApiReq): Future[Session] {.async.} =
   for i in 0 ..< sessionPool.len:
-    if result.isReady(api): break
+    if result.isReady(req): break
     result = sessionPool.sample()
 
-  if not result.isNil and result.isReady(api):
+  if not result.isNil and result.isReady(req):
     inc result.pending
   else:
-    log "no sessions available for API: ", api
+    logSession "no sessions available for API: ", req.cookie.endpoint
     raise noSessionsError()
 
-proc setLimited*(session: Session; api: Api) =
+proc setLimited*(session: Session; req: ApiReq) =
+  let api = req.endpoint(session)
   session.limited = true
   session.limitedAt = epochTime().int
-  log "rate limited by api: ", api, ", reqs left: ", session.apis[api].remaining, ", id: ", session.id
+  logSession "rate limited by api: ", api, ", reqs left: ", session.apis[api].remaining, ", ", session.pretty
 
-proc setRateLimit*(session: Session; api: Api; remaining, reset, limit: int) =
+proc setRateLimit*(session: Session; req: ApiReq; remaining, reset, limit: int) =
   # avoid undefined behavior in race conditions
+  let api = req.endpoint(session)
   if api in session.apis:
     let rateLimit = session.apis[api]
     if rateLimit.reset >= reset and rateLimit.remaining < remaining:
@@ -189,15 +199,15 @@ proc initSessionPool*(cfg: Config; path: string) =
   enableLogging = cfg.enableDebug
 
   if path.endsWith(".json"):
-    log "ERROR: .json is not supported, the file must be a valid JSONL file ending in .jsonl"
+    fatal ".json is not supported, the file must be a valid JSONL file ending in .jsonl"
     quit 1
 
   if not fileExists(path):
-    log "ERROR: ", path, " not found. This file is required to authenticate API requests."
+    fatal path, " not found. This file is required to authenticate API requests."
     quit 1
 
-  log "parsing JSONL account sessions file: ", path
+  logSession "parsing JSONL account sessions file: ", path
   for line in path.lines:
     sessionPool.add parseSession(line)
 
-  log "successfully added ", sessionPool.len, " valid account sessions"
+  logSession "successfully added ", sessionPool.len, " valid account sessions"

src/config.nim

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import parsecfg except Config
-import types, strutils
+import types, strutils, sequtils
 
 proc get*[T](config: parseCfg.Config; section, key: string; default: T): T =
   let val = config.getSectionValue(section, key)
@@ -9,6 +9,7 @@ proc get*[T](config: parseCfg.Config; section, key: string; default: T): T =
   when T is int: parseInt(val)
   elif T is bool: parseBool(val)
   elif T is string: val
+  elif T is seq[string]: val.split(',').mapIt(it.strip())
 
 proc getConfig*(path: string): (Config, parseCfg.Config) =
   var cfg = loadConfig(path)
@@ -39,8 +40,11 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
     minTokens: cfg.get("Config", "tokenCount", 10),
     enableRss: cfg.get("Config", "enableRSS", true),
     enableDebug: cfg.get("Config", "enableDebug", false),
+    logLevel: cfg.get("Config", "logLevel", ""),
     proxy: cfg.get("Config", "proxy", ""),
-    proxyAuth: cfg.get("Config", "proxyAuth", "")
+    proxyAuth: cfg.get("Config", "proxyAuth", ""),
+    defaultFollowedAccounts: cfg.get("Config", "defaultFollowedAccounts", @["eff", "fsf"]),
+    disableTid: cfg.get("Config", "disableTid", false)
   )
 
   return (conf, cfg)

src/consts.nim

@@ -1,29 +1,36 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import uri, strutils
+import strutils
 
 const
   consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
   consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
+  bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
+  bearerToken2* = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
 
-  gql = parseUri("https://api.x.com") / "graphql"
+  graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
-
+  graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery"
-  graphUser* = gql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery"
+  graphUserById* = "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery"
-  graphUserById* = gql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery"
+  graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2"
-  graphUserTweets* = gql / "JLApJKFY0MxGTzCoK6ps8Q/UserWithProfileTweetsQueryV2"
+  graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2"
-  graphUserTweetsAndReplies* = gql / "Y86LQY7KMvxn5tu3hFTyPg/UserWithProfileTweetsAndRepliesQueryV2"
+  graphUserTweets* = "oRJs8SLCRNRbQzuZG93_oA/UserTweets"
-  graphUserMedia* = gql / "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
+  graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
-  graphUserMediaV2* = gql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2"
+  graphUserMedia* = "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
-  graphTweet* = gql / "Vorskcd2tZ-tc4Gx3zbk4Q/ConversationTimelineV2"
+  graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
-  graphTweetResult* = gql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery"
+  graphTweet* = "Y4Erk_-0hObvLpz0Iw3bzA/ConversationTimeline"
-  graphSearchTimeline* = gql / "KI9jCXUx3Ymt-hDKLOZb9Q/SearchTimeline"
+  graphTweetDetail* = "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail"
-  graphListById* = gql / "oygmAig8kjn0pKsx_bUadQ/ListByRestId"
+  graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
-  graphListBySlug* = gql / "88GTz-IPPWLn1EiU8XoNVg/ListBySlug"
+  graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
-  graphListMembers* = gql / "kSmxeqEeelqdHSR7jMnb_w/ListMembers"
+  graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
-  graphListTweets* = gql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline"
+  graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
+  graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
+  graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
 
   gqlFeatures* = """{
+  "android_ad_formats_media_component_render_overlay_enabled": false,
   "android_graphql_skip_api_media_color_palette": false,
+  "android_professional_link_spotlight_display_enabled": false,
   "blue_business_profile_image_shape_enabled": false,
+  "commerce_android_shop_module_enabled": false,
   "creator_subscriptions_subscription_count_enabled": false,
   "creator_subscriptions_tweet_preview_api_enabled": true,
   "freedom_of_speech_not_reach_fetch_enabled": true,
@@ -33,8 +40,9 @@ const
   "interactive_text_enabled": false,
   "longform_notetweets_consumption_enabled": true,
   "longform_notetweets_inline_media_enabled": true,
-  "longform_notetweets_richtext_consumption_enabled": true,
   "longform_notetweets_rich_text_read_enabled": true,
+  "longform_notetweets_richtext_consumption_enabled": true,
+  "mobile_app_spotlight_module_enabled": false,
   "responsive_web_edit_tweet_api_enabled": true,
   "responsive_web_enhance_cards_enabled": false,
   "responsive_web_graphql_exclude_directive_enabled": true,
@@ -43,6 +51,7 @@ const
   "responsive_web_media_download_video_enabled": false,
   "responsive_web_text_conversations_enabled": false,
   "responsive_web_twitter_article_tweet_consumption_enabled": true,
+  "unified_cards_destination_url_params_enabled": false,
   "responsive_web_twitter_blue_verified_badge_is_enabled": true,
   "rweb_lists_timeline_redesign_enabled": true,
   "spaces_2022_h2_clipping": true,
@@ -83,11 +92,21 @@ const
   "payments_enabled": false,
   "responsive_web_profile_redirect_enabled": false,
   "responsive_web_grok_show_grok_translated_post": false,
-  "responsive_web_grok_community_note_auto_translation_is_enabled": false
+  "responsive_web_grok_community_note_auto_translation_is_enabled": false,
+  "profile_label_improvements_pcf_label_in_profile_enabled": false,
+  "grok_android_analyze_trend_fetch_enabled": false,
+  "grok_translations_community_note_auto_translation_is_enabled": false,
+  "grok_translations_post_auto_translation_is_enabled": false,
+  "grok_translations_community_note_translation_is_enabled": false,
+  "grok_translations_timeline_user_bio_auto_translation_is_enabled": false,
+  "subscriptions_feature_can_gift_premium": false,
+  "responsive_web_twitter_article_notes_tab_enabled": false,
+  "subscriptions_verification_info_is_identity_verified_enabled": false,
+  "hidden_profile_subscriptions_enabled": false
 }""".replace(" ", "").replace("\n", "")
 
-  tweetVariables* = """{
+  tweetVars* = """{
-  "focalTweetId": "$1",
+  "postId": "$1",
   $2
   "includeHasBirdwatchNotes": false,
   "includePromotedContent": false,
@@ -96,29 +115,25 @@ const
   "withV2Timeline": true
 }""".replace(" ", "").replace("\n", "")
 
-#   oldUserTweetsVariables* = """{
+  tweetDetailVars* = """{
-#   "userId": "$1", $2
+  "focalTweetId": "$1",
-#   "count": 20,
+  $2
-#   "includePromotedContent": false,
+  "referrer": "profile",
-#   "withDownvotePerspective": false,
+  "with_rux_injections": false,
-#   "withReactionsMetadata": false,
+  "rankingMode": "Relevance",
-#   "withReactionsPerspective": false,
+  "includePromotedContent": true,
-#   "withVoice": false,
+  "withCommunity": true,
-#   "withV2Timeline": true
+  "withQuickPromoteEligibilityTweetFields": true,
-# }
+  "withBirdwatchNotes": true,
-# """
+  "withVoice": true
+}""".replace(" ", "").replace("\n", "")
 
-  userTweetsVariables* = """{
+  restIdVars* = """{
   "rest_id": "$1", $2
   "count": 20
 }"""
 
-  listTweetsVariables* = """{
+  userMediaVars* = """{
-  "rest_id": "$1", $2
-  "count": 20
-}"""
-
-  userMediaVariables* = """{
   "userId": "$1", $2
   "count": 20,
   "includePromotedContent": false,
@@ -126,3 +141,24 @@ const
   "withBirdwatchNotes": false,
   "withVoice": true
 }""".replace(" ", "").replace("\n", "")
+
+  userTweetsVars* = """{
+  "userId": "$1", $2
+  "count": 20,
+  "includePromotedContent": false,
+  "withQuickPromoteEligibilityTweetFields": true,
+  "withVoice": true
+}""".replace(" ", "").replace("\n", "")
+
+  userTweetsAndRepliesVars* = """{
+  "userId": "$1", $2
+  "count": 20,
+  "includePromotedContent": false,
+  "withCommunity": true,
+  "withVoice": true
+}""".replace(" ", "").replace("\n", "")
+
+  fieldToggles* = """{"withArticlePlainText":false,"withViewCounts":true}"""
+  userFieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}"""
+  userTweetsFieldToggles* = """{"withArticlePlainText":false}"""
+  tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}"""

src/experimental/parser/graphql.nim

@@ -1,21 +1,53 @@
-import options
+import options, strutils
 import jsony
-import user, ../types/[graphuser, graphlistmembers]
+import user, utils, ../types/[graphuser, graphlistmembers]
 from ../../types import User, VerifiedType, Result, Query, QueryKind
 
+proc parseUserResult*(userResult: UserResult): User =
+  result = userResult.legacy
+
+  if result.verifiedType == none and userResult.isBlueVerified:
+    result.verifiedType = blue
+
+  if result.username.len == 0 and userResult.core.screenName.len > 0:
+    result.id = userResult.restId
+    result.username = userResult.core.screenName
+    result.fullname = userResult.core.name
+    result.userPic = userResult.avatar.imageUrl.replace("_normal", "")
+
+    if userResult.privacy.isSome:
+      result.protected = userResult.privacy.get.protected
+
+    if userResult.location.isSome:
+      result.location = userResult.location.get.location
+
+    if userResult.core.createdAt.len > 0:
+      result.joinDate = parseTwitterDate(userResult.core.createdAt)
+
+    if userResult.verification.isSome:
+      let v = userResult.verification.get
+      if v.verifiedType != VerifiedType.none:
+        result.verifiedType = v.verifiedType
+
+    if userResult.profileBio.isSome and result.bio.len == 0:
+      result.bio = userResult.profileBio.get.description
+
 proc parseGraphUser*(json: string): User =
   if json.len == 0 or json[0] != '{':
     return
 
-  let raw = json.fromJson(GraphUser)
+  let 
+    raw = json.fromJson(GraphUser)
+    userResult = 
+      if raw.data.userResult.isSome: raw.data.userResult.get.result
+      elif raw.data.user.isSome: raw.data.user.get.result
+      else: UserResult()
 
-  if raw.data.userResult.result.unavailableReason.get("") == "Suspended":
+  if userResult.unavailableReason.get("") == "Suspended" or
+     userResult.reason.get("") == "Suspended":
     return User(suspended: true)
 
-  result = raw.data.userResult.result.legacy
+  result = parseUserResult(userResult)
-  result.id = raw.data.userResult.result.restId
-  if result.verifiedType == VerifiedType.none and raw.data.userResult.result.isBlueVerified:
-    result.verifiedType = blue
 
 proc parseGraphListMembers*(json, cursor: string): Result[User] =
   result = Result[User](
@@ -31,7 +63,7 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] =
         of TimelineTimelineItem:
           let userResult = entry.content.itemContent.userResults.result
           if userResult.restId.len > 0:
-            result.content.add userResult.legacy
+            result.content.add parseUserResult(userResult)
         of TimelineTimelineCursor:
           if entry.content.cursorType == "Bottom":
             result.bottom = entry.content.value

src/experimental/parser/session.nim

@@ -13,6 +13,7 @@ proc parseSession*(raw: string): Session =
     result = Session(
       kind: SessionKind.oauth,
       id: parseBiggestInt(id),
+      username: session.username,
       oauthToken: session.oauthToken,
       oauthSecret: session.oauthTokenSecret
     )
@@ -21,6 +22,7 @@ proc parseSession*(raw: string): Session =
     result = Session(
       kind: SessionKind.cookie,
       id: id,
+      username: session.username,
       authToken: session.authToken,
       ct0: session.ct0
     )

src/experimental/parser/tid.nim

@@ -0,0 +1,8 @@
+import jsony
+import ../types/tid
+export TidPair
+
+proc parseTidPairs*(raw: string): seq[TidPair] =
+  result = raw.fromJson(seq[TidPair])
+  if result.len == 0:
+    raise newException(ValueError, "Parsing pairs failed: " & raw)

src/experimental/parser/unifiedcard.nim

@@ -1,6 +1,9 @@
-import std/[options, tables, strutils, strformat, sugar]
+
+import std/[options, tables, strutils, strformat, sugar, logging]
+
 import jsony
 import user, ../types/unifiedcard
+import ../../formatters
 from ../../types import Card, CardKind, Video
 from ../../utils import twimg, https
 
@@ -77,6 +80,18 @@ proc parseMedia(component: Component; card: UnifiedCard; result: var Card) =
   of model3d:
     result.title = "Unsupported 3D model ad"
 
+proc parseGrokShare(data: ComponentData; card: UnifiedCard; result: var Card) =
+  result.kind = summaryLarge
+
+  data.destination.parseDestination(card, result)
+  result.dest = "Answer by Grok"
+
+  for msg in data.conversationPreview:
+    if msg.sender == "USER":
+      result.title = msg.message.shorten(70)
+    elif msg.sender == "AGENT":
+      result.text = msg.message.shorten(500)
+
 proc parseUnifiedCard*(json: string): Card =
   let card = json.fromJson(UnifiedCard)
 
@@ -92,12 +107,14 @@ proc parseUnifiedCard*(json: string): Card =
       component.parseMedia(card, result)
     of buttonGroup:
       discard
+    of grokShare:
+      component.data.parseGrokShare(card, result)
     of ComponentType.jobDetails:
       component.data.parseJobDetails(card, result)
     of ComponentType.hidden:
       result.kind = CardKind.hidden
     of ComponentType.unknown:
-      echo "ERROR: Unknown component type: ", json
+      error "ERROR: Unknown component type: ", json
 
     case component.kind
     of twitterListDetails:

src/experimental/parser/user.nim

@@ -58,11 +58,13 @@ proc toUser*(raw: RawUser): User =
     media: raw.mediaCount,
     verifiedType: raw.verifiedType,
     protected: raw.protected,
-    joinDate: parseTwitterDate(raw.createdAt),
     banner: getBanner(raw),
     userPic: getImageUrl(raw.profileImageUrlHttps).replace("_normal", "")
   )
 
+  if raw.createdAt.len > 0:
+    result.joinDate = parseTwitterDate(raw.createdAt)
+
   if raw.pinnedTweetIdsStr.len > 0:
     result.pinnedTweet = parseBiggestInt(raw.pinnedTweetIdsStr[0])
 
@@ -72,21 +74,3 @@ proc parseHook*(s: string; i: var int; v: var User) =
   var u: RawUser
   parseHook(s, i, u)
   v = toUser u
-
-proc parseUser*(json: string; username=""): User =
-  handleErrors:
-    case error.code
-    of suspended: return User(username: username, suspended: true)
-    of userNotFound: return
-    else: echo "[error - parseUser]: ", error
-
-  result = json.fromJson(User)
-
-proc parseUsers*(json: string; after=""): Result[User] =
-  result = Result[User](beginning: after.len == 0)
-
-  # starting with '{' means it's an error
-  if json[0] == '[':
-    let raw = json.fromJson(seq[RawUser])
-    for user in raw:
-      result.content.add user.toUser

src/experimental/types/graphuser.nim

@@ -1,15 +1,48 @@
-import options
+import options, strutils
-from ../../types import User
+from ../../types import User, VerifiedType
 
 type
   GraphUser* = object
-    data*: tuple[userResult: UserData]
+    data*: tuple[userResult: Option[UserData], user: Option[UserData]]
 
   UserData* = object
     result*: UserResult
 
-  UserResult = object
+  UserCore* = object
+    name*: string
+    screenName*: string
+    createdAt*: string
+
+  UserBio* = object
+    description*: string
+
+  UserAvatar* = object
+    imageUrl*: string
+
+  Verification* = object
+    verifiedType*: VerifiedType
+
+  Location* = object
+    location*: string
+
+  Privacy* = object
+    protected*: bool
+
+  UserResult* = object
     legacy*: User
     restId*: string
     isBlueVerified*: bool
+    core*: UserCore
+    avatar*: UserAvatar
     unavailableReason*: Option[string]
+    reason*: Option[string]
+    privacy*: Option[Privacy]
+    profileBio*: Option[UserBio]
+    verification*: Option[Verification]
+    location*: Option[Location]
+
+proc enumHook*(s: string; v: var VerifiedType) =
+  v = try:
+    parseEnum[VerifiedType](s)
+  except:
+    VerifiedType.none

src/experimental/types/session.nim

@@ -1,8 +1,8 @@
 type
   RawSession* = object
     kind*: string
-    username*: string
     id*: string
+    username*: string
     oauthToken*: string
     oauthTokenSecret*: string
     authToken*: string

src/experimental/types/tid.nim

@@ -0,0 +1,4 @@
+type
+  TidPair* = object
+    animationKey*: string
+    verification*: string

src/experimental/types/timeline.nim

@@ -1,23 +0,0 @@
-import std/tables
-from ../../types import User
-
-type
-  Search* = object
-    globalObjects*: GlobalObjects
-    timeline*: Timeline
-
-  GlobalObjects = object
-    users*: Table[string, User]
-
-  Timeline = object
-    instructions*: seq[Instructions]
-
-  Instructions = object
-    addEntries*: tuple[entries: seq[Entry]]
-
-  Entry = object
-    entryId*: string
-    content*: tuple[operation: Operation]
-
-  Operation = object
-    cursor*: tuple[value, cursorType: string]

src/experimental/types/unifiedcard.nim

@@ -1,4 +1,6 @@
-import std/[options, tables, times]
+
+import std/[options, tables, times, logging]
+
 import jsony
 from ../../types import VideoType, VideoVariant, User
 
@@ -22,6 +24,7 @@ type
     communityDetails
     mediaWithDetailsHorizontal
     hidden
+    grokShare
     unknown
 
   Component* = object
@@ -42,6 +45,7 @@ type
     topicDetail*: tuple[title: Text]
     profileUser*: User
     shortDescriptionText*: string
+    conversationPreview*: seq[GrokConversation]
 
   MediaItem* = object
     id*: string
@@ -76,6 +80,10 @@ type
     title*: Text
     category*: Text
 
+  GrokConversation* = object
+    message*: string
+    sender*: string
+
   TypeField = Component | Destination | MediaEntity | AppStoreData
 
 converter fromText*(text: Text): string = string(text)
@@ -96,21 +104,22 @@ proc enumHook*(s: string; v: var ComponentType) =
       of "community_details": communityDetails
       of "media_with_details_horizontal": mediaWithDetailsHorizontal
       of "commerce_drop_details": hidden
-      else: echo "ERROR: Unknown enum value (ComponentType): ", s; unknown
+      of "grok_share": grokShare
+      else: error "ERROR: Unknown enum value (ComponentType): ", s; unknown
 
 proc enumHook*(s: string; v: var AppType) =
   v = case s
       of "android_app": androidApp
       of "iphone_app": iPhoneApp
       of "ipad_app": iPadApp
-      else: echo "ERROR: Unknown enum value (AppType): ", s; androidApp
+      else: error "ERROR: Unknown enum value (AppType): ", s; androidApp
 
 proc enumHook*(s: string; v: var MediaType) =
   v = case s
       of "video": video
       of "photo": photo
       of "model3d": model3d
-      else: echo "ERROR: Unknown enum value (MediaType): ", s; photo
+      else: error "ERROR: Unknown enum value (MediaType): ", s; photo
 
 proc parseHook*(s: string; i: var int; v: var DateTime) =
   var str: string

src/formatters.nim

@@ -6,13 +6,18 @@ import types, utils, query
 const
   cards = "cards.twitter.com/cards"
   tco = "https://t.co"
-  twitter = parseUri("https://twitter.com")
+  twitter = parseUri("https://x.com")
 
 let
   twRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?twitter\.com"
   twLinkRegex = re"""<a href="https:\/\/twitter.com([^"]+)">twitter\.com(\S+)</a>"""
   xRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?x\.com"
   xLinkRegex = re"""<a href="https:\/\/x.com([^"]+)">x\.com(\S+)</a>"""
+  imgurRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(i\.)?imgur\.com"
+  imgurLinkRegex = re"""<a href="https://(i\.)?imgur.com([^"]+)">(i\.)?imgur\.com(\S+)</a>"""
+  fandomRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))([a-z0-9-]+)\.fandom\.com"
+  fandomLinkRegex = re"""<a href="https://([a-z0-9-]+)\.fandom\.com([^"]+)">([a-z0-9-]+)\.fandom\.com(\S+)</a>"""
+  soundcloudRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(on\.|www\.)?soundcloud\.com"
 
   ytRegex = re(r"([A-z.]+\.)?youtu(be\.com|\.be)", {reStudy, reIgnoreCase})
 
@@ -33,10 +38,13 @@ proc getUrlPrefix*(cfg: Config): string =
   if cfg.useHttps: https & cfg.hostname
   else: "http://" & cfg.hostname
 
-proc shortLink*(text: string; length=28): string =
+proc shorten*(text: string; length=28): string =
-  result = text.replace(wwwRegex, "")
+  result = text
   if result.len > length:
     result = result[0 ..< length] & "…"
+
+proc shortLink*(text: string; length=28): string =
+  result = text.replace(wwwRegex, "").shorten(length)
     
 proc stripHtml*(text: string; shorten=false): string =
   var html = parseHtml(text)
@@ -56,25 +64,44 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
   result = body
 
   if prefs.replaceYouTube.len > 0 and "youtu" in result:
-    result = result.replace(ytRegex, prefs.replaceYouTube)
+    let youtubeHost = strip(prefs.replaceYouTube, chars={'/'})
+    result = result.replace(ytRegex, youtubeHost)
 
   if prefs.replaceTwitter.len > 0:
+    let twitterHost = strip(prefs.replaceTwitter, chars={'/'})
     if tco in result:
-      result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
+      result = result.replace(tco, https & twitterHost & "/t.co")
     if "x.com" in result:
-      result = result.replace(xRegex, prefs.replaceTwitter)
+      result = result.replace(xRegex, twitterHost)
       result = result.replacef(xLinkRegex, a(
-        prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
+        twitterHost & "$2", href = https & twitterHost & "$1"))
     if "twitter.com" in result:
-      result = result.replace(cards, prefs.replaceTwitter & "/cards")
+      result = result.replace(cards, twitterHost & "/cards")
-      result = result.replace(twRegex, prefs.replaceTwitter)
+      result = result.replace(twRegex, twitterHost)
       result = result.replacef(twLinkRegex, a(
         prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
+    if "imgur.com" in result:
+      result = result.replace(imgurRegex, prefs.replaceImgur)
+      result = result.replacef(imgurLinkRegex, a(
+        prefs.replaceImgur & "$4", href = https & prefs.replaceImgur & "$2"))
+
+  if prefs.replaceFandom.len > 0 and "fandom.com" in result:
+    result = result.replace(fandomRegex, prefs.replaceFandom & "/$1")
+    result = result.replacef(fandomLinkRegex, a(
+      prefs.replaceFandom & "/$1$2", href = https & prefs.replaceFandom & "/$1$2"))
+
+  if prefs.replaceSoundCloud.len > 0 and "soundcloud.com" in result:
+    result = result.replace(soundcloudRegex, prefs.replaceSoundCloud)
+    result = result.replacef(re"""<a href="https://on\.soundcloud\.com([^"]+)">on\.soundcloud\.com(\S+)</a>""", a(
+      prefs.replaceSoundCloud & "/on$2", href = https & prefs.replaceSoundCloud & "/on$1"))
+    result = result.replacef(re"""<a href="https://(www\.)?soundcloud\.com([^"]+)">(www\.)?soundcloud\.com(\S+)</a>""", a(
+      prefs.replaceSoundCloud & "$4", href = https & prefs.replaceSoundCloud & "$2"))
 
   if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result):
-    result = result.replace(rdShortRegex, prefs.replaceReddit & "/comments/")
+    let redditHost = strip(prefs.replaceReddit, chars={'/'})
-    result = result.replace(rdRegex, prefs.replaceReddit)
+    result = result.replace(rdShortRegex, redditHost & "/comments/")
-    if prefs.replaceReddit in result and "/gallery/" in result:
+    result = result.replace(rdRegex, redditHost)
+    if redditHost in result and "/gallery/" in result:
       result = result.replace("/gallery/", "/comments/")
 
   if absolute.len > 0 and "href" in result:

src/nitter.nim

@@ -1,20 +1,48 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strformat, logging
+import asyncdispatch, strformat, logging, terminal, times, strutils
 from net import Port
 from htmlgen import a
 from os import getEnv
 
 import jester
 
-import types, config, prefs, formatters, redis_cache, http_pool, auth
+import types, config, prefs, formatters, redis_cache, http_pool, auth, apiutils
-import views/[general, about]
+import views/[general, about, search, homepage]
+import karax/[vdom]
 import routes/[
   preferences, timeline, status, media, search, rss, list, debug,
-  unsupported, embed, resolver, router_utils]
+  unsupported, embed, resolver, router_utils, follow]
 
 const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
 const issuesUrl = "https://github.com/zedeus/nitter/issues"
 
+type ColoredLogger = ref object of Logger
+
+method log(logger: ColoredLogger, level: Level, args: varargs[string, `$`]) =
+  if level < logger.levelThreshold: return
+  
+  let color = case level
+    of lvlFatal, lvlError: fgRed
+    of lvlWarn: fgYellow
+    of lvlInfo: fgGreen
+    of lvlDebug: fgCyan
+    else: fgWhite
+
+  let levelStr = case level
+    of lvlFatal: "fatal"
+    of lvlError: "error"
+    of lvlWarn: "warn"
+    of lvlInfo: "info"
+    of lvlDebug: "debug"
+    else: "other"
+    
+  let timeStr = format(now(), "HH:mm:ss")
+  stdout.styledWrite(fgWhite, "[", timeStr, "] ", color, levelStr, fgWhite, ": ")
+  for arg in args:
+    stdout.write(arg)
+  stdout.write("\n")
+  stdout.flushFile()
+
 let
   configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
   (cfg, fullCfg) = getConfig(configPath)
@@ -23,13 +51,21 @@ let
 
 initSessionPool(cfg, sessionsPath)
 
-if not cfg.enableDebug:
+addHandler(new(ColoredLogger))
-  # Silence Jester's query warning
-  addHandler(newConsoleLogger())
-  setLogFilter(lvlError)
 
-stdout.write &"Starting Nitter at {getUrlPrefix(cfg)}\n"
+let level = case cfg.logLevel.toLowerAscii
-stdout.flushFile
+  of "debug": lvlDebug
+  of "info": lvlInfo
+  of "warn": lvlWarn
+  of "error": lvlError
+  of "fatal": lvlFatal
+  of "none": lvlNone
+  else:
+    if cfg.enableDebug: lvlDebug else: lvlInfo
+
+setLogFilter(level)
+
+info &"Starting Nitter at {getUrlPrefix(cfg)}"
 
 updateDefaultPrefs(fullCfg)
 setCacheTimes(cfg)
@@ -37,13 +73,14 @@ setHmacKey(cfg.hmacKey)
 setProxyEncoding(cfg.base64Media)
 setMaxHttpConns(cfg.httpMaxConns)
 setHttpProxy(cfg.proxy, cfg.proxyAuth)
+setDisableTid(cfg.disableTid)
 initAboutPage(cfg.staticDir)
 
 waitFor initRedisPool(cfg)
-stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n"
+info &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}"
-stdout.flushFile
 
 createUnsupportedRouter(cfg)
+createFollowRouter(cfg)
 createResolverRouter(cfg)
 createPrefRouter(cfg)
 createTimelineRouter(cfg)
@@ -63,7 +100,41 @@ settings:
 
 routes:
   get "/":
-    resp renderMain(renderSearch(), request, cfg, themePrefs())
+    let prefs = cookiePrefs()
+    
+    if prefs.following.len > 0 or cfg.defaultFollowedAccounts.len > 0:
+      let
+        cursor = getCursor()
+        accounts = if prefs.following.len > 0: prefs.following else: cfg.defaultFollowedAccounts
+        isDefault = prefs.following.len == 0
+        currentPath = if @"f".len > 0: "/?f=" & @"f" else: "/"
+      
+      var homepageQuery = initQuery(params(request))
+      if @"f".len == 0:
+        homepageQuery.kind = tweets
+      
+      homepageQuery.fromUser = accounts
+      
+      let
+        timeline = await getGraphTweetSearch(homepageQuery, cursor)
+        html = if isDefault: 
+          renderDefaultTimeline(timeline, prefs, getPath())
+        else:
+          renderHomepageTimeline(timeline, prefs, getPath())
+      
+      var users: seq[User]
+      for username in accounts:
+        try:
+          let user = await getCachedUser(username)
+          users.add(user)
+        except:
+          continue
+      
+      let homepageHtml = renderHomepage(users, html, prefs, currentPath)
+      
+      resp renderMain(homepageHtml, request, cfg, prefs, "Following")
+    else:
+      resp renderMain(renderSearch(), request, cfg, themePrefs())
 
   get "/about":
     resp renderMain(renderAbout(), request, cfg, themePrefs())
@@ -83,13 +154,13 @@ routes:
     resp Http404, showError("Page not found", cfg)
 
   error InternalError:
-    echo error.exc.name, ": ", error.exc.msg
+    error error.exc.name, ": ", error.exc.msg
     const link = a("open a GitHub issue", href = issuesUrl)
     resp Http500, showError(
       &"An error occurred, please {link} with the URL you tried to visit.", cfg)
 
   error BadClientError:
-    echo error.exc.name, ": ", error.exc.msg
+    error error.exc.name, ": ", error.exc.msg
     resp Http500, showError("Network error occurred, please try again.", cfg)
 
   error RateLimitError:
@@ -111,5 +182,6 @@ routes:
   extend preferences, ""
   extend resolver, ""
   extend embed, ""
+  extend follow, ""
   extend debug, ""
   extend unsupported, ""

src/parser.nim

@@ -1,10 +1,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, options, times, math
+import strutils, options, times, math, tables
 import packedjson, packedjson/deserialiser
 import types, parserutils, utils
 import experimental/parser/unifiedcard
 
-proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet
+proc parseGraphTweet(js: JsonNode): Tweet
 
 proc parseUser(js: JsonNode; id=""): User =
   if js.isNull: return
@@ -21,11 +21,17 @@ proc parseUser(js: JsonNode; id=""): User =
     tweets: js{"statuses_count"}.getInt,
     likes: js{"favourites_count"}.getInt,
     media: js{"media_count"}.getInt,
-    verifiedType: parseEnum[VerifiedType](js{"verified_type"}.getStr("None")),
+    protected: js{"protected"}.getBool(js{"privacy", "protected"}.getBool),
-    protected: js{"protected"}.getBool,
+    sensitive: "sensitive" in js{"profile_interstitial_type"}.getStr("") or js{"possibly_sensitive"}.getBool,
     joinDate: js{"created_at"}.getTime
   )
 
+  if js{"is_blue_verified"}.getBool(false):
+    result.verifiedType = blue
+
+  with verifiedType, js{"verified_type"}:
+    result.verifiedType = parseEnum[VerifiedType](verifiedType.getStr)
+
   result.expandUserEntities(js)
 
 proc parseGraphUser(js: JsonNode): User =
@@ -41,17 +47,20 @@ proc parseGraphUser(js: JsonNode): User =
 
   result = parseUser(user{"legacy"}, user{"rest_id"}.getStr)
 
+  if result.verifiedType == none and user{"is_blue_verified"}.getBool(false):
+    result.verifiedType = blue
+
   # fallback to support UserMedia/recent GraphQL updates
-  if result.username.len == 0 and user{"core", "screen_name"}.notNull:
+  if result.username.len == 0:
     result.username = user{"core", "screen_name"}.getStr
     result.fullname = user{"core", "name"}.getStr
     result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
 
     if user{"is_blue_verified"}.getBool(false):
       result.verifiedType = blue
-    elif user{"verification", "verified_type"}.notNull:
+
-      let verifiedType = user{"verification", "verified_type"}.getStr("None")
+    with verifiedType, user{"verification", "verified_type"}:
-      result.verifiedType = parseEnum[VerifiedType](verifiedType)
+      result.verifiedType = parseEnum[VerifiedType](verifiedType.getStr)
 
 proc parseGraphList*(js: JsonNode): List =
   if js.isNull: return
@@ -90,16 +99,24 @@ proc parsePoll(js: JsonNode): Poll =
   result.leader = result.values.find(max(result.values))
   result.votes = result.values.sum
 
-proc parseGif(js: JsonNode): Gif =
+proc parseVideoVariants(variants: JsonNode): seq[VideoVariant] =
-  result = Gif(
+  result = @[]
-    url: js{"video_info", "variants"}[0]{"url"}.getImageStr,
+  for v in variants:
-    thumb: js{"media_url_https"}.getImageStr
+    let
-  )
+      url = v{"url"}.getStr
+      contentType = parseEnum[VideoType](v{"content_type"}.getStr("video/mp4"))
+      bitrate = v{"bit_rate"}.getInt(v{"bitrate"}.getInt(0))
+
+    result.add VideoVariant(
+      contentType: contentType,
+      bitrate: bitrate,
+      url: url,
+      resolution: if contentType == mp4: getMp4Resolution(url) else: 0
+    )
 
 proc parseVideo(js: JsonNode): Video =
   result = Video(
     thumb: js{"media_url_https"}.getImageStr,
-    views: getVideoViewCount(js),
     available: true,
     title: js{"ext_alt_text"}.getStr,
     durationMs: js{"video_info", "duration_millis"}.getInt
@@ -116,17 +133,62 @@ proc parseVideo(js: JsonNode): Video =
   with description, js{"additional_media_info", "description"}:
     result.description = description.getStr
 
-  for v in js{"video_info", "variants"}:
+  result.variants = parseVideoVariants(js{"video_info", "variants"})
-    let
-      contentType = parseEnum[VideoType](v{"content_type"}.getStr("summary"))
-      url = v{"url"}.getStr
 
-    result.variants.add VideoVariant(
+proc parseLegacyMediaEntities(js: JsonNode; result: var Tweet) =
-      contentType: contentType,
+  with jsMedia, js{"extended_entities", "media"}:
-      bitrate: v{"bitrate"}.getInt,
+    for m in jsMedia:
-      url: url,
+      case m.getTypeName:
-      resolution: if contentType == mp4: getMp4Resolution(url) else: 0
+      of "photo":
-    )
+        result.photos.add m{"media_url_https"}.getImageStr
+      of "video":
+        result.video = some(parseVideo(m))
+        with user, m{"additional_media_info", "source_user"}:
+          if user{"id"}.getInt > 0:
+            result.attribution = some(parseUser(user))
+          else:
+            result.attribution = some(parseGraphUser(user))
+      of "animated_gif":
+        result.gif = some Gif(
+          url: m{"video_info", "variants"}[0]{"url"}.getImageStr,
+          thumb: m{"media_url_https"}.getImageStr
+        )
+      else: discard
+
+      with url, m{"url"}:
+        if result.text.endsWith(url.getStr):
+          result.text.removeSuffix(url.getStr)
+          result.text = result.text.strip()
+
+proc parseMediaEntities(js: JsonNode; result: var Tweet) =
+  with mediaEntities, js{"media_entities"}:
+    for mediaEntity in mediaEntities:
+      with mediaInfo, mediaEntity{"media_results", "result", "media_info"}:
+        case mediaInfo.getTypeName
+        of "ApiImage":
+          result.photos.add mediaInfo{"original_img_url"}.getImageStr
+        of "ApiVideo":
+          let status = mediaEntity{"media_results", "result", "media_availability_v2", "status"}
+          result.video = some Video(
+            available: status.getStr == "Available",
+            thumb: mediaInfo{"preview_image", "original_img_url"}.getImageStr,
+            durationMs: mediaInfo{"duration_millis"}.getInt,
+            variants: parseVideoVariants(mediaInfo{"variants"})
+          )
+        of "ApiGif":
+          result.gif = some Gif(
+            url: mediaInfo{"variants"}[0]{"url"}.getImageStr,
+            thumb: mediaInfo{"preview_image", "original_img_url"}.getImageStr
+          )
+        else: discard
+
+  # Remove media URLs from text
+  with mediaList, js{"legacy", "entities", "media"}:
+    for url in mediaList:
+      let expandedUrl = url.getExpandedUrl
+      if result.text.endsWith(expandedUrl):
+        result.text.removeSuffix(expandedUrl)
+        result.text = result.text.strip()
 
 proc parsePromoVideo(js: JsonNode): Video =
   result = Video(
@@ -206,7 +268,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
 
   for u in ? urls:
     if u{"url"}.getStr == result.url:
-      result.url = u{"expanded_url"}.getStr
+      result.url = u.getExpandedUrl(result.url)
       break
 
   if kind in {videoDirectMessage, imageDirectMessage}:
@@ -218,20 +280,26 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
 
 proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
   if js.isNull: return
+
+  let time =
+    if js{"created_at"}.notNull: js{"created_at"}.getTime
+    else: js{"created_at_ms"}.getTimeFromMs
+
   result = Tweet(
     id: js{"id_str"}.getId,
     threadId: js{"conversation_id_str"}.getId,
     replyId: js{"in_reply_to_status_id_str"}.getId,
     text: js{"full_text"}.getStr,
-    time: js{"created_at"}.getTime,
+    time: time,
     hasThread: js{"self_thread"}.notNull,
+    sensitive: js{"possibly_sensitive"}.getBool,
     available: true,
     user: User(id: js{"user_id_str"}.getStr),
     stats: TweetStats(
       replies: js{"reply_count"}.getInt,
       retweets: js{"retweet_count"}.getInt,
       likes: js{"favorite_count"}.getInt,
-      quotes: js{"quote_count"}.getInt
+      views: js{"views_count"}.getInt
     )
   )
 
@@ -256,6 +324,12 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
       result.retweet = some parseGraphTweet(rt)
       return
 
+  with reposts, js{"repostedStatusResults"}:
+    with rt, reposts{"result"}:
+      if "legacy" in rt:
+        result.retweet = some parseGraphTweet(rt)
+        return
+
   if jsCard.kind != JNull:
     let name = jsCard{"name"}.getStr
     if "poll" in name:
@@ -269,27 +343,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
       result.card = some parseCard(jsCard, js{"entities", "urls"})
 
   result.expandTweetEntities(js)
-
+  parseLegacyMediaEntities(js, result)
-  with jsMedia, js{"extended_entities", "media"}:
-    for m in jsMedia:
-      case m{"type"}.getStr
-      of "photo":
-        result.photos.add m{"media_url_https"}.getImageStr
-      of "video":
-        result.video = some(parseVideo(m))
-        with user, m{"additional_media_info", "source_user"}:
-          if user{"id"}.getInt > 0:
-            result.attribution = some(parseUser(user))
-          else:
-            result.attribution = some(parseGraphUser(user))
-      of "animated_gif":
-        result.gif = some(parseGif(m))
-      else: discard
-
-      with url, m{"url"}:
-        if result.text.endsWith(url.getStr):
-          result.text.removeSuffix(url.getStr)
-          result.text = result.text.strip()
 
   with jsWithheld, js{"withheld_in_countries"}:
     let withheldInCountries: seq[string] =
@@ -305,91 +359,108 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
       result.text.removeSuffix(" Learn more.")
       result.available = false
 
-proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
+proc parseGraphTweet(js: JsonNode): Tweet =
   if js.kind == JNull:
     return Tweet()
 
-  case js{"__typename"}.getStr
+  case js.getTypeName:
   of "TweetUnavailable":
     return Tweet()
   of "TweetTombstone":
-    with text, js{"tombstone", "richText"}:
+    with text, select(js{"tombstone", "richText"}, js{"tombstone", "text"}):
-      return Tweet(text: text.getTombstone)
-    with text, js{"tombstone", "text"}:
       return Tweet(text: text.getTombstone)
     return Tweet()
   of "TweetPreviewDisplay":
     return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.")
   of "TweetWithVisibilityResults":
-    return parseGraphTweet(js{"tweet"}, isLegacy)
+    return parseGraphTweet(js{"tweet"})
   else:
     discard
 
   if not js.hasKey("legacy"):
     return Tweet()
 
-  var jsCard = copy(js{if isLegacy: "card" else: "tweet_card", "legacy"})
+  var jsCard = select(js{"card"}, js{"tweet_card"}, js{"legacy", "tweet_card"})
   if jsCard.kind != JNull:
-    var values = newJObject()
+    let legacyCard = jsCard{"legacy"}
-    for val in jsCard["binding_values"]:
+    if legacyCard.kind != JNull:
-      values[val["key"].getStr] = val["value"]
+      let bindingArray = legacyCard{"binding_values"}
-    jsCard["binding_values"] = values
+      if bindingArray.kind == JArray:
+        var bindingObj: seq[(string, JsonNode)]
+        for item in bindingArray:
+          bindingObj.add((item{"key"}.getStr, item{"value"}))
+        # Create a new card object with flattened structure
+        jsCard = %*{
+          "name": legacyCard{"name"},
+          "url": legacyCard{"url"},
+          "binding_values": %bindingObj
+        }
 
   result = parseTweet(js{"legacy"}, jsCard)
   result.id = js{"rest_id"}.getId
   result.user = parseGraphUser(js{"core"})
 
+  if result.replyId == 0:
+    result.replyId = js{"reply_to_results", "rest_id"}.getId
+
+  with count, js{"views", "count"}:
+    result.stats.views = count.getStr("0").parseInt
+
   with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
     result.expandNoteTweetEntities(noteTweet)
 
+  parseMediaEntities(js, result)
+
   if result.quote.isSome:
-    result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}, isLegacy))
+    result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}))
+
+  with quoted, js{"quotedPostResults", "result"}:
+    result.quote = some(parseGraphTweet(quoted))
 
 proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
-  for t in js{"content", "items"}:
+  for t in ? js{"content", "items"}:
-    let entryId = t{"entryId"}.getStr
+    let entryId = t.getEntryId
     if "cursor-showmore" in entryId:
       let cursor = t{"item", "content", "value"}
       result.thread.cursor = cursor.getStr
       result.thread.hasMore = true
     elif "tweet" in entryId and "promoted" notin entryId:
-      let
+      with tweet, t.getTweetResult("item"):
-        isLegacy = t{"item"}.hasKey("itemContent")
+        result.thread.content.add parseGraphTweet(tweet)
-        (contentKey, resultKey) = if isLegacy: ("itemContent", "tweet_results")
-                                  else: ("content", "tweetResult")
 
-      with content, t{"item", contentKey}:
+        let tweetDisplayType = select(
-        result.thread.content.add parseGraphTweet(content{resultKey, "result"}, isLegacy)
+          t{"item", "content", "tweet_display_type"},
-
+          t{"item", "itemContent", "tweetDisplayType"}
-        if content{"tweetDisplayType"}.getStr == "SelfThread":
+        )
+        if tweetDisplayType.getStr == "SelfThread":
           result.self = true
 
 proc parseGraphTweetResult*(js: JsonNode): Tweet =
   with tweet, js{"data", "tweet_result", "result"}:
-    result = parseGraphTweet(tweet, false)
+    result = parseGraphTweet(tweet)
 
-proc parseGraphConversation*(js: JsonNode; tweetId: string; v2=true): Conversation =
+proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
   result = Conversation(replies: Result[Chain](beginning: true))
 
-  let
+  let instructions = ? select(
-    rootKey = if v2: "timeline_response" else: "threaded_conversation_with_injections_v2"
+    js{"data", "timelineResponse", "instructions"},
-    contentKey = if v2: "content" else: "itemContent"
+    js{"data", "timeline_response", "instructions"},
-    resultKey = if v2: "tweetResult" else: "tweet_results"
+    js{"data", "threaded_conversation_with_injections_v2", "instructions"}
-
+  )
-  let instructions = ? js{"data", rootKey, "instructions"}
   if instructions.len == 0:
     return
 
   for i in instructions:
-    if i{"__typename"}.getStr == "TimelineAddEntries":
+    if i.getTypeName == "TimelineAddEntries":
       for e in i{"entries"}:
-        let entryId = e{"entryId"}.getStr
+        let entryId = e.getEntryId
         if entryId.startsWith("tweet"):
-          with tweetResult, e{"content", contentKey, resultKey, "result"}:
+          let tweetResult = getTweetResult(e)
-            let tweet = parseGraphTweet(tweetResult, not v2)
+          if tweetResult.notNull:
+            let tweet = parseGraphTweet(tweetResult)
 
             if not tweet.available:
-              tweet.id = parseBiggestInt(entryId.getId())
+              tweet.id = entryId.getId
 
             if $tweet.id == tweetId:
               result.tweet = tweet
@@ -402,66 +473,67 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string; v2=true): Conversati
           elif thread.content.len > 0:
             result.replies.content.add thread
         elif entryId.startsWith("tombstone"):
-          let id = entryId.getId()
+          let
-          let tweet = Tweet(
+            content = select(e{"content", "content"}, e{"content", "itemContent"})
-            id: parseBiggestInt(id),
+            tweet = Tweet(
-            available: false,
+              id: entryId.getId,
-            text: e{"content", contentKey, "tombstoneInfo", "richText"}.getTombstone
+              available: false,
-          )
+              text: content{"tombstoneInfo", "richText"}.getTombstone
+            )
 
-          if id == tweetId:
+          if $tweet.id == tweetId:
             result.tweet = tweet
           else:
             result.before.content.add tweet
         elif entryId.startsWith("cursor-bottom"):
-          result.replies.bottom = e{"content", contentKey, "value"}.getStr
+          var cursorValue = select(
+            e{"content", "value"},
+            e{"content", "content", "value"},
+            e{"content", "itemContent", "value"}
+          )
+          result.replies.bottom = cursorValue.getStr
 
-proc extractTweetsFromEntry*(e: JsonNode; entryId: string): seq[Tweet] =
+proc extractTweetsFromEntry*(e: JsonNode): seq[Tweet] =
-  if e{"content", "items"}.notNull:
+  with tweetResult, getTweetResult(e):
-    for item in e{"content", "items"}:
+    var tweet = parseGraphTweet(tweetResult)
-      with tweetResult, item{"item", "itemContent", "tweet_results", "result"}:
+    if not tweet.available:
-        var tweet = parseGraphTweet(tweetResult, false)
+      tweet.id = e.getEntryId.getId
-        if not tweet.available:
+    result.add tweet
-          tweet.id = parseBiggestInt(item{"entryId"}.getStr.getId())
-        result.add tweet
     return
 
-  with tweetResult, e{"content", "content", "tweetResult", "result"}:
+  for item in e{"content", "items"}:
-    var tweet = parseGraphTweet(tweetResult, false)
+    with tweetResult, item.getTweetResult("item"):
-    if not tweet.available:
+      var tweet = parseGraphTweet(tweetResult)
-      tweet.id = parseBiggestInt(entryId.getId())
+      if not tweet.available:
-    result.add tweet
+        tweet.id = item.getEntryId.getId
+      result.add tweet
 
 proc parseGraphTimeline*(js: JsonNode; after=""): Profile =
   result = Profile(tweets: Timeline(beginning: after.len == 0))
 
-  let instructions =
+  let instructions = ? select(
-    if js{"data", "list"}.notNull:
+    js{"data", "list", "timeline_response", "timeline", "instructions"},
-      ? js{"data", "list", "timeline_response", "timeline", "instructions"}
+    js{"data", "user", "result", "timeline", "timeline", "instructions"},
-    elif js{"data", "user"}.notNull:
+    js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
-      ? js{"data", "user", "result", "timeline", "timeline", "instructions"}
+  )
-    else:
-      ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
-
   if instructions.len == 0:
     return
 
   for i in instructions:
-    # TimelineAddToModule instruction is used by UserMedia
     if i{"moduleItems"}.notNull:
       for item in i{"moduleItems"}:
-        with tweetResult, item{"item", "itemContent", "tweet_results", "result"}:
+        with tweetResult, item.getTweetResult("item"):
-          let tweet = parseGraphTweet(tweetResult, false)
+          let tweet = parseGraphTweet(tweetResult)
           if not tweet.available:
-            tweet.id = parseBiggestInt(item{"entryId"}.getStr.getId())
+            tweet.id = item.getEntryId.getId
           result.tweets.content.add tweet
       continue
 
     if i{"entries"}.notNull:
       for e in i{"entries"}:
-        let entryId = e{"entryId"}.getStr
+        let entryId = e.getEntryId
         if entryId.startsWith("tweet") or entryId.startsWith("profile-grid"):
-          for tweet in extractTweetsFromEntry(e, entryId):
+          for tweet in extractTweetsFromEntry(e):
             result.tweets.content.add tweet
         elif "-conversation-" in entryId or entryId.startsWith("homeConversation"):
           let (thread, self) = parseGraphThread(e)
@@ -469,36 +541,31 @@ proc parseGraphTimeline*(js: JsonNode; after=""): Profile =
         elif entryId.startsWith("cursor-bottom"):
           result.tweets.bottom = e{"content", "value"}.getStr
 
-    if after.len == 0 and i{"__typename"}.getStr == "TimelinePinEntry":
+    if after.len == 0:
-      with tweetResult, i{"entry", "content", "content", "tweetResult", "result"}:
+      if i.getTypeName == "TimelinePinEntry":
-        let tweet = parseGraphTweet(tweetResult, false)
+        let tweets = extractTweetsFromEntry(i{"entry"})
-        tweet.pinned = true
+        if tweets.len > 0:
-        if not tweet.available and tweet.tombstone.len == 0:
+          var tweet = tweets[0]
-          let entryId = i{"entry", "entryId"}.getEntryId
+          tweet.pinned = true
-          if entryId.len > 0:
+          result.pinned = some tweet
-            tweet.id = parseBiggestInt(entryId)
-        result.pinned = some tweet
 
 proc parseGraphPhotoRail*(js: JsonNode): PhotoRail =
   result = @[]
 
-  let instructions =
+  let instructions = select(
-    if js{"data", "user"}.notNull:
+    js{"data", "user", "result", "timeline", "timeline", "instructions"},
-      ? js{"data", "user", "result", "timeline", "timeline", "instructions"}
+    js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
-    else:
+  )
-      ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
-
   if instructions.len == 0:
     return
 
   for i in instructions:
-    # TimelineAddToModule instruction is used by MediaTimelineV2
     if i{"moduleItems"}.notNull:
       for item in i{"moduleItems"}:
-        with tweetResult, item{"item", "itemContent", "tweet_results", "result"}:
+        with tweetResult, item.getTweetResult("item"):
-          let t = parseGraphTweet(tweetResult, false)
+          let t = parseGraphTweet(tweetResult)
           if not t.available:
-            t.id = parseBiggestInt(item{"entryId"}.getStr.getId())
+            t.id = item.getEntryId.getId
 
           let photo = extractGalleryPhoto(t)
           if photo.url.len > 0:
@@ -508,14 +575,13 @@ proc parseGraphPhotoRail*(js: JsonNode): PhotoRail =
             return
       continue
 
-    let instrType = i{"type"}.getStr(i{"__typename"}.getStr)
+    if i.getTypeName != "TimelineAddEntries":
-    if instrType != "TimelineAddEntries":
       continue
 
     for e in i{"entries"}:
-      let entryId = e{"entryId"}.getStr
+      let entryId = e.getEntryId
       if entryId.startsWith("tweet") or entryId.startsWith("profile-grid"):
-        for t in extractTweetsFromEntry(e, entryId):
+        for t in extractTweetsFromEntry(e):
           let photo = extractGalleryPhoto(t)
           if photo.url.len > 0:
             result.add photo
@@ -526,21 +592,24 @@ proc parseGraphPhotoRail*(js: JsonNode): PhotoRail =
 proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] =
   result = Result[T](beginning: after.len == 0)
 
-  let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
+  let instructions = select(
+    js{"data", "search", "timeline_response", "timeline", "instructions"},
+    js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
+  )
   if instructions.len == 0:
     return
 
   for instruction in instructions:
-    let typ = instruction{"type"}.getStr
+    let typ = getTypeName(instruction)
     if typ == "TimelineAddEntries":
       for e in instruction{"entries"}:
-        let entryId = e{"entryId"}.getStr
+        let entryId = e.getEntryId
         when T is Tweets:
           if entryId.startsWith("tweet"):
-            with tweetRes, e{"content", "itemContent", "tweet_results", "result"}:
+            with tweetRes, getTweetResult(e):
               let tweet = parseGraphTweet(tweetRes)
               if not tweet.available:
-                tweet.id = parseBiggestInt(entryId.getId())
+                tweet.id = entryId.getId
               result.content.add tweet
         elif T is User:
           if entryId.startsWith("user"):

src/parserutils.nim

@@ -36,6 +36,12 @@ template `?`*(js: JsonNode): untyped =
   if j.isNull: return
   j
 
+template select*(a, b: JsonNode): untyped =
+  if a.notNull: a else: b
+
+template select*(a, b, c: JsonNode): untyped =
+  if a.notNull: a elif b.notNull: b else: c
+
 template with*(ident, value, body): untyped =
   if true:
     let ident {.inject.} = value
@@ -44,8 +50,7 @@ template with*(ident, value, body): untyped =
 template with*(ident; value: JsonNode; body): untyped =
   if true:
     let ident {.inject.} = value
-    # value.notNull causes a compilation error for versions < 1.6.14
+    if value.notNull: body
-    if notNull(value): body
 
 template getCursor*(js: JsonNode): string =
   js{"content", "operation", "cursor", "value"}.getStr
@@ -54,6 +59,20 @@ template getError*(js: JsonNode): Error =
   if js.kind != JArray or js.len == 0: null
   else: Error(js[0]{"code"}.getInt)
 
+proc getTweetResult*(js: JsonNode; root="content"): JsonNode =
+  select(
+    js{root, "content", "tweet_results", "result"},
+    js{root, "itemContent", "tweet_results", "result"},
+    js{root, "content", "tweetResult", "result"}
+  )
+
+template getTypeName*(js: JsonNode): string =
+  js{"__typename"}.getStr(js{"type"}.getStr)
+
+template getEntryId*(e: JsonNode): string =
+  e{"entryId"}.getStr(e{"entry_id"}.getStr)
+
+
 template parseTime(time: string; f: static string; flen: int): DateTime =
   if time.len != flen: return
   parse(time, f, utc())
@@ -64,29 +83,24 @@ proc getDateTime*(js: JsonNode): DateTime =
 proc getTime*(js: JsonNode): DateTime =
   parseTime(js.getStr, "ddd MMM dd hh:mm:ss \'+0000\' yyyy", 30)
 
-proc getId*(id: string): string {.inline.} =
+proc getTimeFromMs*(js: JsonNode): DateTime =
+  let ms = js.getInt(0)
+  if ms == 0: return
+  let seconds = ms div 1000
+  return fromUnix(seconds).utc()
+
+proc getId*(id: string): int64 {.inline.} =
   let start = id.rfind("-")
-  if start < 0: return id
+  if start < 0:
-  id[start + 1 ..< id.len]
+    return parseBiggestInt(id)
+  return parseBiggestInt(id[start + 1 ..< id.len])
 
 proc getId*(js: JsonNode): int64 {.inline.} =
   case js.kind
-  of JString: return parseBiggestInt(js.getStr("0"))
+  of JString: return js.getStr("0").getId
   of JInt: return js.getBiggestInt()
   else: return 0
 
-proc getEntryId*(js: JsonNode): string {.inline.} =
-  let entry = js{"entryId"}.getStr
-  if entry.len == 0: return
-
-  if "tweet" in entry or "sq-I-t" in entry:
-    return entry.getId
-  elif "tombstone" in entry:
-    return js{"content", "item", "content", "tombstone", "tweet", "id"}.getStr
-  else:
-    echo "unknown entry: ", entry
-    return
-
 template getStrVal*(js: JsonNode; default=""): string =
   js{"string_value"}.getStr(default)
 
@@ -98,6 +112,9 @@ proc getImageStr*(js: JsonNode): string =
 template getImageVal*(js: JsonNode): string =
   js{"image_value", "url"}.getImageStr
 
+template getExpandedUrl*(js: JsonNode; fallback=""): string =
+  js{"expanded_url"}.getStr(js{"url"}.getStr(fallback))
+
 proc getCardUrl*(js: JsonNode; kind: CardKind): string =
   result = js{"website_url"}.getStrVal
   if kind == promoVideoConvo:
@@ -157,19 +174,13 @@ proc getMp4Resolution*(url: string): int =
     # cannot determine resolution (e.g. m3u8/non-mp4 video)
     return 0
 
-proc getVideoViewCount*(js: JsonNode): string =
-  with stats, js{"ext_media_stats"}:
-    return stats{"view_count"}.getStr($stats{"viewCount"}.getInt)
-
-  return $js{"mediaStats", "viewCount"}.getInt(0)
-
 proc extractSlice(js: JsonNode): Slice[int] =
   result = js["indices"][0].getInt ..< js["indices"][1].getInt
 
 proc extractUrls(result: var seq[ReplaceSlice]; js: JsonNode;
                  textLen: int; hideTwitter = false) =
   let
-    url = js["expanded_url"].getStr
+    url = js.getExpandedUrl
     slice = js.extractSlice
 
   if hideTwitter and slice.b.succ >= textLen and url.isTwitterUrl:
@@ -230,7 +241,7 @@ proc expandUserEntities*(user: var User; js: JsonNode) =
     ent = ? js{"entities"}
 
   with urls, ent{"url", "urls"}:
-    user.website = urls[0]{"expanded_url"}.getStr
+    user.website = urls[0].getExpandedUrl
 
   var replacements = newSeq[ReplaceSlice]()
 
@@ -260,7 +271,7 @@ proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlic
       replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink)
 
       if hasCard and u{"url"}.getStr == get(tweet.card).url:
-        get(tweet.card).url = u{"expanded_url"}.getStr
+        get(tweet.card).url = u.getExpandedUrl
 
   with media, entities{"media"}:
     for m in media:

src/prefs.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import tables
+import tables, strutils, sequtils
 import types, prefs_impl
 from config import get
 from parsecfg import nil
@@ -13,6 +13,8 @@ proc updateDefaultPrefs*(cfg: parsecfg.Config) =
 
 proc getPrefs*(cookies: Table[string, string]): Prefs =
   result = defaultPrefs
+  if "nitter_following" in cookies:
+    result.following = cookies["nitter_following"].split(',').filterIt(it.len > 0)
   genCookiePrefs(cookies)
 
 template getPref*(cookies: Table[string, string], pref): untyped =

src/prefs_impl.nim

@@ -54,6 +54,10 @@ genPrefs:
     theme(select, "Nitter"):
       "Theme"
 
+    verifiedBadge(select, "Show all"):
+      "Verified badges"
+      options: @["Show all", "Show official only", "Hide all"]
+
     infiniteScroll(checkbox, false):
       "Infinite scrolling (experimental, requires JavaScript)"
 
@@ -78,6 +82,9 @@ genPrefs:
     squareAvatars(checkbox, false):
       "Square profile pictures"
 
+    hideNsfw(checkbox, true):
+      "Hide NSFW content"
+
   Media:
     mp4Playback(checkbox, true):
       "Enable mp4 video playback (only for gifs)"
@@ -107,6 +114,18 @@ genPrefs:
       "Reddit -> Teddit/Libreddit"
       placeholder: "Teddit hostname"
 
+    replaceImgur(input, ""):
+      "Imgur -> Rimgo"
+      placeholder: "Rimgo hostname"
+
+    replaceFandom(input, ""):
+      "Fandom -> Phantom"
+      placeholder: "Phantom hostname"
+
+    replaceSoundCloud(input, ""):
+      "SoundCloud -> SoundCloak"
+      placeholder: "SoundCloak hostname"
+
 iterator allPrefs*(): Pref =
   for k, v in prefList:
     for pref in v:
@@ -206,6 +225,7 @@ macro genPrefsType*(): untyped =
   let name = nnkPostfix.newTree(ident("*"), ident("Prefs"))
   result = quote do:
     type `name` = object
+      following*: seq[string]
       discard
 
   for pref in allPrefs():

src/query.nim

@@ -6,10 +6,9 @@ import types
 const
   validFilters* = @[
     "media", "images", "twimg", "videos",
-    "native_video", "consumer_video", "pro_video",
+    "native_video", "consumer_video", "spaces",
     "links", "news", "quote", "mentions",
-    "replies", "retweets", "nativeretweets",
+    "replies", "retweets", "nativeretweets"
-    "verified", "safe"
   ]
 
   emptyQuery* = "include:nativeretweets"
@@ -18,6 +17,11 @@ template `@`(param: string): untyped =
   if param in pms: pms[param]
   else: ""
 
+proc validateNumber(value: string): string =
+  if value.anyIt(not it.isDigit):
+    return ""
+  return value
+
 proc initQuery*(pms: Table[string, string]; name=""): Query =
   result = Query(
     kind: parseEnum[QueryKind](@"f", tweets),
@@ -26,7 +30,8 @@ proc initQuery*(pms: Table[string, string]; name=""): Query =
     excludes: validFilters.filterIt("e-" & it in pms),
     since: @"since",
     until: @"until",
-    near: @"near"
+    near: @"near",
+    minLikes: validateNumber(@"min_faves")
   )
 
   if name.len > 0:
@@ -59,8 +64,11 @@ proc genQueryParam*(query: Query): string =
     if i < query.fromUser.high:
       param &= "OR "
 
-  if query.fromUser.len > 0 and query.kind in {posts, media}:
+  if query.fromUser.len > 0:
-    param &= "filter:self_threads OR -filter:replies "
+    if query.kind in {posts, media}:
+      param &= "filter:self_threads OR -filter:replies "
+    elif query.kind == tweets and query.fromUser.len > 1:
+      param &= "filter:self_threads OR -filter:replies "
 
   if "nativeretweets" notin query.excludes:
     param &= "include:nativeretweets "
@@ -79,7 +87,9 @@ proc genQueryParam*(query: Query): string =
   if query.until.len > 0:
     result &= " until:" & query.until
   if query.near.len > 0:
-    result &= &" near:\"{query.near}\" within:15mi"
+    result &= " near:\"" & query.near & "\""
+  if query.minLikes.len > 0:
+    result &= " min_faves:" & query.minLikes
   if query.text.len > 0:
     if result.len > 0:
       result &= " " & query.text
@@ -105,6 +115,8 @@ proc genQueryUrl*(query: Query): string =
     params.add "until=" & query.until
   if query.near.len > 0:
     params.add "near=" & query.near
+  if query.minLikes.len > 0:
+    params.add "min_faves=" & query.minLikes
 
   if params.len > 0:
     result &= params.join("&")

src/redis_cache.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, times, strformat, strutils, tables, hashes
+import asyncdispatch, times, strformat, strutils, tables, hashes, logging
 import redis, redpool, flatty, supersnappy
 
 import types, api
@@ -59,8 +59,7 @@ proc initRedisPool*(cfg: Config) {.async.} =
       await r.configSet("hash-max-ziplist-entries", "1000")
 
   except OSError:
-    stdout.write "Failed to connect to Redis.\n"
+    fatal "Failed to connect to Redis."
-    stdout.flushFile
     quit(1)
 
 template uidKey(name: string): string = "pid:" & $(hash(name) div 1_000_000)
@@ -112,7 +111,7 @@ template deserialize(data, T) =
   try:
     result = fromFlatty(uncompress(data), T)
   except:
-    echo "Decompression failed($#): '$#'" % [astToStr(T), data]
+    error "Decompression failed($#): '$#'" % [astToStr(T), data]
 
 proc getUserId*(username: string): Future[string] {.async.} =
   let name = toLower(username)
@@ -189,6 +188,6 @@ proc getCachedRss*(key: string): Future[Rss] {.async.} =
         let feed = await r.hGet(k, "rss")
         if feed.len > 0 and feed != redisNil:
           try: result.feed = uncompress feed
-          except: echo "Decompressing RSS failed: ", feed
+          except: error "Decompressing RSS failed: ", feed
     else:
       result.cursor.setLen 0

src/routes/follow.nim

@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import strutils, sequtils
+import jester
+import router_utils
+import ".."/[types]
+
+proc createFollowRouter*(cfg: Config) =
+  router follow:
+    post "/follow":
+      let user = @"user"
+      var prefs = cookiePrefs()
+      if user.len > 0 and user notin prefs.following:
+        prefs.following.add(user)
+        setCookie("nitter_following", prefs.following.join(","), daysForward(360), path="/", httpOnly=true, secure=cfg.useHttps, sameSite=None)
+      redirect(refPath())
+
+    post "/unfollow":
+      let user = @"user"
+      var prefs = cookiePrefs()
+      if user.len > 0 and user in prefs.following:
+        prefs.following.keepItIf(it != user)
+        setCookie("nitter_following", prefs.following.join(","), daysForward(360), path="/", httpOnly=true, secure=cfg.useHttps, sameSite=None)
+      redirect(refPath())

src/routes/media.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import uri, strutils, httpclient, os, hashes, base64, re
+import uri, strutils, httpclient, os, hashes, base64, re, logging
 import asynchttpserver, asyncstreams, asyncfile, asyncnet
 
 import jester
@@ -38,7 +38,7 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
     let res = await client.get(url)
     if res.status != "200 OK":
       if res.status != "404 Not Found":
-        echo "[media] Proxying failed, status: $1, url: $2" % [res.status, url]
+        warn "[media] Proxying failed, status: $1, url: $2" % [res.status, url]
       return Http404
 
     let hashed = $hash(url)
@@ -67,7 +67,7 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
         await request.client.send(data)
     data.setLen 0
   except HttpRequestError, ProtocolError, OSError:
-    echo "[media] Proxying exception, error: $1, url: $2" % [getCurrentExceptionMsg(), url]
+    error "[media] Proxying exception, error: $1, url: $2" % [getCurrentExceptionMsg(), url]
     result = Http404
   finally:
     client.close()

src/routes/search.nim

@@ -32,7 +32,7 @@ proc createSearchRouter*(cfg: Config) =
           users = await getGraphUserSearch(query, getCursor())
         except InternalError:
           users = Result[User](beginning: true, query: query)
-        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title)
+        resp renderMain(renderUserSearch(users, prefs, getPath()), request, cfg, prefs, title)
       of tweets:
         let
           tweets = await getGraphTweetSearch(query, getCursor())

src/routes/status.nim

@@ -31,8 +31,6 @@ proc createStatusRouter*(cfg: Config) =
         resp $renderReplies(replies, prefs, getPath())
 
       let conv = await getTweet(id, getCursor())
-      if conv == nil:
-        echo "nil conv"
 
       if conv == nil or conv.tweet == nil or conv.tweet.id == 0:
         var error = "Tweet not found"
@@ -68,7 +66,7 @@ proc createStatusRouter*(cfg: Config) =
 
     get "/@name/@s/@id/@m/?@i?":
       cond @"s" in ["status", "statuses"]
-      cond @"m" in ["video", "photo"]
+      cond @"m" in ["video", "photo", "history"]
       redirect("/$1/status/$2" % [@"name", @"id"])
 
     get "/@name/statuses/@id/?":
@@ -76,6 +74,6 @@ proc createStatusRouter*(cfg: Config) =
 
     get "/i/web/status/@id":
       redirect("/i/status/" & @"id")
-      
+
     get "/@name/thread/@id/?":
       redirect("/$1/status/$2" % [@"name", @"id"])

src/routes/timeline.nim

@@ -105,6 +105,12 @@ proc createTimelineRouter*(cfg: Config) =
     get "/intent/user":
       respUserId()
 
+    get "/intent/follow/?":
+      let username = request.params.getOrDefault("screen_name")
+      if username.len == 0:
+        resp Http400, showError("Missing screen_name parameter", cfg)
+      redirect("/" & username)
+
     get "/@name/?@tab?/?":
       cond '.' notin @"name"
       cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]

src/routes/unsupported.nim

@@ -17,7 +17,7 @@ proc createUnsupportedRouter*(cfg: Config) =
     get "/@name/lists/?": feature()
 
     get "/intent/?@i?": 
-      cond @"i" notin ["user"]
+      cond @"i" notin ["user", "follow"]
       feature()
 
     get "/i/@i?/?@j?":

src/sass/general.scss

@@ -37,3 +37,52 @@
         height: unset;
     }
 }
+
+.follow-form {
+    display: inline-block;
+    pointer-events: all;
+}
+
+.follow-button {
+    background-color: transparent;
+    color: var(--accent);
+    border: 1px solid var(--accent);
+    border-radius: 9999px;
+    padding: 4px 12px;
+    font-weight: 700;
+    font-size: 14px;
+    cursor: pointer;
+    line-height: 1.2;
+    transition: background-color 0.2s;
+    outline: none;
+
+    &:hover {
+        background-color: var(--bg_hover);
+    }
+
+    &.unfollow {
+        color: #e0245e;
+        border-color: #e0245e;
+
+        &:hover {
+             background-color: rgba(224, 36, 94, 0.1);
+        }
+    }
+}
+
+.profile-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: flex-start;
+    flex: 1;
+    min-width: 0;
+}
+
+.tweet-name-row .follow-form {
+    margin-left: 10px;
+}
+
+.tweet-name-row .follow-button {
+    padding: 2px 10px;
+    font-size: 13px;
+}

src/sass/homepage.scss

@@ -0,0 +1,53 @@
+@import '_variables';
+@import '_mixins';
+
+.homepage-container {
+    display: flex;
+    gap: 5px;
+    max-width: 1200px;
+    margin: 0 auto;
+    justify-content: center;
+}
+
+.following-column,
+.timeline-users-column {
+    width: 280px;
+    flex-shrink: 0;
+}
+
+.timeline-users-column {
+    .timeline-item {
+        flex-direction: column;
+
+    }
+}
+
+.profile-cards {
+    display: flex;
+    flex-direction: column;
+    gap: 5px;
+}
+
+.timeline-users-column > :first-child,
+.homepage-container .timeline-container > .timeline > :first-child {
+    margin-top: 0;
+}
+
+@media (max-width: 1200px) {
+    .homepage-container {
+        max-width: 100%;
+        padding: 0 10px;
+    }
+}
+
+@media (max-width: 1100px) {
+    .following-column {
+        display: none;
+    }
+}
+
+@media (max-width: 800px) {
+    .timeline-users-column {
+        display: none;
+    }
+}

src/sass/include/_mixins.css

@@ -61,23 +61,8 @@
 
 @mixin search-resize($width, $rows) {
     @media(max-width: $width) {
-        .search-toggles {
-            grid-template-columns: repeat($rows, auto);
-        }
-
         #search-panel-toggle:checked ~ .search-panel {
-            @if $rows == 6 {
+            max-height: 80vh !important;
-                max-height: 200px !important;
-            }
-            @if $rows == 5 {
-                max-height: 300px !important;
-            }
-            @if $rows == 4 {
-                max-height: 300px !important;
-            }
-            @if $rows == 3 {
-                max-height: 365px !important;
-            }
         }
     }
 }

src/sass/index.scss

@@ -7,6 +7,7 @@
 @import 'inputs';
 @import 'timeline';
 @import 'search';
+@import 'homepage';
 
 body {
     // colors
@@ -51,7 +52,7 @@ body {
     background-color: var(--bg_color);
     color: var(--fg_color);
     font-family: $font_stack;
-    font-size: 14px;
+    font-size: 15px;
     line-height: 1.3;
     margin: 0;
 }
@@ -114,7 +115,7 @@ ul {
     display: flex;
     flex-wrap: wrap;
     box-sizing: border-box;
-    padding-top: 50px;
+    padding-top: 55px;
     margin: auto;
     min-height: 100vh;
 }
@@ -127,7 +128,6 @@ ul {
     max-width: 600px;
     width: 100%;
     margin: 0 auto;
-    margin-top: 10px;
     background-color: var(--bg_overlays);
     padding: 10px 15px;
     align-self: start;
@@ -169,6 +169,14 @@ ul {
     }
 }
 
+body.hide-verified-all .verified-icon {
+    display: none !important;
+}
+
+body.hide-verified-blue .verified-icon.blue {
+    display: none !important;
+}
+
 @media(max-width: 600px) {
     .preferences-container {
         max-width: 95vw;

src/sass/inputs.scss

@@ -14,6 +14,7 @@ button {
 
 input[type="text"],
 input[type="date"],
+input[type="number"],
 select {
     @include input-colors;
     background-color: var(--bg_elements);
@@ -24,7 +25,12 @@ select {
     font-size: 14px;
 }
 
-input[type="text"] {
+input[type="number"] {
+    -moz-appearance: textfield;
+}
+
+input[type="text"],
+input[type="number"] {
     height: 16px;
 }
 
@@ -38,6 +44,17 @@ input[type="date"]::-webkit-inner-spin-button {
     display: none;
 }
 
+input[type="number"] {
+    -moz-appearance: textfield;
+}
+
+input[type="number"]::-webkit-inner-spin-button,
+input[type="number"]::-webkit-outer-spin-button {
+    display: none;
+    -webkit-appearance: none;
+    margin: 0;
+}
+
 input[type="date"]::-webkit-clear-button {
     margin-left: 17px;
     filter: grayscale(100%);
@@ -137,7 +154,7 @@ input::-webkit-datetime-edit-year-field:focus {
         bottom: 0;
         font-size: 13px;
         font-family: $font_icon;
-        content: '\e803';
+        content: '\e804';
     }
 }
 
@@ -162,9 +179,22 @@ input::-webkit-datetime-edit-year-field:focus {
         -moz-appearance: none;
         -webkit-appearance: none;
         appearance: none;
+        padding-right: 18px;
     }
 
-    input[type="text"] {
+    .pref-select:after {
+        content: '\e80b'; /* icon-down */
+        font-family: $font_icon;
+        font-size: 10px;
+        pointer-events: none;
+        position: absolute;
+        right: 4px;
+        top: 5px;
+        color: var(--fg_color);
+    }
+
+    input[type="text"],
+    input[type="number"] {
         position: absolute;
         right: 0;
         max-width: 140px;

src/sass/navbar.scss

@@ -25,6 +25,14 @@ nav {
     align-items: center;
     flex-basis: 920px;
     height: 50px;
+    gap: 15px;
+}
+
+.inner-nav .search-field {
+    flex: 1;
+    margin: 0 auto;
+    max-width: 350px;
+    min-width: 0;
 }
 
 .site-name {
@@ -46,11 +54,11 @@ nav {
 
 .nav-item {
     display: flex;
-    flex: 1;
+    flex: 0 0 auto;
     line-height: 50px;
     height: 50px;
     overflow: hidden;
-    flex-wrap: wrap;
+    flex-wrap: nowrap;
     align-items: center;
 
     &.right {

src/sass/profile/_base.scss

@@ -15,7 +15,7 @@
 }
 
 .profile-banner {
-    margin: 4px 0 4px 0;
+    margin-bottom: 5px;
     background-color: var(--bg_panel);
 
     a {

src/sass/search.scss

@@ -9,70 +9,111 @@
 
 .search-field {
     display: flex;
-    flex-wrap: wrap;
+    flex-wrap: nowrap;
+    position: relative;
 
     button {
         margin: 0 2px 0 0;
         height: 23px;
         display: flex;
         align-items: center;
+        flex: 0 0 auto;
     }
 
     .pref-input {
         margin: 0 4px 0 0;
         flex-grow: 1;
         height: 23px;
+        min-width: 0;
     }
 
-    input[type="text"] {
+    input[type="text"],
+    input[type="number"] {
         height: calc(100% - 4px);
         width: calc(100% - 8px);
     }
 
     > label {
-        display: inline;
+        display: flex;
+        align-items: center;
+        height: 23px;
         background-color: var(--bg_elements);
         color: var(--fg_color);
         border: 1px solid var(--accent_border);
-        padding: 1px 6px 2px 6px;
+        padding: 0 6px;
         font-size: 14px;
         cursor: pointer;
-        margin-bottom: 2px;
+        margin-bottom: 0;
+        box-sizing: border-box;
 
         @include input-colors;
     }
 
-    @include create-toggle(search-panel, 200px);
+    @include create-toggle(search-panel, 600px);
 }
 
 .search-panel {
     width: 100%;
+    margin-top: 5px;
+    border: 1px solid var(--accent_border);
     max-height: 0;
-    overflow: hidden;
+    overflow-y: auto;
+    overflow-x: hidden;
     transition: max-height 0.4s;
+    
+    position: absolute;
+    top: 100%;
+    left: 0;
+    background-color: var(--bg_overlays);
+    box-shadow: 0 4px 6px rgba(0,0,0,0.3);
+    border-radius: 0 0 4px 4px;
+    z-index: 2000;
+    padding: 5px 10px;
+    box-sizing: border-box;
+
+    opacity: 0;
+    visibility: hidden;
+    transition: max-height 0.4s, opacity 0.4s, visibility 0.4s;
 
-    flex-grow: 1;
     font-weight: initial;
     text-align: left;
+    font-size: 13px;
 
     > div {
-        line-height: 1.7em;
+        line-height: 1.4em;
+        margin-bottom: 4px;
     }
 
     .checkbox-container {
-        display: inline;
+        display: inline-flex;
+        align-items: center;
         padding-right: unset;
-        margin-bottom: unset;
+        margin-bottom: 2px;
-        margin-left: 23px;
+        margin-left: 18px;
+        margin-right: 0;
+        white-space: nowrap;
     }
 
     .checkbox {
         right: unset;
-        left: -22px;
+        left: -18px;
+        width: 15px;
+        height: 15px;
     }
 
     .checkbox-container .checkbox:after {
-        top: -4px;
+        top: 0;
+        left: 1px;
+        bottom: unset;
+        font-size: 13px;
+        width: auto;
+        height: auto;
+    }
+    
+    .search-title {
+        font-size: 13px;
+        margin-top: 2px;
+        margin-bottom: 2px;
     }
 }
 
@@ -92,11 +133,11 @@
 
     .pref-input {
         display: block;
-        padding-bottom: 5px;
+        padding-bottom: 2px;
 
         input {
             height: 21px;
-            margin-top: 1px;
+            margin-top: 0;
         }
     }
 }
@@ -104,19 +145,36 @@
 .search-toggles {
     flex-grow: 1;
     display: grid;
-    grid-template-columns: repeat(6, auto);
+    grid-template-columns: repeat(auto-fill, minmax(110px, 1fr));
-    grid-column-gap: 10px;
+    gap: 5px 10px;
+}
+
+#search-panel-toggle:checked ~ .search-panel {
+    opacity: 1;
+    visibility: visible;
 }
 
 .profile-tabs {
     @include search-resize(820px, 5);
-    @include search-resize(725px, 4);
+    @include search-resize(715px, 4);
-    @include search-resize(600px, 6);
+    @include search-resize(700px, 5);
-    @include search-resize(560px, 5);
+    @include search-resize(485px, 4);
-    @include search-resize(480px, 4);
     @include search-resize(410px, 3);
 }
 
-@include search-resize(560px, 5);
+@include search-resize(700px, 5);
-@include search-resize(480px, 4);
+@include search-resize(485px, 4);
 @include search-resize(410px, 3);
+
+@media(max-width: 600px) {
+    .search-panel {
+        position: fixed;
+        top: 50px;
+        left: 0;
+        right: 0;
+        width: 100%;
+        border-radius: 0;
+        border-top: 1px solid var(--accent_border);
+        box-shadow: 0 4px 6px rgba(0,0,0,0.3);
+    }
+}

src/sass/timeline.scss

@@ -2,13 +2,14 @@
 
 .timeline-container {
     @include panel(100%, 600px);
+    background-color: transparent;
 }
 
 .timeline {
-    background-color: var(--bg_panel);
+    background-color: transparent;
 
     > div:not(:first-child) {
-        border-top: 1px solid var(--border_grey);
+        border-top: none;
     }
 }
 
@@ -25,6 +26,25 @@
     button {
         float: unset;
     }
+
+    &.timeline-default {
+        background-color: var(--bg_panel);
+        border-bottom: 1px solid var(--bg_elements);
+        margin-bottom: 5px;
+
+        .timeline-default-message {
+            color: var(--fg_color);
+            font-size: 16px;
+            font-weight: normal;
+            margin: 0;
+            padding: 5px;
+
+            strong {
+                font-weight: bold;
+                color: var(--accent);
+            }
+        }
+    }
 }
 
 .timeline-banner img {
@@ -159,4 +179,6 @@
     padding: .75em;
     display: flex;
     position: relative;
+    margin-top: 5px;
+    background-color: var(--bg_panel);
 }

src/sass/tweet/_base.scss

@@ -72,6 +72,7 @@
     display: flex;
     flex-shrink: 0;
     margin-left: 4px;
+    font-size: smaller;
 }
 
 .tweet-date a, .username, .show-more a {
@@ -83,6 +84,7 @@
     margin-top: 5px;
     color: var(--grey);
     pointer-events: all;
+    font-size: smaller;
 }
 
 .tweet-avatar {
@@ -172,7 +174,8 @@
 
 .replying-to {
     color: var(--fg_faded);
-    margin: -2px 0 4px;
+    margin: -2px 0 5px;
+    font-size: smaller;
 
     a {
         pointer-events: all;
@@ -200,15 +203,22 @@
 
 .tweet-stats {
     margin-bottom: -3px;
+    padding-top: 5px;
     -webkit-user-select: none;
 
     a {
         pointer-events: all;
     }
+
+    .tweet-published {
+        margin-left: auto;
+        margin-top: 0;
+        font-weight: 400;
+        align-self: center;
+    }
 }
 
 .tweet-stat {
-    padding-top: 5px;
     min-width: 1em;
     margin-right: 0.8em;
 }

src/sass/tweet/card.scss

@@ -42,6 +42,7 @@
 
 .card-description {
     margin: 0.3em 0;
+    white-space: pre-wrap;
 }
 
 .card-destination {

src/sass/tweet/thread.scss

@@ -10,15 +10,10 @@
 }
 
 .main-thread {
-    margin-bottom: 20px;
+    margin-bottom: 5px;
     background-color: var(--bg_panel);
 }
 
-.main-tweet, .replies {
-    padding-top: 50px;
-    margin-top: -50px;
-}
-
 .main-tweet .tweet-content {
     font-size: 18px;
 }
@@ -31,13 +26,13 @@
 
 .reply {
     background-color: var(--bg_panel);
-    margin-bottom: 10px;
+    margin-bottom: 5px;
 }
 
 .thread-line {
     .timeline-item::before,
     &.timeline-item::before {
-        background: var(--accent_dark);
+        background: var(--dark_grey);
         content: '';
         position: relative;
         min-width: 3px;

src/tid.nim

@@ -0,0 +1,62 @@
+import std/[asyncdispatch, base64, httpclient, random, strutils, sequtils, times]
+import nimcrypto
+import experimental/parser/tid
+
+randomize()
+
+const defaultKeyword = "obfiowerehiring";
+const pairsUrl =
+  "https://raw.githubusercontent.com/fa0311/x-client-transaction-id-pair-dict/refs/heads/main/pair.json";
+
+var
+  cachedPairs: seq[TidPair] = @[]
+  lastCached = 0
+  # refresh every hour
+  ttlSec = 60 * 60
+
+proc getPair(): Future[TidPair] {.async.} =
+  if cachedPairs.len == 0 or int(epochTime()) - lastCached > ttlSec:
+    lastCached = int(epochTime())
+
+    let client = newAsyncHttpClient()
+    defer: client.close()
+
+    let resp = await client.get(pairsUrl)
+    if resp.status == $Http200:
+      cachedPairs = parseTidPairs(await resp.body)
+
+  return sample(cachedPairs)
+
+proc encodeSha256(text: string): array[32, byte] =
+  let
+    data = cast[ptr byte](addr text[0])
+    dataLen = uint(len(text))
+    digest = sha256.digest(data, dataLen)
+  return digest.data
+
+proc encodeBase64[T](data: T): string =
+  return encode(data).replace("=", "")
+
+proc decodeBase64(data: string): seq[byte] =
+  return cast[seq[byte]](decode(data))
+
+proc genTid*(path: string): Future[string] {.async.} =
+  let 
+    pair = await getPair()
+
+    timeNow = int(epochTime() - 1682924400)
+    timeNowBytes = @[
+      byte(timeNow and 0xff),
+      byte((timeNow shr 8) and 0xff),
+      byte((timeNow shr 16) and 0xff),
+      byte((timeNow shr 24) and 0xff)
+    ]
+
+    data = "GET!" & path & "!" & $timeNow & defaultKeyword & pair.animationKey
+    hashBytes = encodeSha256(data)
+    keyBytes = decodeBase64(pair.verification)
+    bytesArr = keyBytes & timeNowBytes & hashBytes[0 ..< 16] & @[3'u8]
+    randomNum = byte(rand(256))
+    tid = @[randomNum] & bytesArr.mapIt(it xor randomNum)
+
+  return encodeBase64(tid)

src/types.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import times, sequtils, options, tables, uri
+import times, sequtils, options, tables
 import prefs_impl
 
 genPrefsType()
@@ -13,19 +13,13 @@ type
   TimelineKind* {.pure.} = enum
     tweets, replies, media
 
-  Api* {.pure.} = enum
+  ApiUrl* = object
-    tweetDetail
+    endpoint*: string
-    tweetResult
+    params*: seq[(string, string)]
-    search
+
-    list
+  ApiReq* = object
-    listBySlug
+    oauth*: ApiUrl
-    listMembers
+    cookie*: ApiUrl
-    listTweets
-    userRestId
-    userScreenName
-    userTweets
-    userTweetsAndReplies
-    userMedia
 
   RateLimit* = object
     limit*: int
@@ -38,10 +32,11 @@ type
 
   Session* = ref object
     id*: int64
+    username*: string
     pending*: int
     limited*: bool
     limitedAt*: int
-    apis*: Table[Api, RateLimit]
+    apis*: Table[string, RateLimit]
     case kind*: SessionKind
     of oauth:
       oauthToken*: string
@@ -50,10 +45,6 @@ type
       authToken*: string
       ct0*: string
 
-  SessionAwareUrl* = object
-    oauthUrl*: Uri
-    cookieUrl*: Uri
-
   Error* = enum
     null = 0
     noUserMatches = 17
@@ -102,6 +93,7 @@ type
     media*: int
     verifiedType*: VerifiedType
     protected*: bool
+    sensitive*: bool
     suspended*: bool
     joinDate*: DateTime
 
@@ -120,7 +112,6 @@ type
     durationMs*: int
     url*: string
     thumb*: string
-    views*: string
     available*: bool
     reason*: string
     title*: string
@@ -141,6 +132,7 @@ type
     since*: string
     until*: string
     near*: string
+    minLikes*: string
     sep*: string
 
   Gif* = object
@@ -201,7 +193,7 @@ type
     replies*: int
     retweets*: int
     likes*: int
-    quotes*: int
+    views*: int
 
   Tweet* = ref object
     id*: int64
@@ -212,6 +204,7 @@ type
     time*: DateTime
     reply*: seq[string]
     pinned*: bool
+    sensitive*: bool
     hasThread*: bool
     available*: bool
     tombstone*: string
@@ -283,8 +276,11 @@ type
     minTokens*: int
     enableRss*: bool
     enableDebug*: bool
+    logLevel*: string
     proxy*: string
     proxyAuth*: string
+    defaultFollowedAccounts*: seq[string]
+    disableTid*: bool
 
     rssCacheTime*: int
     listCacheTime*: int

src/views/general.nim

@@ -2,8 +2,8 @@
 import uri, strutils, strformat
 import karax/[karaxdsl, vdom]
 
-import renderutils
+import renderutils, search_panel
-import ../utils, ../types, ../prefs, ../formatters
+import ../utils, ../types, ../prefs, ../formatters, ../query, tables
 
 import jester
 
@@ -20,25 +20,27 @@ proc renderNavbar(cfg: Config; req: Request; rss, canonical: string): VNode =
     path = $(parseUri(req.path) ? filterParams(req.params))
     if "/status/" in path: path.add "#m"
 
+  let query = initQuery(req.params)
+
   buildHtml(nav):
     tdiv(class="inner-nav"):
       tdiv(class="nav-item"):
         a(class="site-name", href="/"): text cfg.title
+        a(href="/about"): text "(donate)"
 
-      a(href="/"): img(class="site-logo", src="/logo.png", alt="Logo")
+      renderSearchPanel(query)
 
       tdiv(class="nav-item right"):
-        icon "search", title="Search", href="/search"
         if cfg.enableRss and rss.len > 0:
-          icon "rss-feed", title="RSS Feed", href=rss
+          icon "rss", title="RSS Feed", href=rss
-        icon "bird", title="Open in Twitter", href=canonical
+        icon "bird", title="Open in X", href=canonical
-        a(href="https://liberapay.com/zedeus"): verbatim lp
+        a(href="https://buymeacoffee.com/kuu7o"): verbatim lp
         icon "info", title="About", href="/about"
         icon "cog", title="Preferences", href=("/settings?referer=" & encodeUrl(path))
 
 proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
                  video=""; images: seq[string] = @[]; banner=""; ogTitle="";
-                 rss=""; canonical=""): VNode =
+                 rss=""; alternate=""): VNode =
   var theme = prefs.theme.toTheme
   if "theme" in req.params:
     theme = req.params["theme"].toTheme
@@ -52,8 +54,8 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
   let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
 
   buildHtml(head):
-    link(rel="stylesheet", type="text/css", href="/css/style.css?v=19")
+    link(rel="stylesheet", type="text/css", href="/css/style.css?v=21")
-    link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
+    link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=3")
 
     if theme.len > 0:
       link(rel="stylesheet", type="text/css", href=(&"/css/themes/{theme}.css"))
@@ -66,8 +68,8 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
     link(rel="search", type="application/opensearchdescription+xml", title=cfg.title,
                             href=opensearchUrl)
 
-    if canonical.len > 0:
+    if alternate.len > 0:
-      link(rel="canonical", href=canonical)
+      link(rel="alternate", href=alternate, title="View on X")
 
     if cfg.enableRss and rss.len > 0:
       link(rel="alternate", type="application/rss+xml", href=rss, title="RSS feed")
@@ -119,20 +121,26 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
     # this is last so images are also preloaded
     # if this is done earlier, Chrome only preloads one image for some reason
     link(rel="preload", type="font/woff2", `as`="font",
-         href="/fonts/fontello.woff2?21002321", crossorigin="anonymous")
+         href="/fonts/fontello.woff2?61663884", crossorigin="anonymous")
+
+    if prefs.hideNsfw:
+      style:
+        verbatim ".nsfw { display: none !important; }"
 
 proc renderMain*(body: VNode; req: Request; cfg: Config; prefs=defaultPrefs;
                  titleText=""; desc=""; ogTitle=""; rss=""; video="";
                  images: seq[string] = @[]; banner=""): string =
 
-  let canonical = getTwitterLink(req.path, req.params)
+  let twitterLink = getTwitterLink(req.path, req.params)
 
   let node = buildHtml(html(lang="en")):
     renderHead(prefs, cfg, req, titleText, desc, video, images, banner, ogTitle,
-               rss, canonical)
+               rss, twitterLink)
 
-    body:
+    body(class=if prefs.verifiedBadge == "Hide all": "hide-verified-all"
-      renderNavbar(cfg, req, rss, canonical)
+               elif prefs.verifiedBadge == "Show official only": "hide-verified-blue"
+               else: ""):
+      renderNavbar(cfg, req, rss, twitterLink)
 
       tdiv(class="container"):
         body

src/views/homepage.nim

@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import karax/[karaxdsl, vdom]
+
+import timeline, profile
+import ".."/[types]
+
+proc renderHomepage*(users: seq[User]; timeline: VNode; prefs: Prefs; path: string): VNode =
+  buildHtml(tdiv(class="homepage-container")):
+    tdiv(class="timeline-users-column"):
+      for user in users:
+        renderUser(user, prefs, path)
+
+    tdiv(class="timeline"):
+      timeline
+
+    tdiv(class="following-column"):
+      tdiv(class="profile-cards"):
+        for user in users:
+          renderUserCard(user, prefs, path)

src/views/profile.nim

@@ -12,8 +12,9 @@ proc renderStat(num: int; class: string; text=""): VNode =
     span(class="profile-stat-num"):
       text insertSep($num, ',')
 
-proc renderUserCard*(user: User; prefs: Prefs): VNode =
+proc renderUserCard*(user: User; prefs: Prefs; path: string): VNode =
-  buildHtml(tdiv(class="profile-card")):
+  let class = if user.sensitive: "profile-card nsfw" else: "profile-card"
+  buildHtml(tdiv(class=class)):
     tdiv(class="profile-card-info"):
       let
         url = getPicUrl(user.getUserPic())
@@ -24,9 +25,12 @@ proc renderUserCard*(user: User; prefs: Prefs): VNode =
       a(class="profile-card-avatar", href=url, target="_blank"):
         genImg(user.getUserPic(size))
 
-      tdiv(class="profile-card-tabs-name"):
+      tdiv(class="profile-header"):
-        linkUser(user, class="profile-card-fullname")
+        tdiv(class="profile-card-tabs-name"):
-        linkUser(user, class="profile-card-username")
+          linkUser(user, class="profile-card-fullname")
+          linkUser(user, class="profile-card-username")
+
+        renderFollowButton(user, prefs, path)
 
     tdiv(class="profile-card-extra"):
       if user.bio.len > 0:
@@ -109,7 +113,7 @@ proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
 
     let sticky = if prefs.stickyProfile: " sticky" else: ""
     tdiv(class=("profile-tab" & sticky)):
-      renderUserCard(profile.user, prefs)
+      renderUserCard(profile.user, prefs, path)
       if profile.photoRail.len > 0:
         renderPhotoRail(profile)
 

src/views/renderutils.nim

@@ -77,7 +77,7 @@ proc genInput*(pref, label, state, placeholder: string; class=""; autofocus=true
     input(name=pref, `type`="text", placeholder=p, value=state, autofocus=(autofocus and state.len == 0))
 
 proc genSelect*(pref, label, state: string; options: seq[string]): VNode =
-  buildHtml(tdiv(class="pref-group pref-input")):
+  buildHtml(tdiv(class="pref-group pref-input pref-select")):
     label(`for`=pref): text label
     select(name=pref):
       for opt in options:
@@ -89,6 +89,13 @@ proc genDate*(pref, state: string): VNode =
     input(name=pref, `type`="date", value=state)
     icon "calendar"
 
+proc genNumberInput*(pref, label, state, placeholder: string; class=""; autofocus=true; min="0"): VNode =
+  let p = placeholder
+  buildHtml(tdiv(class=("pref-group pref-input " & class))):
+    if label.len > 0:
+      label(`for`=pref): text label
+    input(name=pref, `type`="number", placeholder=p, value=state, autofocus=(autofocus and state.len == 0), min=min, step="1")
+
 proc genImg*(url: string; class=""): VNode =
   buildHtml():
     img(src=getPicUrl(url), class=class, alt="", loading="lazy")
@@ -100,3 +107,15 @@ proc getTabClass*(query: Query; tab: QueryKind): string =
 proc getAvatarClass*(prefs: Prefs): string =
   if prefs.squareAvatars: "avatar"
   else: "avatar round"
+
+proc renderFollowButton*(user: User; prefs: Prefs; path: string): VNode =
+  let
+    isFollowing = user.username in prefs.following
+    action = if isFollowing: "/unfollow" else: "/follow"
+    text = if isFollowing: "Unfollow" else: "Follow"
+    class = if isFollowing: "follow-button unfollow" else: "follow-button"
+
+  buildHtml(form(action=action, `method`="post", class="follow-form")):
+    refererField path
+    hiddenField("user", user.username)
+    button(class=class, `type`="submit"): text text

src/views/rss.nimf

@@ -9,9 +9,9 @@
 #elif tweet.reply.len > 0: result = &"R to @{tweet.reply[0]}: "
 #end if
 #var text = stripHtml(tweet.text)
-##if unicode.runeLen(text) > 32:
+#if unicode.runeLen(text) > 100:
-##  text = unicode.runeSubStr(text, 0, 32) & "..."
+#  text = unicode.runeSubStr(text, 0, 64) & "..."
-##end if
+#end if
 #result &= xmltree.escape(text)
 #if result.len > 0: return
 #end if
@@ -25,7 +25,7 @@
 #end proc
 #
 #proc getDescription(desc: string; cfg: Config): string =
-Twitter feed for: ${desc}. Generated by ${cfg.hostname}
+Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)}
 #end proc
 #
 #proc getTweetsWithPinned(profile: Profile): seq[Tweets] =
@@ -56,7 +56,10 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
 <img src="${urlPrefix}${getPicUrl(photo)}" style="max-width:250px;" />
 #  end for
 #elif tweet.video.isSome:
-<img src="${urlPrefix}${getPicUrl(get(tweet.video).thumb)}" style="max-width:250px;" />
+<a href="${urlPrefix}${tweet.getLink}">
+<br>Video<br>
+  <img src="${urlPrefix}${getPicUrl(get(tweet.video).thumb)}" style="max-width:250px;" />
+</a>
 #elif tweet.gif.isSome:
 #  let thumb = &"{urlPrefix}{getPicUrl(get(tweet.gif).thumb)}"
 #  let url = &"{urlPrefix}{getPicUrl(get(tweet.gif).url)}"
@@ -69,14 +72,17 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
 #  end if
 #end if
 #if tweet.quote.isSome and get(tweet.quote).available:
-#  let quoteLink = getLink(get(tweet.quote))
+#  let quoteTweet = get(tweet.quote)
+#  let quoteLink = urlPrefix & getLink(quoteTweet)
+<hr/>
 <blockquote>
-  <p>
+<b>${quoteTweet.user.fullname} (@${quoteTweet.user.username})</b>
-    ${renderRssTweet(get(tweet.quote), cfg)}
+<p>
-  </p>
+${renderRssTweet(quoteTweet, cfg)}
-  <footer>
+</p>
-    — <cite><a href="${urlPrefix}${quoteLink}">${cfg.hostname}${quoteLink}</a></cite>
+<footer>
-  </footer>
+— <cite><a href="${quoteLink}">${quoteLink}</a>
+</footer>
 </blockquote>
 #end if
 #end proc
@@ -101,6 +107,18 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
         <title>${getTitle(tweet, retweet)}</title>
         <dc:creator>@${tweet.user.username}</dc:creator>
         <description><![CDATA[${renderRssTweet(tweet, cfg).strip(chars={'\n'})}]]></description>
+#    let desc = renderRssTweet(tweet, cfg).strip(chars={'\n'})
+#    if retweet.len > 0:
+        <description><![CDATA[
+<blockquote>
+<b>${tweet.user.fullname} (@${tweet.user.username})</b>
+<p>${desc}</p>
+<footer>— <cite><a href="${urlPrefix & link}">${urlPrefix & link}</a></footer>
+</blockquote>
+]]></description>
+#    else:
+        <description><![CDATA[${desc}]]></description>
+#    end if
         <pubDate>${getRfc822Time(tweet)}</pubDate>
         <guid>${urlPrefix & link}</guid>
         <link>${urlPrefix & link}</link>

src/views/search.nim

@@ -1,33 +1,13 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, strformat, sequtils, unicode, tables, options
+import strutils, options
 import karax/[karaxdsl, vdom]
 
 import renderutils, timeline
 import ".."/[types, query]
 
-const toggles = {
-  "nativeretweets": "Retweets",
-  "media": "Media",
-  "videos": "Videos",
-  "news": "News",
-  "verified": "Verified",
-  "native_video": "Native videos",
-  "replies": "Replies",
-  "links": "Links",
-  "images": "Images",
-  "safe": "Safe",
-  "quote": "Quotes",
-  "pro_video": "Pro videos"
-}.toOrderedTable
-
 proc renderSearch*(): VNode =
   buildHtml(tdiv(class="panel-container")):
-    tdiv(class="search-bar"):
+    discard
-      form(`method`="get", action="/search", autocomplete="off"):
-        hiddenField("f", "tweets")
-        input(`type`="text", name="q", autofocus="",
-              placeholder="Search...", dir="auto")
-        button(`type`="submit"): icon "search"
 
 proc renderProfileTabs*(query: Query; username: string): VNode =
   let link = "/" & username
@@ -51,43 +31,6 @@ proc renderSearchTabs*(query: Query): VNode =
       q.kind = users
       a(href=("?" & genQueryUrl(q))): text "Users"
 
-proc isPanelOpen(q: Query): bool =
-  q.fromUser.len == 0 and (q.filters.len > 0 or q.excludes.len > 0 or
-  @[q.near, q.until, q.since].anyIt(it.len > 0))
-
-proc renderSearchPanel*(query: Query): VNode =
-  let user = query.fromUser.join(",")
-  let action = if user.len > 0: &"/{user}/search" else: "/search"
-  buildHtml(form(`method`="get", action=action,
-                 class="search-field", autocomplete="off")):
-    hiddenField("f", "tweets")
-    genInput("q", "", query.text, "Enter search...", class="pref-inline")
-    button(`type`="submit"): icon "search"
-
-    input(id="search-panel-toggle", `type`="checkbox", checked=isPanelOpen(query))
-    label(`for`="search-panel-toggle"): icon "down"
-
-    tdiv(class="search-panel"):
-      for f in @["filter", "exclude"]:
-        span(class="search-title"): text capitalize(f)
-        tdiv(class="search-toggles"):
-          for k, v in toggles:
-            let state =
-              if f == "filter": k in query.filters
-              else: k in query.excludes
-            genCheckbox(&"{f[0]}-{k}", v, state)
-
-      tdiv(class="search-row"):
-        tdiv:
-          span(class="search-title"): text "Time range"
-          tdiv(class="date-range"):
-            genDate("since", query.since)
-            span(class="search-title"): text "-"
-            genDate("until", query.until)
-        tdiv:
-          span(class="search-title"): text "Near"
-          genInput("near", "", query.near, "Location...", autofocus=false)
-
 proc renderTweetSearch*(results: Timeline; prefs: Prefs; path: string;
                         pinned=none(Tweet)): VNode =
   let query = results.query
@@ -100,21 +43,42 @@ proc renderTweetSearch*(results: Timeline; prefs: Prefs; path: string;
       renderProfileTabs(query, query.fromUser.join(","))
 
     if query.fromUser.len == 0 or query.kind == tweets:
-      tdiv(class="timeline-header"):
+      discard
-        renderSearchPanel(query)
 
     if query.fromUser.len == 0:
       renderSearchTabs(query)
 
     renderTimelineTweets(results, prefs, path, pinned)
 
-proc renderUserSearch*(results: Result[User]; prefs: Prefs): VNode =
+proc renderHomepageTabs*(query: Query): VNode =
-  buildHtml(tdiv(class="timeline-container")):
+  buildHtml(ul(class="tab")):
-    tdiv(class="timeline-header"):
+    li(class=if query.kind == tweets: "tab-item active" else: "tab-item"):
-      form(`method`="get", action="/search", class="search-field", autocomplete="off"):
+      a(href="/?f=tweets"): text "Tweets"
-        hiddenField("f", "users")
+    
-        genInput("q", "", results.query.text, "Enter username...", class="pref-inline")
+    li(class=if query.kind == replies: "tab-item active" else: "tab-item"):
-        button(`type`="submit"): icon "search"
+      a(href="/?f=replies"): text "Tweets & Replies"
 
+proc renderHomepageTimeline*(results: Timeline; prefs: Prefs; path: string): VNode =
+  let query = results.query
+  buildHtml(tdiv(class="timeline-container")):
+    renderHomepageTabs(query)
+    
+    renderTimelineTweets(results, prefs, path)
+
+proc renderDefaultTimeline*(results: Timeline; prefs: Prefs; path: string): VNode =
+  let query = results.query
+  buildHtml(tdiv(class="timeline-container")):
+    tdiv(class="timeline-header timeline-default"):
+      h2(class="timeline-default-message"):
+        text "Follow people to populate your "
+        strong: text "own"
+        text " feed"
+    
+    renderHomepageTabs(query)
+    
+    renderTimelineTweets(results, prefs, path)
+
+proc renderUserSearch*(results: Result[User]; prefs: Prefs; path: string): VNode =
+  buildHtml(tdiv(class="timeline-container")):
     renderSearchTabs(results.query)
-    renderTimelineUsers(results, prefs)
+    renderTimelineUsers(results, prefs, path)

src/views/search_panel.nim

@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import strutils, strformat, sequtils, unicode, tables
+import karax/[karaxdsl, vdom]
+
+import renderutils
+import ".."/[types]
+
+const toggles = {
+  "nativeretweets": "Retweets",
+  "media": "Media",
+  "videos": "Videos",
+  "news": "News",
+  "verified": "Verified",
+  "native_video": "Native videos",
+  "replies": "Replies",
+  "links": "Links",
+  "images": "Images",
+  "safe": "Safe",
+  "quote": "Quotes",
+  "pro_video": "Pro videos"
+}.toOrderedTable
+
+proc isPanelOpen(q: Query): bool =
+  q.fromUser.len == 0 and (q.filters.len > 0 or q.excludes.anyIt(it != "nativeretweets") or
+  @[q.near, q.until, q.since].anyIt(it.len > 0))
+
+proc renderSearchPanel*(query: Query): VNode =
+  let user = query.fromUser.join(",")
+  let action = if user.len > 0: &"/{user}/search" else: "/search"
+  buildHtml(form(`method`="get", action=action,
+                 class="search-field", autocomplete="off")):
+    hiddenField("f", "tweets")
+    genInput("q", "", query.text, "Enter search...", class="pref-inline")
+    button(`type`="submit"): icon "search"
+
+    input(id="search-panel-toggle", `type`="checkbox", checked=isPanelOpen(query))
+    label(`for`="search-panel-toggle"): icon "down"
+
+    tdiv(class="search-panel"):
+      for f in @["filter", "exclude"]:
+        span(class="search-title"): text capitalize(f)
+        tdiv(class="search-toggles"):
+          for k, v in toggles:
+            let state =
+              if f == "filter": k in query.filters
+              else: k in query.excludes
+            genCheckbox(&"{f[0]}-{k}", v, state)
+
+      tdiv(class="search-row"):
+        tdiv:
+          span(class="search-title"): text "Time range"
+          tdiv(class="date-range"):
+            genDate("since", query.since)
+            span(class="search-title"): text "-"
+            genDate("until", query.until)
+        tdiv:
+          span(class="search-title"): text "Near"
+          genInput("near", "", query.near, "Location...", autofocus=false)

src/views/timeline.nim

@@ -55,8 +55,9 @@ proc renderThread(thread: Tweets; prefs: Prefs; path: string): VNode =
       renderTweet(tweet, prefs, path, class=(header & "thread"),
                   index=i, last=(i == thread.high), showThread=show)
 
-proc renderUser(user: User; prefs: Prefs): VNode =
+proc renderUser*(user: User; prefs: Prefs; path: string): VNode =
-  buildHtml(tdiv(class="timeline-item")):
+  let class = if user.sensitive: "timeline-item nsfw" else: "timeline-item"
+  buildHtml(tdiv(class=class, data-username=user.username)):
     a(class="tweet-link", href=("/" & user.username))
     tdiv(class="tweet-body profile-result"):
       tdiv(class="tweet-header"):
@@ -66,6 +67,7 @@ proc renderUser(user: User; prefs: Prefs): VNode =
         tdiv(class="tweet-name-row"):
           tdiv(class="fullname-and-username"):
             linkUser(user, class="fullname")
+          renderFollowButton(user, prefs, path)
         linkUser(user, class="username")
 
       tdiv(class="tweet-content media-body", dir="auto"):
@@ -78,7 +80,7 @@ proc renderTimelineUsers*(results: Result[User]; prefs: Prefs; path=""): VNode =
 
     if results.content.len > 0:
       for user in results.content:
-        renderUser(user, prefs)
+        renderUser(user, prefs, path)
       if results.bottom.len > 0:
         renderMore(results.query, results.bottom)
       renderToTop()

src/views/tweet.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, sequtils, strformat, options, algorithm, uri
+import strutils, sequtils, strformat, options, algorithm
 import karax/[karaxdsl, vdom, vstyles]
 from jester import Request
 
@@ -178,14 +178,12 @@ func formatStat(stat: int): string =
   if stat > 0: insertSep($stat, ',')
   else: ""
 
-proc renderStats(tweet_id: int64; stats: TweetStats; views: string): VNode =
+proc renderStats(stats: TweetStats): VNode =
   buildHtml(tdiv(class="tweet-stats")):
     span(class="tweet-stat"): icon "comment", formatStat(stats.replies)
     span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets)
-    a(class="tweet-stat", href=("/search?q=" & encodeUrl(&"-from:quotedreplies url:{tweet_id}") & "&e-nativeretweets=on")): icon "quote", formatStat(stats.quotes)
     span(class="tweet-stat"): icon "heart", formatStat(stats.likes)
-    if views.len > 0:
+    span(class="tweet-stat"): icon "views", formatStat(stats.views)
-      span(class="tweet-stat"): icon "play", insertSep(views, ',')
 
 proc renderReply(tweet: Tweet): VNode =
   buildHtml(tdiv(class="replying-to")):
@@ -273,8 +271,11 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
   if index == -1 or last:
     divClass = "thread-last " & class
 
+  if tweet.sensitive:
+    divClass.add " nsfw"
+
   if not tweet.available:
-    return buildHtml(tdiv(class=divClass & "unavailable timeline-item")):
+    return buildHtml(tdiv(class=divClass & "unavailable timeline-item", data-username=tweet.user.username)):
       tdiv(class="unavailable-box"):
         if tweet.tombstone.len > 0:
           text tweet.tombstone
@@ -296,12 +297,11 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
     tweet = tweet.retweet.get
     retweet = fullTweet.user.fullname
 
-  buildHtml(tdiv(class=("timeline-item " & divClass))):
+  buildHtml(tdiv(class=("timeline-item " & divClass), data-username=tweet.user.username)):
     if not mainTweet:
       a(class="tweet-link", href=getLink(tweet))
 
     tdiv(class="tweet-body"):
-      var views = ""
       renderHeader(tweet, retweet, pinned, prefs)
 
       if not afterTweet and index == 0 and tweet.reply.len > 0 and
@@ -325,10 +325,8 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         renderAlbum(tweet)
       elif tweet.video.isSome:
         renderVideo(tweet.video.get(), prefs, path)
-        views = tweet.video.get().views
       elif tweet.gif.isSome:
         renderGif(tweet.gif.get(), prefs)
-        views = "GIF"
 
       if tweet.poll.isSome:
         renderPoll(tweet.poll.get())
@@ -336,14 +334,13 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       if tweet.quote.isSome:
         renderQuote(tweet.quote.get(), prefs, path)
 
-      if mainTweet:
+      let published = if mainTweet: getTime(tweet) else: ""
-        p(class="tweet-published"): text &"{getTime(tweet)}"
 
       if tweet.mediaTags.len > 0:
         renderMediaTags(tweet.mediaTags)
 
       if not prefs.hideTweetStats:
-        renderStats(tweet.id, tweet.stats, views)
+        renderStats(tweet.stats)
 
       if showThread:
         a(class="show-thread", href=("/i/status/" & $tweet.threadId)):

tests/test_card.py

@@ -11,12 +11,7 @@ card = [
     ['voidtarget/status/1094632512926605312',
      'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too)',
      'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim',
-     'gist.github.com', True],
+     'gist.github.com', True]
-
-    ['nim_lang/status/1082989146040340480',
-     'Nim in 2018: A short recap',
-     'There were several big news in the Nim world in 2018 – two new major releases, partnership with Status, and much more. But let us go chronologically.',
-     'nim-lang.org', True]
 ]
 
 no_thumb = [

tools/create_session_browser.py

@@ -1,23 +1,20 @@
 #!/usr/bin/env python3
 """
-Authenticates with X.com/Twitter and extracts session cookies for use with Nitter.
-Handles 2FA, extracts user info, and outputs clean JSON for sessions.jsonl.
-
 Requirements:
   pip install -r tools/requirements.txt
 
 Usage:
-  python3 tools/get_web_session.py <username> <password> [totp_seed] [--append sessions.jsonl] [--headless]
+  python3 tools/create_session_browser.py <username> <password> [totp_seed] [--append sessions.jsonl] [--headless]
 
 Examples:
   # Output to terminal
-  python3 tools/get_web_session.py myusername mypassword TOTP_BASE32_SECRET
+  python3 tools/create_session_browser.py myusername mypassword TOTP_SECRET
 
   # Append to sessions.jsonl
-  python3 tools/get_web_session.py myusername mypassword TOTP_SECRET --append sessions.jsonl
+  python3 tools/create_session_browser.py myusername mypassword TOTP_SECRET --append sessions.jsonl
 
   # Headless mode (may increase detection risk)
-  python3 tools/get_web_session.py myusername mypassword TOTP_SECRET --headless
+  python3 tools/create_session_browser.py myusername mypassword TOTP_SECRET --headless
 
 Output:
   {"kind": "cookie", "username": "...", "id": "...", "auth_token": "...", "ct0": "..."}
@@ -97,7 +94,7 @@ async def login_and_get_cookies(username, password, totp_seed=None, headless=Fal
 
 async def main():
     if len(sys.argv) < 3:
-        print('Usage: python3 twitter-auth.py username password [totp_seed] [--append sessions.jsonl] [--headless]')
+        print('Usage: python3 create_session_browser.py username password [totp_seed] [--append file.jsonl] [--headless]')
         sys.exit(1)
 
     username = sys.argv[1]

tools/create_session_curl.py

@@ -0,0 +1,328 @@
+#!/usr/bin/env python3
+"""
+Requirements:
+  pip install curl_cffi pyotp
+
+Usage:
+  python3 tools/create_session_curl.py <username> <password> [totp_seed] [--append sessions.jsonl]
+
+Examples:
+  # Output to terminal
+  python3 tools/create_session_curl.py myusername mypassword TOTP_SECRET
+
+  # Append to sessions.jsonl
+  python3 tools/create_session_curl.py myusername mypassword TOTP_SECRET --append sessions.jsonl
+
+Output:
+  {"kind": "cookie", "username": "...", "id": "...", "auth_token": "...", "ct0": "..."}
+"""
+
+import sys
+import json
+import pyotp
+from curl_cffi import requests
+
+BEARER_TOKEN = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
+BASE_URL = "https://api.x.com/1.1/onboarding/task.json"
+GUEST_ACTIVATE_URL = "https://api.x.com/1.1/guest/activate.json"
+
+# Subtask versions required by API
+SUBTASK_VERSIONS = {
+    "action_list": 2, "alert_dialog": 1, "app_download_cta": 1,
+    "check_logged_in_account": 2, "choice_selection": 3,
+    "contacts_live_sync_permission_prompt": 0, "cta": 7, "email_verification": 2,
+    "end_flow": 1, "enter_date": 1, "enter_email": 2, "enter_password": 5,
+    "enter_phone": 2, "enter_recaptcha": 1, "enter_text": 5, "generic_urt": 3,
+    "in_app_notification": 1, "interest_picker": 3, "js_instrumentation": 1,
+    "menu_dialog": 1, "notifications_permission_prompt": 2, "open_account": 2,
+    "open_home_timeline": 1, "open_link": 1, "phone_verification": 4,
+    "privacy_options": 1, "security_key": 3, "select_avatar": 4,
+    "select_banner": 2, "settings_list": 7, "show_code": 1, "sign_up": 2,
+    "sign_up_review": 4, "tweet_selection_urt": 1, "update_users": 1,
+    "upload_media": 1, "user_recommendations_list": 4,
+    "user_recommendations_urt": 1, "wait_spinner": 3, "web_modal": 1
+}
+
+
+def get_base_headers(guest_token=None):
+    """Build base headers for API requests."""
+    headers = {
+        "Authorization": f"Bearer {BEARER_TOKEN}",
+        "Content-Type": "application/json",
+        "Accept": "*/*",
+        "Accept-Language": "en-US",
+        "X-Twitter-Client-Language": "en-US",
+        "Origin": "https://x.com",
+        "Referer": "https://x.com/",
+    }
+    if guest_token:
+        headers["X-Guest-Token"] = guest_token
+    return headers
+
+
+def get_cookies_dict(session):
+    """Extract cookies from session."""
+    return session.cookies.get_dict() if hasattr(session.cookies, 'get_dict') else dict(session.cookies)
+
+
+def make_request(session, headers, flow_token, subtask_data, print_msg):
+    """Generic request handler for flow steps."""
+    print(f"[*] {print_msg}...", file=sys.stderr)
+
+    payload = {
+        "flow_token": flow_token,
+        "subtask_inputs": [subtask_data] if isinstance(subtask_data, dict) else subtask_data
+    }
+
+    response = session.post(BASE_URL, json=payload, headers=headers)
+    response.raise_for_status()
+
+    data = response.json()
+    new_flow_token = data.get('flow_token')
+    if not new_flow_token:
+        raise Exception(f"Failed to get flow token: {print_msg}")
+
+    return new_flow_token, data
+
+
+def get_guest_token(session):
+    """Get guest token for unauthenticated requests."""
+    print("[*] Getting guest token...", file=sys.stderr)
+    response = session.post(GUEST_ACTIVATE_URL, headers={"Authorization": f"Bearer {BEARER_TOKEN}"})
+    response.raise_for_status()
+
+    guest_token = response.json().get('guest_token')
+    if not guest_token:
+        raise Exception("Failed to obtain guest token")
+
+    print(f"[*] Got guest token: {guest_token}", file=sys.stderr)
+    return guest_token
+
+
+def init_flow(session, guest_token):
+    """Initialize the login flow."""
+    print("[*] Initializing login flow...", file=sys.stderr)
+
+    headers = get_base_headers(guest_token)
+    payload = {
+        "input_flow_data": {
+            "flow_context": {
+                "debug_overrides": {},
+                "start_location": {"location": "manual_link"}
+            },
+            "subtask_versions": SUBTASK_VERSIONS
+        }
+    }
+
+    response = session.post(f"{BASE_URL}?flow_name=login", json=payload, headers=headers)
+    response.raise_for_status()
+
+    flow_token = response.json().get('flow_token')
+    if not flow_token:
+        raise Exception("Failed to get initial flow token")
+
+    print("[*] Got initial flow token", file=sys.stderr)
+    return flow_token, headers
+
+
+def submit_username(session, flow_token, headers, guest_token, username):
+    """Submit username."""
+    headers = headers.copy()
+    headers["X-Guest-Token"] = guest_token
+
+    subtask = {
+        "subtask_id": "LoginEnterUserIdentifierSSO",
+        "settings_list": {
+            "setting_responses": [{
+                "key": "user_identifier",
+                "response_data": {"text_data": {"result": username}}
+            }],
+            "link": "next_link"
+        }
+    }
+
+    flow_token, data = make_request(session, headers, flow_token, subtask, "Submitting username")
+
+    # Check for denial (suspicious activity)
+    if data.get('subtasks') and 'cta' in data['subtasks'][0]:
+        error_msg = data['subtasks'][0]['cta'].get('primary_text', {}).get('text')
+        if error_msg:
+            raise Exception(f"Login denied: {error_msg}")
+
+    return flow_token
+
+
+def submit_password(session, flow_token, headers, guest_token, password):
+    """Submit password and detect if 2FA is needed."""
+    headers = headers.copy()
+    headers["X-Guest-Token"] = guest_token
+
+    subtask = {
+        "subtask_id": "LoginEnterPassword",
+        "enter_password": {"password": password, "link": "next_link"}
+    }
+
+    flow_token, data = make_request(session, headers, flow_token, subtask, "Submitting password")
+
+    needs_2fa = any(s.get('subtask_id') == 'LoginTwoFactorAuthChallenge' for s in data.get('subtasks', []))
+    if needs_2fa:
+        print("[*] 2FA required", file=sys.stderr)
+
+    return flow_token, needs_2fa
+
+
+def submit_2fa(session, flow_token, headers, guest_token, totp_seed):
+    """Submit 2FA code."""
+    if not totp_seed:
+        raise Exception("2FA required but no TOTP seed provided")
+
+    code = pyotp.TOTP(totp_seed).now()
+    print("[*] Generating 2FA code...", file=sys.stderr)
+
+    headers = headers.copy()
+    headers["X-Guest-Token"] = guest_token
+
+    subtask = {
+        "subtask_id": "LoginTwoFactorAuthChallenge",
+        "enter_text": {"text": code, "link": "next_link"}
+    }
+
+    flow_token, _ = make_request(session, headers, flow_token, subtask, "Submitting 2FA code")
+    return flow_token
+
+
+def submit_js_instrumentation(session, flow_token, headers, guest_token):
+    """Submit JS instrumentation response."""
+    headers = headers.copy()
+    headers["X-Guest-Token"] = guest_token
+
+    subtask = {
+        "subtask_id": "LoginJsInstrumentationSubtask",
+        "js_instrumentation": {
+            "response": '{"rf":{"a4fc506d24bb4843c48a1966940c2796bf4fb7617a2d515ad3297b7df6b459b6":121,"bff66e16f1d7ea28c04653dc32479cf416a9c8b67c80cb8ad533b2a44fee82a3":-1,"ac4008077a7e6ca03210159dbe2134dea72a616f03832178314bb9931645e4f7":-22,"c3a8a81a9b2706c6fec42c771da65a9597c537b8e4d9b39e8e58de9fe31ff239":-12},"s":"ZHYaDA9iXRxOl2J3AZ9cc23iJx-Fg5E82KIBA_fgeZFugZGYzRtf8Bl3EUeeYgsK30gLFD2jTQx9fAMsnYCw0j8ahEy4Pb5siM5zD6n7YgOeWmFFaXoTwaGY4H0o-jQnZi5yWZRAnFi4lVuCVouNz_xd2BO2sobCO7QuyOsOxQn2CWx7bjD8vPAzT5BS1mICqUWyjZDjLnRZJU6cSQG5YFIHEPBa8Kj-v1JFgkdAfAMIdVvP7C80HWoOqYivQR7IBuOAI4xCeLQEdxlGeT-JYStlP9dcU5St7jI6ExyMeQnRicOcxXLXsan8i5Joautk2M8dAJFByzBaG4wtrPhQ3QAAAZEi-_t7"}',
+            "link": "next_link"
+        }
+    }
+
+    flow_token, _ = make_request(session, headers, flow_token, subtask, "Submitting JS instrumentation")
+    return flow_token
+
+
+def complete_flow(session, flow_token, headers):
+    """Complete the login flow."""
+    cookies = get_cookies_dict(session)
+
+    headers = headers.copy()
+    headers["X-Twitter-Auth-Type"] = "OAuth2Session"
+    if cookies.get('ct0'):
+        headers["X-Csrf-Token"] = cookies['ct0']
+
+    subtask = {
+        "subtask_id": "AccountDuplicationCheck",
+        "check_logged_in_account": {"link": "AccountDuplicationCheck_false"}
+    }
+
+    make_request(session, headers, flow_token, subtask, "Completing login flow")
+
+
+def extract_user_id(cookies_dict):
+    """Extract user ID from twid cookie."""
+    twid = cookies_dict.get('twid', '').strip('"')
+
+    for prefix in ['u=', 'u%3D']:
+        if prefix in twid:
+            return twid.split(prefix)[1].split('&')[0].strip('"')
+
+    return None
+
+
+def login_and_get_cookies(username, password, totp_seed=None):
+    """Authenticate with X.com and extract session cookies."""
+    session = requests.Session(impersonate="chrome")
+
+    try:
+        guest_token = get_guest_token(session)
+        flow_token, headers = init_flow(session, guest_token)
+        flow_token = submit_js_instrumentation(session, flow_token, headers, guest_token)
+        flow_token = submit_username(session, flow_token, headers, guest_token, username)
+        flow_token, needs_2fa = submit_password(session, flow_token, headers, guest_token, password)
+
+        if needs_2fa:
+            flow_token = submit_2fa(session, flow_token, headers, guest_token, totp_seed)
+
+        complete_flow(session, flow_token, headers)
+
+        cookies_dict = get_cookies_dict(session)
+        cookies_dict['username'] = username
+
+        user_id = extract_user_id(cookies_dict)
+        if user_id:
+            cookies_dict['id'] = user_id
+
+        print("[*] Successfully authenticated", file=sys.stderr)
+        return cookies_dict
+
+    finally:
+        session.close()
+
+
+def main():
+    if len(sys.argv) < 3:
+        print('Usage: python3 create_session_curl.py username password [totp_seed] [--append sessions.jsonl]', file=sys.stderr)
+        sys.exit(1)
+
+    username = sys.argv[1]
+    password = sys.argv[2]
+    totp_seed = None
+    append_file = None
+
+    # Parse optional arguments
+    i = 3
+    while i < len(sys.argv):
+        arg = sys.argv[i]
+        if arg == '--append':
+            if i + 1 < len(sys.argv):
+                append_file = sys.argv[i + 1]
+                i += 2
+            else:
+                print('[!] Error: --append requires a filename', file=sys.stderr)
+                sys.exit(1)
+        elif not arg.startswith('--'):
+            if totp_seed is None:
+                totp_seed = arg
+            i += 1
+        else:
+            print(f'[!] Warning: Unknown argument: {arg}', file=sys.stderr)
+            i += 1
+
+    try:
+        cookies = login_and_get_cookies(username, password, totp_seed)
+
+        session = {
+            'kind': 'cookie',
+            'username': cookies['username'],
+            'id': cookies.get('id'),
+            'auth_token': cookies['auth_token'],
+            'ct0': cookies['ct0']
+        }
+
+        output = json.dumps(session)
+
+        if append_file:
+            with open(append_file, 'a') as f:
+                f.write(output + '\n')
+            print(f'✓ Session appended to {append_file}', file=sys.stderr)
+        else:
+            print(output)
+
+        sys.exit(0)
+
+    except Exception as error:
+        print(f'[!] Error: {error}', file=sys.stderr)
+        import traceback
+        traceback.print_exc(file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

tools/requirements.txt

@@ -1,2 +1,3 @@
 nodriver>=0.48.0
 pyotp
+curl_cffi