Merge remote-tracking branch 'upstream/master'

* Add experimental x-client-transaction-id support

* Remove broken test

nitter.example.conf

@@ -28,6 +28,7 @@ logLevel = "info"  # log level (debug, info, warn, error, fatal)
 proxy = ""  # http/https url, SOCKS proxies are not supported
 proxyAuth = ""
 defaultFollowedAccounts = "eff,fsf" # default accounts to show when user follows none
+disableTid = false # enable this if cookie-based auth is failing
 
 # Change default preferences here, see src/prefs_impl.nim for a complete list
 [Preferences]

src/api.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, httpclient, uri, strutils, sequtils, sugar, tables
+import asyncdispatch, httpclient, strutils, sequtils, sugar
 import packedjson
 import types, query, formatters, consts, apiutils, parser
 import experimental/parser as newParser
@@ -11,88 +11,91 @@ proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
   if fieldToggles.len > 0:
     result.add ("fieldToggles", fieldToggles)
 
-proc mediaUrl(id: string; cursor: string): SessionAwareUrl =
+proc apiUrl(endpoint, variables: string; fieldToggles = ""): ApiUrl =
-  let
+  return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles))
-    cookieVariables = userMediaVariables % [id, cursor]
+
-    oauthVariables = restIdVariables % [id, cursor]
+proc apiReq(endpoint, variables: string; fieldToggles = ""): ApiReq =
-  result = SessionAwareUrl(
+  let url = apiUrl(endpoint, variables, fieldToggles)
-    cookieUrl: graphUserMedia ? genParams(cookieVariables),
+  return ApiReq(cookie: url, oauth: url)
-    oauthUrl: graphUserMediaV2 ? genParams(oauthVariables)
+
+proc mediaUrl(id: string; cursor: string): ApiReq =
+  result = ApiReq(
+    cookie: apiUrl(graphUserMedia, userMediaVars % [id, cursor]),
+    oauth: apiUrl(graphUserMediaV2, restIdVars % [id, cursor])
   )
 
-proc userTweetsUrl(id: string; cursor: string): SessionAwareUrl =
+proc userTweetsUrl(id: string; cursor: string): ApiReq =
-  let
+  result = ApiReq(
-    cookieVariables = userTweetsVariables % [id, cursor]
+    # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
-    oauthVariables = restIdVariables % [id, cursor]
+    oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor])
-  result = SessionAwareUrl(
-    # cookieUrl: graphUserTweets ? genParams(cookieVariables, fieldToggles),
-    oauthUrl: graphUserTweetsV2 ? genParams(oauthVariables)
   )
   # might change this in the future pending testing
-  result.cookieUrl = result.oauthUrl
+  result.cookie = result.oauth
 
-proc userTweetsAndRepliesUrl(id: string; cursor: string): SessionAwareUrl =
+proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq =
-  let
+  let cookieVars = userTweetsAndRepliesVars % [id, cursor]
-    cookieVariables = userTweetsAndRepliesVariables % [id, cursor]
+  result = ApiReq(
-    oauthVariables = restIdVariables % [id, cursor]
+    cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
-  result = SessionAwareUrl(
+    oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor])
-    cookieUrl: graphUserTweetsAndReplies ? genParams(cookieVariables, fieldToggles),
-    oauthUrl: graphUserTweetsAndRepliesV2 ? genParams(oauthVariables)
   )
 
-proc tweetDetailUrl(id: string; cursor: string): SessionAwareUrl =
+proc tweetDetailUrl(id: string; cursor: string): ApiReq =
-  let
+  let cookieVars = tweetDetailVars % [id, cursor]
-    cookieVariables = tweetDetailVariables % [id, cursor]
+  result = ApiReq(
-    oauthVariables = tweetVariables % [id, cursor]
+    cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
-  result = SessionAwareUrl(
+    oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
-    cookieUrl: graphTweetDetail ? genParams(cookieVariables, tweetDetailFieldToggles),
+  )
-    oauthUrl: graphTweet ? genParams(oauthVariables)
+
+proc userUrl(username: string): ApiReq =
+  let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
+  result = ApiReq(
+    cookie: apiUrl(graphUser, cookieVars, tweetDetailFieldToggles),
+    oauth: apiUrl(graphUserV2, """{"screen_name": "$1"}""" % username)
   )
 
 proc getGraphUser*(username: string): Future[User] {.async.} =
   if username.len == 0: return
-  let
+  let js = await fetchRaw(userUrl(username))
-    url = graphUser ? genParams("""{"screen_name": "$1"}""" % username)
-    js = await fetchRaw(url, Api.userScreenName)
   result = parseGraphUser(js)
 
 proc getGraphUserById*(id: string): Future[User] {.async.} =
   if id.len == 0 or id.any(c => not c.isDigit): return
   let
-    url = graphUserById ? genParams("""{"rest_id": "$1"}""" % id)
+    url = apiReq(graphUserById, """{"rest_id": "$1"}""" % id)
-    js = await fetchRaw(url, Api.userRestId)
+    js = await fetchRaw(url)
   result = parseGraphUser(js)
 
 proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
   if id.len == 0: return
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    js = case kind
+    url = case kind
-      of TimelineKind.tweets:
+      of TimelineKind.tweets: userTweetsUrl(id, cursor)
-        await fetch(userTweetsUrl(id, cursor), Api.userTweets)
+      of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor)
-      of TimelineKind.replies:
+      of TimelineKind.media: mediaUrl(id, cursor)
-        await fetch(userTweetsAndRepliesUrl(id, cursor), Api.userTweetsAndReplies)
+    js = await fetch(url)
-      of TimelineKind.media:
-        await fetch(mediaUrl(id, cursor), Api.userMedia)
   result = parseGraphTimeline(js, after)
 
 proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
   if id.len == 0: return
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    url = graphListTweets ? genParams(restIdVariables % [id, cursor])
+    url = apiReq(graphListTweets, restIdVars % [id, cursor])
-  result = parseGraphTimeline(await fetch(url, Api.listTweets), after).tweets
+    js = await fetch(url)
+  result = parseGraphTimeline(js, after).tweets
 
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
   let
     variables = %*{"screenName": name, "listSlug": list}
-    url = graphListBySlug ? genParams($variables)
+    url = apiReq(graphListBySlug, $variables)
-  result = parseGraphList(await fetch(url, Api.listBySlug))
+    js = await fetch(url)
+  result = parseGraphList(js)
 
 proc getGraphList*(id: string): Future[List] {.async.} =
   let 
-    url = graphListById ? genParams("""{"listId": "$1"}""" % id)
+    url = apiReq(graphListById, """{"listId": "$1"}""" % id)
-  result = parseGraphList(await fetch(url, Api.list))
+    js = await fetch(url)
+  result = parseGraphList(js)
 
 proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
   if list.id.len == 0: return
@@ -106,22 +109,23 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
     }
   if after.len > 0:
     variables["cursor"] = % after
-  let url = graphListMembers ? genParams($variables)
+  let 
-  result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
+    url = apiReq(graphListMembers, $variables)
+    js = await fetchRaw(url)
+  result = parseGraphListMembers(js, after)
 
 proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
   if id.len == 0: return
   let
-    variables = """{"rest_id": "$1"}""" % id
+    url = apiReq(graphTweetResult, """{"rest_id": "$1"}""" % id)
-    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(url)
-    js = await fetch(graphTweetResult ? params, Api.tweetResult)
   result = parseGraphTweetResult(js)
 
 proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
   if id.len == 0: return
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    js = await fetch(tweetDetailUrl(id, cursor), Api.tweetDetail)
+    js = await fetch(tweetDetailUrl(id, cursor))
   result = parseGraphConversation(js, id)
 
 proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
@@ -150,8 +154,10 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
     }
   if after.len > 0:
     variables["cursor"] = % after
-  let url = graphSearchTimeline ? genParams($variables)
+  let 
-  result = parseGraphSearch[Tweets](await fetch(url, Api.search), after)
+    url = apiReq(graphSearchTimeline, $variables)
+    js = await fetch(url)
+  result = parseGraphSearch[Tweets](js, after)
   result.query = query
 
 proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} =
@@ -172,13 +178,15 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
     variables["cursor"] = % after
     result.beginning = false
 
-  let url = graphSearchTimeline ? genParams($variables)
+  let 
-  result = parseGraphSearch[User](await fetch(url, Api.search), after)
+    url = apiReq(graphSearchTimeline, $variables)
+    js = await fetch(url)
+  result = parseGraphSearch[User](js, after)
   result.query = query
 
 proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
   if id.len == 0: return
-  let js = await fetch(mediaUrl(id, ""), Api.userMedia)
+  let js = await fetch(mediaUrl(id, ""))
   result = parseGraphPhotoRail(js)
 
 proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =

src/apiutils.nim

@@ -1,16 +1,30 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import httpclient, asyncdispatch, options, strutils, uri, times, math, tables, logging
 import jsony, packedjson, zippy, oauth1
-import types, auth, consts, parserutils, http_pool
+import types, auth, consts, parserutils, http_pool, tid
 import experimental/types/common
 
 const
   rlRemaining = "x-rate-limit-remaining"
   rlReset = "x-rate-limit-reset"
   rlLimit = "x-rate-limit-limit"
-  errorsToSkip = {doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
+  errorsToSkip = {null, doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
 
-var pool: HttpPool
+var 
+  pool: HttpPool
+  disableTid: bool
+
+proc setDisableTid*(disable: bool) =
+  disableTid = disable
+
+proc toUrl(req: ApiReq; sessionKind: SessionKind): Uri =
+  case sessionKind
+  of oauth:  
+    let o = req.oauth
+    parseUri("https://api.x.com/graphql")   / o.endpoint ? o.params
+  of cookie: 
+    let c = req.cookie
+    parseUri("https://x.com/i/api/graphql") / c.endpoint ? c.params
 
 proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
   let
@@ -32,15 +46,15 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
 proc getCookieHeader(authToken, ct0: string): string =
   "auth_token=" & authToken & "; ct0=" & ct0
 
-proc genHeaders*(session: Session, url: string): HttpHeaders =
+proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
   result = newHttpHeaders({
     "connection": "keep-alive",
     "content-type": "application/json",
     "x-twitter-active-user": "yes",
     "x-twitter-client-language": "en",
-    "authority": "api.x.com",
+    "origin": "https://x.com",
     "accept-encoding": "gzip",
-    "accept-language": "en-US,en;q=0.9",
+    "accept-language": "en-US,en;q=0.5",
     "accept": "*/*",
     "DNT": "1",
     "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
@@ -48,15 +62,20 @@ proc genHeaders*(session: Session, url: string): HttpHeaders =
 
   case session.kind
   of SessionKind.oauth:
-    result["authorization"] = getOauthHeader(url, session.oauthToken, session.oauthSecret)
+    result["authority"] = "api.x.com"
+    result["authorization"] = getOauthHeader($url, session.oauthToken, session.oauthSecret)
   of SessionKind.cookie:
-    result["authorization"] = "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
     result["x-twitter-auth-type"] = "OAuth2Session"
     result["x-csrf-token"] = session.ct0
     result["cookie"] = getCookieHeader(session.authToken, session.ct0)
+    if disableTid:
+      result["authorization"] = bearerToken2
+    else:
+      result["authorization"] = bearerToken
+      result["x-client-transaction-id"] = await genTid(url.path)
 
-proc getAndValidateSession*(api: Api): Future[Session] {.async.} =
+proc getAndValidateSession*(req: ApiReq): Future[Session] {.async.} =
-  result = await getSession(api)
+  result = await getSession(req)
   case result.kind
   of SessionKind.oauth:
     if result.oauthToken.len == 0:
@@ -73,7 +92,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
 
   try:
     var resp: AsyncResponse
-    pool.use(genHeaders(session, $url)):
+    pool.use(await genHeaders(session, url)):
       template getContent =
         resp = await c.get($url)
         result = await resp.body
@@ -89,7 +108,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
         remaining = parseInt(resp.headers[rlRemaining])
         reset = parseInt(resp.headers[rlReset])
         limit = parseInt(resp.headers[rlLimit])
-      session.setRateLimit(api, remaining, reset, limit)
+      session.setRateLimit(req, remaining, reset, limit)
 
     if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":
@@ -98,24 +117,22 @@ template fetchImpl(result, fetchBody) {.dirty.} =
       if result.startsWith("{\"errors"):
         let errors = result.fromJson(Errors)
         if errors notin errorsToSkip:
-          error "Fetch error, API: ", api, ", errors: ", errors
+          error "Fetch error, API: ", url.path, ", errors: ", errors
           if errors in {expiredToken, badToken, locked}:
             invalidate(session)
             raise rateLimitError()
           elif errors in {rateLimited}:
             # rate limit hit, resets after 24 hours
-            setLimited(session, api)
+            setLimited(session, req)
             raise rateLimitError()
       elif result.startsWith("429 Too Many Requests"):
-        warn "[sessions] 429 error, API: ", api, ", session: ", session.pretty
+        warn "[sessions] 429 error, API: ", url.path, ", session: ", session.pretty
-        session.apis[api].remaining = 0
-        # rate limit hit, resets after the 15 minute window
         raise rateLimitError()
 
     fetchBody
 
     if resp.status == $Http400:
-      error "ERROR 400, ", api, ": ", result
+      error "ERROR 400, ", url.path, ": ", result
       raise newException(InternalError, $url)
   except InternalError as e:
     raise e
@@ -137,19 +154,16 @@ template retry(bod) =
   try:
     bod
   except RateLimitError:
-    info "[sessions] Rate limited, retrying ", api, " request..."
+    info "[sessions] Rate limited, retrying ", req.cookie.endpoint, " request..."
     bod
 
-proc fetch*(url: Uri | SessionAwareUrl; api: Api): Future[JsonNode] {.async.} =
+proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
   retry:
     var 
       body: string
-      session = await getAndValidateSession(api)
+      session = await getAndValidateSession(req)
 
-    when url is SessionAwareUrl:
+    let url = req.toUrl(session.kind)
-      let url = case session.kind
-        of SessionKind.oauth: url.oauthUrl
-        of SessionKind.cookie: url.cookieUrl
 
     fetchImpl body:
       if body.startsWith('{') or body.startsWith('['):
@@ -158,21 +172,17 @@ proc fetch*(url: Uri | SessionAwareUrl; api: Api): Future[JsonNode] {.async.} =
         warn resp.status, ": ", body, " --- url: ", url
         result = newJNull()
 
-      let apiErr = result.getError
+      let error = result.getError
-      if apiErr != null and apiErr notin errorsToSkip:
+      if error != null and error notin errorsToSkip:
-        error "Fetch error, API: ", api, ", error: ", apiErr
+        error "Fetch error, API: ", url.path, ", error: ", error
-        if apiErr in {expiredToken, badToken, locked}:
+        if error in {expiredToken, badToken, locked}:
           invalidate(session)
           raise rateLimitError()
 
-proc fetchRaw*(url: Uri | SessionAwareUrl; api: Api): Future[string] {.async.} =
+proc fetchRaw*(req: ApiReq): Future[string] {.async.} =
   retry:
-    var session = await getAndValidateSession(api)
+    var session = await getAndValidateSession(req)
-
+    let url = req.toUrl(session.kind)
-    when url is SessionAwareUrl:
-      let url = case session.kind
-        of SessionKind.oauth: url.oauthUrl
-        of SessionKind.cookie: url.cookieUrl
 
     fetchImpl result:
       if not (result.startsWith('{') or result.startsWith('[')):

src/auth.nim

@@ -1,6 +1,6 @@
 #SPDX-License-Identifier: AGPL-3.0-only
 import std/[asyncdispatch, times, json, random, strutils, tables, packedsets, os, logging]
-import types
+import types, consts
 import experimental/parser/session
 
 # max requests at a time per session to avoid race conditions
@@ -18,6 +18,11 @@ proc logSession(args: varargs[string, `$`]) =
     s.add arg
   info s
 
+proc endpoint(req: ApiReq; session: Session): string =
+  case session.kind
+  of oauth: req.oauth.endpoint
+  of cookie: req.cookie.endpoint
+
 proc pretty*(session: Session): string =
   if session.isNil:
     return "<null>"
@@ -125,11 +130,12 @@ proc rateLimitError*(): ref RateLimitError =
 proc noSessionsError*(): ref NoSessionsError =
   newException(NoSessionsError, "no sessions available")
 
-proc isLimited(session: Session; api: Api): bool =
+proc isLimited(session: Session; req: ApiReq): bool =
   if session.isNil:
     return true
 
-  if session.limited and api != Api.userTweets:
+  let api = req.endpoint(session)
+  if session.limited and api != graphUserTweetsV2:
     if (epochTime().int - session.limitedAt) > hourInSeconds:
       session.limited = false
       logSession "resetting limit: ", session.pretty
@@ -143,8 +149,8 @@ proc isLimited(session: Session; api: Api): bool =
   else:
     return false
 
-proc isReady(session: Session; api: Api): bool =
+proc isReady(session: Session; req: ApiReq): bool =
-  not (session.isNil or session.pending > maxConcurrentReqs or session.isLimited(api))
+  not (session.isNil or session.pending > maxConcurrentReqs or session.isLimited(req))
 
 proc invalidate*(session: var Session) =
   if session.isNil: return
@@ -159,24 +165,26 @@ proc release*(session: Session) =
   if session.isNil: return
   dec session.pending
 
-proc getSession*(api: Api): Future[Session] {.async.} =
+proc getSession*(req: ApiReq): Future[Session] {.async.} =
   for i in 0 ..< sessionPool.len:
-    if result.isReady(api): break
+    if result.isReady(req): break
     result = sessionPool.sample()
 
-  if not result.isNil and result.isReady(api):
+  if not result.isNil and result.isReady(req):
     inc result.pending
   else:
-    logSession "no sessions available for API: ", api
+    logSession "no sessions available for API: ", req.cookie.endpoint
     raise noSessionsError()
 
-proc setLimited*(session: Session; api: Api) =
+proc setLimited*(session: Session; req: ApiReq) =
+  let api = req.endpoint(session)
   session.limited = true
   session.limitedAt = epochTime().int
   logSession "rate limited by api: ", api, ", reqs left: ", session.apis[api].remaining, ", ", session.pretty
 
-proc setRateLimit*(session: Session; api: Api; remaining, reset, limit: int) =
+proc setRateLimit*(session: Session; req: ApiReq; remaining, reset, limit: int) =
   # avoid undefined behavior in race conditions
+  let api = req.endpoint(session)
   if api in session.apis:
     let rateLimit = session.apis[api]
     if rateLimit.reset >= reset and rateLimit.remaining < remaining:

src/config.nim

@@ -43,7 +43,8 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
     logLevel: cfg.get("Config", "logLevel", ""),
     proxy: cfg.get("Config", "proxy", ""),
     proxyAuth: cfg.get("Config", "proxyAuth", ""),
-    defaultFollowedAccounts: cfg.get("Config", "defaultFollowedAccounts", @["eff", "fsf"])
+    defaultFollowedAccounts: cfg.get("Config", "defaultFollowedAccounts", @["eff", "fsf"]),
+    disableTid: cfg.get("Config", "disableTid", false)
   )
 
   return (conf, cfg)

src/consts.nim

@@ -1,28 +1,29 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import uri, strutils
+import strutils
 
 const
   consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
   consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
+  bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
+  bearerToken2* = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
 
-  gql = parseUri("https://api.x.com") / "graphql"
+  graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
-
+  graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery"
-  graphUser* = gql / "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery"
+  graphUserById* = "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery"
-  graphUserById* = gql / "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery"
+  graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2"
-  graphUserTweetsV2* = gql / "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2"
+  graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2"
-  graphUserTweetsAndRepliesV2* = gql / "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2"
+  graphUserTweets* = "oRJs8SLCRNRbQzuZG93_oA/UserTweets"
-  graphUserTweets* = gql / "oRJs8SLCRNRbQzuZG93_oA/UserTweets"
+  graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
-  graphUserTweetsAndReplies* = gql / "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
+  graphUserMedia* = "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
-  graphUserMedia* = gql / "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
+  graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
-  graphUserMediaV2* = gql / "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
+  graphTweet* = "Y4Erk_-0hObvLpz0Iw3bzA/ConversationTimeline"
-  graphTweet* = gql / "Y4Erk_-0hObvLpz0Iw3bzA/ConversationTimeline"
+  graphTweetDetail* = "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail"
-  graphTweetDetail* = gql / "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail"
+  graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
-  graphTweetResult* = gql / "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
+  graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
-  graphSearchTimeline* = gql / "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
+  graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
-  graphListById* = gql / "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
+  graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
-  graphListBySlug* = gql / "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
+  graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
-  graphListMembers* = gql / "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
+  graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
-  graphListTweets* = gql / "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
 
   gqlFeatures* = """{
   "android_ad_formats_media_component_render_overlay_enabled": false,
@@ -97,10 +98,14 @@ const
   "grok_translations_community_note_auto_translation_is_enabled": false,
   "grok_translations_post_auto_translation_is_enabled": false,
   "grok_translations_community_note_translation_is_enabled": false,
-  "grok_translations_timeline_user_bio_auto_translation_is_enabled": false
+  "grok_translations_timeline_user_bio_auto_translation_is_enabled": false,
+  "subscriptions_feature_can_gift_premium": false,
+  "responsive_web_twitter_article_notes_tab_enabled": false,
+  "subscriptions_verification_info_is_identity_verified_enabled": false,
+  "hidden_profile_subscriptions_enabled": false
 }""".replace(" ", "").replace("\n", "")
 
-  tweetVariables* = """{
+  tweetVars* = """{
   "postId": "$1",
   $2
   "includeHasBirdwatchNotes": false,
@@ -110,7 +115,7 @@ const
   "withV2Timeline": true
 }""".replace(" ", "").replace("\n", "")
 
-  tweetDetailVariables* = """{
+  tweetDetailVars* = """{
   "focalTweetId": "$1",
   $2
   "referrer": "profile",
@@ -123,12 +128,12 @@ const
   "withVoice": true
 }""".replace(" ", "").replace("\n", "")
 
-  restIdVariables* = """{
+  restIdVars* = """{
   "rest_id": "$1", $2
   "count": 20
 }"""
 
-  userMediaVariables* = """{
+  userMediaVars* = """{
   "userId": "$1", $2
   "count": 20,
   "includePromotedContent": false,
@@ -137,7 +142,7 @@ const
   "withVoice": true
 }""".replace(" ", "").replace("\n", "")
 
-  userTweetsVariables* = """{
+  userTweetsVars* = """{
   "userId": "$1", $2
   "count": 20,
   "includePromotedContent": false,
@@ -145,7 +150,7 @@ const
   "withVoice": true
 }""".replace(" ", "").replace("\n", "")
 
-  userTweetsAndRepliesVariables* = """{
+  userTweetsAndRepliesVars* = """{
   "userId": "$1", $2
   "count": 20,
   "includePromotedContent": false,
@@ -154,4 +159,6 @@ const
 }""".replace(" ", "").replace("\n", "")
 
   fieldToggles* = """{"withArticlePlainText":false,"withViewCounts":true}"""
-  tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false,"withViewCounts":true}"""
+  userFieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}"""
+  userTweetsFieldToggles* = """{"withArticlePlainText":false}"""
+  tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}"""

src/experimental/parser/graphql.nim

@@ -1,6 +1,6 @@
 import options, strutils
 import jsony
-import user, ../types/[graphuser, graphlistmembers]
+import user, utils, ../types/[graphuser, graphlistmembers]
 from ../../types import User, VerifiedType, Result, Query, QueryKind
 
 proc parseUserResult*(userResult: UserResult): User =
@@ -15,22 +15,36 @@ proc parseUserResult*(userResult: UserResult): User =
     result.fullname = userResult.core.name
     result.userPic = userResult.avatar.imageUrl.replace("_normal", "")
 
+    if userResult.privacy.isSome:
+      result.protected = userResult.privacy.get.protected
+
+    if userResult.location.isSome:
+      result.location = userResult.location.get.location
+
+    if userResult.core.createdAt.len > 0:
+      result.joinDate = parseTwitterDate(userResult.core.createdAt)
+
     if userResult.verification.isSome:
       let v = userResult.verification.get
       if v.verifiedType != VerifiedType.none:
         result.verifiedType = v.verifiedType
 
-    if userResult.profileBio.isSome:
+    if userResult.profileBio.isSome and result.bio.len == 0:
       result.bio = userResult.profileBio.get.description
 
 proc parseGraphUser*(json: string): User =
   if json.len == 0 or json[0] != '{':
     return
 
-  let raw = json.fromJson(GraphUser)
+  let 
-  let userResult = raw.data.userResult.result
+    raw = json.fromJson(GraphUser)
+    userResult = 
+      if raw.data.userResult.isSome: raw.data.userResult.get.result
+      elif raw.data.user.isSome: raw.data.user.get.result
+      else: UserResult()
 
-  if userResult.unavailableReason.get("") == "Suspended":
+  if userResult.unavailableReason.get("") == "Suspended" or
+     userResult.reason.get("") == "Suspended":
     return User(suspended: true)
 
   result = parseUserResult(userResult)

src/experimental/parser/tid.nim

@@ -0,0 +1,8 @@
+import jsony
+import ../types/tid
+export TidPair
+
+proc parseTidPairs*(raw: string): seq[TidPair] =
+  result = raw.fromJson(seq[TidPair])
+  if result.len == 0:
+    raise newException(ValueError, "Parsing pairs failed: " & raw)

src/experimental/parser/user.nim

@@ -58,11 +58,13 @@ proc toUser*(raw: RawUser): User =
     media: raw.mediaCount,
     verifiedType: raw.verifiedType,
     protected: raw.protected,
-    joinDate: parseTwitterDate(raw.createdAt),
     banner: getBanner(raw),
     userPic: getImageUrl(raw.profileImageUrlHttps).replace("_normal", "")
   )
 
+  if raw.createdAt.len > 0:
+    result.joinDate = parseTwitterDate(raw.createdAt)
+
   if raw.pinnedTweetIdsStr.len > 0:
     result.pinnedTweet = parseBiggestInt(raw.pinnedTweetIdsStr[0])
 

src/experimental/types/graphuser.nim

@@ -3,7 +3,7 @@ from ../../types import User, VerifiedType
 
 type
   GraphUser* = object
-    data*: tuple[userResult: UserData]
+    data*: tuple[userResult: Option[UserData], user: Option[UserData]]
 
   UserData* = object
     result*: UserResult
@@ -22,15 +22,24 @@ type
   Verification* = object
     verifiedType*: VerifiedType
 
+  Location* = object
+    location*: string
+
+  Privacy* = object
+    protected*: bool
+
   UserResult* = object
     legacy*: User
     restId*: string
     isBlueVerified*: bool
-    unavailableReason*: Option[string]
     core*: UserCore
     avatar*: UserAvatar
+    unavailableReason*: Option[string]
+    reason*: Option[string]
+    privacy*: Option[Privacy]
     profileBio*: Option[UserBio]
     verification*: Option[Verification]
+    location*: Option[Location]
 
 proc enumHook*(s: string; v: var VerifiedType) =
   v = try:

src/experimental/types/tid.nim

@@ -0,0 +1,4 @@
+type
+  TidPair* = object
+    animationKey*: string
+    verification*: string

src/formatters.nim

@@ -6,7 +6,7 @@ import types, utils, query
 const
   cards = "cards.twitter.com/cards"
   tco = "https://t.co"
-  twitter = parseUri("https://twitter.com")
+  twitter = parseUri("https://x.com")
 
 let
   twRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?twitter\.com"
@@ -64,18 +64,20 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
   result = body
 
   if prefs.replaceYouTube.len > 0 and "youtu" in result:
-    result = result.replace(ytRegex, prefs.replaceYouTube)
+    let youtubeHost = strip(prefs.replaceYouTube, chars={'/'})
+    result = result.replace(ytRegex, youtubeHost)
 
   if prefs.replaceTwitter.len > 0:
+    let twitterHost = strip(prefs.replaceTwitter, chars={'/'})
     if tco in result:
-      result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
+      result = result.replace(tco, https & twitterHost & "/t.co")
     if "x.com" in result:
-      result = result.replace(xRegex, prefs.replaceTwitter)
+      result = result.replace(xRegex, twitterHost)
       result = result.replacef(xLinkRegex, a(
-        prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
+        twitterHost & "$2", href = https & twitterHost & "$1"))
     if "twitter.com" in result:
-      result = result.replace(cards, prefs.replaceTwitter & "/cards")
+      result = result.replace(cards, twitterHost & "/cards")
-      result = result.replace(twRegex, prefs.replaceTwitter)
+      result = result.replace(twRegex, twitterHost)
       result = result.replacef(twLinkRegex, a(
         prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
     if "imgur.com" in result:
@@ -96,9 +98,10 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
       prefs.replaceSoundCloud & "$4", href = https & prefs.replaceSoundCloud & "$2"))
 
   if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result):
-    result = result.replace(rdShortRegex, prefs.replaceReddit & "/comments/")
+    let redditHost = strip(prefs.replaceReddit, chars={'/'})
-    result = result.replace(rdRegex, prefs.replaceReddit)
+    result = result.replace(rdShortRegex, redditHost & "/comments/")
-    if prefs.replaceReddit in result and "/gallery/" in result:
+    result = result.replace(rdRegex, redditHost)
+    if redditHost in result and "/gallery/" in result:
       result = result.replace("/gallery/", "/comments/")
 
   if absolute.len > 0 and "href" in result:

src/nitter.nim

@@ -6,7 +6,7 @@ from os import getEnv
 
 import jester
 
-import types, config, prefs, formatters, redis_cache, http_pool, auth, query
+import types, config, prefs, formatters, redis_cache, http_pool, auth, apiutils
 import views/[general, about, search, homepage]
 import karax/[vdom]
 import routes/[
@@ -73,6 +73,7 @@ setHmacKey(cfg.hmacKey)
 setProxyEncoding(cfg.base64Media)
 setMaxHttpConns(cfg.httpMaxConns)
 setHttpProxy(cfg.proxy, cfg.proxyAuth)
+setDisableTid(cfg.disableTid)
 initAboutPage(cfg.staticDir)
 
 waitFor initRedisPool(cfg)

src/parser.nim

@@ -21,7 +21,7 @@ proc parseUser(js: JsonNode; id=""): User =
     tweets: js{"statuses_count"}.getInt,
     likes: js{"favourites_count"}.getInt,
     media: js{"media_count"}.getInt,
-    protected: js{"protected"}.getBool,
+    protected: js{"protected"}.getBool(js{"privacy", "protected"}.getBool),
     sensitive: "sensitive" in js{"profile_interstitial_type"}.getStr("") or js{"possibly_sensitive"}.getBool,
     joinDate: js{"created_at"}.getTime
   )
@@ -185,7 +185,7 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) =
   # Remove media URLs from text
   with mediaList, js{"legacy", "entities", "media"}:
     for url in mediaList:
-      let expandedUrl = url{"expanded_url"}.getStr
+      let expandedUrl = url.getExpandedUrl
       if result.text.endsWith(expandedUrl):
         result.text.removeSuffix(expandedUrl)
         result.text = result.text.strip()
@@ -268,7 +268,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
 
   for u in ? urls:
     if u{"url"}.getStr == result.url:
-      result.url = u{"expanded_url"}.getStr
+      result.url = u.getExpandedUrl(result.url)
       break
 
   if kind in {videoDirectMessage, imageDirectMessage}:
@@ -487,6 +487,7 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
             result.before.content.add tweet
         elif entryId.startsWith("cursor-bottom"):
           var cursorValue = select(
+            e{"content", "value"},
             e{"content", "content", "value"},
             e{"content", "itemContent", "value"}
           )

src/parserutils.nim

@@ -112,6 +112,9 @@ proc getImageStr*(js: JsonNode): string =
 template getImageVal*(js: JsonNode): string =
   js{"image_value", "url"}.getImageStr
 
+template getExpandedUrl*(js: JsonNode; fallback=""): string =
+  js{"expanded_url"}.getStr(js{"url"}.getStr(fallback))
+
 proc getCardUrl*(js: JsonNode; kind: CardKind): string =
   result = js{"website_url"}.getStrVal
   if kind == promoVideoConvo:
@@ -177,7 +180,7 @@ proc extractSlice(js: JsonNode): Slice[int] =
 proc extractUrls(result: var seq[ReplaceSlice]; js: JsonNode;
                  textLen: int; hideTwitter = false) =
   let
-    url = js["expanded_url"].getStr
+    url = js.getExpandedUrl
     slice = js.extractSlice
 
   if hideTwitter and slice.b.succ >= textLen and url.isTwitterUrl:
@@ -238,7 +241,7 @@ proc expandUserEntities*(user: var User; js: JsonNode) =
     ent = ? js{"entities"}
 
   with urls, ent{"url", "urls"}:
-    user.website = urls[0]{"expanded_url"}.getStr
+    user.website = urls[0].getExpandedUrl
 
   var replacements = newSeq[ReplaceSlice]()
 
@@ -268,7 +271,7 @@ proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlic
       replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink)
 
       if hasCard and u{"url"}.getStr == get(tweet.card).url:
-        get(tweet.card).url = u{"expanded_url"}.getStr
+        get(tweet.card).url = u.getExpandedUrl
 
   with media, entities{"media"}:
     for m in media:

src/query.nim

@@ -6,10 +6,9 @@ import types
 const
   validFilters* = @[
     "media", "images", "twimg", "videos",
-    "native_video", "consumer_video", "pro_video",
+    "native_video", "consumer_video", "spaces",
     "links", "news", "quote", "mentions",
-    "replies", "retweets", "nativeretweets",
+    "replies", "retweets", "nativeretweets"
-    "verified", "safe"
   ]
 
   emptyQuery* = "include:nativeretweets"
@@ -18,6 +17,11 @@ template `@`(param: string): untyped =
   if param in pms: pms[param]
   else: ""
 
+proc validateNumber(value: string): string =
+  if value.anyIt(not it.isDigit):
+    return ""
+  return value
+
 proc initQuery*(pms: Table[string, string]; name=""): Query =
   result = Query(
     kind: parseEnum[QueryKind](@"f", tweets),
@@ -26,7 +30,8 @@ proc initQuery*(pms: Table[string, string]; name=""): Query =
     excludes: validFilters.filterIt("e-" & it in pms),
     since: @"since",
     until: @"until",
-    near: @"near"
+    near: @"near",
+    minLikes: validateNumber(@"min_faves")
   )
 
   if name.len > 0:
@@ -82,7 +87,9 @@ proc genQueryParam*(query: Query): string =
   if query.until.len > 0:
     result &= " until:" & query.until
   if query.near.len > 0:
-    result &= &" near:\"{query.near}\" within:15mi"
+    result &= " near:\"" & query.near & "\""
+  if query.minLikes.len > 0:
+    result &= " min_faves:" & query.minLikes
   if query.text.len > 0:
     if result.len > 0:
       result &= " " & query.text
@@ -108,6 +115,8 @@ proc genQueryUrl*(query: Query): string =
     params.add "until=" & query.until
   if query.near.len > 0:
     params.add "near=" & query.near
+  if query.minLikes.len > 0:
+    params.add "min_faves=" & query.minLikes
 
   if params.len > 0:
     result &= params.join("&")

src/routes/status.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, sequtils, uri, options, sugar, logging
+import asyncdispatch, strutils, sequtils, uri, options, sugar
 
 import jester, karax/vdom
 
@@ -31,8 +31,6 @@ proc createStatusRouter*(cfg: Config) =
         resp $renderReplies(replies, prefs, getPath())
 
       let conv = await getTweet(id, getCursor())
-      if conv == nil:
-        warn "nil conv"
 
       if conv == nil or conv.tweet == nil or conv.tweet.id == 0:
         var error = "Tweet not found"
@@ -68,7 +66,7 @@ proc createStatusRouter*(cfg: Config) =
 
     get "/@name/@s/@id/@m/?@i?":
       cond @"s" in ["status", "statuses"]
-      cond @"m" in ["video", "photo"]
+      cond @"m" in ["video", "photo", "history"]
       redirect("/$1/status/$2" % [@"name", @"id"])
 
     get "/@name/statuses/@id/?":

src/routes/timeline.nim

@@ -105,6 +105,12 @@ proc createTimelineRouter*(cfg: Config) =
     get "/intent/user":
       respUserId()
 
+    get "/intent/follow/?":
+      let username = request.params.getOrDefault("screen_name")
+      if username.len == 0:
+        resp Http400, showError("Missing screen_name parameter", cfg)
+      redirect("/" & username)
+
     get "/@name/?@tab?/?":
       cond '.' notin @"name"
       cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]

src/routes/unsupported.nim

@@ -17,7 +17,7 @@ proc createUnsupportedRouter*(cfg: Config) =
     get "/@name/lists/?": feature()
 
     get "/intent/?@i?": 
-      cond @"i" notin ["user"]
+      cond @"i" notin ["user", "follow"]
       feature()
 
     get "/i/@i?/?@j?":

src/sass/index.scss

@@ -52,7 +52,7 @@ body {
     background-color: var(--bg_color);
     color: var(--fg_color);
     font-family: $font_stack;
-    font-size: 14px;
+    font-size: 15px;
     line-height: 1.3;
     margin: 0;
 }

src/sass/inputs.scss

@@ -14,6 +14,7 @@ button {
 
 input[type="text"],
 input[type="date"],
+input[type="number"],
 select {
     @include input-colors;
     background-color: var(--bg_elements);
@@ -24,7 +25,12 @@ select {
     font-size: 14px;
 }
 
-input[type="text"] {
+input[type="number"] {
+    -moz-appearance: textfield;
+}
+
+input[type="text"],
+input[type="number"] {
     height: 16px;
 }
 
@@ -38,6 +44,17 @@ input[type="date"]::-webkit-inner-spin-button {
     display: none;
 }
 
+input[type="number"] {
+    -moz-appearance: textfield;
+}
+
+input[type="number"]::-webkit-inner-spin-button,
+input[type="number"]::-webkit-outer-spin-button {
+    display: none;
+    -webkit-appearance: none;
+    margin: 0;
+}
+
 input[type="date"]::-webkit-clear-button {
     margin-left: 17px;
     filter: grayscale(100%);
@@ -176,7 +193,8 @@ input::-webkit-datetime-edit-year-field:focus {
         color: var(--fg_color);
     }
 
-    input[type="text"] {
+    input[type="text"],
+    input[type="number"] {
         position: absolute;
         right: 0;
         max-width: 140px;

src/sass/search.scss

@@ -27,7 +27,8 @@
         min-width: 0;
     }
 
-    input[type="text"] {
+    input[type="text"],
+    input[type="number"] {
         height: calc(100% - 4px);
         width: calc(100% - 8px);
     }
@@ -155,15 +156,14 @@
 
 .profile-tabs {
     @include search-resize(820px, 5);
-    @include search-resize(725px, 4);
+    @include search-resize(715px, 4);
-    @include search-resize(600px, 6);
+    @include search-resize(700px, 5);
-    @include search-resize(560px, 5);
+    @include search-resize(485px, 4);
-    @include search-resize(480px, 4);
     @include search-resize(410px, 3);
 }
 
-@include search-resize(560px, 5);
+@include search-resize(700px, 5);
-@include search-resize(480px, 4);
+@include search-resize(485px, 4);
 @include search-resize(410px, 3);
 
 @media(max-width: 600px) {

src/tid.nim

@@ -0,0 +1,62 @@
+import std/[asyncdispatch, base64, httpclient, random, strutils, sequtils, times]
+import nimcrypto
+import experimental/parser/tid
+
+randomize()
+
+const defaultKeyword = "obfiowerehiring";
+const pairsUrl =
+  "https://raw.githubusercontent.com/fa0311/x-client-transaction-id-pair-dict/refs/heads/main/pair.json";
+
+var
+  cachedPairs: seq[TidPair] = @[]
+  lastCached = 0
+  # refresh every hour
+  ttlSec = 60 * 60
+
+proc getPair(): Future[TidPair] {.async.} =
+  if cachedPairs.len == 0 or int(epochTime()) - lastCached > ttlSec:
+    lastCached = int(epochTime())
+
+    let client = newAsyncHttpClient()
+    defer: client.close()
+
+    let resp = await client.get(pairsUrl)
+    if resp.status == $Http200:
+      cachedPairs = parseTidPairs(await resp.body)
+
+  return sample(cachedPairs)
+
+proc encodeSha256(text: string): array[32, byte] =
+  let
+    data = cast[ptr byte](addr text[0])
+    dataLen = uint(len(text))
+    digest = sha256.digest(data, dataLen)
+  return digest.data
+
+proc encodeBase64[T](data: T): string =
+  return encode(data).replace("=", "")
+
+proc decodeBase64(data: string): seq[byte] =
+  return cast[seq[byte]](decode(data))
+
+proc genTid*(path: string): Future[string] {.async.} =
+  let 
+    pair = await getPair()
+
+    timeNow = int(epochTime() - 1682924400)
+    timeNowBytes = @[
+      byte(timeNow and 0xff),
+      byte((timeNow shr 8) and 0xff),
+      byte((timeNow shr 16) and 0xff),
+      byte((timeNow shr 24) and 0xff)
+    ]
+
+    data = "GET!" & path & "!" & $timeNow & defaultKeyword & pair.animationKey
+    hashBytes = encodeSha256(data)
+    keyBytes = decodeBase64(pair.verification)
+    bytesArr = keyBytes & timeNowBytes & hashBytes[0 ..< 16] & @[3'u8]
+    randomNum = byte(rand(256))
+    tid = @[randomNum] & bytesArr.mapIt(it xor randomNum)
+
+  return encodeBase64(tid)

src/types.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import times, sequtils, options, tables, uri
+import times, sequtils, options, tables
 import prefs_impl
 
 genPrefsType()
@@ -13,19 +13,13 @@ type
   TimelineKind* {.pure.} = enum
     tweets, replies, media
 
-  Api* {.pure.} = enum
+  ApiUrl* = object
-    tweetDetail
+    endpoint*: string
-    tweetResult
+    params*: seq[(string, string)]
-    search
+
-    list
+  ApiReq* = object
-    listBySlug
+    oauth*: ApiUrl
-    listMembers
+    cookie*: ApiUrl
-    listTweets
-    userRestId
-    userScreenName
-    userTweets
-    userTweetsAndReplies
-    userMedia
 
   RateLimit* = object
     limit*: int
@@ -42,7 +36,7 @@ type
     pending*: int
     limited*: bool
     limitedAt*: int
-    apis*: Table[Api, RateLimit]
+    apis*: Table[string, RateLimit]
     case kind*: SessionKind
     of oauth:
       oauthToken*: string
@@ -51,10 +45,6 @@ type
       authToken*: string
       ct0*: string
 
-  SessionAwareUrl* = object
-    oauthUrl*: Uri
-    cookieUrl*: Uri
-
   Error* = enum
     null = 0
     noUserMatches = 17
@@ -142,6 +132,7 @@ type
     since*: string
     until*: string
     near*: string
+    minLikes*: string
     sep*: string
 
   Gif* = object
@@ -289,6 +280,7 @@ type
     proxy*: string
     proxyAuth*: string
     defaultFollowedAccounts*: seq[string]
+    disableTid*: bool
 
     rssCacheTime*: int
     listCacheTime*: int

src/views/general.nim

@@ -33,14 +33,14 @@ proc renderNavbar(cfg: Config; req: Request; rss, canonical: string): VNode =
       tdiv(class="nav-item right"):
         if cfg.enableRss and rss.len > 0:
           icon "rss", title="RSS Feed", href=rss
-        icon "bird", title="Open in Twitter", href=canonical
+        icon "bird", title="Open in X", href=canonical
         a(href="https://buymeacoffee.com/kuu7o"): verbatim lp
         icon "info", title="About", href="/about"
         icon "cog", title="Preferences", href=("/settings?referer=" & encodeUrl(path))
 
 proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
                  video=""; images: seq[string] = @[]; banner=""; ogTitle="";
-                 rss=""; canonical=""): VNode =
+                 rss=""; alternate=""): VNode =
   var theme = prefs.theme.toTheme
   if "theme" in req.params:
     theme = req.params["theme"].toTheme
@@ -54,7 +54,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
   let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
 
   buildHtml(head):
-    link(rel="stylesheet", type="text/css", href="/css/style.css?v=20")
+    link(rel="stylesheet", type="text/css", href="/css/style.css?v=21")
     link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=3")
 
     if theme.len > 0:
@@ -68,8 +68,8 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
     link(rel="search", type="application/opensearchdescription+xml", title=cfg.title,
                             href=opensearchUrl)
 
-    if canonical.len > 0:
+    if alternate.len > 0:
-      link(rel="canonical", href=canonical)
+      link(rel="alternate", href=alternate, title="View on X")
 
     if cfg.enableRss and rss.len > 0:
       link(rel="alternate", type="application/rss+xml", href=rss, title="RSS feed")
@@ -131,16 +131,16 @@ proc renderMain*(body: VNode; req: Request; cfg: Config; prefs=defaultPrefs;
                  titleText=""; desc=""; ogTitle=""; rss=""; video="";
                  images: seq[string] = @[]; banner=""): string =
 
-  let canonical = getTwitterLink(req.path, req.params)
+  let twitterLink = getTwitterLink(req.path, req.params)
 
   let node = buildHtml(html(lang="en")):
     renderHead(prefs, cfg, req, titleText, desc, video, images, banner, ogTitle,
-               rss, canonical)
+               rss, twitterLink)
 
     body(class=if prefs.verifiedBadge == "Hide all": "hide-verified-all"
                elif prefs.verifiedBadge == "Show official only": "hide-verified-blue"
                else: ""):
-      renderNavbar(cfg, req, rss, canonical)
+      renderNavbar(cfg, req, rss, twitterLink)
 
       tdiv(class="container"):
         body

src/views/renderutils.nim

@@ -89,6 +89,13 @@ proc genDate*(pref, state: string): VNode =
     input(name=pref, `type`="date", value=state)
     icon "calendar"
 
+proc genNumberInput*(pref, label, state, placeholder: string; class=""; autofocus=true; min="0"): VNode =
+  let p = placeholder
+  buildHtml(tdiv(class=("pref-group pref-input " & class))):
+    if label.len > 0:
+      label(`for`=pref): text label
+    input(name=pref, `type`="number", placeholder=p, value=state, autofocus=(autofocus and state.len == 0), min=min, step="1")
+
 proc genImg*(url: string; class=""): VNode =
   buildHtml():
     img(src=getPicUrl(url), class=class, alt="", loading="lazy")

src/views/rss.nimf

@@ -9,9 +9,9 @@
 #elif tweet.reply.len > 0: result = &"R to @{tweet.reply[0]}: "
 #end if
 #var text = stripHtml(tweet.text)
-##if unicode.runeLen(text) > 32:
+#if unicode.runeLen(text) > 100:
-##  text = unicode.runeSubStr(text, 0, 32) & "..."
+#  text = unicode.runeSubStr(text, 0, 64) & "..."
-##end if
+#end if
 #result &= xmltree.escape(text)
 #if result.len > 0: return
 #end if
@@ -25,7 +25,7 @@
 #end proc
 #
 #proc getDescription(desc: string; cfg: Config): string =
-Twitter feed for: ${desc}. Generated by ${cfg.hostname}
+Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)}
 #end proc
 #
 #proc getTweetsWithPinned(profile: Profile): seq[Tweets] =
@@ -56,7 +56,10 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
 <img src="${urlPrefix}${getPicUrl(photo)}" style="max-width:250px;" />
 #  end for
 #elif tweet.video.isSome:
-<img src="${urlPrefix}${getPicUrl(get(tweet.video).thumb)}" style="max-width:250px;" />
+<a href="${urlPrefix}${tweet.getLink}">
+<br>Video<br>
+  <img src="${urlPrefix}${getPicUrl(get(tweet.video).thumb)}" style="max-width:250px;" />
+</a>
 #elif tweet.gif.isSome:
 #  let thumb = &"{urlPrefix}{getPicUrl(get(tweet.gif).thumb)}"
 #  let url = &"{urlPrefix}{getPicUrl(get(tweet.gif).url)}"
@@ -69,14 +72,17 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
 #  end if
 #end if
 #if tweet.quote.isSome and get(tweet.quote).available:
-#  let quoteLink = getLink(get(tweet.quote))
+#  let quoteTweet = get(tweet.quote)
+#  let quoteLink = urlPrefix & getLink(quoteTweet)
+<hr/>
 <blockquote>
-  <p>
+<b>${quoteTweet.user.fullname} (@${quoteTweet.user.username})</b>
-    ${renderRssTweet(get(tweet.quote), cfg)}
+<p>
-  </p>
+${renderRssTweet(quoteTweet, cfg)}
-  <footer>
+</p>
-    — <cite><a href="${urlPrefix}${quoteLink}">${cfg.hostname}${quoteLink}</a></cite>
+<footer>
-  </footer>
+— <cite><a href="${quoteLink}">${quoteLink}</a>
+</footer>
 </blockquote>
 #end if
 #end proc
@@ -101,6 +107,18 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
         <title>${getTitle(tweet, retweet)}</title>
         <dc:creator>@${tweet.user.username}</dc:creator>
         <description><![CDATA[${renderRssTweet(tweet, cfg).strip(chars={'\n'})}]]></description>
+#    let desc = renderRssTweet(tweet, cfg).strip(chars={'\n'})
+#    if retweet.len > 0:
+        <description><![CDATA[
+<blockquote>
+<b>${tweet.user.fullname} (@${tweet.user.username})</b>
+<p>${desc}</p>
+<footer>— <cite><a href="${urlPrefix & link}">${urlPrefix & link}</a></footer>
+</blockquote>
+]]></description>
+#    else:
+        <description><![CDATA[${desc}]]></description>
+#    end if
         <pubDate>${getRfc822Time(tweet)}</pubDate>
         <guid>${urlPrefix & link}</guid>
         <link>${urlPrefix & link}</link>

src/views/timeline.nim

@@ -57,7 +57,7 @@ proc renderThread(thread: Tweets; prefs: Prefs; path: string): VNode =
 
 proc renderUser*(user: User; prefs: Prefs; path: string): VNode =
   let class = if user.sensitive: "timeline-item nsfw" else: "timeline-item"
-  buildHtml(tdiv(class=class)):
+  buildHtml(tdiv(class=class, data-username=user.username)):
     a(class="tweet-link", href=("/" & user.username))
     tdiv(class="tweet-body profile-result"):
       tdiv(class="tweet-header"):

src/views/tweet.nim

@@ -275,7 +275,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
     divClass.add " nsfw"
 
   if not tweet.available:
-    return buildHtml(tdiv(class=divClass & "unavailable timeline-item")):
+    return buildHtml(tdiv(class=divClass & "unavailable timeline-item", data-username=tweet.user.username)):
       tdiv(class="unavailable-box"):
         if tweet.tombstone.len > 0:
           text tweet.tombstone
@@ -297,7 +297,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
     tweet = tweet.retweet.get
     retweet = fullTweet.user.fullname
 
-  buildHtml(tdiv(class=("timeline-item " & divClass))):
+  buildHtml(tdiv(class=("timeline-item " & divClass), data-username=tweet.user.username)):
     if not mainTweet:
       a(class="tweet-link", href=getLink(tweet))
 

tests/test_card.py

@@ -11,12 +11,7 @@ card = [
     ['voidtarget/status/1094632512926605312',
      'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too)',
      'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim',
-     'gist.github.com', True],
+     'gist.github.com', True]
-
-    ['nim_lang/status/1082989146040340480',
-     'Nim in 2018: A short recap',
-     'There were several big news in the Nim world in 2018 – two new major releases, partnership with Status, and much more. But let us go chronologically.',
-     'nim-lang.org', True]
 ]
 
 no_thumb = [

tools/create_session_browser.py

@@ -94,7 +94,7 @@ async def login_and_get_cookies(username, password, totp_seed=None, headless=Fal
 
 async def main():
     if len(sys.argv) < 3:
-        print('Usage: python3 twitter-auth.py username password [totp_seed] [--append sessions.jsonl] [--headless]')
+        print('Usage: python3 create_session_browser.py username password [totp_seed] [--append file.jsonl] [--headless]')
         sys.exit(1)
 
     username = sys.argv[1]