GraphQL timeline (#812)

* Update deps

* Replace profile timeline with GraphQL endpoint

* Update GraphQL endpoint versions

* Use GraphQL for profile media tab

* Fix UserByRestId request

* Improve routing, fixes #814

* Fix token pool JSON

* Deduplicate GraphQL timeline endpoints

* Update list endpoints

* Use GraphQL for list tweets

* Remove debug leftover

* Replace old pinned tweet endpoint with GraphQL

* Validate tweet ID

* Minor token handling fix

* Hide US-only commerce cards

* Update config example

* Remove http pool and gzip from token pool

* Support tombstoned tweets in threads

* Retry GraphQL timeout errors

* Remove unnecessary 401 retry

* Remove broken timeout retry

* Update karax, use new bool attribute feature

* Update card test

* Fix odd edgecase with broken retweets

* Replace search endpoints, switch Bearer token

* Only parse user search if it's a list

* Fix quoted tweet crash

* Fix empty search query handling

* Fix invalid user search errors again

src/tokens.nim

@@ -1,8 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import asyncdispatch, httpclient, times, sequtils, json, random
 import strutils, tables
-import zippy
-import types, consts, http_pool
+import types, consts
 
 const
   maxConcurrentReqs = 5  # max requests at a time per token, to avoid race conditions
@@ -11,11 +10,12 @@ const
   failDelay = initDuration(minutes=30)
 
 var
-  clientPool: HttpPool
   tokenPool: seq[Token]
   lastFailed: Time
   enableLogging = false
 
+let headers = newHttpHeaders({"authorization": auth})
+
 template log(str) =
   if enableLogging: echo "[tokens] ", str
 
@@ -41,10 +41,12 @@ proc getPoolJson*(): JsonNode =
       let
         maxReqs =
           case api
-          of Api.listMembers, Api.listBySlug, Api.list,
-             Api.userRestId, Api.userScreenName, Api.tweetDetail: 500
           of Api.timeline: 187
-          else: 180
+          of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets,
+             Api.userTweets, Api.userTweetsAndReplies, Api.userMedia,
+             Api.userRestId, Api.userScreenName,
+             Api.tweetDetail, Api.tweetResult, Api.search: 500
+          of Api.userSearch: 900
         reqs = maxReqs - token.apis[api].remaining
 
       reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
@@ -65,18 +67,12 @@ proc fetchToken(): Future[Token] {.async.} =
   if getTime() - lastFailed < failDelay:
     raise rateLimitError()
 
-  let headers = newHttpHeaders({
-    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-    "accept-encoding": "gzip",
-    "accept-language": "en-US,en;q=0.5",
-    "connection": "keep-alive",
-    "authorization": auth
-  })
+  let client = newAsyncHttpClient(headers=headers)
 
   try:
     let
-      resp = clientPool.use(headers): await c.postContent(activate)
-      tokNode = parseJson(uncompress(resp))["guest_token"]
+      resp = await client.postContent(activate)
+      tokNode = parseJson(resp)["guest_token"]
       tok = tokNode.getStr($(tokNode.getInt))
       time = getTime()
 
@@ -86,6 +82,8 @@ proc fetchToken(): Future[Token] {.async.} =
     if "Try again" notin e.msg:
       echo "[tokens] fetching tokens paused, resuming in 30 minutes"
       lastFailed = getTime()
+  finally:
+    client.close()
 
 proc expired(token: Token): bool =
   let time = getTime()
@@ -158,7 +156,6 @@ proc poolTokens*(amount: int) {.async.} =
       tokenPool.add newToken
 
 proc initTokenPool*(cfg: Config) {.async.} =
-  clientPool = HttpPool()
   enableLogging = cfg.enableDebug
 
   while true: