GraphQL timeline (#812)

* Update deps

* Replace profile timeline with GraphQL endpoint

* Update GraphQL endpoint versions

* Use GraphQL for profile media tab

* Fix UserByRestId request

* Improve routing, fixes #814

* Fix token pool JSON

* Deduplicate GraphQL timeline endpoints

* Update list endpoints

* Use GraphQL for list tweets

* Remove debug leftover

* Replace old pinned tweet endpoint with GraphQL

* Validate tweet ID

* Minor token handling fix

* Hide US-only commerce cards

* Update config example

* Remove http pool and gzip from token pool

* Support tombstoned tweets in threads

* Retry GraphQL timeout errors

* Remove unnecessary 401 retry

* Remove broken timeout retry

* Update karax, use new bool attribute feature

* Update card test

* Fix odd edgecase with broken retweets

* Replace search endpoints, switch Bearer token

* Only parse user search if it's a list

* Fix quoted tweet crash

* Fix empty search query handling

* Fix invalid user search errors again

src/consts.nim

@@ -1,30 +1,28 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import uri, sequtils
+import uri, sequtils, strutils
 
 const
-  auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
+  auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
 
   api = parseUri("https://api.twitter.com")
   activate* = $(api / "1.1/guest/activate.json")
 
-  userShow* = api / "1.1/users/show.json"
   photoRail* = api / "1.1/statuses/media_timeline.json"
-  status* = api / "1.1/statuses/show"
-  search* = api / "2/search/adaptive.json"
-
-  timelineApi = api / "2/timeline"
-  timeline* = timelineApi / "profile"
-  mediaTimeline* = timelineApi / "media"
-  listTimeline* = timelineApi / "list.json"
-  tweet* = timelineApi / "conversation"
+  userSearch* = api / "1.1/users/search.json"
 
   graphql = api / "graphql"
-  graphTweet* = graphql / "6lWNh96EXDJCXl05SAtn_g/TweetDetail"
-  graphUser* = graphql / "7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName"
-  graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
-  graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
-  graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
-  graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
+  graphUser* = graphql / "8mPfHBetXOg-EHAyeVxUoA/UserByScreenName"
+  graphUserById* = graphql / "nI8WydSd-X-lQIVo6bdktQ/UserByRestId"
+  graphUserTweets* = graphql / "9rys0A7w1EyqVd2ME0QCJg/UserTweets"
+  graphUserTweetsAndReplies* = graphql / "ehMCHF3Mkgjsfz_aImqOsg/UserTweetsAndReplies"
+  graphUserMedia* = graphql / "MA_EP2a21zpzNWKRkaPBMg/UserMedia"
+  graphTweet* = graphql / "6I7Hm635Q6ftv69L8VrSeQ/TweetDetail"
+  graphTweetResult* = graphql / "rt-rHeSJ-2H9O9gxWQcPcg/TweetResultByRestId"
+  graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline"
+  graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
+  graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug"
+  graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
+  graphListTweets* = graphql / "jZntL0oVJSdjhmPcdbw_eA/ListLatestTweetsTimeline"
 
   timelineParams* = {
     "include_profile_interstitial_type": "0",
@@ -35,58 +33,85 @@ const
     "include_mute_edge": "0",
     "include_can_dm": "0",
     "include_can_media_tag": "1",
+    "include_ext_is_blue_verified": "1",
     "skip_status": "1",
     "cards_platform": "Web-12",
     "include_cards": "1",
-    "include_composer_source": "false",
+    "include_composer_source": "0",
     "include_reply_count": "1",
     "tweet_mode": "extended",
-    "include_entities": "true",
-    "include_user_entities": "true",
-    "include_ext_media_color": "false",
-    "send_error_codes": "true",
-    "simple_quoted_tweet": "true",
-    "include_quote_count": "true"
+    "include_entities": "1",
+    "include_user_entities": "1",
+    "include_ext_media_color": "0",
+    "send_error_codes": "1",
+    "simple_quoted_tweet": "1",
+    "include_quote_count": "1"
   }.toSeq
 
-  searchParams* = {
-    "query_source": "typed_query",
-    "pc": "1",
-    "spelling_corrections": "1"
-  }.toSeq
-  ## top: nothing
-  ## latest: "tweet_search_mode: live"
-  ## user:   "result_filter: user"
-  ## photos: "result_filter: photos"
-  ## videos: "result_filter: videos"
+  gqlFeatures* = """{
+  "blue_business_profile_image_shape_enabled": false,
+  "freedom_of_speech_not_reach_fetch_enabled": false,
+  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
+  "interactive_text_enabled": false,
+  "longform_notetweets_consumption_enabled": true,
+  "longform_notetweets_richtext_consumption_enabled": true,
+  "longform_notetweets_rich_text_read_enabled": false,
+  "responsive_web_edit_tweet_api_enabled": false,
+  "responsive_web_enhance_cards_enabled": false,
+  "responsive_web_graphql_exclude_directive_enabled": true,
+  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+  "responsive_web_graphql_timeline_navigation_enabled": false,
+  "responsive_web_text_conversations_enabled": false,
+  "responsive_web_twitter_blue_verified_badge_is_enabled": true,
+  "spaces_2022_h2_clipping": true,
+  "spaces_2022_h2_spaces_communities": true,
+  "standardized_nudges_misinfo": false,
+  "tweet_awards_web_tipping_enabled": false,
+  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
+  "tweetypie_unmention_optimization_enabled": false,
+  "verified_phone_label_enabled": false,
+  "vibe_api_enabled": false,
+  "view_counts_everywhere_api_enabled": false
+}""".replace(" ", "").replace("\n", "")
 
   tweetVariables* = """{
   "focalTweetId": "$1",
   $2
-  "includePromotedContent": false,
   "withBirdwatchNotes": false,
+  "includePromotedContent": false,
+  "withDownvotePerspective": false,
+  "withReactionsMetadata": false,
+  "withReactionsPerspective": false,
+  "withVoice": false
+}"""
+
+  tweetResultVariables* = """{
+  "tweetId": "$1",
+  "includePromotedContent": false,
+  "withDownvotePerspective": false,
+  "withReactionsMetadata": false,
+  "withReactionsPerspective": false,
+  "withVoice": false,
+  "withCommunity": false
+}"""
+
+  userTweetsVariables* = """{
+  "userId": "$1", $2
+  "count": 20,
+  "includePromotedContent": false,
   "withDownvotePerspective": false,
   "withReactionsMetadata": false,
   "withReactionsPerspective": false,
-  "withSuperFollowsTweetFields": false,
-  "withSuperFollowsUserFields": false,
   "withVoice": false,
   "withV2Timeline": true
 }"""
 
-  tweetFeatures* = """{
-  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
-  "responsive_web_graphql_timeline_navigation_enabled": false,
-  "standardized_nudges_misinfo": false,
-  "verified_phone_label_enabled": false,
-  "responsive_web_twitter_blue_verified_badge_is_enabled": false,
-  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
-  "view_counts_everywhere_api_enabled": false,
-  "responsive_web_edit_tweet_api_enabled": false,
-  "tweetypie_unmention_optimization_enabled": false,
-  "vibe_api_enabled": false,
-  "longform_notetweets_consumption_enabled": true,
-  "responsive_web_text_conversations_enabled": false,
-  "responsive_web_enhance_cards_enabled": false,
-  "interactive_text_enabled": false
+  listTweetsVariables* = """{
+  "listId": "$1", $2
+  "count": 20,
+  "includePromotedContent": false,
+  "withDownvotePerspective": false,
+  "withReactionsMetadata": false,
+  "withReactionsPerspective": false,
+  "withVoice": false
 }"""