kuu7o/nitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix everything (#927)

Browse Source 
* Switch bearer token and endpoints, update parser

* Enable user search, disable tweet search

* Disable multi-user timelines for now

* Fix parsing of pinned tombstone
This commit is contained in:
Zed
2023-07-10 11:25:34 +02:00
committed by GitHub
parent dcf73354ff
commit 0bc3c153d9
20 changed files with 260 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
src/parser.nim
View File

@@ -29,7 +29,7 @@ proc parseUser(js: JsonNode; id=""): User =
result.expandUserEntities(js)
proc parseGraphUser(js: JsonNode): User =
let user = ? js{"user_results", "result"}
let user = ? js{"user_result", "result"}
result = parseUser(user{"legacy"})
if "is_blue_verified" in user:
@@ -262,6 +262,11 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
result.gif = some(parseGif(m))
else: discard
with url, m{"url"}:
if result.text.endsWith(url.getStr):
result.text.removeSuffix(url.getStr)
result.text = result.text.strip()
with jsWithheld, js{"withheld_in_countries"}:
let withheldInCountries: seq[string] =
if jsWithheld.kind != JArray: @[]
@@ -294,16 +299,6 @@ proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
else:
result.retweet = some Tweet()
proc parsePin(js: JsonNode; global: GlobalObjects): Tweet =
let pin = js{"pinEntry", "entry", "entryId"}.getStr
if pin.len == 0: return
let id = pin.getId
if id notin global.tweets: return
global.tweets[id].pinned = true
return finalizeTweet(global, id)
proc parseGlobalObjects(js: JsonNode): GlobalObjects =
result = GlobalObjects()
let
@@ -314,7 +309,7 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects =
result.users[k] = parseUser(v, k)
for k, v in tweets:
var tweet = parseTweet(v, v{"card"})
var tweet = parseTweet(v, v{"tweet_card"})
if tweet.user.id in result.users:
tweet.user = result.users[tweet.user.id]
result.tweets[k] = tweet
@@ -324,11 +319,6 @@ proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNod
return
for i in js:
when T is Tweet:
if res.beginning and i{"pinEntry"}.notNull:
with pin, parsePin(i, global):
res.content.add pin
with r, i{"replaceEntry", "entry"}:
if "top" in r{"entryId"}.getStr:
res.top = r.getCursor
@@ -369,7 +359,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
proc parsePhotoRail*(js: JsonNode): PhotoRail =
for tweet in js:
let
t = parseTweet(tweet, js{"card"})
t = parseTweet(tweet, js{"tweet_card"})
url = if t.photos.len > 0: t.photos[0]
elif t.video.isSome: get(t.video).thumb
elif t.gif.isSome: get(t.gif).thumb
@@ -387,13 +377,17 @@ proc parseGraphTweet(js: JsonNode): Tweet =
of "TweetUnavailable":
return Tweet()
of "TweetTombstone":
return Tweet(text: js{"tombstone", "text"}.getTombstone)
with text, js{"tombstone", "richText"}:
return Tweet(text: text.getTombstone)
with text, js{"tombstone", "text"}:
return Tweet(text: text.getTombstone)
return Tweet()
of "TweetPreviewDisplay":
return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.")
of "TweetWithVisibilityResults":
return parseGraphTweet(js{"tweet"})
var jsCard = copy(js{"card", "legacy"})
var jsCard = copy(js{"tweet_card", "legacy"})
if jsCard.kind != JNull:
var values = newJObject()
for val in jsCard["binding_values"]:
@@ -401,6 +395,7 @@ proc parseGraphTweet(js: JsonNode): Tweet =
jsCard["binding_values"] = values
result = parseTweet(js{"legacy"}, jsCard)
result.id = js{"rest_id"}.getId
result.user = parseGraphUser(js{"core"})
with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
@@ -414,32 +409,31 @@ proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
for t in js{"content", "items"}:
let entryId = t{"entryId"}.getStr
if "cursor-showmore" in entryId:
let cursor = t{"item", "itemContent", "value"}
let cursor = t{"item", "content", "value"}
result.thread.cursor = cursor.getStr
result.thread.hasMore = true
elif "tweet" in entryId:
let tweet = parseGraphTweet(t{"item", "itemContent", "tweet_results", "result"})
let tweet = parseGraphTweet(t{"item", "content", "tweetResult", "result"})
result.thread.content.add tweet
if t{"item", "itemContent", "tweetDisplayType"}.getStr == "SelfThread":
if t{"item", "content", "tweetDisplayType"}.getStr == "SelfThread":
result.self = true
proc parseGraphTweetResult*(js: JsonNode): Tweet =
with tweet, js{"data", "tweetResult", "result"}:
with tweet, js{"data", "tweet_result", "result"}:
result = parseGraphTweet(tweet)
proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
result = Conversation(replies: Result[Chain](beginning: true))
let instructions = ? js{"data", "threaded_conversation_with_injections", "instructions"}
let instructions = ? js{"data", "timeline_response", "instructions"}
if instructions.len == 0:
return
for e in instructions[0]{"entries"}:
let entryId = e{"entryId"}.getStr
# echo entryId
if entryId.startsWith("tweet"):
with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
with tweetResult, e{"content", "content", "tweetResult", "result"}:
let tweet = parseGraphTweet(tweetResult)
if not tweet.available:
@@ -454,7 +448,7 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
let tweet = Tweet(
id: parseBiggestInt(id),
available: false,
text: e{"content", "itemContent", "tombstoneInfo", "richText"}.getTombstone
text: e{"content", "content", "tombstoneInfo", "richText"}.getTombstone
)
if id == tweetId:
@@ -468,34 +462,42 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
else:
result.replies.content.add thread
elif entryId.startsWith("cursor-bottom"):
result.replies.bottom = e{"content", "itemContent", "value"}.getStr
result.replies.bottom = e{"content", "content", "value"}.getStr
proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Timeline =
result = Timeline(beginning: after.len == 0)
proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile =
result = Profile(tweets: Timeline(beginning: after.len == 0))
let instructions =
if root == "list": ? js{"data", "list", "tweets_timeline", "timeline", "instructions"}
else: ? js{"data", "user", "result", "timeline_v2", "timeline", "instructions"}
if root == "list": ? js{"data", "list", "timeline_response", "timeline", "instructions"}
else: ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
if instructions.len == 0:
return
for i in instructions:
if i{"type"}.getStr == "TimelineAddEntries":
if i{"__typename"}.getStr == "TimelineAddEntries":
for e in i{"entries"}:
let entryId = e{"entryId"}.getStr
if entryId.startsWith("tweet"):
with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
with tweetResult, e{"content", "content", "tweetResult", "result"}:
let tweet = parseGraphTweet(tweetResult)
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
result.content.add tweet
elif entryId.startsWith("profile-conversation") or entryId.startsWith("homeConversation"):
result.tweets.content.add tweet
elif "-conversation-" in entryId or entryId.startsWith("homeConversation"):
let (thread, self) = parseGraphThread(e)
for tweet in thread.content:
result.content.add tweet
result.tweets.content.add thread
elif entryId.startsWith("cursor-bottom"):
result.bottom = e{"content", "value"}.getStr
result.tweets.bottom = e{"content", "value"}.getStr
if after.len == 0 and i{"__typename"}.getStr == "TimelinePinEntry":
with tweetResult, i{"entry", "content", "content", "tweetResult", "result"}:
let tweet = parseGraphTweet(tweetResult)
tweet.pinned = true
if not tweet.available and tweet.tombstone.len == 0:
let entryId = i{"entry", "entryId"}.getEntryId
if entryId.len > 0:
tweet.id = parseBiggestInt(entryId)
result.pinned = some tweet
proc parseGraphSearch*(js: JsonNode; after=""): Timeline =
result = Timeline(beginning: after.len == 0)