Refactor api code

src/api/media.nim

@@ -0,0 +1,168 @@
+import httpclient, asyncdispatch, times, sequtils, strutils, json, uri
+
+import ".."/[types, parser, formatters]
+import utils, consts
+
+var
+  guestToken = ""
+  tokenUses = 0
+  tokenMaxUses = 230
+  tokenUpdated: Time
+  tokenLifetime = initDuration(minutes=20)
+
+macro genMediaGet(media: untyped; token=false) =
+  let
+    mediaName = capitalizeAscii($media)
+    multi = ident("get" & mediaName & "s")
+    convo = ident("getConversation" & mediaName & "s")
+    single = ident("get" & mediaName)
+
+  quote do:
+    proc `multi`*(thread: Thread | Timeline; agent: string; token="") {.async.} =
+      if thread == nil: return
+      var `media` = thread.content.filterIt(it.`media`.isSome)
+      when `token`:
+        var gToken = token
+        if gToken.len == 0: gToken = await getGuestToken(agent)
+        await all(`media`.mapIt(`single`(it, token, agent)))
+      else:
+        await all(`media`.mapIt(`single`(it, agent)))
+
+    proc `convo`*(convo: Conversation; agent: string) {.async.} =
+      var futs: seq[Future[void]]
+      when `token`:
+        var token = await getGuestToken(agent)
+        futs.add `single`(convo.tweet, agent, token)
+        futs.add `multi`(convo.before, agent, token=token)
+        futs.add `multi`(convo.after, agent, token=token)
+        futs.add convo.replies.mapIt(`multi`(it, agent, token=token))
+      else:
+        futs.add `single`(convo.tweet, agent)
+        futs.add `multi`(convo.before, agent)
+        futs.add `multi`(convo.after, agent)
+        futs.add convo.replies.mapIt(`multi`(it, agent))
+      await all(futs)
+
+proc getGuestToken(agent: string; force=false): Future[string] {.async.} =
+  if getTime() - tokenUpdated < tokenLifetime and
+     not force and tokenUses < tokenMaxUses:
+    return guestToken
+
+  tokenUpdated = getTime()
+  tokenUses = 0
+
+  let headers = newHttpHeaders({
+    "Accept": jsonAccept,
+    "Referer": $base,
+    "User-Agent": agent,
+    "Authorization": auth
+  })
+
+  newClient()
+
+  let
+    url = apiBase / tokenUrl
+    json = parseJson(await client.postContent($url))
+
+  result = json["guest_token"].to(string)
+  guestToken = result
+
+proc getVideoFetch(tweet: Tweet; agent, token: string) {.async.} =
+  if tweet.video.isNone(): return
+
+  let headers = newHttpHeaders({
+    "Accept": jsonAccept,
+    "Referer": $(base / getLink(tweet)),
+    "User-Agent": agent,
+    "Authorization": auth,
+    "x-guest-token": token
+  })
+
+  let url = apiBase / (videoUrl % tweet.id)
+  let json = await fetchJson(url, headers)
+
+  if json == nil:
+    if getTime() - tokenUpdated > initDuration(seconds=1):
+      tokenUpdated = getTime()
+      discard await getGuestToken(agent, force=true)
+    await getVideoFetch(tweet, agent, guestToken)
+    return
+
+  if tweet.card.isNone:
+    tweet.video = some(parseVideo(json, tweet.id))
+  else:
+    get(tweet.card).video = some(parseVideo(json, tweet.id))
+    tweet.video = none(Video)
+  tokenUses.inc
+
+proc getVideoVar(tweet: Tweet): var Option[Video] =
+  if tweet.card.isSome():
+    return get(tweet.card).video
+  else:
+    return tweet.video
+
+proc getVideo*(tweet: Tweet; agent, token: string; force=false) {.async.} =
+  withDb:
+    try:
+      getVideoVar(tweet) = some(Video.getOne("videoId = ?", tweet.id))
+    except KeyError:
+      await getVideoFetch(tweet, agent, token)
+      var video = getVideoVar(tweet)
+      if video.isSome():
+        get(video).insert()
+
+proc getPoll*(tweet: Tweet; agent: string) {.async.} =
+  if tweet.poll.isNone(): return
+
+  let headers = newHttpHeaders({
+    "Accept": htmlAccept,
+    "Referer": $(base / getLink(tweet)),
+    "User-Agent": agent,
+    "Authority": "twitter.com",
+    "Accept-Language": lang,
+  })
+
+  let url = base / (pollUrl % tweet.id)
+  let html = await fetchHtml(url, headers)
+  if html == nil: return
+
+  tweet.poll = some(parsePoll(html))
+
+proc getCard*(tweet: Tweet; agent: string) {.async.} =
+  if tweet.card.isNone(): return
+
+  let headers = newHttpHeaders({
+    "Accept": htmlAccept,
+    "Referer": $(base / getLink(tweet)),
+    "User-Agent": agent,
+    "Authority": "twitter.com",
+    "Accept-Language": lang,
+  })
+
+  let query = get(tweet.card).query.replace("sensitive=true", "sensitive=false")
+  let html = await fetchHtml(base / query, headers)
+  if html == nil: return
+
+  parseCard(get(tweet.card), html)
+
+proc getPhotoRail*(username, agent: string): Future[seq[GalleryPhoto]] {.async.} =
+  let headers = newHttpHeaders({
+    "Accept": jsonAccept,
+    "Referer": $(base / username),
+    "User-Agent": agent,
+    "X-Requested-With": "XMLHttpRequest"
+  })
+
+  let params = {
+    "for_photo_rail": "true",
+    "oldest_unread_id": "0"
+  }
+
+  let url = base / (timelineMediaUrl % username) ? params
+  let html = await fetchHtml(url, headers, jsonKey="items_html")
+
+  result = parsePhotoRail(html)
+
+genMediaGet(video, token=true)
+genMediaGet(poll)
+genMediaGet(card)