Out with the old

2020-06-01 02:14:29 +02:00
parent 4bb92a5201
commit 4167ce458b
12 changed files with 0 additions and 1230 deletions
--- a/src/parser.nim
+++ b/src/parser.nim
@@ -1,290 +0,0 @@
-import xmltree, sequtils, strutils, json, options
-
-import types, parserutils, formatters
-
-proc parseJsonData*(node: XmlNode): JsonNode =
-  let jsonData = node.selectAttr("input.json-data", "value")
-  if jsonData.len > 0:
-    return parseJson(jsonData)
-
-proc parseTimelineProfile*(node: XmlNode): Profile =
-  let profile = node.select(".ProfileHeaderCard")
-  if profile == nil:
-    let data = parseJsonData(node)
-    if data != nil and data{"sectionName"}.getStr == "suspended":
-      let username = data{"internalReferer"}.getStr.strip(chars={'/'})
-      return Profile(username: username, suspended: true)
-    return
-
-  let pre = ".ProfileHeaderCard-"
-  let username = profile.getUsername(pre & "screenname")
-  result = Profile(
-    fullname:  profile.getName(pre & "nameLink"),
-    username:  username,
-    lowername: toLower(username),
-    joinDate:  profile.getDate(pre & "joinDateText"),
-    website:   profile.selectAttr(pre & "urlText a", "title"),
-    bio:       profile.getBio(pre & "bio"),
-    location:  getLocation(profile),
-    userpic:   node.getAvatar(".profile-picture img"),
-    verified:  isVerified(profile),
-    protected: isProtected(profile),
-    banner:    getTimelineBanner(node),
-    media:     getMediaCount(node)
-  )
-
-  result.getProfileStats(node.select(".ProfileNav-list"))
-
-proc parsePopupProfile*(node: XmlNode; selector=".profile-card"): Profile =
-  let profile = node.select(selector)
-  if profile == nil: return
-
-  let username = profile.getUsername(".username")
-  result = Profile(
-    fullname:  profile.getName(".fullname"),
-    username:  username,
-    lowername: toLower(username),
-    bio:       profile.getBio(".bio", fallback=".ProfileCard-bio"),
-    userpic:   profile.getAvatar(".ProfileCard-avatarImage"),
-    verified:  isVerified(profile),
-    protected: isProtected(profile),
-    banner:    getBanner(profile)
-  )
-
-  result.getPopupStats(profile)
-
-proc parseListProfile*(profile: XmlNode): Profile =
-  result = Profile(
-    fullname:  profile.getName(".fullname"),
-    username:  profile.getUsername(".username"),
-    bio:       profile.getBio(".bio").stripText(),
-    userpic:   profile.getAvatar(".avatar"),
-    verified:  isVerified(profile),
-    protected: isProtected(profile),
-  )
-
-proc parseIntentProfile*(profile: XmlNode): Profile =
-  result = Profile(
-    fullname:  profile.getName("a.fn.url.alternate-context"),
-    username:  profile.getUsername(".nickname"),
-    bio:       profile.getBio("p.note"),
-    userpic:   profile.select(".profile.summary").getAvatar("img.photo"),
-    verified:  profile.select("li.verified") != nil,
-    protected: profile.select("li.protected") != nil,
-    banner:    getBanner(profile)
-  )
-
-  result.getIntentStats(profile)
-
-proc parseTweetProfile*(profile: XmlNode): Profile =
-  result = Profile(
-    fullname: profile.attr("data-name").stripText(),
-    username: profile.attr("data-screen-name"),
-    userpic:  profile.getAvatar(".avatar"),
-    verified: isVerified(profile)
-  )
-
-proc parseQuote*(quote: XmlNode): Quote =
-  result = Quote(
-    id:    parseBiggestInt(quote.attr("data-item-id")),
-    text:  getQuoteText(quote),
-    reply: parseTweetReply(quote),
-    hasThread: quote.select(".self-thread-context") != nil,
-    available: true
-  )
-
-  result.profile = Profile(
-    fullname: quote.selectText(".QuoteTweet-fullname").stripText(),
-    username: quote.attr("data-screen-name"),
-    verified: isVerified(quote)
-  )
-
-  result.getQuoteMedia(quote)
-
-proc parseTweet*(node: XmlNode): Tweet =
-  if node == nil:
-    return Tweet()
-
-  if "withheld" in node.attr("class"):
-    return Tweet(tombstone: getTombstone(node.selectText(".Tombstone-label")))
-
-  let tweet = node.select(".tweet")
-  if tweet == nil:
-    return Tweet()
-
-  result = Tweet(
-    id:        parseBiggestInt(tweet.attr("data-item-id")),
-    threadId:  parseBiggestInt(tweet.attr("data-conversation-id")),
-    text:      getTweetText(tweet),
-    time:      getTimestamp(tweet),
-    shortTime: getShortTime(tweet),
-    profile:   parseTweetProfile(tweet),
-    stats:     parseTweetStats(tweet),
-    reply:     parseTweetReply(tweet),
-    mediaTags: getMediaTags(tweet),
-    location:  getTweetLocation(tweet),
-    hasThread: tweet.select(".content > .self-thread-context") != nil,
-    pinned:    "pinned" in tweet.attr("class"),
-    available: true
-  )
-
-  result.getTweetMedia(tweet)
-  result.getTweetCard(tweet)
-
-  let by = tweet.selectText(".js-retweet-text > a > b")
-  if by.len > 0:
-    result.retweet = some Retweet(
-      by: stripText(by),
-      id: parseBiggestInt(tweet.attr("data-retweet-id"))
-    )
-
-  let quote = tweet.select(".QuoteTweet-innerContainer")
-  if quote != nil:
-    result.quote = some parseQuote(quote)
-
-  let tombstone = tweet.select(".Tombstone")
-  if tombstone != nil:
-    if "unavailable" in tombstone.innerText():
-      let quote = Quote(tombstone: getTombstone(node.selectText(".Tombstone-label")))
-      result.quote = some quote
-
-proc parseChain*(nodes: XmlNode): Chain =
-  if nodes == nil: return
-  result = Chain()
-  for n in nodes.filterIt(it.kind != xnText):
-    let class = n.attr("class").toLower()
-    if "tombstone" in class or "unavailable" in class or "withheld" in class:
-      result.content.add Tweet()
-    elif "morereplies" in class:
-      result.more = getMoreReplies(n)
-    else:
-      result.content.add parseTweet(n)
-
-proc parseReplies*(replies: XmlNode; skipFirst=false): Result[Chain] =
-  new(result)
-  for i, reply in replies.filterIt(it.kind != xnText):
-    if skipFirst and i == 0: continue
-    let class = reply.attr("class").toLower()
-    if "lone" in class:
-      result.content.add parseChain(reply)
-    elif "showmore" in class:
-      result.minId = reply.selectAttr("button", "data-cursor")
-      result.hasMore = true
-    else:
-      result.content.add parseChain(reply.select(".stream-items"))
-
-proc parseConversation*(node: XmlNode; after: string): Conversation =
-  let tweet = node.select(".permalink-tweet-container")
-
-  if tweet == nil:
-    return Conversation(tweet: parseTweet(node.select(".permalink-tweet-withheld")))
-
-  result = Conversation(
-    tweet:  parseTweet(tweet),
-    before: parseChain(node.select(".in-reply-to .stream-items")),
-  )
-
-  if result.before != nil:
-    let maxId = node.selectAttr(".in-reply-to .stream-container", "data-max-position")
-    if maxId.len > 0:
-      result.before.more = -1
-
-  let replies = node.select(".replies-to .stream-items")
-  if replies == nil: return
-
-  let nodes = replies.filterIt(it.kind != xnText and "self" in it.attr("class"))
-  if nodes.len > 0 and "self" in nodes[0].attr("class"):
-    result.after = parseChain(nodes[0].select(".stream-items"))
-
-  result.replies = parseReplies(replies, result.after != nil)
-
-  result.replies.beginning = after.len == 0
-  if result.replies.minId.len == 0:
-    result.replies.minId = node.selectAttr(".replies-to .stream-container", "data-min-position")
-    result.replies.hasMore = node.select(".stream-footer .has-more-items") != nil
-
-proc parseTimeline*(node: XmlNode; after: string): Timeline =
-  if node == nil: return Timeline()
-  result = Timeline(
-    content: parseChain(node.select(".stream > .stream-items")).content,
-    minId: node.attr("data-min-position"),
-    maxId: node.attr("data-max-position"),
-    hasMore: node.select(".has-more-items") != nil,
-    beginning: after.len == 0
-  )
-
-proc parseVideo*(node: JsonNode; tweetId: int64): Video =
-  let
-    track = node{"track"}
-    cType = track["contentType"].to(string)
-    pType = track["playbackType"].to(string)
-
-  case cType
-  of "media_entity":
-    result = Video(
-      playbackType: if "mp4" in pType: mp4 else: m3u8,
-      contentId: track["contentId"].to(string),
-      durationMs: track["durationMs"].to(int),
-      views: track["viewCount"].to(string),
-      url: track["playbackUrl"].to(string),
-      available: track{"mediaAvailability"}["status"].to(string) == "available",
-      reason: track{"mediaAvailability"}["reason"].to(string))
-  of "vmap":
-    result = Video(
-      playbackType: vmap,
-      durationMs: track.getOrDefault("durationMs").getInt(0),
-      url: track["vmapUrl"].to(string),
-      available: true)
-  else:
-    echo "Can't parse video of type ", cType, " ", tweetId
-
-  result.videoId = $tweetId
-  result.thumb = node["posterImage"].to(string)
-
-proc parsePoll*(node: XmlNode): Poll =
-  let
-    choices = node.selectAll(".PollXChoice-choice")
-    votes = node.selectText(".PollXChoice-footer--total")
-
-  result.votes = votes.strip().split(" ")[0]
-  result.status = node.selectText(".PollXChoice-footer--time")
-
-  for choice in choices:
-    for span in choice.select(".PollXChoice-choice--text").filterIt(it.kind != xnText):
-      if span.attr("class").len == 0:
-        result.options.add span.innerText()
-      elif "progress" in span.attr("class"):
-        result.values.add parseInt(span.innerText()[0 .. ^2])
-
-  var highest = 0
-  for i, n in result.values:
-    if n > highest:
-      highest = n
-      result.leader = i
-
-proc parsePhotoRail*(node: XmlNode): seq[GalleryPhoto] =
-  for img in node.selectAll(".tweet-media-img-placeholder"):
-    result.add GalleryPhoto(
-      url: img.attr("data-image-url"),
-      tweetId: img.attr("data-tweet-id"),
-      color: img.attr("background-color").replace("style: ", "")
-    )
-
-proc parseCard*(card: var Card; node: XmlNode) =
-  card.title = node.selectText("h2.TwitterCard-title")
-  card.text = node.selectText("p.tcu-resetMargin")
-  card.dest = node.selectText("span.SummaryCard-destination")
-
-  if card.url.len == 0:
-    card.url = node.selectAttr("a", "href")
-  if card.url.len == 0:
-    card.url = node.selectAttr(".ConvoCard-thankYouContent", "data-thank-you-url")
-
-  let image = node.select(".tcu-imageWrapper img")
-  if image != nil:
-    # workaround for issue 11713
-    card.image = some image.attr("data-src").replace("gname", "g&name")
-
-  if card.kind == liveEvent:
-    card.text = card.title
-    card.title = node.selectText(".TwitterCard-attribution--category")