Add list support

src/api/consts.nim

@@ -11,6 +11,8 @@ const
 
   timelineUrl* = "i/profiles/show/$1/timeline/tweets"
   timelineMediaUrl* = "i/profiles/show/$1/media_timeline"
+  listUrl* = "$1/lists/$2/timeline"
+  listMembersUrl* = "$1/lists/$2/members"
   profilePopupUrl* = "i/profiles/popup"
   profileIntentUrl* = "intent/user"
   searchUrl* = "i/search/timeline"

src/api/list.nim

@@ -0,0 +1,83 @@
+import httpclient, asyncdispatch, htmlparser, strformat
+import sequtils, strutils, json, uri
+
+import ".."/[types, parser, parserutils, query]
+import utils, consts, timeline, search
+
+proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.async.} =
+  let url = base / (listUrl % [username, list])
+
+  let headers = newHttpHeaders({
+    "Accept": jsonAccept,
+    "Referer": $url,
+    "User-Agent": agent,
+    "X-Twitter-Active-User": "yes",
+    "X-Requested-With": "XMLHttpRequest",
+    "Accept-Language": lang
+  })
+
+  var params = toSeq({
+    "include_available_features": "1",
+    "include_entities": "1",
+    "reset_error_state": "false"
+  })
+
+  if after.len > 0:
+    params.add {"max_position": after}
+
+  let json = await fetchJson(url ? params, headers)
+  result = await finishTimeline(json, Query(), after, agent)
+  if result.content.len > 0:
+    result.minId = result.content[^1].id
+
+proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} =
+  let url = base / (listMembersUrl % [username, list])
+
+  let headers = newHttpHeaders({
+    "Accept": htmlAccept,
+    "Referer": $(base / &"{username}/lists/{list}/members"),
+    "User-Agent": agent,
+    "Accept-Language": lang
+  })
+
+  let html = await fetchHtml(url, headers)
+
+  result = Result[Profile](
+    minId: html.selectAttr(".stream-container", "data-min-position"),
+    hasMore: html.select(".has-more-items") != nil,
+    beginning: true,
+    query: Query(kind: users),
+    content: html.selectAll(".account").map(parseListProfile)
+  )
+
+proc getListMembersSearch*(username, list, agent, after: string): Future[Result[Profile]] {.async.} =
+  let url = base / ((listMembersUrl & "/timeline") % [username, list])
+
+  let headers = newHttpHeaders({
+    "Accept": jsonAccept,
+    "Referer": $(base / &"{username}/lists/{list}/members"),
+    "User-Agent": agent,
+    "X-Twitter-Active-User": "yes",
+    "X-Requested-With": "XMLHttpRequest",
+    "X-Push-With": "XMLHttpRequest",
+    "Accept-Language": lang
+  })
+
+  var params = toSeq({
+    "include_available_features": "1",
+    "include_entities": "1",
+    "reset_error_state": "false"
+  })
+
+  if after.len > 0:
+    params.add {"max_position": after}
+
+  let json = await fetchJson(url ? params, headers)
+
+  result = getResult[Profile](json, Query(kind: users), after)
+  if json == nil or not json.hasKey("items_html"): return
+
+  let html = json["items_html"].to(string)
+  result.hasMore = html != "\n"
+  for p in parseHtml(html).selectAll(".account"):
+    result.content.add parseListProfile(p)

src/api/search.nim

@@ -7,7 +7,7 @@ import utils, consts, timeline
 proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
   if json == nil: return Result[T](beginning: true, query: query)
   Result[T](
-    hasMore: json["has_more_items"].to(bool),
+    hasMore: json.getOrDefault("has_more_items").getBool(false),
     maxId: json.getOrDefault("max_position").getStr(""),
     minId: json.getOrDefault("min_position").getStr("").cleanPos(),
     query: query,
@@ -16,7 +16,7 @@ proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
 
 proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.async.} =
   let
-    kind = if query.kind == users: "users" else: "tweets"
+    kind = if query.kind == userSearch: "users" else: "tweets"
     pos = when T is Tweet: genPos(after) else: after
 
     param = genQueryParam(query)
@@ -46,10 +46,9 @@ proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.asyn
     return Result[T](query: query, beginning: true)
 
   let json = await fetchJson(base / searchUrl ? params, headers)
-  if json == nil: return Result[T](query: query, beginning: true)
 
   result = getResult[T](json, query, after)
-  if not json.hasKey("items_html"): return
+  if json == nil or not json.hasKey("items_html"): return
 
   when T is Tweet:
     result = await finishTimeline(json, query, after, agent)

src/api/timeline.nim

@@ -1,4 +1,4 @@
-import httpclient, asyncdispatch, htmlparser
+import httpclient, asyncdispatch, htmlparser, strformat
 import sequtils, strutils, json, xmltree, uri
 
 import ".."/[types, parser, parserutils, formatters, query]