kuu7o/nitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize RSS requests by disabling media fetching

Browse Source
This commit is contained in:
Zed
2019-10-21 23:12:40 +02:00
parent 453beff09d
commit 6fb039dd79
5 changed files with 21 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/api/list.nim
View File

@@ -4,7 +4,7 @@ import sequtils, strutils, json, uri
import ".."/[types, parser, parserutils, query]
import utils, consts, timeline, search
proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.async.} =
proc getListTimeline*(username, list, agent, after: string; media=true): Future[Timeline] {.async.} =
let url = base / (listUrl % [username, list])
var params = toSeq({
@@ -17,7 +17,7 @@ proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.
params.add {"max_position": after}
let json = await fetchJson(url ? params, genHeaders(agent, url))
result = await finishTimeline(json, Query(), after, agent)
result = await finishTimeline(json, Query(), after, agent, media)
if result.content.len == 0:
return

4
src/api/search.nim
View File

@@ -14,7 +14,7 @@ proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
beginning: after.len == 0
)
proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.async.} =
proc getSearch*[T](query: Query; after, agent: string; media=true): Future[Result[T]] {.async.} =
let
kind = if query.kind == users: "users" else: "tweets"
@@ -44,7 +44,7 @@ proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.asyn
if json == nil or not json.hasKey("items_html"): return
when T is Tweet:
result = await finishTimeline(json, query, after, agent)
result = await finishTimeline(json, query, after, agent, media)
elif T is Profile:
let html = json["items_html"].to(string)
result.hasMore = html != "\n"

13
src/api/timeline.nim
View File

@@ -9,7 +9,8 @@ proc getMedia(thread: Chain | Timeline; agent: string) {.async.} =
getCards(thread, agent),
getPolls(thread, agent))
proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future[Timeline] {.async.} =
proc finishTimeline*(json: JsonNode; query: Query; after, agent: string;
media=true): Future[Timeline] {.async.} =
result = getResult[Tweet](json, query, after)
if json == nil: return
@@ -19,10 +20,10 @@ proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future
let html = parseHtml(json["items_html"].to(string))
let thread = parseChain(html)
await getMedia(thread, agent)
if media: await getMedia(thread, agent)
result.content = thread.content
proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
proc getTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} =
var params = toSeq({
"include_available_features": "1",
"include_entities": "1",
@@ -36,9 +37,9 @@ proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
let headers = genHeaders(agent, base / username, xml=true)
let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
result = await finishTimeline(json, Query(), after, agent)
result = await finishTimeline(json, Query(), after, agent, media)
proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} =
proc getProfileAndTimeline*(username, agent, after: string; media=true): Future[(Profile, Timeline)] {.async.} =
var url = base / username
if after.len > 0:
url = url ? {"max_position": after}
@@ -49,5 +50,5 @@ proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Ti
timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
profile = parseTimelineProfile(html)
await getMedia(timeline, agent)
if media: await getMedia(timeline, agent)
result = (profile, timeline)