Improve rate limit handling, minor refactor

src/apiutils.nim

@@ -61,13 +61,6 @@ proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders =
     "DNT": "1"
   })
 
-template updateAccount() =
-  if resp.headers.hasKey(rlRemaining):
-    let
-      remaining = parseInt(resp.headers[rlRemaining])
-      reset = parseInt(resp.headers[rlReset])
-    account.setRateLimit(api, remaining, reset)
-
 template fetchImpl(result, fetchBody) {.dirty.} =
   once:
     pool = HttpPool()
@@ -89,28 +82,46 @@ template fetchImpl(result, fetchBody) {.dirty.} =
         badClient = true
         raise newException(BadClientError, "Bad client")
 
+    if resp.headers.hasKey(rlRemaining):
+      let
+        remaining = parseInt(resp.headers[rlRemaining])
+        reset = parseInt(resp.headers[rlReset])
+      account.setRateLimit(api, remaining, reset)
+
     if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":
         result = uncompress(result, dfGzip)
-      else:
-        echo "non-gzip body, url: ", url, ", body: ", result
+
+      if result.startsWith("{\"errors"):
+        let errors = result.fromJson(Errors)
+        if errors in {expiredToken, badToken}:
+          echo "fetch error: ", errors
+          invalidate(account)
+          raise rateLimitError()
+        elif errors in {rateLimited}:
+          # rate limit hit, resets after 24 hours
+          setLimited(account, api)
+          raise rateLimitError()
+      elif result.startsWith("429 Too Many Requests"):
+        account.apis[api].remaining = 0
+        # rate limit hit, resets after the 15 minute window
+        raise rateLimitError()
 
     fetchBody
 
-    release(account, used=true)
-
     if resp.status == $Http400:
       raise newException(InternalError, $url)
   except InternalError as e:
     raise e
   except BadClientError as e:
-    release(account, used=true)
+    raise e
+  except OSError as e:
     raise e
   except Exception as e:
     echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", account.id, ", url: ", url
-    if "length" notin e.msg and "descriptor" notin e.msg:
-      release(account, invalid=true)
     raise rateLimitError()
+  finally:
+    release(account)
 
 proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
   var body: string
@@ -121,36 +132,14 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
       echo resp.status, ": ", body, " --- url: ", url
       result = newJNull()
 
-    updateAccount()
-
     let error = result.getError
-    if error in {invalidToken, badToken}:
-      echo "fetch error: ", result.getError
-      release(account, invalid=true)
+    if error in {expiredToken, badToken}:
+      echo "fetchBody error: ", error
+      invalidate(account)
       raise rateLimitError()
 
-    if body.startsWith("{\"errors"):
-      let errors = body.fromJson(Errors)
-      if errors in {invalidToken, badToken}:
-        echo "fetch error: ", errors
-        release(account, invalid=true)
-        raise rateLimitError()
-      elif errors in {rateLimited}:
-        account.apis[api].limited = true
-        account.apis[api].limitedAt = epochTime().int
-        echo "[accounts] rate limited, api: ", api, ", reqs left: ", account.apis[api].remaining, ", id: ", account.id
-
 proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
   fetchImpl result:
     if not (result.startsWith('{') or result.startsWith('[')):
       echo resp.status, ": ", result, " --- url: ", url
       result.setLen(0)
-
-    updateAccount()
-
-    if result.startsWith("{\"errors"):
-      let errors = result.fromJson(Errors)
-      if errors in {invalidToken, badToken}:
-        echo "fetch error: ", errors
-        release(account, invalid=true)
-        raise rateLimitError()