|
11 | 11 | (def graph-api-url "https://graph.facebook.com/") |
12 | 12 | (def default-version "v20.0") |
13 | 13 |
|
14 | | -(defn- parse-unix-ts |
15 | | - "Parse a 10-digit Unix timestamp from a string, return nil if not valid." |
16 | | - [value] |
17 | | - (when (and value (string? value) (re-matches #"\d{10}" value)) |
18 | | - (Long/parseLong value))) |
19 | | - |
20 | | -(defn- extract-url-param |
21 | | - "Extract a query parameter value from a URL string." |
22 | | - [url param-name] |
23 | | - (when url |
24 | | - (second (re-find (re-pattern (str param-name "=([^&]+)")) url)))) |
25 | | - |
26 | | -(defn- remove-url-param |
27 | | - "Remove a query parameter from a URL string." |
28 | | - [url param-name] |
29 | | - (when url |
30 | | - (-> url |
31 | | - (string/replace (re-pattern (str "&" param-name "=[^&]+")) "") |
32 | | - (string/replace (re-pattern (str "\\?" param-name "=[^&]+&")) "?") |
33 | | - (string/replace (re-pattern (str "\\?" param-name "=[^&]+$")) "")))) |
34 | | - |
35 | | -(defn- handle-future-timestamps |
36 | | - "Handle Meta bug: pagination URLs sometimes have future timestamps. |
37 | | - Returns the URL (possibly modified) or nil if pagination should stop." |
38 | | - [url] |
39 | | - (when url |
40 | | - (let [until-str (extract-url-param url "until") |
41 | | - until-ts (parse-unix-ts until-str) |
42 | | - now-ts (quot (System/currentTimeMillis) 1000)] |
43 | | - (if (and until-ts (> until-ts now-ts)) |
44 | | - (let [since-str (extract-url-param url "since") |
45 | | - since-ts (parse-unix-ts since-str)] |
46 | | - (if (and since-ts (> since-ts now-ts)) |
47 | | - (do |
48 | | - (log-strings "Skipping future-only pagination range. URL:" url) |
49 | | - nil) |
50 | | - (do |
51 | | - (log-strings "Removing future 'until'=" until-str ". URL:" url) |
52 | | - (remove-url-param url "until")))) |
53 | | - url)))) |
54 | | - |
55 | 14 | (s/fdef make-url |
56 | 15 | :args (s/or :path-only (s/cat :path string?) |
57 | 16 | :path-and-version (s/cat :path string? :version string?)) |
|
162 | 121 | "return url to the next page from @response param" |
163 | 122 | [response time-base-pagination? stop-on-empty-response?] |
164 | 123 | (let [next-url (get-in response [:paging :next]) |
165 | | - processed-url (handle-future-timestamps next-url) |
166 | 124 | time-base-pagination-valid (or (not time-base-pagination?) |
167 | | - (and processed-url |
168 | | - (not (clojure.string/includes? processed-url "since=")) |
169 | | - (not (clojure.string/includes? processed-url "until=")))) |
| 125 | + (and next-url |
| 126 | + (not (clojure.string/includes? next-url "since=")) |
| 127 | + (not (clojure.string/includes? next-url "until=")))) |
170 | 128 | stop-on-empty-response-valid (or (not stop-on-empty-response?) |
171 | 129 | (not (-> response :data empty?)))] |
172 | 130 | (if (not stop-on-empty-response-valid) (log-strings "Found empty data response, stopping pagintaion.")) |
173 | 131 | (if (and time-base-pagination-valid stop-on-empty-response-valid) |
174 | | - processed-url))) |
| 132 | + next-url))) |
175 | 133 |
|
176 | 134 | (defn get-next-page-data |
177 | 135 | "if response contains next page url then call it and wait for new repsonse |
|
0 commit comments