Skip to content

Commit d6f7cc3

Browse files
fix(SUPPORT-14907): handle future timestamps in pagination URLs
Fixes Facebook Pages API v2 returning 400/403 errors for post_insights requests using insights.since(now).metric(...). Root cause: Facebook returns paging.next URLs with Unix timestamps pointing to the future, which fail with error: '(#100) since param is not valid. Metrics data is available for the last 2 years.' Changes: - Add parse-unix-ts helper to parse 10-digit Unix timestamps - Add extract-url-param helper to extract query params from URLs - Add remove-url-param helper to remove query params from URLs - Add handle-future-timestamps to process pagination URLs: - If both since and until are in future -> skip (end of data) - If only until is in future -> remove it, API uses 'now' implicitly - Update get-next-page-url to use handle-future-timestamps This fix is similar to the one applied in component-meta PR #19 for Instagram insights. Co-Authored-By: Zora Jelínková <zora.jelinkova@keboola.com>
1 parent 21bff5c commit d6f7cc3

1 file changed

Lines changed: 46 additions & 4 deletions

File tree

src/keboola/facebook/api/request.clj

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,47 @@
1111
(def graph-api-url "https://graph.facebook.com/")
1212
(def default-version "v20.0")
1313

14+
(defn- parse-unix-ts
15+
"Parse a 10-digit Unix timestamp from a string, return nil if not valid."
16+
[value]
17+
(when (and value (string? value) (re-matches #"\d{10}" value))
18+
(Long/parseLong value)))
19+
20+
(defn- extract-url-param
21+
"Extract a query parameter value from a URL string."
22+
[url param-name]
23+
(when url
24+
(second (re-find (re-pattern (str param-name "=([^&]+)")) url))))
25+
26+
(defn- remove-url-param
27+
"Remove a query parameter from a URL string."
28+
[url param-name]
29+
(when url
30+
(-> url
31+
(string/replace (re-pattern (str "&" param-name "=[^&]+")) "")
32+
(string/replace (re-pattern (str "\\?" param-name "=[^&]+&")) "?")
33+
(string/replace (re-pattern (str "\\?" param-name "=[^&]+$")) ""))))
34+
35+
(defn- handle-future-timestamps
36+
"Handle Meta bug: pagination URLs sometimes have future timestamps.
37+
Returns the URL (possibly modified) or nil if pagination should stop."
38+
[url]
39+
(when url
40+
(let [until-str (extract-url-param url "until")
41+
until-ts (parse-unix-ts until-str)
42+
now-ts (quot (System/currentTimeMillis) 1000)]
43+
(if (and until-ts (> until-ts now-ts))
44+
(let [since-str (extract-url-param url "since")
45+
since-ts (parse-unix-ts since-str)]
46+
(if (and since-ts (> since-ts now-ts))
47+
(do
48+
(log-strings "Skipping future-only pagination range. URL:" url)
49+
nil)
50+
(do
51+
(log-strings "Removing future 'until'=" until-str ". URL:" url)
52+
(remove-url-param url "until"))))
53+
url))))
54+
1455
(s/fdef make-url
1556
:args (s/or :path-only (s/cat :path string?)
1657
:path-and-version (s/cat :path string? :version string?))
@@ -121,15 +162,16 @@
121162
"return url to the next page from @response param"
122163
[response time-base-pagination? stop-on-empty-response?]
123164
(let [next-url (get-in response [:paging :next])
165+
processed-url (handle-future-timestamps next-url)
124166
time-base-pagination-valid (or (not time-base-pagination?)
125-
(and next-url
126-
(not (clojure.string/includes? next-url "since="))
127-
(not (clojure.string/includes? next-url "until="))))
167+
(and processed-url
168+
(not (clojure.string/includes? processed-url "since="))
169+
(not (clojure.string/includes? processed-url "until="))))
128170
stop-on-empty-response-valid (or (not stop-on-empty-response?)
129171
(not (-> response :data empty?)))]
130172
(if (not stop-on-empty-response-valid) (log-strings "Found empty data response, stopping pagintaion."))
131173
(if (and time-base-pagination-valid stop-on-empty-response-valid)
132-
next-url)))
174+
processed-url)))
133175

134176
(defn get-next-page-data
135177
"if response contains next page url then call it and wait for new repsonse

0 commit comments

Comments
 (0)