From d6f7cc36e911feb59cebf0bf254d9d91bcc6c9ba Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 12 Jan 2026 15:15:01 +0000 Subject: [PATCH 1/3] fix(SUPPORT-14907): handle future timestamps in pagination URLs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes Facebook Pages API v2 returning 400/403 errors for post_insights requests using insights.since(now).metric(...). Root cause: Facebook returns paging.next URLs with Unix timestamps pointing to the future, which fail with error: '(#100) since param is not valid. Metrics data is available for the last 2 years.' Changes: - Add parse-unix-ts helper to parse 10-digit Unix timestamps - Add extract-url-param helper to extract query params from URLs - Add remove-url-param helper to remove query params from URLs - Add handle-future-timestamps to process pagination URLs: - If both since and until are in future -> skip (end of data) - If only until is in future -> remove it, API uses 'now' implicitly - Update get-next-page-url to use handle-future-timestamps This fix is similar to the one applied in component-meta PR #19 for Instagram insights. Co-Authored-By: Zora Jelínková --- src/keboola/facebook/api/request.clj | 50 +++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/src/keboola/facebook/api/request.clj b/src/keboola/facebook/api/request.clj index 18ff560..cd27b41 100644 --- a/src/keboola/facebook/api/request.clj +++ b/src/keboola/facebook/api/request.clj @@ -11,6 +11,47 @@ (def graph-api-url "https://graph.facebook.com/") (def default-version "v20.0") +(defn- parse-unix-ts + "Parse a 10-digit Unix timestamp from a string, return nil if not valid." + [value] + (when (and value (string? value) (re-matches #"\d{10}" value)) + (Long/parseLong value))) + +(defn- extract-url-param + "Extract a query parameter value from a URL string." + [url param-name] + (when url + (second (re-find (re-pattern (str param-name "=([^&]+)")) url)))) + +(defn- remove-url-param + "Remove a query parameter from a URL string." + [url param-name] + (when url + (-> url + (string/replace (re-pattern (str "&" param-name "=[^&]+")) "") + (string/replace (re-pattern (str "\\?" param-name "=[^&]+&")) "?") + (string/replace (re-pattern (str "\\?" param-name "=[^&]+$")) "")))) + +(defn- handle-future-timestamps + "Handle Meta bug: pagination URLs sometimes have future timestamps. + Returns the URL (possibly modified) or nil if pagination should stop." + [url] + (when url + (let [until-str (extract-url-param url "until") + until-ts (parse-unix-ts until-str) + now-ts (quot (System/currentTimeMillis) 1000)] + (if (and until-ts (> until-ts now-ts)) + (let [since-str (extract-url-param url "since") + since-ts (parse-unix-ts since-str)] + (if (and since-ts (> since-ts now-ts)) + (do + (log-strings "Skipping future-only pagination range. URL:" url) + nil) + (do + (log-strings "Removing future 'until'=" until-str ". URL:" url) + (remove-url-param url "until")))) + url)))) + (s/fdef make-url :args (s/or :path-only (s/cat :path string?) :path-and-version (s/cat :path string? :version string?)) @@ -121,15 +162,16 @@ "return url to the next page from @response param" [response time-base-pagination? stop-on-empty-response?] (let [next-url (get-in response [:paging :next]) + processed-url (handle-future-timestamps next-url) time-base-pagination-valid (or (not time-base-pagination?) - (and next-url - (not (clojure.string/includes? next-url "since=")) - (not (clojure.string/includes? next-url "until=")))) + (and processed-url + (not (clojure.string/includes? processed-url "since=")) + (not (clojure.string/includes? processed-url "until=")))) stop-on-empty-response-valid (or (not stop-on-empty-response?) (not (-> response :data empty?)))] (if (not stop-on-empty-response-valid) (log-strings "Found empty data response, stopping pagintaion.")) (if (and time-base-pagination-valid stop-on-empty-response-valid) - next-url))) + processed-url))) (defn get-next-page-data "if response contains next page url then call it and wait for new repsonse From 907b61bf226dfd28ae9720cca5c6034fa729d490 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 13 Jan 2026 21:38:07 +0000 Subject: [PATCH 2/3] fix(SUPPORT-14907): log errors when page token lookup fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, exceptions during page token lookup were silently swallowed, causing the component to fall back to user token without any indication of what went wrong. This led to confusing 403 Forbidden errors when the user token lacked permissions for page insights. Changes: - Replace swallow-exceptions macro with try-with-warning that logs errors - Add detailed error message when both /me/accounts and page details fail - Only attempt page details API if accounts list lookup didn't find token - Improve logging to help diagnose token permission issues Co-Authored-By: Zora Jelínková --- src/keboola/facebook/extractor/query.clj | 28 +++++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/keboola/facebook/extractor/query.clj b/src/keboola/facebook/extractor/query.clj index cdd45aa..f5c8b6a 100644 --- a/src/keboola/facebook/extractor/query.clj +++ b/src/keboola/facebook/extractor/query.clj @@ -23,17 +23,29 @@ all-rows (apply concat nested-data)] (output/write-rows all-rows out-dir prefix false (incremental? query)))) -(defmacro swallow-exceptions [& body] - `(try ~@body (catch Exception e#))) +(defmacro try-with-warning [context & body] + `(try ~@body + (catch Exception e# + (runtime/log-error (str "Warning: " ~context " failed: " (.getMessage e#))) + nil))) (defn retrieve-page-access-token [id token version] - (let [accounts (swallow-exceptions (request/get-accounts token :version version)) - page-token-from-accounts-list (if (some? accounts) (:access_token (first (filter #(= id (:id %)) accounts)))) - page-token-from-page-details (swallow-exceptions (request/get-page-token-via-page-details token id)) + (let [accounts (try-with-warning (str "fetching accounts list for page " id) + (request/get-accounts token :version version)) + page-token-from-accounts-list (when (some? accounts) + (:access_token (first (filter #(= id (:id %)) accounts)))) + page-token-from-page-details (when (nil? page-token-from-accounts-list) + (try-with-warning (str "fetching page token via page details for " id) + (request/get-page-token-via-page-details token id))) page-token (or page-token-from-accounts-list page-token-from-page-details)] - (if (nil? page-token) - (runtime/log-strings "Could not find page access token for" id ". Token from the configuration will be used instead.") - (runtime/log-strings "Using page access token to retrieve data for" id)) + (cond + (some? page-token) + (runtime/log-strings "Using page access token to retrieve data for" id) + (and (nil? accounts) (nil? page-token-from-page-details)) + (runtime/log-error (str "Page token failed for " id ": both /me/accounts and page details API failed. " + "The user token will be used but may lack permissions for insights queries.")) + :else + (runtime/log-strings "Could not find page access token for" id ". Token from the configuration will be used instead.")) page-token)) (defn choose-token [id user-token version] From 2e84f4184b68fa74f081e10e2afcfa8bad48307f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 14 Jan 2026 08:34:43 +0000 Subject: [PATCH 3/3] revert: remove future timestamp handling from pagination MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per user request, removing the future timestamp handling code as it was not the correct fix for SUPPORT-14907. The actual issue is related to page token lookup failures, which is addressed by the token logging fix in query.clj. Co-Authored-By: Zora Jelínková --- src/keboola/facebook/api/request.clj | 50 +++------------------------- 1 file changed, 4 insertions(+), 46 deletions(-) diff --git a/src/keboola/facebook/api/request.clj b/src/keboola/facebook/api/request.clj index cd27b41..18ff560 100644 --- a/src/keboola/facebook/api/request.clj +++ b/src/keboola/facebook/api/request.clj @@ -11,47 +11,6 @@ (def graph-api-url "https://graph.facebook.com/") (def default-version "v20.0") -(defn- parse-unix-ts - "Parse a 10-digit Unix timestamp from a string, return nil if not valid." - [value] - (when (and value (string? value) (re-matches #"\d{10}" value)) - (Long/parseLong value))) - -(defn- extract-url-param - "Extract a query parameter value from a URL string." - [url param-name] - (when url - (second (re-find (re-pattern (str param-name "=([^&]+)")) url)))) - -(defn- remove-url-param - "Remove a query parameter from a URL string." - [url param-name] - (when url - (-> url - (string/replace (re-pattern (str "&" param-name "=[^&]+")) "") - (string/replace (re-pattern (str "\\?" param-name "=[^&]+&")) "?") - (string/replace (re-pattern (str "\\?" param-name "=[^&]+$")) "")))) - -(defn- handle-future-timestamps - "Handle Meta bug: pagination URLs sometimes have future timestamps. - Returns the URL (possibly modified) or nil if pagination should stop." - [url] - (when url - (let [until-str (extract-url-param url "until") - until-ts (parse-unix-ts until-str) - now-ts (quot (System/currentTimeMillis) 1000)] - (if (and until-ts (> until-ts now-ts)) - (let [since-str (extract-url-param url "since") - since-ts (parse-unix-ts since-str)] - (if (and since-ts (> since-ts now-ts)) - (do - (log-strings "Skipping future-only pagination range. URL:" url) - nil) - (do - (log-strings "Removing future 'until'=" until-str ". URL:" url) - (remove-url-param url "until")))) - url)))) - (s/fdef make-url :args (s/or :path-only (s/cat :path string?) :path-and-version (s/cat :path string? :version string?)) @@ -162,16 +121,15 @@ "return url to the next page from @response param" [response time-base-pagination? stop-on-empty-response?] (let [next-url (get-in response [:paging :next]) - processed-url (handle-future-timestamps next-url) time-base-pagination-valid (or (not time-base-pagination?) - (and processed-url - (not (clojure.string/includes? processed-url "since=")) - (not (clojure.string/includes? processed-url "until=")))) + (and next-url + (not (clojure.string/includes? next-url "since=")) + (not (clojure.string/includes? next-url "until=")))) stop-on-empty-response-valid (or (not stop-on-empty-response?) (not (-> response :data empty?)))] (if (not stop-on-empty-response-valid) (log-strings "Found empty data response, stopping pagintaion.")) (if (and time-base-pagination-valid stop-on-empty-response-valid) - processed-url))) + next-url))) (defn get-next-page-data "if response contains next page url then call it and wait for new repsonse