Zyte API cookies are no longer experimental #243
Description
Cookie support in Zyte API used to be experimental, but it seems it's no longer the case. For a sample code:
ZYTE_API_KEY = "..."
import json
from base64 import b64decode
import parsel
import requests
api_response = requests.post(
"https://api.zyte.com/v1/extract",
auth=(ZYTE_API_KEY, ""),
json={
"url": "https://toscrape.com",
"browserHtml": True,
"responseCookies": True,
},
)
print(api_response.json()["responseCookies"])I obtain:
$ python /tmp/test.py
[{'name': 'tmx_guid', 'value': 'AAxKgYWXIeB2XYr2ahqcPp5yPhWPhivNbZuv7MJFc1WjZxiYf0FO9woP7kqsFEM2kUADEC5L72Aq7LY5ItYg916UmNZg2g', 'domain': 'customer.homedepot.com', 'path': '/', 'expires': 1772497083, 'httpOnly': True, 'secure': True}, {'name': '_px', 'value': '1PjydhMTn0yatrTCE6ZS38f8Nl85MCvSGzMugjuHOrkYubrk6VmPxmCbaGLoNHAPqG5t2PM1jmuaqhozXT0E3g==:1000:h+VVoXosQ1LN8kmGyJSf7ufWlCoBP3viBeYOKVjo4rIxmIVm5VYGA1Z/YQMnmdNk7DY/ewsqcXCESTW1D0yRZvNPwqauaGEO9GDeUdgZmNHylqQT4UZs4E2yYxtmrOjuo59DhYZZTSgYjeBgT8/eaejUuHSOH6j1O3Bg/ipmbOrkc/LV4GLPOmOUmk/0XWH+m9Fgrv3f9qslfPwySSGFXITBpN9MJTH4EBb0ZZlEg6DUGYsnYvOh2sgt3KCg4mDUEREUlRnjdsgFKXE4NBNThQ==', 'domain': 'www.homedepot.com', 'path': '/', 'expires': 1738109886, 'httpOnly': False, 'secure': False}, {'name': 'RT', 'value': '"z=1&dm=www.homedepot.com&si=1c87f7f3-8ede-4d21-af5d-be0e5cca602c&ss=m6eav6lj&sl=1&tt=2xv&rl=1&ld=2xx"', 'domain': '.www.homedepot.com', 'path': '/', 'expires': 1738541885, 'httpOnly': False, 'secure': False}, {'name': 'forterToken', 'value': '34028bcd8d454845954468b87d45659c_1737937085393___23ck', 'domain': '.homedepot.com', 'path': '/', 'expires': 1772497085, 'httpOnly': False, 'secure': False}, {'name': '_px_f394gi7Fvmc43dfg_user_id', 'value': 'MmRhYjQyYTAtZGM0NC0xMWVmLTliOTQtNzlhYmJhNDQ3YjRj', 'domain': '.homedepot.com', 'path': '/', 'expires': 1772497085, 'httpOnly': False, 'secure': False}, {'name': 'bm_sv', 'value': 'EEC03943036A8983090C6CE52AF19BF0~YAAQRQw0F3aY2IyUAQAAu2AfpRp8HBSGE0FL/3wpsoIxe71Vjkweyv4IY9Nfz06sO5ieajHuHNSd7zxM4z+TyPWEgN/TsQM5s5ASJP04T6SXuluxB+3ZiVT0IGKsHBSM2xri/yh3HQzgDM1jRk9zPnHIsF/jCBbdYmV/Ic6BuMvTloTqYC8sSaGXx1U2e/dGTXRn/7whG3iaUvTgNHddIOfD8R0+SWYQbCp+j4L1LDDRwb6pIG0pRuFRh5QDRBDJ5/Vx~1', 'domain': '.homedepot.com', 'path': '/', 'expires': 1737944284, 'httpOnly': False, 'secure': True}, {'name': 'kndctr_F6421253512D2C100A490D45_AdobeOrg_identity', 'value': 'CiY2MjYwOTY0NDU1NzMyNTYxMjYyNDU0MDEyNjYwNzY0Njc3MzYzMlISCLG0%5FajKMhABGAEqA09SMjAA8AGZwP2oyjI%3D', 'domain': '.homedepot.com', 'path': '/', 'expires': 1772065084, 'httpOnly': False, 'secure': True}, {'name': 'DELIVERY_ZIP_TYPE', 'value': 'DEFAULT', 'domain': '.homedepot.com', 'path': '/', 'expires': 1769473084, 'httpOnly': False, 'secure': False}, {'name': 'DELIVERY_ZIP', 'value': '60607', 'domain': '.homedepot.com', 'path': '/', 'expires': 1769473084, 'httpOnly': False, 'secure': False}, {'name': '_pxde', 'value': '7cf99386d6fb610ae1bfd38956a51164c18af8920acaf3f2642fcb8da7181644:eyJ0aW1lc3RhbXAiOjE3Mzc5MzcwODU0MjQsImluY19pZCI6WyI1ZWY3OTQ5Y2VjZjQ4ZmQ1MGU3MjlmNGIzZjY0NGNiZiJdfQ==', 'domain': 'www.homedepot.com', 'path': '/', 'expires': 1738109886, 'httpOnly': False, 'secure': False}, {'name': '_pxvid', 'value': '2c750dee-dc44-11ef-85e7-082cf353950e', 'domain': 'www.homedepot.com', 'path': '/', 'expires': 1769473084, 'httpOnly': False, 'secure': False}, {'name': 'kndctr_F6421253512D2C100A490D45_AdobeOrg_cluster', 'value': 'or2', 'domain': '.homedepot.com', 'path': '/', 'expires': 1737938884, 'httpOnly': False, 'secure': True}, {'name': 'ak_bmsc', 'value': '86D1594384445EDDFE66153898A75296~000000000000000000000000000000~YAAQWgw0F+59E5aUAQAApGEfpRrie/WhN4bx0xZlmENDwU6O3j4icR/ZlAmEkcJjhj9mp8EnavDVxoc8CzUrH4PKAY56OA1a8GVGtpasFUUreC22C8BZhA0nTbW5WXioZP0MpvaSi6D/a+HwVw0h/q+kaT1IZt0f95SyhgWYL4KdnrAkf7v6tCiDvR2A7CK1dm/1X8JWKOi/EZSrj5jlbegjoUwZ6Z2/lYwCW+ohqPxolOjLzrPYB1Ad1vD5tPCgntqgPuNfJRRuZWxglUJOf4k/H6RoR6seB0XClHkYl7MnfIHnxIEQtrNxYAcjmveZc/NY2YhRNGp2oOdhKLrtKKCoI0pOXvIQ2tZbIAXzfQ6ceVNu8JIyvftnhrQPJjs58JZUuQFhW5n1ry7U/9B+81fY4CdH4aGsSU8tbeULFNBpSsHkSNaleOWN/sQxxlGaCmKBFA==', 'domain': '.homedepot.com', 'path': '/', 'expires': 1737944282, 'httpOnly': False, 'secure': False}, {'name': 'thda.u', 'value': 'c4f0965e-cb0c-28af-912f-a403bc780fb1', 'domain': '.homedepot.com', 'path': '/', 'expires': 1772497084, 'httpOnly': False, 'secure': False}, {'name': 'bm_ss', 'value': 'ab8e18ef4e', 'domain': '.homedepot.com', 'path': '/', 'expires': 1737940682, 'httpOnly': True, 'secure': True}, {'name': 'THD_CACHE_NAV_PERSIST', 'value': '', 'domain': '.homedepot.com', 'path': '/', 'expires': 1772497083, 'httpOnly': False, 'secure': False}, {'name': 'THD_PERSIST', 'value': '', 'domain': '.homedepot.com', 'path': '/', 'expires': 1772497083, 'httpOnly': False, 'secure': False}, {'name': 'bm_s', 'value': 'YAAQWgw0F8l+E5aUAQAAj2QfpQJ/dcQasG11rNISq1f/c27Zkf97blV7ylGFQDkSD0+UYeX1WNkWXuFVCC90pRH1Fkv8NM9Qm9Y8E7afLYknnvEFKpHzsq0NK+vOYsGUCMilH9WJpfteMBal8lNpVVnLm9YjrKBiCZM4khn2KsjJm9t7ngnqKxY28fQo1P+4oiJ4aXMatRJXsg2b8VEIOjdD8U+pKCKzAwpbuFTlM7OkPsXfuuylsX5VxNFB/RaqFYYQJTk8xj/FRMxnTtDDKNznqwUB8fCBeEY0FEPxYN+9ga1G3qVNnhJUAx1Yq4XJ0Eg1W1cgNQV2ssIe/QyIk5XJ/Vj22iqfO4PSTWJn4z3gd1OUQ/z8b3nijMIGiy78J9u0gOV4/nQUiPglclGoCY5s7GIBHuPim9rORfEfx0xnnQQu+GwyZj+kCROk4yjKZ6q1w2VO2BZayBXJefE=', 'domain': '.homedepot.com', 'path': '/', 'expires': 1740615485, 'httpOnly': True, 'secure': True}, {'name': 'HD_DC', 'value': 'origin', 'domain': '.homedepot.com', 'path': '/', 'httpOnly': False, 'secure': True}, {'name': 'bm_sz', 'value': '59188A0ADCE3BE1C9DDE282159C1753E~YAAQWgw0F/Z7E5aUAQAA0lYfpRqr2gR9odBl1Ru3opdT9KPpL/RH1maFKiOJjfiLPerEbmoNsHWwYDKKsFRiCuMQXZYrOyViDomLw+4dm1iFo9LoMtn6vVCd8JJJPKh7LZVNbLhjjRAAHbOj3nfyF3rwqibG7O4pms7O+Y85YmbIGC7OHiR2nXVWl95pLZx8kaZAVJzjPqtlNk0tpO1AKNOYyt/32Ev8fE8RDX1jHIw+U63YQoyARMsSy/22UKzATSDD/ll/JIczQZjEPexivqkWB2ZPTSXxylkMzUGPO1Z/QyMOEEspyMhuTem+sFy5oeRqCiTmz1G1ku/cMSSRsCGJGb6jLGdm5MxTcr+l2Sh/1+lOG3ujK3TE5OSFT+/kw6L+F6i8z4vO2BGkPx805Q==~4539954~4600881', 'domain': '.homedepot.com', 'path': '/', 'expires': 1737951481, 'httpOnly': False, 'secure': False}, {'name': 'IN_STORE_API_SESSION', 'value': 'TRUE', 'domain': 'www.homedepot.com', 'path': '/', 'expires': 1737937383, 'httpOnly': False, 'secure': False}, {'name': 'THD_CACHE_NAV_SESSION', 'value': '', 'domain': '.homedepot.com', 'path': '/', 'expires': 1738023483, 'httpOnly': False, 'secure': False}, {'name': '_abck', 'value': 'BFE716FF6BB611B2F6008A2402D93C99~-1~YAAQWgw0FzB+E5aUAQAAfWIfpQ1Rl2ZCVesn+jPHRbbOiTBQnrd9VxChlEAlgChG4OYB2N5hOnKY4MeH86LJgNM6JE8iTaDUmdlxcT+67GHY3P47eylxc6DxwM3TEodcw+J31xXc1rbvxzgs3OFqvHVJpMLwAKpSwjG/dJGwHdys14+5pWxkznp4A1Rth4kX5cteen16o41ZTjO0eOOCQqEeZq7knn21lQWEqEf8EJRG3JO3d2MO7WEJoRPWsoOK6yoipKJJKDfTFv6v80wxTtD9J8/NSuv3OeoOCYOa+uOrPHiznkggcN1k4SptHK6grMOtdK7O/7ODRTAPYFHnd4TY1O+8u+vl91qqp6gVKyPNXNkOpInMQ8GTuoiGG2ACNbbqqaFJyNTB0HRNw2hJScUDZU+VOZevTdTRa8qZoZNVSQlgvnNvDnQDtbm2QhfC3MIa5CCfN7+z2whtdP3V3ovTN1OmjHn3xhnUasMzE9A=~-1~||0||~-1', 'domain': '.homedepot.com', 'path': '/', 'expires': 1769473085, 'httpOnly': False, 'secure': True}, {'name': 'bm_lso', 'value': 'CC42994AC2A3455C21548B1B030A6C329CC1F54D7F1566A01626D3AD3BB85264~YAAQWgw0F/V7E5aUAQAA0lYfpQKQ9OL93fxmj1UlQFh6JkXhA7ssHYZF7Ek8O9+lnalB8rm56qSMFoZ/JlD/PSmnd7E1cmq+Z4lcge/iusGnlz7Ev0GGzrQ5mC9kDhzBb0HNtMVgWhV+lKqZHe7b++2jB7JKn1p9uWsH1bMGPhBY3MkzzrlAEoqv3UagGca8Bii81uC4KCFabC4WyJzGm7iwhgU4WvLeaZMgsTJUOQZsmG7L9L2BUn0UC0fisWtpDS7UpajXm7vikqwH4+QIfHAAqWGp0K8l0KeucyoNAgjk70DxEaaXuNoymaqA4eAHDDGWh7nsOLqMjiaSHRchEL5vgUoMHzi+/xnmaYvHJ6FG/mudp7YjgF8FqSlVBhbcsji+jVfHRfmUjI5qmcSOZN5pBIwB6kbi4uwEILV/EaP2jynmoYga2215U/A2WyHM++4WG/To1OscDn+EYH+stIg=^1737937083589', 'domain': '.www.homedepot.com', 'path': '/', 'expires': 1740529083, 'httpOnly': False, 'secure': False}, {'name': 'THD_NR', 'value': '1', 'domain': '.homedepot.com', 'path': '/', 'expires': 1738005482, 'httpOnly': False, 'secure': False}, {'name': 'AKA_A2', 'value': 'A', 'domain': '.homedepot.com', 'path': '/', 'expires': 1737940682, 'httpOnly': True, 'secure': True}, {'name': 'bm_so', 'value': 'CC42994AC2A3455C21548B1B030A6C329CC1F54D7F1566A01626D3AD3BB85264~YAAQWgw0F/V7E5aUAQAA0lYfpQKQ9OL93fxmj1UlQFh6JkXhA7ssHYZF7Ek8O9+lnalB8rm56qSMFoZ/JlD/PSmnd7E1cmq+Z4lcge/iusGnlz7Ev0GGzrQ5mC9kDhzBb0HNtMVgWhV+lKqZHe7b++2jB7JKn1p9uWsH1bMGPhBY3MkzzrlAEoqv3UagGca8Bii81uC4KCFabC4WyJzGm7iwhgU4WvLeaZMgsTJUOQZsmG7L9L2BUn0UC0fisWtpDS7UpajXm7vikqwH4+QIfHAAqWGp0K8l0KeucyoNAgjk70DxEaaXuNoymaqA4eAHDDGWh7nsOLqMjiaSHRchEL5vgUoMHzi+/xnmaYvHJ6FG/mudp7YjgF8FqSlVBhbcsji+jVfHRfmUjI5qmcSOZN5pBIwB6kbi4uwEILV/EaP2jynmoYga2215U/A2WyHM++4WG/To1OscDn+EYH+stIg=', 'domain': '.homedepot.com', 'path': '/', 'expires': 1738023482, 'httpOnly': False, 'secure': True}, {'name': 'thx_guid', 'value': 'fa351ead85e9f9f0735a9e0f0ef971b1', 'domain': 'customer.homedepot.com', 'path': '/', 'expires': 1772497083, 'httpOnly': True, 'secure': True}, {'name': 'thda.s', 'value': '3b8bc050-7416-be6f-e2ec-3a7a6789f3f2', 'domain': '.homedepot.com', 'path': '/', 'httpOnly': False, 'secure': False}, {'name': 'akacd_usbeta', 'value': '3915389880~rv=45~id=230aa35cd260bfe3bbd7f36cd59b3289', 'domain': 'www.homedepot.com', 'path': '/', 'httpOnly': False, 'secure': True}, {'name': 'AMCV_F6421253512D2C100A490D45%40AdobeOrg', 'value': 'MCMID|62609644557325612624540126607646773632', 'domain': '.homedepot.com', 'path': '/', 'expires': 1771633083, 'httpOnly': False, 'secure': True}, {'name': 'akavpau_prod', 'value': '1737937382~id=33f63ed9a90eb59c56ae00d7253e737a', 'domain': 'www.homedepot.com', 'path': '/', 'httpOnly': True, 'secure': True}, {'name': 'THD_LOCALIZER', 'value': '%7B%22WORKFLOW%22%3A%22LOC_HISTORY_BY_IP%22%2C%22THD_FORCE_LOC%22%3A%221%22%2C%22THD_INTERNAL%22%3A%220%22%2C%22THD_LOCSTORE%22%3A%221950%2BSouth%20Loop%20-%20Chicago%2C%20IL%2B%22%2C%22THD_STRFINDERZIP%22%3A%2260607%22%2C%22THD_STORE_HOURS%22%3A%221%3B7%3A00-20%3A00%3B2%3B6%3A00-22%3A00%3B3%3B6%3A00-22%3A00%3B4%3B6%3A00-22%3A00%3B5%3B6%3A00-22%3A00%3B6%3B6%3A00-22%3A00%3B7%3B6%3A00-22%3A00%22%2C%22THD_STORE_HOURS_EXPIRY%22%3A1737940684%7D', 'domain': '.homedepot.com', 'path': '/', 'httpOnly': False, 'secure': False}, {'name': 'THD_SESSION', 'value': '', 'domain': '.homedepot.com', 'path': '/', 'expires': 1738023483, 'httpOnly': False, 'secure': False}]
scrapy-zyte-api however assumes presence of experimental in return data when querying for cookies:
# responses.py
@classmethod
def _prepare_headers(cls, api_response: Dict[str, Any]):
# ...
response_cookies: Optional[List[Dict[str, str]]] = api_response.get(
"experimental", {}
).get("responseCookies")
# ...
if response_cookies:
result["Set-Cookie"] = []
for cookie in response_cookies:
result["Set-Cookie"].append(
cls._response_cookie_to_header_value(cookie)
)
return resultSo I'm forced to use experimental myself (inside params method of my session handler) even though it's no longer necessary.
I'd like to have a proper handling of responseCookies whenever experimental was used or not. ZYTE_API_EXPERIMENTAL_COOKIES_ENABLED should likely be discontinued (in favor of ZYTE_API_COOKIES_ENABLED...?)