← back to Kravet Sheet Sync 2026 04 20

__pycache__/bulk_image_check.cpython-314.pyc

70 lines

+
UK�jo�
�4�Rt^RIt^RIt^RIt^RIt^RIt^RIt^RIH	t	H
t
RtRR.tRR/t
RR	ltR
Rlt]!R4]!R
4]!R4]!R4t]P%4P'R4Uu.uFq'gKVP'R4NK	upt]!R]!]4R24RRlt]!R4.t^t]P!4tRt]	!^(R7;_uu_4t]UUUu.uFwrp]P9]WV4NK	upppt]
!]4Fzt]P?]PA44]^,
t]R,^8XgK;]]!!^]P!4],
4,t"]!R]R]!]4R]"R
R24K|	RRR4]#!R]44t$]!]4]$,
t%]#!R]44t&]#!R ]44t']!R!]$R"]&R#]'R$]%R%]P!4],
R&
R'24]!R(4^RI(t(^RI)t)](PT!R)R*R+R,R-7;_uu_4t+])PX!]+4t-]Fwt.t/t0t1t2]-Pg].]/]0]1]2.4K!	]+Pht5RRR4]Pl!R.]5R/.R0R17]!R24]!R34]Pl!]R4.,R0R0R57t7]!]7Pp4]!R64R#uupiuupppi +'giELb;i +'giL�;i)7ux
Bulk-validate Kravet image URLs for every wallcovering row (~10,184).
URL pattern: https://www.kravet.com/media/catalog/product{image_file_hires}
Runs HEAD requests, persists result to kravet_sheet_fields:
  - image_url_hires (computed)
  - image_url_lores (computed)
  - image_hires_status (http code)
  - image_checked_at
Concurrency: 20 threads, ~20 rps → ~10min total.
N)�ThreadPoolExecutor�as_completedz,https://www.kravet.com/media/catalog/product�sshzroot@45.61.58.125z
User-AgentzMozilla/5.0 (dw-sync bot)c�0�V^8�dQhR\R\/#)��sql�return��str)�formats"�bulk_image_check.py�__annotate__r
s��������c��\P!\R\R4PPRR4RVR2.,RRRR7pVP#)�PGPASSWORD=�os�
PGPASSWORD�z< psql -h 127.0.0.1 -U dw_admin -d dw_unified -At -F "|" -c "�"T��capture_output�text�check)�
subprocess�run�SSH�
__import__�environ�get�stdout)r�rs& r�pg_cmdr!sr�������Z��-�5�5�9�9�,�r�J�K�LH�IL�HM�MN�O�P�	P��$�d�	4�A�
�8�8�Orc�(�V^8�dQhR\RR/#)rrrNr	)rs"rr
r
s��4�4��4��4rc��\P!\R\R4PPRR4RVR2.,RRRR7R#)	rrrrz1 psql -h 127.0.0.1 -U dw_admin -d dw_unified -c "rTrN)rrrrrr)rs&r�pg_execr$sa���N�N���Z��-�5�5�9�9�,�r�J�K�K|�~A�}B�BC�D�E�	E��$�d�4rz/[1/4] adding columns to kravet_sheet_fields ...z�
  ALTER TABLE kravet_sheet_fields
    ADD COLUMN IF NOT EXISTS image_url_hires text,
    ADD COLUMN IF NOT EXISTS image_url_lores text,
    ADD COLUMN IF NOT EXISTS image_hires_status int,
    ADD COLUMN IF NOT EXISTS image_checked_at timestamptz
z2[2/4] pulling all rows with any image filename ...z�
  SELECT mfr_sku_norm, COALESCE(image_file_hires,''), COALESCE(image_file_lores,'')
  FROM kravet_sheet_fields
  WHERE COALESCE(image_file_hires,'') <> '' OR COALESCE(image_file_lores,'') <> ''
�
�|z       z rows to checkc�0�V^8�dQhR\R\/#)r�urlr)r
�int)rs"rr
r
6s����c��c�rc��\PPV\RR7p\PP	V^
R7;_uu_4pVP
uuRRR4# +'giR#;i \PPdpTPuRp?#Rp?i\d^#i;i)�HEAD)�headers�method)�timeoutN)
�urllib�request�Request�UA�urlopen�status�error�	HTTPError�code�	Exception)r(�reqr �es&   r�headr;6s�����n�n�$�$�S�"�V�$�D��
�^�^�
#�
#�C��
#�
4�
4���8�8�5�
4�
4�
4���<�<�!�!���v�v�
������sH�AA<�A(�
A<�(A9	�3A<�9A<�<B>�B,�&B>�,B>�9B>�=B>uA[3/4] validating URLs (40 concurrent, hires→lores fallback) ...c���V'd\V,MRpV'd\V,MRp^pRpV'd\V4pRpV^�8wd+V'd#\V4pV^�8XdTpRpMV^8XdTpRpWWEV3#)r�hires�lores)�BASEr;)�normr=r>�hu�lur4�kind�lss&&&     r�	check_onerEHst������B�B�����B�B�
�F�r��	��b���'�4�
��}��
�"�X��
��9��F���
�q�[��F����b�$�'�'r)�max_workersi��/z  z.1fz/sc#�D"�TFq^,^�8XgK^x�K	R#5i)�N���.0r s& r�	<genexpr>rM`s���+�G�q��t�s�{���G��� �
 c#�D"�TFq^,R8XgK^x�K	R#5i)�r=NrJrKs& rrMrMb����8�W��!����!�!�W�rNc#�D"�TFq^,R8XgK^x�K	R#5i)rPr>NrJrKs& rrMrMcrQrNz
       DONE: z OK (z	 hires / z	 lores), z	 failed, z.0f�sz[4/4] writing back to PG ...�wz.csvFr)�suffix�delete�newline�scpz-root@45.61.58.125:/tmp/kravet_image_check.csvT)rz�
  DROP TABLE IF EXISTS _img_tmp;
  CREATE TEMP TABLE _img_tmp (mfr_sku_norm text, hu text, lu text, status int) ON COMMIT DROP;
zMALTER TABLE kravet_sheet_fields ADD COLUMN IF NOT EXISTS image_best_kind textauPGPASSWORD={__import__("os").environ.get("PGPASSWORD","")} psql -h 127.0.0.1 -U dw_admin -d dw_unified <<'SQL'
CREATE TEMP TABLE _img_tmp (mfr_sku_norm text, hu text, lu text, status int, kind text);
\copy _img_tmp FROM '/tmp/kravet_image_check.csv' WITH (FORMAT csv);
UPDATE kravet_sheet_fields s
   SET image_url_hires    = NULLIF(t.hu,''),
       image_url_lores    = NULLIF(t.lu,''),
       image_hires_status = t.status,
       image_best_kind    = NULLIF(t.kind,''),
       image_checked_at   = now()
  FROM _img_tmp t
 WHERE s.mfr_sku_norm = t.mfr_sku_norm;
SELECT COUNT(*) FILTER (WHERE image_hires_status=200) AS ok,
       COUNT(*) FILTER (WHERE image_best_kind='hires') AS hires,
       COUNT(*) FILTER (WHERE image_best_kind='lores') AS lores,
       COUNT(*) FILTER (WHERE image_hires_status<>200 AND image_hires_status IS NOT NULL) AS bad
  FROM kravet_sheet_fields;
SQL
)rrzdone.)9�__doc__�jsonr�urllib.requestr/�urllib.error�sys�time�concurrent.futuresrrr?rr2r!r$�print�rows_out�strip�split�rows�lenr;�results�done�t0rE�ex�submit�futs�f�append�result�max�rps�sum�ok�err�hires_count�lores_count�tempfile�csv�NamedTemporaryFile�tf�writerrTr@rArBr4rC�writerow�name�tmprr r)�line�n�h�ls0000r�<module>r�s��	�A�@�@�?�5��
�"�#���1�2���4��7�8��	���:�;�����
%-�N�N�$4�$:�$:�4�$@�I�$@�D�D���
�
�3��$@�I�����D�	�{�.�)�*���I�J�
����	�Y�Y�[��(��B�'�'�2�9=�>��g�a�A�B�I�I�i��q�)��>�D�
�$�
�����q�x�x�z�"���	���$�;�!����Q��	�	��b� 0�1�1�C��G�D�6��3�t�9�+�R��C�y��;�<� �(�	�+�G�+�+��	�'�l�R����8�W�8�8���8�W�8�8���
�b�T��{�m�9�[�M��3�%�y�Y]�Yb�Yb�Yd�eg�Yg�hk�Xl�lm�n�o��$�%��
� � ��V�E�2�N�N�RT��
�
�2��A�&-�"��b�"�f�d�	�
�
�D�"�b�&�$�/�0�'.�
�'�'�C�	O�
����s�K�L�TX�Y��	��
�W�X��N�N�3����"��#&��$�a�h�h���g���oJ��L?�(�'�'��&O�N�s?�:	K'�K'�)K3�0K,�AK3�AK3�;AL�,K3�3L	�L