← back to Nationalrealestate

scripts/__pycache__/scrape-asc-appraisers.cpython-314.pyc

69 lines

+
%Bj�����Rt^RIt^RIt^RIt^RIt^RIt^RIt^RItRt	Rt
]PP]PP]4RR4t]PP]R4t]PP]R4tRtR	tR
tRRlt]P,!R]P.4tR
tRtRtRtRtR0tRt] R8Xd
]!4R#R#)u�
TK-10560 — Scrape the ASC National Appraiser Registry (asc.gov) for California,
filter to the SoCal directory footprint (within 300mi of LA), classify commercial
relevance, and stage rows for rentv_licensed_targets (role='Appraiser').

VERIFIED PAYOFF FIRST: the ASC results carry state/city/zip/license#/license-type/
status per record (confirmed on a small sample before this full run — see TK-10560).

Scrape path (server-side; the browser spike only discovered the working param set):
  GET /appraiser?field_state_name_value=California&field_license_type__value=All
      &field_first_name__value=&field_last_name__value=&items_per_page=250&submit=Apply&page=N

Output: JSONL of staged rows (SoCal only) -> stdout. Read-only vs asc.gov; NO DB writes.
Nz�https://www.asc.gov/appraiser?field_state_name_value=California&field_license_type__value=All&field_first_name__value=&field_last_name__value=&items_per_page=250&submit=Apply&page=z[Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/151 Safari/537.36z..�datazsocal_city_market.psvzsocal_zip_market.psvc�4�/p\V4;_uu_4pVF_pVPR4pRV9gKVPR^4wrEVP4WP4P	4&Ka	RRR4V# +'giT#;i)�
�|N)�open�rstrip�split�strip�lower)�path�m�f�line�k�vs&     �N/Users/macstudio3/Projects/nationalrealestate/scripts/scrape-asc-appraisers.py�load_psvrsy��
�A�	
�d���q��D��;�;�t�$�D��d�{��z�z�#�q�)���'(�w�w�y��'�'�)�/�/�#�$�	�
�
�H�

��
�H�s�B�AB�B	c� �\\4#�N)r�
CITY_MAP_PATH��r�load_city_marketr$s
���M�"�"rc� �\\4#r)r�ZIP_MAP_PATHrrr�load_zip_marketr's
���L�!�!rc	�f�\\V4,p\V4Fp\PPVR\/R7p\PPV^-R7;_uu_4pVP4PRR4uuRRR4u#	R	# +'giK�;i \dkpY1^,
8Xd.\PPRTRTR24Rp?R	#\P!^T^,,4Rp?EK	Rp?ii;i)
z
User-Agent)�headers)�timeoutzutf-8�replaceNz
[fetch fail pz] r�)�BASE�str�range�urllib�request�Request�UA�urlopen�read�decode�	Exception�sys�stderr�write�time�sleep)�page�tries�url�t�req�r�es&&     r�fetchr8*s���
��T��
�C�
�5�\��	$��.�.�(�(��|�R�6H�(�I�C����'�'��R�'�8�8�A��v�v�x���w�	�:�9�8���
9�8�8���	$��A�I�~��
�
� � �=���b���2�!>�?���J�J�q�A��E�{�#�#��	$�s<�AB;�6 B'�
B;�'B8�2B;�8B;�;D0�3D+�$D+�+D0z.<tr[^>]*class="[A-Za-z0-9_\- ]+"\s*>(.*?)</tr>c�R�\P!RRV\PR7#)z
<!--.*?-->r )�flags)�re�sub�S��ss&r�strip_commentsr@8s��b�f�f�]�B�����F�Frc�v�\P!\P!RRV44P	4#)z<[^>]+>� )�html�unescaper;r<r	r>s&r�txtrE9s&��4�=�=����
�C��!;�<�B�B�D�Drc���.p\PV4EFLp\VP^44p\P
!RV4pV'gK?VP^4p\P!VP^44P4p\P
!RV\P4pR;p;r�V'EdR\P!RVP^444Uu.uFq�'gKVNK	ppVF�p\P!RV4'd)\P!RRV4P4p	KHVP4R8XdR	pKaV
'dKkVPR
4'dK�Tp
K�	V
'g<VF5pVP4R8wgKVPR
4'dK3Tp
M	V'gR	p\P
!RV4;'g\P
!RV4p
V
'd\V
P^44MRp\P
!R
V4p\P
!RV4pV'd\VP^44MRpR;p;ppV'd�\VP^44P!R4P4p\P!R\VP^444pV'd#VP^4pVP^4pTP#RTRTRT	RT
RTRTRTRT;'gTRV/	4EKO	V#uupi)�z4/individual-appraiser-public/(\d+)"[^>]*>([^<]+)</a>z+middle-appraiser-columns1"[^>]*>(.*?)</div>r c3�8"�TFp\V4x�K	R#5ir)rE)�.0�xs& r�	<genexpr>�parse_page.<locals>.<genexpr>Gs��� U�1T�A��Q���1T�s�z	<br\s*/?>z^#\s*[A-Z]{1,3}\d+z^#\s*�
california�CA�#z2class="isactive">([^<]+)</span>\s*</div>\s*</span>z:class="isactive">\s*<div[^>]*>\s*<span[^>]*>([^<]+)</span>z"optionalstreet"[^>]*>([^<]*)</div>z.optionalcity"[^>]*>([^<]*)</span>([^<]*)</div>z, z([A-Z]{2})\s+([\d\-]+)�	detail_id�name_raw�
license_no�license_type�license_status�address�city�state�zip)�ROW_RE�finditerr@�groupr;�searchrCrDr	r=r�matchr<r
�
startswithrEr�append)�hbody�rowsr�cell�nmrPrQ�c1rW�lic�ltype�p�parts�st�status�streetrV�street_v�city_v�zip_v�state_v�mms&                     r�
parse_pagerq;s���
�D�
�_�_�U�
#���a�g�g�a�j�)��
�Y�Y�N�PT�
U�����H�H�Q�K�	��=�=����!��-�3�3�5��
�Y�Y�E�t�R�T�T�
R�� � �� ��
�2� U����,����QR��1T� U�[� U�1�YZ�Q�Q� U�E�[����8�8�1�1�5�5��&�&��2�q�1�7�7�9�C��W�W�Y�,�.� �E���q�|�|�C�'8�'8��E�
���A��w�w�y�L�0����c�9J�9J� !��5�����
�Y�Y�L�d�
S�_�_��	�	�W�Y]�^�	�%'��R�X�X�a�[�!�R�����@�$�G���y�y�J�D�Q��+1�3�v�|�|�A��'�r��#%�%��%��������A��'�.�.�t�4�:�:�<�F����3�S����A��5G�H�B���(�(�1�+��r�x�x��{�u����[�)�Z��<�QT�#�U�,<�f�i�QY��V�W�g�.>�.>���u�N�	O�I$�N�K��;\s�>	M+�M+c��RV9dFVPR^4wrVP4RVP42P4#V#)�,rB)rr	)rQ�last�firsts&  r�name_lfrvfsF��
�h���n�n�S�!�,����+�+�-���$�*�*�,��0�6�6�8�8��Orc���\P!RT;'gR4p\V4;'d/R\VP	^44u;8*;'dR8*#u#)z^(\d{3})r i�i�)r;r]�bool�intr[)�zrs& r�	zip_socalr{lsI��
����a�g�g�2�&�A���7�4�4�s�c�!�'�'�!�*�o�4�4��4�4�4�4rzcertified generalc
��\4p\4p\4.r2R^R^R^R^R^/p^^re\V4pV'd\	V4M.pV'gV^,
pV^8�dEM�V^,
pKF^pVR;;,^,
uu&VR;;,\V4,
uu&VEF	p	V	R,;'g
V	R,p
W�9dVR;;,^,
uu&K;\P!RV	R	,;'gR
4pV'd!VPVP^44MRpV'g1TPV	R,;'gR
P44pV'd\V	R	,4'gK�VPV
4VR;;,^,
uu&V	R
,;'gR
P4P4\9p
V
'dVR;;,^,
uu&TP/RRbRRbR\!V	R,4bR\!V	R,4bRV	R,bR
V	R
,bRV	R,bRV	R,bRV	R,bRRbRV	R,;'gRbR	V	R	,bRVbRV
bRRbRRV	R,2bRV	b4EK	VR,^
,^8Xd8\"P$P'RVR VR,R!VR,R"24V^,
p\(P*!R#4EK�VF$p\-\.P0!VR$R%74K&	\"P$P'R&\.P0!V4R"24R#)'�pages�raw�socal�
commercial�dupTrRrPz^(\d{5})rXr NrVrS�sourcezASC National Appraiser Registry�role�	Appraiser�entity_namerQ�contact_namerTrU�countyrWrN�market�commercial_flag�within_300mi�
source_urlz0https://www.asc.gov/individual-appraiser-public/z[progress] page=z raw=z socal=rg�������?F)�ensure_asciiz[DONE] )rr�setr8rq�lenr;r]�getr[r
r{�addr	�COMMERCIAL_TYPESr_rvr,r-r.r/r0�print�json�dumps)�city_market�
zip_market�seen�out�statsr1�empty�bodyrar6�key�z5r��comm�recs               r�mainr�rsO��"�$�K� �"�J���r�#�
�a���7�A�|�Q��q�I�E��Q�%�
��T�{��#'�z�$��R����Q�J�E���z���A�I�D�x���
�g��!���U�5�\�S��Y�6�\��A��L�/�3�3�Q�{�^�C��{��e���!��8����+�q��x�~�~�2�6�B�46�Z�^�^�B�H�H�Q�K�0�D�F��$���!�F�)�/�/�r�)@�)@�)B�C���y��5��2�2���H�H�S�M�5��>�Q�.�>��n�%�+�+��2�2�4�:�:�<�@P�P�D���l�#�q�(�#��J�J�	��;�	�=C�[�	��w�q��}�5�	�7E�w�q�Q[�}�G]�	��a��o�	�0>�q��?P�	�!�!�$4�"5�		�8A�!�I�,�		�
��&�	�	�
$,�T�	�
4;�A�g�J�<N�<N�$�	��q��x�
	�"*�6�
	�4E�d�
	���	�� P�QR�S^�Q_�P`�a�	��q�	�	
��0��>�B��!�#��J�J���/��v�U�5��<�.��PU�V]�P^�O_�_a�b�c���	��4�:�:�c�?���
�d�j�j��5�1�2���J�J���w�t�z�z�%�0�1��4�5r�__main__)�)!�__doc__r,r;r�r/rC�urllib.requestr$�osr!r'r�join�dirname�__file__�DATA_DIRrrrrrr8�compiler=rYr@rErqrvr{r�r��__name__rrr�<module>r�s���
�5�4�4�4�	I��c��
�7�7�<�<�������1�4��@�������X�'>�?�
��w�w�|�|�H�&<�=��
�#�"��
���E�r�t�t�	L��F�D�)�V�5�(�(��-6�^�z���F�r