o
    vOAjw%                     @   s   d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZ d dl	m
Z
 g dZg dZdd Zdd
dZdd ZdddZdd ZdS )    N)urlparse
urlunparse)async_playwright)BeautifulSoup)	zmc.yandex.ruzgoogle-analytics.comzyandex.ru/adszdzen.ruzprivacy-cs.mail.ruzr.mradx.net/imgspixel1x1ztransparent.gif)imagemediafont
stylesheetc                 C   sP   | sdS |  drd|  } n	|  dsd|  } t| }t|j|j|jdddfS )N /zhttps://ok.ruhttpzhttps://ok.ru/)
startswithr   r   schemenetlocpath)hrefparsed r   H/var/www/www-root/data/www/avtoreputaciya.ru/cron/ok_parser/ok_parser.py	clean_url$   s
   r      c                    s   g }| j dddD ]?}t||kr n6|d d|dg } s$q	d| v r+q	t fdd	d
D r7q	d v s?d v rH |vrH|  q	|d | S )NimgTsrcr    classavatarc                 3   s    | ]	}|   v V  qd S N)lower).0xr   r   r   	<genexpr>J       z!extract_photos.<locals>.<genexpr>)r   iconspriter   r   transparentzokcdn.ruzmycdn.me)find_alllengetjoinr    anyappend)block
max_photosphotosr   classesr   r   r   extract_photos4   s"   

r2   c           	   
      s   zZ| j |dddI dH  | dI dH  |  I dH }t|d}|jddd	}|rY|jd
d}dD ]#}| |rU|d}t	|dkrQd
|dd  } |W S q2|W S W dS  tyx } ztjd| d W Y d}~dS d}~ww )u]   Открывает пост по прямой ссылке и возвращает текстdomcontentloadedi:  
wait_untiltimeoutN  html.parserdivzmedia-text_cnt_tx)class_Tstrip)u   продажаu   продамu
   куплюu
   обмен.   zError getting full text: 
)gotowait_for_timeoutcontentr   findget_textr    r   splitr)   r+   r<   	Exceptionsysstderrwrite)	pagepost_urlhtmlsouptext_eltextprefix	sentenceser   r   r   get_full_text_from_postZ   s0   

rS      c           !         s^  dd}i }t  4 I d H s}|jjdg ddI d H }dddd	d
d}	|r.d|i|	d< |jd?i |	I d H }
|
dI d H  |
 I d H }fdd}dd }|d| |d|I d H  zU|j| dddI d H  |	dI d H  d}|dk r|
dI d H }||krn-|jddI d H  |	dI d H  |d7 }|r|
d| dI d H }|rd}n|dk s{W n ty } ztjd| d  W Y d }~nd }~ww | I d H }t|d!}|d"}g }|d | D ]?}|d#d$ |r |krd} n.|d%}|rd&| v rq|d'}|r+|d(r+t|d( }| ||d) q|D ]B}zt||d* I d H }|rKt|d+krK|||d, < W q. typ } ztjd-|d,  d.| d  W Y d }~q.d }~ww | I d H  W d   I d H  n1 I d H sw   Y  g }|D ]}|d,  |d/ }t fd0d1|D rq d$d$|d* g d2} |v r|  |d3< n*|d4}|s|d5}|r|jdd6}d7|v r||d7d8 d  }||d3< |d9} | r| jdd6|d:< t|||d;< |d3 rt|d3 d+kr|| q|d | |r$|d d, nd td< d=|d>S )@Nr   FT)z--no-sandboxz--disable-setuid-sandboxz---disable-blink-features=AutomationControlledz--disable-dev-shm-usage)headlessargsz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36i  i8  )widthheightzru-RU)
user_agentviewportlocaleserverproxyzFObject.defineProperty(navigator, 'webdriver', {get: () => undefined});c                    s0   zt | jdd} |7  W d S    Y d S )Nzcontent-lengthr   )intheadersr*   )responsecl)traffic_bytesr   r   count_traffic   s   z"parse_group.<locals>.count_trafficc                    sd   | j j}| j j}|tv r|  I d H  d S tD ]}||v r(|  I d H   d S q|  I d H  d S r   )requesturlresource_typeBLOCKED_RESOURCE_TYPESabortBLOCKED_PATTERNS	continue_)routere   rtypepatternr   r   r   route_handler   s   z"parse_group.<locals>.route_handlerr`   z**/*r3   i0u  r4   i     zr() => {

                    return document.querySelectorAll('div.feed[data-feed-id]').length;

                }r7   r>   z() => {

                        const posts = document.querySelectorAll('div.feed[data-feed-id]');

                        for (let post of posts) {

                            if (post.getAttribute('data-feed-id') === 'z') {

                                return true;

                            }

                        }

                        return false;

                    }zNavigation error: r?   r8   zdiv.feed[data-feed-id]zdata-feed-idr   zdiv.feed-captionu   Закрепленоza[href*="/topic/"]r   )feed_idre   r.   re   
   rp   zError processing post z: r.   c                 3   s    | ]	}|d   kV  qdS )rp   Nr   )r!   p)rp   r   r   r#     r$   zparse_group.<locals>.<genexpr>)rp   rO   	timestampre   r0   rO   zdiv.media-text_cnt_txzdiv.media-text_cntr;   u   Комментировать   timers   r0   i   r   )postslatest_post_id
traffic_mbstopped_by_last_idr   ) r   chromiumlaunchnew_contextadd_init_scriptnew_pageonrk   r@   rA   evaluatemousewheelrF   rG   rH   rI   rB   r   selectr*   
select_onerD   r   r-   rS   r)   closer,   rC   r2   round)!re   	max_postsr/   last_post_id	proxy_urlfound_last_post
full_textsrr   browserctx_argscontextrJ   rc   rn   scrollscurrent_postsfoundrR   rL   rM   post_blocksposts_to_processr.   captionlinkrK   	post_info	full_text	all_postspostrN   rO   time_elr   )rp   rb   r   parse_group   s   

4 



, * k






r   c                  C   s:   t  } | jddd | jdtdd | jdtdd d S )	Nz--urlT)requiredz--max-postsrT   )typedefaultz--max-photosr   )argparseArgumentParseradd_argumentr^   )parserr   r   r   main  s   r   )r   )rT   r   NN)asyncior   jsonrG   urllib.parser   r   playwright.async_apir   bs4r   ri   rg   r   r2   rS   r   r   r   r   r   r   <module>   s    

&
:  X