
    6jJ                     2   U d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	m
Z
mZ ddlmZ ddlmZ ddlmZmZ ddlmZ ddlmZmZ ddlZd	d
lmZ d	dlmZ d	dlmZmZ d	dlm Z  d	dl!m"Z" d	dl#m$Z$ d	dl%m&Z&  ej'        e(          Z)dZ*dZ+dZ,da-ej.        dz  e/d<   da0e
dz  e/d<    ej1                    Z2da3ej4        dz  e/d<   da5ej6        dz  e/d<   ddZ7 ej8        e7           de
fdZ9dej.        fdZ:dej4        fdZ; G d d          Z<dS ) zDDGS class implementation.    N)ThreadPoolExecutorwait)ceil)Path)randomshuffle)TracebackType)AnyClassVar   )BaseSearchEngine)ENGINES)DDGSExceptionTimeoutException)
HttpClient)ResultsAggregator)SimpleFilterRanker)_expand_proxy_tb_aliaszhttp://localhost:4479      $@g      ?_http_client_cache_executor_async_loop_async_threadreturnc                     t           j        J	 t           j                                        ht                              dt           j        j                   t           j                                         t           j                            d           n# t          $ r} t          	                    d|            	 t           j        
                                 t           j                            d           n2# t          $ r%} t          	                    d|            Y d} ~ nd} ~ ww xY wY d} ~ nd} ~ ww xY wdt           _        n# dt           _        w xY wt          t                                          r	 t                              t          j                   t          t                              d           n2# t          $ r%} t          	                    d|            Y d} ~ nd} ~ ww xY wdadan	# dadaw xY wt"          Z	 t"                              d	
           n2# t          $ r%} t          	                    d|            Y d} ~ nd} ~ ww xY wdadS # daw xY wdS )z/Cleanup any spawned API server process on exit.Nz%Stopping spawned API server (PID: %d)g      @timeoutzFailed to stop API server: %rg       @zFailed to kill API server: %rzFailed to stop async loop: %rT)r   z%Failed to shutdown cache executor: %r)DDGS_api_processpollloggerinfopid	terminater   	Exceptiondebugkillr   
is_runningcall_soon_threadsafestopr   joinr   shutdown)exs    K/root/.hermes/hermes-agent/venv/lib64/python3.11/site-packages/ddgs/ddgs.py_cleanup_api_processr/   *   s   $	% %%''/CTEVEZ[[[!++---!&&s&333 	B 	B 	BLL8"===B!&&(((!&&s&3333 B B B<bAAAAAAAAB	B !%DD$$$$ ;#9#9#;#;	!,,[-=>>>(""3"/// 	> 	> 	>LL8"========	> K MM K M     "	#$$$$//// 	F 	F 	FLL@"EEEEEEEE	F #OOOdO"""" #"s   BB E 
D7 D2<>C;:D2;
D*D% D2%D**D2-E 2D77E E9AG  ?G7  
G/
G*%G7 *G//G7 7G=H$ #I $
I.I	I II Ic                  @    t           t          dd          a t           S )z@Get shared thread pool executor for background cache operations.N   z
DDGS-Cachemax_workersthread_name_prefix)r   r        r.   _get_cache_executorr7   W   s"     ,|\\\r6   c                  H    t           t          j        d          a t           S )z+Get or create the shared primp HTTP client.N   r   )r   primpClientr5   r6   r.   _get_http_clientr<   `   s"     |A...r6   c                      t           lt          5  t           Ft          j                    a dd} t	          j        | d          at                                           ddd           n# 1 swxY w Y   t           S )z<Get or create shared async loop running in dedicated thread.Nr   c                  j    t          j        t                     t                                           d S N)asyncioset_event_loopr   run_foreverr5   r6   r.   	_run_loopz"_get_async_loop.<locals>._run_loopp   s+    *;777++-----r6   T)targetdaemonr   N)r   _network_lockr@   new_event_loop	threadingThreadr   start)rC   s    r.   _get_async_looprL   h   s      		& 		&"%466. . . . !* 0	$ O O O##%%%		& 		& 		& 		& 		& 		& 		& 		& 		& 		& 		& 		& 		& 		& 		& s   AA))A-0A-c                      e Zd ZU dZdZeedz           ed<   dZee	         ed<   dZ
eej        e         dz           ed<   	 	 d5dddd	d
edz  dedz  deez  dedz  deddfdZd6dZ	 	 	 d7dee         dz  dedz  dedz  ddfdZd8dZde	fdZdedeeee	f                  deddfdZdededeee	                  fdZ	 d9dddd d!d"d#deded$edz  d%ed&ed'edz  d(edz  d)eded*edeeee	f                  fd+Zded*e	deeee	f                  fd,Zded*e	deeee	f                  fd-Zded*e	deeee	f                  fd.Zded*e	deeee	f                  fd/Z ded*e	deeee	f                  fd0Z!d:d2ed3edeeee"z  f         fd4Z#dS );r   a  DDGS | Dux Distributed Global Search.

    A metasearch library that aggregates results from diverse web search services.

    Args:
        proxy: The proxy to use for the search. Defaults to None.
        timeout: The timeout for the search. Defaults to 5.
        verify: bool (True to verify, False to skip) or str path to a PEM file. Defaults to True.

    Attributes:
        threads: The maximum number of threads per search. Defaults to None (automatic, based on max_results).

    Raises:
        DDGSException: If an error occurs during the search.

    Example:
        >>> from ddgs import DDGS
        >>> results = DDGS().search("python")

    Nthreads_network_clientr   r9   TF)verifyapi_url	spawn_apiproxyr   rP   rQ   rR   r   c                   t          |          pt          j                            d          | _        || _        || _        || _        || _        i | _	        | j        r"t          j        |                                  d S d S d S )N
DDGS_PROXY)r   osenvironget_proxy_timeout_verify_api_url
_spawn_api_engines_cacher   rO   _ensure_network_running)selfrS   r   rP   rQ   rR   s         r.   __init__zDDGS.__init__   s     -U33Srz~~l7S7S#  	
 = 	+T19((*****	+ 	+99r6   c                     | S )z7Enter the context manager and return the DDGS instance.r5   r`   s    r.   	__enter__zDDGS.__enter__   s    r6   exc_typeexc_valexc_tbc                     dS )zExit the context manager.Nr5   )r`   re   rf   rg   s       r.   __exit__zDDGS.__exit__   s      r6   c           	         t           j        dS t          5  t           j        	 ddd           dS | j        pt          }	 t                                          | d          }|j        dk    rEddlm	}  ||          t           _        t                              d|           	 ddd           dS nW# t          $ r t                              d           Y n1t          $ r%}t                              d	|           Y d}~nd}~ww xY w| j        rGt           j        t           j                                        	 t%          t'          t(          j                  j                  }i t.          j        d
| t.          j         t.          j                            d
d           i}t5          j        t(          j        ddddg|t4          j        t4          j        dd          t           _        t                              dt           j        j                   n># t          $ r1}t                              d|           Y d}~ddd           dS d}~ww xY wt?          j                    }t?          j                    |z
  t@          k     r	 t                                          | d          }|j        dk    rnn# t          $ r Y nw xY wt@          t?          j                    |z
  z
  }|dk    rnnt?          j!        tE          tF          |                     t?          j                    |z
  t@          k     t                              d           	 ddd           dS 	 ddlm	}  ||          t           _        t                              d|           n*# t          $ r t                              d           Y nw xY wddd           dS # 1 swxY w Y   dS )zEnsure the network service is running.

        If not running, spawn a new ddgs api process only if spawn_api is True.
        Nz/health   r   )	DhtClient)rQ   zNetwork client ready: %sz7DHT dependencies not available - network cache disabledzAPI health check failed: %rPATH z-mddgsapiz-dT)envstdoutstderrstart_new_sessiontextz"Spawned ddgs api service (PID: %d)zFailed to spawn ddgs api: %rr   z/Timed out waiting for ddgs api service to start)$r   rO   rG   r\   DEFAULT_API_URLr<   rX   status_codedhtrl   r!   r"   ImportErrorr&   r%   r]   r   r    strr   sys
executableparentrV   rW   pathsep
subprocessPopenDEVNULLr#   warningtimeNETWORK_START_TIMEOUTsleepminNETWORK_CHECK_INTERVAL)	r`   rQ   resprl   r-   venv_binrq   
start_time	remainings	            r.   r_   zDDGS._ensure_network_running   s   
 +F A	X A	X#/A	X A	X A	X A	X A	X A	X A	X A	X m6G@'))--.A.A.ABB#s**......+49W+E+E+ED(KK :GDDDA	X A	X A	X A	X A	X A	X A	X A	X +  X X XVWWWWW @ @ @:B????????@  D$5$=ARAWAWAYAYAe"4#7#7#>??HgRZgH1fbj1f"*..Y_acJdJd1f1fggC(2(8vudC)1)1*.!) ) )D% KK DdFWF[\\\\    NN#A2FFFFFFMA	X A	X A	X A	X A	X A	X A	X A	XH J)++
*-BBB+--11W2E2E2EFFD'3.. /    D 2TY[[:5MN	>>
35yAABBB )++
*-BBB PQQQqA	X A	X A	X A	X A	X A	X A	X A	XvX******'0y'A'A'A$6@@@@ X X XVWWWWWXAA	X A	X A	X A	X A	X A	X A	X A	X A	X A	X A	X A	X A	X A	X A	X A	X A	X A	Xs   OO A&B54O5$D	O	D	$D?OD		6O CHO
I'IOI5O/J?=O?
K	OKB	O#7NO$O?OOOOOc                     t           j        S )z|Get the network client for caching.

        Returns:
            DhtClient instance if available, None otherwise.

        )r   rO   rc   s    r.   _get_network_clientzDDGS._get_network_client   s     ##r6   queryresultscategoryc                     |                                  sdS dfd}t                      }|                    |           dS )z$Cache search results asynchronously.Nr   c                  B   	 t                      } t          j                                      |           }|                    d           t
                              d           d S # t          $ r&}t
                              d|           Y d }~d S d }~ww xY w)Nr   r   zCached results: %szCache failed: %r)rL   r@   run_coroutine_threadsafecacheresultr!   r&   r%   )loopfuturer-   r   networkr   r   s      r.   _cache_workerz0DDGS._cache_results_async.<locals>._cache_worker  s    5&(( 9'--wX`:a:acghhd+++1599999 5 5 5/4444444445s   A)A. .
B8BBrF   )r   r7   submit)r`   r   r   r   r   executorr   s    ```  @r.   _cache_results_asynczDDGS._cache_results_async	  sx     **,, 	F	5 	5 	5 	5 	5 	5 	5 	5 	5 '((&&&&&r6   backendc           	      ,   t          |t                    rd                    |          }d |                    d          D             }t          t          |                                                   }t          |           d|v sd|v r|}|dk    rddgd |D             z   }n|}g }g }|D ]O}t          |                             |          x}	r|                    |	           :|                    |           P|r\t          
                    d	d
                    t          |                    d
                    t          |                               g }
|D ]h}	|	| j        v r!|
                    | j        |	                    , |	| j        | j        | j                  }|| j        |	<   |
                    |           i|
s0t          
                    d           |                     |d          S |
                    d d           |
S )a  Retrieve a list of search engine instances for a given category and backend.

        Args:
            category: The category of search engines (e.g., 'text', 'images', etc.).
            backend: A single or comma-delimited backends. Defaults to "auto".

        Returns:
            A list of initialized search engine instances corresponding to the specified
            category and backend. Instances are cached for reuse.

        ,c                 6    g | ]}|                                 S r5   )strip).0xs     r.   
<listcomp>z%DDGS._get_engines.<locals>.<listcomp>2  s     >>>a		>>>r6   autoallru   	wikipedia
grokipediac                     g | ]}|d v|	S ))r   r   r5   )r   ks     r.   r   z%DDGS._get_engines.<locals>.<listcomp>8  s#    5n5n5nARmImImaImImImr6   z9%s - backends do not exist or are disabled. Available: %sz, rS   r   rP   z backend is not set. Using 'auto'c                     | j         t          fS r?   )priorityr   )es    r.   <lambda>z#DDGS._get_engines.<locals>.<lambda>\  s    aj&%9 r6   T)keyreverse)
isinstancelistr+   splitr   keysr   rX   appendr!   r   sortedr^   rY   rZ   r[   _get_enginessort)r`   r   r   backend_listengine_keysr   engine_classesinvalid_keysr   engine_class	instancesengine_instances               r.   r   zDDGS._get_engines   sK     gt$$ 	(hhw''G>>7==+=+=>>>78,113344\!!Ul%:%:D6!!#\25n5n5n5n5nnD 	) 	)C&x044S999| )%%l3333##C(((( 	NNK		&..//		&--..   	* 	2 	2Lt222  !4\!BCCCC #/,T[$-`d`l"m"m"m4C#L1  1111 	7NN=>>>$$Xv666 	994HHHr6   zus-enmoderate
   r   r   )region
safesearch	timelimitmax_resultspager   keywordsr   r   r   r   r   kwargsc          
        ! |p|}|sd}t          |          |                                 }|r	 t                      }t          j        |                    ||          |          }|                    d          }|rt                              d|           |S n2# t          $ r%}t                              d|           Y d}~nd}~ww xY w| 
                    ||	          }t          d |D                       }t                      }t          h d          }|r#t          |t          |d	z            d
z             n|}t           j        rt          |t           j                  }i dc!}t%          |d          5 }t'          |d
          D ]%\  }}|j        |v r |j        |j        |f||||d|
}|!|<   t          !          |k    s||k    rt/          !| j        d          \  }}!                                D ]\  }}||v r	 |                                x}r/|                    |           |                    |j                   P# t          $ r-}|}t                              d|j        |           Y d}~d}~ww xY w!fd|D             !|rt          |          |k    r n'ddd           n# 1 swxY w Y   |                                }t?                      } |                      ||          }|r'|r| !                    |||           |r
|d|         n|S d| v rtE          |          t          |pd          )a  Perform a search across engines in the given category.

        Args:
            category: The category of search engines (e.g., 'text', 'images', etc.).
            query: The search query.
            keywords: Deprecated alias for `query`.
            region: The region to use for the search (e.g., us-en, uk-en, ru-ru, etc.).
            safesearch: The safesearch setting (e.g., on, moderate, off).
            timelimit: The timelimit for the search (e.g., d, w, m, y) or custom date range.
            max_results: The maximum number of results to return. Defaults to 10.
            page: The page of results to return. Defaults to 1.
            backend: A single or comma-delimited backends. Defaults to "auto".
            **kwargs: Additional keyword arguments to pass to the search engines.

        Returns:
            A list of dictionaries containing the search results.

        zquery is mandatory.g      ?r   zCache hit: %szCache check failed: %rNc                     h | ]	}|j         
S r5   )provider)r   engines     r.   	<setcomp>z$DDGS._search_sync.<locals>.<setcomp>  s    #J#J#JFO#J#J#Jr6   >   urlhrefimage	embed_urlr   r   r   r2   )rK   )r   r   r   r   FIRST_EXCEPTION)r   return_whenzError in engine %s: %rc                 "    i | ]}||         S r5   r5   )r   ffuturess     r.   
<dictcomp>z%DDGS._search_sync.<locals>.<dictcomp>  s    ???q'!*???r6   z	timed outzNo results found.)#r   r   rL   r@   r   
get_cachedr   r!   r&   r%   r   lensetr   r   r   r   rN   r   	enumerater   r   searchr   rZ   itemsextendaddr"   nameextract_dictsr   rankr   r   )"r`   r   r   r   r   r   r   r   r   r   r   msgr   r   r   cachedr-   engineslen_unique_providersseen_providersresults_aggregatorr3   errr   ir   donenot_doner   f_enginerr   rankerr   s"                                    @r.   _search_synczDDGS._search_sync_  sU   @ !E 	%'C$$$**,, 
	;	;&(( 9':L:LUT\:]:]_cdds33 "LL%888!M"  ; ; ;5r::::::::; ##Hg66"#J#J'#J#J#JKK#&55 ;LLqLqLq:r:rOZtc.[25E0F0F0JKKK`t< 	9k4<88K4KFSSS 	W_&wa888  	6?n44(M ")'    #)w<<;..!{2B2B%)'4=Vg%h%h%hND('.}} Y Y899Y()

?1 !J$6$=$=a$@$@$@$2$6$6x7H$I$I$I#, Y Y Y&( &,DhmUW X X X X X X X XY % @???h???G 3'9#:#:k#I#IE;	 	 	 	 	 	 	 	 	 	 	 	 	 	 	> %2244#%%++gu-- 	E D))%(CCC,7D7<K<((WDS("""3'''C6#6777sV   A+B 
C'CC BKAIK
J	!#J		K	J	+KK	K	c                       | j         d|fi |S )zPerform a text search.ru   r   r`   r   r   s      r.   ru   z	DDGS.text       t 99&999r6   c                       | j         d|fi |S )zPerform an image search.imagesr   r   s      r.   r   zDDGS.images       t 5;;F;;;r6   c                       | j         d|fi |S )zPerform a news search.newsr   r   s      r.   r   z	DDGS.news  r   r6   c                       | j         d|fi |S )zPerform a video search.videosr   r   s      r.   r   zDDGS.videos  r   r6   c                       | j         d|fi |S )zPerform a book search.booksr   r   s      r.   r   z
DDGS.books  s     t %::6:::r6   text_markdownr   fmtc                 :   t          | j        | j        | j                  }|                    |          }|j        dk    rd| d|j         }t          |          |j        |j        |j	        |j
        |j        d}||                    ||j                  dS )a9  Fetch a URL and extract its content.

        Args:
            url: The URL to fetch and extract content from.
            fmt: Output format: "text_markdown", "text_plain", "text_rich", "text" (raw HTML), "content" (raw bytes).

        Returns:
            A dictionary with 'url' and 'content' keys.

        r   rk   zFailed to fetch z: HTTP )r   
text_plain	text_richru   content)r   r  )r   rY   rZ   r[   rX   rw   r   r   r  r  ru   r  )r`   r   r   clientr   r   content_maps          r.   extractzDDGS.extract  s     $+t}T\ZZZzz#s""CSCC1ACCC$$$ "//I|/
 /
 {sD<N'O'OPPPr6   )Nr9   )r   r   )NNNrF   r?   )r   )$__name__
__module____qualname____doc__rN   r   int__annotations__rO   r
   r   r   r   rz   boolra   rd   typeBaseExceptionr	   ri   r_   r   r   dictr   r   r   r   ru   r   r   r   r   bytesr  r5   r6   r.   r   r   y   sF         * %)GXcDj!(((%)OXc])));?L(:+C0478??? !+
 ""+ + +Tz+ t+
 s
+ t+ + 
+ + + +,    04(,'+	( (}%,( %( $	(
 
( ( ( (IX IX IX IXV$S $ $ $ $'' d38n%' 	'
 
' ' ' '.== = 
s#	$	= = = =F  $	g8 $ $"$g8 g8 g8g8 g8 *	g8 g8 g8 :g8 4Zg8 g8 g8 g8 
d38n	g8 g8 g8 g8R:# : :d38n1E : : : :<C <3 <4S#X3G < < < <:# : :d38n1E : : : :<C <3 <4S#X3G < < < <;3 ;# ;$tCH~2F ; ; ; ;Q Q3 QS QtCuDT?U Q Q Q Q Q Qr6   r   rF   )=r  r@   atexitloggingrV   r   r{   rI   r   concurrent.futuresr   r   mathr   pathlibr   r   r   typesr	   typingr
   r   r:   baser   r   r   
exceptionsr   r   http_clientr   r   r   
similarityr   utilsr   	getLoggerr  r!   rv   r   r   r   r;   r  r   LockrG   r   AbstractEventLoopr   rJ   r/   registerr7   r<   rL   r   r5   r6   r.   <module>r#     s            				     



      7 7 7 7 7 7 7 7             " " " " " " " "                        " " " " " "       7 7 7 7 7 7 7 7 # # # # # # & & & & & & * * * * * * ) ) ) ) ) )		8	$	$)   %)elT! ( ( (-1#d* 1 1 1	  04W&- 4 4 4)-y$& - - -&# &# &# &#T $ % % %/    %,    2    "{Q {Q {Q {Q {Q {Q {Q {Q {Q {Qr6   