
    6j                        d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlmZ dZdZdd	Z G d
 d          ZddZdS )zASlow request diagnostics for latency-sensitive browser API paths.    )annotationsN)Anyg      @(   returnfloatc                     t          j        dd                                          } | st          S 	 t	          |           }n# t
          $ r
 t          cY S w xY wt          d|          S )N!HERMES_WEBUI_SLOW_REQUEST_SECONDS         )osgetenvstripDEFAULT_SLOW_REQUEST_SECONDSr   
ValueErrormax)rawvalues     -/root/hermes-webui/api/request_diagnostics.py_slow_request_secondsr      su    
)7
<
<
B
B
D
DC ,++,c

 , , ,++++,sE??s   A AAc                  b    e Zd ZdZddddddZedddd            ZddZddZddZ	ddZ
dS ) RequestDiagnosticszDTrack request stages and emit a watchdog record if a request wedges.NT)loggertimeout_seconds
auto_startmethodstrpathr   logging.Logger | Noner   float | Noner   boolr   Nonec               "   t          j                    j        d d         | _        t	          |pd          | _        t	          |pd                              dd          d         | _        |pt          j	        t                    | _        |t                      nt          dt          |                    | _        t!          j                    | _        t!          j                    | _        t)          j                    | _        g | _        d| _        | j        | _        d| _        d| _        d | _        |rV| j        dk    rMt)          j        | j        | j                  | _        d	| j        _        | j                                          d S d S d S )
N
   -?   r   r   startFT)!uuiduuid4hex
request_idr   r   splitr   logging	getLogger__name__r   r   r   r   r   time	monotonicstarted_monotonicstarted_wall	threadingLock_lock_stages_current_stage_current_stage_started	_finished_watchdog_logged_timerTimer_on_timeoutdaemonr'   )selfr   r   r   r   r   s         r   __init__zRequestDiagnostics.__init__"   s]    *,,*3B3/&-C(($$**32215	; 1( ; ;:I:Q4666WZ[^`efu`v`vWwWw!%!1!1 IKK^%%
-/%&*&<# %.2 	 $.22#/$*>@PQQDK!%DKK	  	 22    r   'RequestDiagnostics | None'c                   t          |pd                              dd          d         }|                                |fdvrd S  | |||          S )Nr
   r%   r&   r   >   GETz/api/sessionsPOSTz/api/chat/startrC   )r   r,   upper)clsr   r   r   
clean_paths        r   maybe_startzRequestDiagnostics.maybe_start>   se     __**32215
LLNNJ' 0
 
 
 4s6:f5555rB   namec                r   t          j                    }t          |pd                                          pd}| j        5  | j        r	 d d d            d S | j                            | j        t          || j
        z
  dz  d          d           || _        || _
        d d d            d S # 1 swxY w Y   d S )Nunknown  r&   rN   ms)r0   r1   r   r   r6   r:   r7   appendr8   roundr9   )r@   rN   nowcleans       r   stagezRequestDiagnostics.stageN   s+   nD%I&&,,..;)Z 
	. 
	.~ 
	. 
	. 
	. 
	. 
	. 
	. 
	. 
	. L /t'B!Bd JANN    #(D*-D'
	. 
	. 
	. 
	. 
	. 
	. 
	. 
	. 
	. 
	. 
	. 
	. 
	. 
	. 
	. 
	. 
	. 
	.s    	B,A	B,,B03B0c                   d }d }| j         5  | j        r	 d d d            d S d| _        | j        }|                     d          }d d d            n# 1 swxY w Y   ||                                 |rP| j        dk    rG|d         | j        dz  k    r5| j                            dt          j	        |d                     d S d S d S d S )	NTFinclude_stacksr   
elapsed_msrQ   z Slow WebUI request completed: %s	sort_keys)
r6   r:   r<   _build_record_lockedcancelr   r   warningjsondumps)r@   timerrecords      r   finishzRequestDiagnostics.finish]   sn   Z 	E 	E~ 	E 	E 	E 	E 	E 	E 	E 	E "DNKE..e.DDF	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E LLNNN 	d*Q..6,3G4K_bfKf3f3fK2
6T222    	 	..3f3fs   	A$AAAc                   | j         5  | j        s| j        r	 d d d            d S d| _        |                     d          }d d d            n# 1 swxY w Y   | j                            dt          j        |d                     d S )NTrZ   z$Slow WebUI request still running: %sr]   )r6   r:   r;   r_   r   ra   rb   rc   )r@   re   s     r   r>   zRequestDiagnostics._on_timeoutn   s   Z 	D 	D~ !6 	D 	D 	D 	D 	D 	D 	D 	D %)D!..d.CCF		D 	D 	D 	D 	D 	D 	D 	D 	D 	D 	D 	D 	D 	D 	D
 	2Jv...	
 	
 	
 	
 	
s   AAAAr[   dict[str, Any]c          	        t          j                    }t          | j                  }|                    | j        t          || j        z
  dz  d          d           | j        | j	        | j
        t          j        dt          j        | j                            t          || j        z
  dz  d          | j        |d}|rt                      |d<   |S )NrQ   r&   rR   z%Y-%m-%dT%H:%M:%SZ)r+   r   r   
started_atr\   current_stagestagesthread_stacks)r0   r1   listr7   rT   r8   rU   r9   r+   r   r   strftimegmtimer3   r2   _thread_stack_snapshot)r@   r[   rV   rl   re   s        r   r_   z'RequestDiagnostics._build_record_lockedy   s    ndl##+S4#>>$FJJ 	
 	
 	
 /kI-(<dk$J[>\>\]]t'=!= EqII!0"
 "
  	?&<&>&>F?#rB   )r   r   r   r   r   r   r   r   r   r    r   r!   )r   r   r   r   r   r   r   rD   )rN   r   r   r!   )r   r!   )r[   r    r   rh   )r/   
__module____qualname____doc__rA   classmethodrM   rX   rf   r>   r_    rB   r   r   r      s        NN )-(,           8  )-6 6 6 6 6 [6. . . .   "	
 	
 	
 	
     rB   r   list[dict[str, Any]]c                    t          j                    } d t          j                    D             }g }|                                 D ]x\  }}|                    |          }t          j        |t                    }|	                    ||r|j
        nd|rt          |j                  nd d |D             d           y|                    d            |S )Nc                    i | ]
}|j         |S rv   )ident).0threads     r   
<dictcomp>z*_thread_stack_snapshot.<locals>.<dictcomp>   s    HHHv|VHHHrB   )limitr
   c                6    g | ]}|                                 S rv   )rstrip)r{   lines     r   
<listcomp>z*_thread_stack_snapshot.<locals>.<listcomp>   s     :::D$++--:::rB   )	thread_idthread_namer?   stackc                J    t          |                     d          pd          S )Nr   r
   )r   get)items    r   <lambda>z(_thread_stack_snapshot.<locals>.<lambda>   s    3txx'>'>'D"#E#E rB   )key)sys_current_framesr4   	enumerateitemsr   	tracebackformat_stackMAX_STACK_FRAMES_PER_THREADrT   rN   r    r?   sort)framesthreadssnapshotrz   framer|   r   s          r   rq   rq      s     ""FHH)2E2G2GHHHG%'H 

 

uU##&u4OPPP".4<v{{"17A$v}---T::E:::	 	
 	
 	
 	
 MMEEMFFFOrB   )r   r   )r   rw   )rt   
__future__r   rb   r-   r   r   r4   r0   r   r(   typingr   r   r   r   r   rq   rv   rB   r   <module>r      s    G G " " " " " "   				 



                  #      n n n n n n n nb     rB   