Fi2KddlZddlZddlZddlZddlZddlmZddlmZddlm Z m Z m Z ddl m Z GddZGdd Zed k(red ed eZ ej'Zed dededddededddededdddededdddededddd ed!ed"d#d$ded%ed"d&d$ded'rLed(ed'j+D]-\ZZed)ed*ed+d,d-ed.dd/ed0d1d2ed3d,d4 /ned5ej0d6y#e$r ed7YywxYw)8N)datetime)Path)LOGGERMACOSRANK)check_requirementsczeZdZdZddZdZdZdZdZdZ d Z Gd d Z Gd d e jZy) ConsoleLoggera Console output capture with batched streaming to file, API, or custom callback. Captures stdout/stderr output and streams it with intelligent deduplication and configurable batching. Attributes: destination (str | Path | None): Target destination for streaming (URL, Path, or None for callback-only). batch_size (int): Number of lines to batch before flushing (default: 1 for immediate). flush_interval (float): Seconds between automatic flushes (default: 5.0). on_flush (callable | None): Optional callback function called with batched content on flush. active (bool): Whether console capture is currently active. Examples: File logging (immediate): >>> logger = ConsoleLogger("training.log") >>> logger.start_capture() >>> print("This will be logged") >>> logger.stop_capture() API streaming with batching: >>> logger = ConsoleLogger("https://api.example.com/logs", batch_size=10) >>> logger.start_capture() Custom callback with batching: >>> def my_handler(content, line_count, chunk_id): ... print(f"Received {line_count} lines") >>> logger = ConsoleLogger(on_flush=my_handler, batch_size=5) >>> logger.start_capture() Nc||_t|txr|jd|_||jst ||_t d||_||_||_ tj|_ tj|_d|_d|_g|_t%j&|_d|_d|_d|_d|_d|_d|_y)aInitialize console logger with optional batching. Args: destination (str | Path | None): API endpoint URL (http/https), local file path, or None. batch_size (int): Lines to accumulate before flush (1 = immediate, higher = batched). flush_interval (float): Max seconds between flushes when batching. on_flush (callable | None): Callback(content: str, line_count: int, chunk_id: int) for custom handling. )zhttp://zhttps://NFrg) destination isinstancestr startswithis_apirmax batch_sizeflush_intervalon_flushsysstdoutoriginal_stdoutstderroriginal_stderractive _log_handlerbuffer threadingLock buffer_lock flush_threadchunk_id last_line last_timelast_progress_linelast_was_progress)selfrrrrs Z/var/www/pod-logistic/pod-ai/venv/lib/python3.12/site-packages/ultralytics/utils/logger.py__init__zConsoleLogger.__init__-s' c2f{7M7MNe7f  "4;;#K0D a,,   #zz"zz   $>>+  "$!&cT|jstdvryd|_|j|j|jt _|j|j|jt _ |j|j|_ tjdj|j|jdkDrAt!j"|j$d|_|j&j)yy#t$rY\wxYw)zStart capturing console output and redirect stdout/stderr. Notes: In DDP training, only activates on rank 0/-1 to prevent duplicate logging. >rNT ultralyticsr )targetdaemon)rr_ConsoleCapturer _queue_logrrrr _LogHandlerrlogging getLogger addHandler ExceptionrrThread _flush_workerr"startr(s r) start_capturezConsoleLogger.start_captureRs ;;$g-  ))$*>*>P ))$*>*>P   $ 0 0 AD    m , 7 78I8I J ??Q  ) 0 08J8JSW XD     # # %     s>t?P?PQ!%D     s .B BBcX|jsytj}d|vr|jdd}|jd}|r|ddk(r|j|D]}|j }d|vrd|vrd|v}|s$|j}d}|r`d |d vr)|d j d dj r|d }n0|d d k(rt|d kDr|d d |d }n |d dvr|d }|r|j|dk(r|r |d|_d|_ n|s|jrd|_ d|_ ||jk(r||jz dkr||_ ||_ |jds+tjjd}d|d|}d} |j 5|j"j%|t|j"|j&k\rd} ddd| s|j)y#1swY!xYw)z?Queue console text with deduplication and timestamp processing.N r- r u─u ━━z100%/rClassr _)ztrain:zval:z:doneTF皙?z[20z%Y-%m-%d %H:%M:%S[z] )rtimesplitpoprstripreplaceisdigitlenr&r'r$r%rrnowstrftimer!rappendrr?) r(text current_timelinesline is_completepartsseq_key timestamp should_flushs r)r2zConsoleLogger._queue_logsE{{ yy{  4<::d#B'D 4  U2Y"_ IIK@ %D;;=D}D $n # eAh58+;+;C+D+L+L+N"'(qW,Ua%*1XJaaz":q%77"'(t66WIU:KK18 .?D+)-& 6 6-2D*).&t~~%,*G#*M!DN)DN??5)$LLN334GH 9+Rv.!L!! ( ""4(t{{#t6#'L ( ""$A@ %r ( (s AH  H) c|jrItj|j|jr|j |jrHyy)z3Background worker that flushes buffer periodically.N)rrIsleeprr?r;s r)r9zConsoleLogger._flush_workers7kk JJt** +{{""$kkr+c|j5|js dddy|jj}|jj|xjdz c_|j}ddddj }t |}|jr |j|||j|j|yy#1swYdxYw#t$rY6wxYw)z4Flush buffered lines to destination and/or callback.Nr rC) r!rcopyclearr#joinrOrr7r_write_destination)r(rUr#content line_counts r)r?zConsoleLogger._flush_buffers    %;; % %KK$$&E KK    MMQ M}}H  %))E"Z  ==  gz8<    '  # #G , (' % %  s#CAC*C(C%( C43C4c |jrQddl}tjj |d}|j t |j|dy|jjjdd|jjdd 5}|j|d zdddy#1swYyxYw#t$r$}td ||j Yd}~yd}~wwxYw)z)Write content to file or API destination.rN)rZmessage)jsontimeoutT)parentsexist_okazutf-8)encodingrCzConsole logger write error: )file)rrequestsrrP isoformatpostrrparentmkdiropenwriter7printr)r(rcropayloadfes r)rbz ConsoleLogger._write_destinations Q{{(0 (@(@(BwW c$"2"23'1 M  ''--dT-J%%**3*A,QGGGdN+,,, Q 044;O;O P P Qs=AC AC #C8C C C C C:C55C:c,eZdZdZdZdZdZdZdZy)ConsoleLogger._ConsoleCapturez"Lightweight stdout/stderr capture.)callbackoriginalc ||_||_y)z^Initialize a stream wrapper that redirects writes to a callback while preserving the original.N)r}r|)r(r}r|s r)r*z&ConsoleLogger._ConsoleCapture.__init__s$DM$DMr+c\|jj||j|y)zIWrite text to the original stream and forward it to the capture callback.N)r}rur|)r(rSs r)ruz#ConsoleLogger._ConsoleCapture.writes MM   % MM$ r+c8|jjy)zVFlush the wrapped stream to propagate buffered output promptly during console capture.N)r}flushr;s r)rz#ConsoleLogger._ConsoleCapture.flush s MM   !r+c6|jjS)z-Delegate isatty check to the original stream.)r}isattyr;s r)rz$ConsoleLogger._ConsoleCapture.isattys=='') )r+N) __name__ __module__ __qualname____doc__ __slots__r*rurrr+r)r1r{s0,  %   " *r+r1c,eZdZdZdZfdZdZxZS)ConsoleLogger._LogHandlerzLightweight logging handler.)r|c0t|||_y)z\Initialize a lightweight logging.Handler that forwards log records to the provided callback.N)superr*r|)r(r| __class__s r)r*z"ConsoleLogger._LogHandler.__init__s G  $DMr+cJ|j|j|dzy)zXFormat and forward LogRecord messages to the capture callback for unified log streaming.rCN)r|format)r(records r)emitzConsoleLogger._LogHandler.emits MM$++f-4 5r+)rrrrrr*r __classcell__)rs@r)r3rs*!  %  6r+r3)Nr g@N)rrrrr*r<r@r2r9r?rbr1r4Handlerr3rr+r)r r sH:#'J&2(O%b%-0 Q**. 6goo 6r+r c*eZdZdZdZdZddZdZy) SystemLoggeruLog dynamic system metrics for training monitoring. Captures real-time system metrics including CPU, RAM, disk I/O, network I/O, and NVIDIA GPU statistics for training performance monitoring and analysis. Attributes: pynvml: NVIDIA pynvml module instance if successfully imported, None otherwise. nvidia_initialized (bool): Whether NVIDIA GPU monitoring is available and initialized. net_start: Initial network I/O counters for calculating cumulative usage. disk_start: Initial disk I/O counters for calculating cumulative usage. Examples: Basic usage: >>> logger = SystemLogger() >>> metrics = logger.get_metrics() >>> print(f"CPU: {metrics['cpu']}%, RAM: {metrics['ram']}%") >>> if metrics["gpus"]: ... gpu0 = metrics["gpus"]["0"] ... print(f"GPU0: {gpu0['usage']}% usage, {gpu0['temp']}°C") Training loop integration: >>> system_logger = SystemLogger() >>> for epoch in range(epochs): ... # Training code here ... metrics = system_logger.get_metrics() ... # Log to database/file cddl}d|_|j|_|j |_|j |_|j |_|j|_ tj|_ y)zInitialize the system logger.rN) psutilpynvml _init_nvidianvidia_initializednet_io_counters net_startdisk_io_counters disk_start _prev_net _prev_diskrI _prev_time)r(rs r)r*zSystemLogger.__init__Asb "&"3"3"5//1 113//))+r+ctry tdtd|_|jj y#t $r@}ddl}|jjrtjd|Yd}~yd}~wwxYw)z-Initialize NVIDIA GPU monitoring with pynvml.Fznvidia-ml-py>=12.0.0rTrNzSystemLogger NVML init failed: ) rr __import__rnvmlInitr7torchcuda is_availablerwarning)r(ryrs r)rzSystemLogger._init_nvidiaOso   5 6$X.DK KK "  zz&&(!@DE  s5? B6BBcddl}|j}|j}|j}t j d}t j }t|jdt|jdid}td||jz } |rttd|j|jjz dz | z dttd|j|jjz dz | z dt|jdz dd |d <ttd|j |j"j z dz | z dttd|j$|j"j$z dz | z dd |d <nt|j|j&jz dz dt|j|j&jz dz dt|jdz dd |d <t|j |j(j z dz dt|j$|j(j$z dz dd|d <||_||_ ||_ |j*r"|dj-|j/|S)a Get current system metrics including CPU, RAM, disk, network, and GPU usage. Collects comprehensive system metrics including CPU usage, RAM usage, disk I/O statistics, network I/O statistics, and GPU metrics (if available). Example output (rates=False, default): ```python { "cpu": 45.2, "ram": 78.9, "disk": {"read_mb": 156.7, "write_mb": 89.3, "used_gb": 256.8}, "network": {"recv_mb": 157.2, "sent_mb": 89.1}, "gpus": { "0": {"usage": 95.6, "memory": 85.4, "temp": 72, "power": 285}, "1": {"usage": 94.1, "memory": 82.7, "temp": 70, "power": 278}, }, } ``` Example output (rates=True): ```python { "cpu": 45.2, "ram": 78.9, "disk": {"read_mbs": 12.5, "write_mbs": 8.3, "used_gb": 256.8}, "network": {"recv_mbs": 5.2, "sent_mbs": 1.1}, "gpus": { "0": {"usage": 95.6, "memory": 85.4, "temp": 72, "power": 285}, }, } ``` Args: rates (bool): If True, return disk/network as MB/s rates instead of cumulative MB. Returns: (dict): Metrics dictionary with cpu, ram, disk, network, and gpus keys. Examples: >>> logger = SystemLogger() >>> logger.get_metrics()["cpu"] # CPU percentage >>> logger.get_metrics(rates=True)["network"]["recv_mbs"] # MB/s download rate rNrD)cpuramgpusrGii@)read_mbs write_mbsused_gbdisk)recv_mbssent_mbsnetwork)read_mbwrite_mbr)recv_mbsent_mbr)rrrvirtual_memoryshutil disk_usagerIround cpu_percentpercentrr read_bytesr write_bytesused bytes_recvr bytes_sentrrrupdate_get_nvidia_metrics) r(ratesrnetrmemoryrrPmetricselapseds r) get_metricszSystemLogger.get_metrics`sX $$&&&(&&(&&s+ iik++-q1+ c301 "#a$//DOOY>Y+Y^e*fip*p#qstu G!>;T;T*TY`)adk)k"lnop!#a#..4>>;T;T*TY`)adk)k"lnop"GI !$//DOO4N4N"NSZ![]^_!4#3#3doo6Q6Q#QV]"^`ab G!>3L3L"LQX!Y[\] #..4>>3L3L"LQX!Y[\]"GI    " " FO " "4#;#;#= >r+ci}|jr |js|S |jj}t|D]}|jj |}|jj |}|jj |}|jj||jj}|jj|dz}t|jdt|j|jz dzd||d|t|< |S#t$rY|SwxYw)zMGet NVIDIA GPU metrics including utilization, memory, temperature, and power.ird)usagertemppower)rrnvmlDeviceGetCountrangenvmlDeviceGetHandleByIndexnvmlDeviceGetUtilizationRatesnvmlDeviceGetMemoryInfonvmlDeviceGetTemperatureNVML_TEMPERATURE_GPUnvmlDeviceGetPowerUsagergpurtotalrr7) r(r device_countihandleutilrrrs r)rz SystemLogger._get_nvidia_metricss&&dkkK ;;99;L<( ??B{{@@H<rs  117R6R6jqqh z 23 "# ^F )((*G .b ) E'%..a0 1 E'%..a0 1 K :4@D E L!