
    Nܜi                        d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	m	Z	m
Z
mZ  ej                  dd       	 ddlZd	Zd
ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZ dZ!dZ"dZ#dZ$ G d d      Z% G d d      Z&d4d Z'd4d!Z(d4d"Z)d#a*d5d$Z+d% Z,d& Z-d' Z.d( Z/d) Z0d* Z1d+ Z2d, Z3d- Z4d. Z5d/ Z6d0 Z7d1 Z8d2 Z9e:d3k(  r e9        yy# e$ r  ed        ej                   d       Y w xY w)6u  
AEI System Operational Audit — Automated Health Check

53 checks across remote (AWS), local (WSL2), and aei-webserv2 servers.
Read-only — no uploads, no writes, no file modifications.

Phases:
  1  Remote Services (1-5)        — httpd, mysqld, crond, postfix, python3.6
  2  Remote Cron Health (6-11)    — scheduler cron, retry queue, ticketcron, PDF cleanup
  3  Remote Disk & Storage (12-15)— disk free, log sizes, uploads size
  4  Remote Security (16-19)      — known exposures, SSL, PHP extensions
  5  Remote Photo API (20-25)     — upload.php, getimagelisting, fetch_image, delete endpoints
  6  Remote Database (26-28)      — connection, meter_files, jobs table
  7  Local Services (29-33)       — apache, mariadb, fail2ban, cron, /mnt/dropbox
  8  Local Firewall (34-38)       — ipset, fail2ban, jail count, HTTP, SSL
  9  Local Cron Health (39-42)    — DB sync, process_queue, backups, disk
  10 Cross-System (43-44)         — scheduler login, photo API
  11 aei-webserv2 Health (45-51)  — apache, mysql, endpoints, DB, dropbox mount

Usage:
  python3 audit/system_audit.py                 # Full audit (both servers)
  python3 audit/system_audit.py --remote-only   # Remote only
  python3 audit/system_audit.py --local-only    # Local only
  python3 audit/system_audit.py --json          # JSON output
  python3 audit/system_audit.py --warn-only     # Only WARN + FAIL

Prerequisites:
  - pip3 install requests paramiko
  - SSH key at /root/.ssh/aei_production.pem
  - Network access to aeihawaii.com and upload.aeihawaii.com
    N)datetime	timedeltatimezoneignorezUnverified HTTPS request)messagezERROR: pip3 install requests   z/root/.ssh/aei_production.pemJulianz18.225.0.90	schedularzM1gif9!6mandhdesign_schedularzhttps://aeihawaii.com/photoapiz)https://aeihawaii.com/scheduler/login.phpzhttps://upload.aeihawaii.comz//var/www/vhosts/aeihawaii.com/httpdocs/photoapiz0/var/www/vhosts/aeihawaii.com/httpdocs/schedulerzaei@89806849aeiuserz192.168.141.21955222z/var/www/html/uploadupload_userzP@55w02d778899	Schedularc                   $    e Zd ZdZdZdZdZdZdZy)Colorsz[92mz[91mz[93mz[94mz[0mz[1mN)	__name__
__module____qualname__PASSFAILWARNINFORESETBOLD     ./var/www/html/AEI_REMOTE/audit/system_audit.pyr   r   P   s     DDDDEDr   r   c                       e Zd ZdZddZd Zy)ResultzSingle check result.c                 J    || _         || _        || _        || _        || _        y )Nstepnamestatusdetailphase)selfr"   r#   r$   r%   r&   s         r   __init__zResult.__init__Z   s%    		
r   c                 v    | j                   | j                  | j                  | j                  | j                  dS )Nr!   r!   )r'   s    r   to_dictzResult.to_dicta   s/    IIIIkkkkZZ
 	
r   N) r+   )r   r   r   __doc__r(   r*   r   r   r   r   r   X   s    
r   r   c                 \   	 t        j                  ddt        ddddddt         dt         | gdd|	      }|j
                  j                         |j                  j                         |j                  fS # t         j                  $ r Y y
t        $ r}dt        |      dfcY d}~S d}~ww xY w)z%Run command on remote server via SSH.sshz-i-oConnectTimeout=10StrictHostKeyChecking=noBatchMode=yes@Tcapture_outputtexttimeoutr+   zSSH command timed outr   r+   r   N)
subprocessrunSSH_KEYSSH_USERSSH_HOSTstdoutstripstderr
returncodeTimeoutExpired	Exceptionstrcmdr7   resultes       r   ssh_cmdrI   j   s    D'4)<-t_z8*%s,  dG	
 }}""$fmm&9&9&;V=N=NNN$$ .- 3q61}$   A3A6 6B+B+B& B+&B+c                 &   	 t        j                  | ddd|      }|j                  j                         |j                  j                         |j
                  fS # t         j                  $ r Y yt        $ r}dt        |      dfcY d}~S d}~ww xY w)zRun command locally.T)shellr5   r6   r7   )r+   zCommand timed outr   r+   r   N)	r9   r:   r>   r?   r@   rA   rB   rC   rD   rE   s       r   	local_cmdrM   y   s    tDtW
 }}""$fmm&9&9&;V=N=NNN$$ *) 3q61}s$   AA B0B8BBBc                 \   	 t        j                  ddt        ddddddt         dt         | gdd|	      }|j
                  j                         |j                  j                         |j                  fS # t         j                  $ r Y y
t        $ r}dt        |      dfcY d}~S d}~ww xY w)z6Run command on aei-webserv2 (192.168.141.219) via SSH.r.   z-pr/   r0   r1   r2   r3   Tr4   r8   r+   r   N)r9   r:   LOCAL_SSH_PORTLOCAL_SSH_USERLOCAL_SSH_HOSTr>   r?   r@   rA   rB   rC   rD   rE   s       r   local_server_ssh_cmdrR      s    D.&-t_q 0138  dG
 }}""$fmm&9&9&;V=N=NNN$$ .- 3q61}rJ   Fc                    t         ry|r| j                  dk(  ryt        j                  t        j                  t        j
                  t        j                  dj                  | j                  t        j                        }| j                  rd| j                   nd}t        d| | j                  dt        j                   d| j                  d	d
| j                   | 	       y)zPrint a formatted result line.Nr   )r   r   r   SKIP    — r+   z  [4sz] >2. )_quiet_moder$   r   r   r   r   r   getr   r%   printr"   r#   )r	warn_onlycolor
detail_strs       r   print_resultr`      s    QXX'V[[V[[ 
c!((FLL! 
 ()xx5
#RJ	CwqxxmFLL>AFF2;b
UVr   c                 n    t         ryt        dt        j                   |  t        j                          y)zPrint phase header.N
)rY   r[   r   r   r   )r#   s    r   print_phaserc      s'    	Bv{{mD6&,,
01r   c                 `   | j                         } | syt        j                  d| t        j                        }|rWt	        |j                  d            }|j                  d      j                         }ddddddd	}||j                  |d      z  S 	 t	        |       dz  S # t        $ r Y yw xY w)
z3Parse du/df output to MB. Handles K, M, G suffixes.r   z^([\d.]+)([KMGTP]?)$r      gMbP?   i   i   @)r+   KMGTP)	r?   rematch
IGNORECASEfloatgroupupperrZ   
ValueError)size_strmvalunit
multipliers        r   parse_size_mbrx      s    ~~H
((BMMBAAGGAJwwqz!%adzZ
Z^^D!,,,X%% s   B! !	B-,B-c           
         t        d       d}t        |d      \  }}}|dk7  r9d|v r5t        g dd      D ]#  \  }}| j                  t	        ||d	d
d             % y|j                  d      fd} |d      }|j                         xr t        |      dkD  }	| j                  t	        dd|	rdnd	|	r| dndd              |d      }|j                         xr t        |      dkD  }	| j                  t	        dd|	rdnd	|	r| dndd              |d      }|j                         xr t        |      dkD  }	| j                  t	        dd|	rdnd	|	r| dndd              |d      }|j                         xr t        |      dkD  }	| j                  t	        dd|	rdnd	|	r| dndd              |d      }
d|
v }	| j                  t	        dd |	rdnd	|	r|
nd!d             y)"z,Checks 1-5: Core services running on remote.u!   Phase 1 — Remote Services (1-5)aJ  echo '===HTTPD==='; ps aux | grep httpd | grep -v grep | wc -l; echo '===MYSQL==='; ps aux | grep mysqld | grep -v grep | wc -l; echo '===CROND==='; ps aux | grep crond | grep -v grep | wc -l; echo '===POSTFIX==='; ps aux | grep 'postfix/master' | grep -v grep | wc -l; echo '===PYTHON==='; /usr/local/bin/python3.6 --version 2>&1   r7   r   	timed out)zApache httpdzMySQL mysqldcrondPostfixz
Python 3.6r   r   zSSH connection failedzRemote ServicesN===c                     t              D ]B  \  }}|j                         | k(  s|dz   t              k  s,|dz      j                         c S  yNr   r+   	enumerater?   lenr#   issectionss      r   sectionz&phase_remote_services.<locals>.section   T    h' 	/DAqwwyD QUS]%:A,,..	/ r   HTTPDzApache httpd runningr   
 processesnot runningMYSQLre   zMySQL mysqld runningCROND   zcrond runningPOSTFIX   zPostfix runningPYTHONz3.6   zPython 3.6 availablez	not found)rc   rI   r   appendr   splitisdigitint)resultsrF   outerrrcr   r#   r   countokpyverr   s              @r   phase_remote_servicesr      s   34
		2  3+LCb	Qw;#% !cefg 	`GAtNN6!T63JL]^_	`yyH GE		+SZ!^BNN6!3rVv24UG:.-IZ\ ] GE		+SZ!^BNN6!3rVv24UG:.-IZ\ ] GE		+SZ!^BNN6!_f24UG:.-IZ\ ] IE		+SZ!^BNN6!."&24UG:.-IZ\ ] HE	%BNN6!3rVv#%%;8IK Lr   c                 B
   t        d       d}t        |d      \  }}}|dk7  r7d|v r3t        dd      D ]#  }| j                  t	        |d	| d
dd             % y|j                  d      fd} |d      }|r	 |j                         }|d    d|d    d|d    }	t        j                         }
t        j                  |
j                   d|	 d      }|
|z
  j                         dz  }|dk  r#| j                  t	        ddd|ddd             n"| j                  t	        ddd|ddd             n| j                  t	        ddd
dd              |d      }|r	 |j                         }|d    d|d    d|d    }	t        j                         }
t        j                  |
j                   d|	 d      }|
|z
  j                         dz  }|d k  r#| j                  t	        d!d"d|ddd             n"| j                  t	        d!d"d|dd#d             n| j                  t	        d!d"d
dd              |d%      }|r	 |j                         }|d    d|d    d|d    }	t        j                         }
t        j                  |
j                   d|	 d      }|
|z
  j                         dz  }|dk  r#| j                  t	        d&d'd|ddd             n"| j                  t	        d&d'd|ddd             n| j                  t	        d&d'ddd              |d(      }|r"| j                  t	        d)d*d|dd d             n| j                  t	        d)d*dd+d              |d,      }	 t        |      }|dk(  r| j                  t	        dd-dd.d             n!| j                  t	        dd-d| d/d              |d1      }	 t        |      }|dk  r"| j                  t	        d2d3d| d4d             y| j                  t	        d2d3d| d5d             y# t        t        f$ r( | j                  t	        dddd|dd  d             Y w xY w# t        t        f$ r" | j                  t	        d!d"dd$d             Y Pw xY w# t        t        f$ r" | j                  t	        d&d'dd$d             Y w xY w# t        $ r" | j                  t	        dd-dd0d             Y )w xY w# t        $ r! | j                  t	        d2d3dd6d             Y yw xY w)7z+Checks 6-11: Cron jobs running on schedule.u%   Phase 2 — Remote Cron Health (6-11)a  echo '===SCHEDULER_CRON==='; sudo grep -a 'scheduler/cron' /var/log/cron 2>/dev/null | tail -1; echo '===RETRY_QUEUE==='; sudo grep -a 'process_retry_queue' /var/log/cron 2>/dev/null | tail -1; echo '===TICKETCRON==='; sudo grep -a 'ticketcron' /var/log/cron 2>/dev/null | tail -1; echo '===PDF_CLEANUP==='; crontab -l 2>/dev/null | grep -iE 'pdf|cleanup|tmp' || echo ''; echo '===FAILED_QUEUE==='; find /var/www/vhosts/aeihawaii.com/httpdocs/photoapi/queue/failed/ -type f -mtime +7 2>/dev/null | wc -l; echo '===QUEUE_SIZE==='; ls /var/www/vhosts/aeihawaii.com/httpdocs/photoapi/queue/ 2>/dev/null | grep -v failed | grep -c '.json' 2>/dev/null || echo '0'rz   r{   r   r|         zCron check r   
SSH failedzRemote CronNr   c                     t              D ]B  \  }}|j                         | k(  s|dz   t              k  s,|dz      j                         c S  yr   r   r   s      r   r   z"phase_remote_cron.<locals>.section  r   r   SCHEDULER_CRON r   re   z%Y %b %d %H:%M:%S<   
   zscheduler/cron last ranr   .0fz min agor   z min ago (>10 min)parse error: zno cron log entry foundRETRY_QUEUE      zprocess_retry_queue.py last ranz min ago (>20 min)zparse error
TICKETCRON   zticketcron last ranPDF_CLEANUP	   zPDF cleanup cron existsznot found in Julian's crontabFAILED_QUEUEzNo stale failed queue itemsznone >7 days oldz items >7 days oldcould not check
QUEUE_SIZE   zRetry queue sizez itemsz items (>10)z0 items)rc   rI   ranger   r   r   r   nowstrptimeyeartotal_secondsrr   
IndexErrorr   )r   rF   r   r   r   r   r   	cron_linepartsts_strr   	cron_timeage_min
retry_lineticket_linepdf_cron
failed_oldr   
queue_sizer   s                      @r   phase_remote_cronr      s   78	K  3+LCb	Qw;#%q" 	^ANN6!{1#%6m\]	^yyH ()I	zOO%Eaz58*AeAhZ8F,,.C ))SXXJax*@BUVIY557"<G|va)BFwWZm[cLdfstuva)BFwWZm[mLnp}~ 	va!:FD]_lmn 'J	p$$&Eaz58*AeAhZ8F,,.C ))SXXJax*@BUVIY557"<G|va)JFW^_bVcckTln{|}va)JFW^_bVccuTv  yF   G  H 	va!BFLegtuv ,'K	c%%'Eaz58*AeAhZ8F,,.C ))SXXJax*@BUVIY557"<G|va)>7SV-W_H`bopqva)>7SV-WiHjlyz{ 	va!6@Y[hij }%Hva!:FHSbMS`abva!:FDcerst (JlJA:NN6"&CVM_anopNN6"&CVPUwVhMikxyz
 &JYJB;NN6"&8&UG6BRTabcNN6"&8&UG<BXZghiE J' 	zNN6!%>-XabeceXfWgHhjwxy	z$ J' 	pNN6!%FR]`mno	p$ J' 	cNN6!%:FMS`ab	c(  lvb"?IZ\ijkl  Yvb"4fiWXYsd   =B<P$ #B<Q 	B<R 7AS 1S4 !S4 $3QQ-RR-SS'S10S14'TTc           
         t        d       d}t        |d      \  }}}|dk7  r7d|v r3t        dd      D ]#  }| j                  t	        |d	| d
dd             % y|j                  d      fd} |d      j                  dd      }	 t        |      }|dk\  r#| j                  t	        ddd|ddd             nJ|dk\  r#| j                  t	        ddd|ddd             n"| j                  t	        ddd
|ddd              |d      }	t        |	      }
|
dk  r#| j                  t	        dd d|	xs d!d             n!| j                  t	        dd d|	 d"d              |d#      }| j                  t	        d$d%d|xs d&d              |d'      j                  dd      }	 t        |      }|d(k\  r#| j                  t	        d)d*d|dd+d             y| j                  t	        d)d*d|dd,d             y# t        $ r% | j                  t	        dddd| d             Y "w xY w# t        $ r$ | j                  t	        d)d*dd| d             Y yw xY w)-z1Checks 12-15: Disk space and log sizes on remote.u)   Phase 3 — Remote Disk & Storage (12-15)as  
    echo '===DISK_FREE==='
    df -BG / 2>/dev/null | awk 'NR==2{print $4}'

    echo '===MAILLOG==='
    du -sh /var/log/maillog 2>/dev/null | awk '{print $1}'

    echo '===UPLOADS==='
    du -sh /var/www/vhosts/aeihawaii.com/httpdocs/scheduler/uploads/ 2>/dev/null | awk '{print $1}'

    echo '===TMP_FREE==='
    df -BG /tmp 2>/dev/null | awk 'NR==2{print $4}'
       r{   r   r|   r      zDisk check r   r   zRemote DiskNr   c                     t              D ]B  \  }}|j                         | k(  s|dz   t              k  s,|dz      j                         c S  yr   r   r   s      r   r   z"phase_remote_disk.<locals>.section  r   r   	DISK_FREEri   r+   2   zDisk free > 50GBr   r   zGB freer   r   zGB free (<50GB)zGB free (<20GB)r   MAILLOGrf      zMaillog size < 1GB0z (>=1GB)UPLOADS   zscheduler/uploads/ sizeunknownTMP_FREEr   rz   z/tmp free > 1GBGBz	GB (<1GB))
rc   rI   r   r   r   r   replacero   rr   rx   )r   rF   r   r   r   r   r   	disk_freegbmaillogml_mbuploadstmp_freer   s                @r   phase_remote_diskr   t  s   ;<C 3+LCb	Qw;#%r2 	^ANN6!{1#%6m\]	^yyH $,,S"5I	k98NN6"&8&RHGBTVcde2XNN6"&8&RHOB\^klmNN6"&8&RHOB\^klm
 i G'"Et|vb"63P]^_vb"67)8@TVcde i GNN6"7AUIWdef z"**33Hi8_7NN6"&7Bs82P]^_NN6"&7Bs89AUWdef-  kvb"4fi[>Y[hijk.  ivb"3V}XJ=WYfghis+   A=G= '2H. "H. =*H+*H+.*IIc                 P   t        d       d}t        |d      \  }}}|dk7  r7d|v r3t        dd      D ]#  }| j                  t	        |d| d	d
d             % y|j                  d      fd} |d      }|dk(  r| j                  t	        ddddd             n| j                  t	        ddd	dd              |d      }|dk(  r| j                  t	        ddddd             n| j                  t	        ddd	dd              |d      }	|	rd|	v r	 |	j                  dd      d   }
t        j                  |
d      }|t        j                  t        j                        j                  d      z
  j                  }|dkD  r"| j                  t	        d d!d| d"d             nE|dkD  r"| j                  t	        d d!d#| d$d             n| j                  t	        d d!d	d%d             n| j                  t	        d d!d#d(d              |d)      j                  d*      j                  d*      }h d+}|D ch c]1  }|j!                         s|j!                         j#                         3 }}||z
  }|s7| j                  t	        d,d-dd.j%                  t'        |            d             y| j                  t	        d,d-d	d/d.j%                  t'        |             d             y# t        t        f$ r( | j                  t	        d d!d#d&|	dd'  d             Y w xY wc c}w )0z/Checks 16-19: Known security exposures and SSL.u#   Phase 4 — Remote Security (16-19)a  
    echo '===SLSQL==='
    [ -f /var/www/vhosts/aeihawaii.com/httpdocs/photoapi/sl.sql ] && echo 'EXISTS' || echo 'GONE'

    echo '===MYSQLDBNE==='
    [ -d /var/www/vhosts/aeihawaii.com/httpdocs/mysqldbne ] && echo 'EXISTS' || echo 'GONE'

    echo '===SSL==='
    echo | openssl s_client -servername aeihawaii.com -connect aeihawaii.com:443 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null

    echo '===PHP_EXT==='
    php -m 2>/dev/null | grep -iE '^(mysqli|curl|gd|json|mbstring)$' | sort | tr '\n' ','
    r   r{   r   r|   r   zSecurity check r   r   zRemote SecurityNr   c                     t              D ]B  \  }}|j                         | k(  s|dz   t              k  s,|dz      j                         c S  yr   r   r   s      r   r   z&phase_remote_security.<locals>.section  r   r   SLSQLGONEzphotoapi/sl.sql removedr   znot presentu.   STILL EXISTS — publicly accessible SQL dump!	MYSQLDBNE   zmysqldbne/ removedu#   STILL EXISTS — old DB admin tool!SSL	notAfter==r   %b %d %H:%M:%S %Y %Ztzinfor      zSSL cert > 30 days
 days leftr    days left (<30)EXPIREDr   r   zcould not check SSLPHP_EXT,>   gdcurljsonmysqlimbstring   zPHP extensions present, z	missing: )rc   rI   r   r   r   r   r   r   r   r   utcr   daysrr   r   rstripr?   lowerjoinsorted)r   rF   r   r   r   r   r   slsqldbnessl_linedate_strexpiry	days_leftextsexpectedrH   foundmissingr   s                     @r   phase_remote_securityr    s'   56C 3+LCb	Qw;#%r2 	fANN6!qc%:FLRcde	fyyH GEvb";V]Tefgvb";VEu  xI  J  	K ;Dv~vb"6O`abvb"6@egxyz u~HK8+	y~~c1-a0H&&x1GHF(,,x||"<"D"DD"D"QQWWI2~vb*>9+U_H`bstuQvb*>9+UeHfhyz{vb*>	Sdef 	vb"6@UWhij 9$$S)//4D;H(,:1	QWWY__:E:Gvb":FDIIfUZmD\^opqvb":FiPTPYPYZ`ahZiPjOkDln  A  	B J' 	yNN6"&:FmT\]`^`TaSbDcevwx	y ;s   CK) 9L# L#)3L L c                 "   t        d       	 t        j                  t         dddidd      }	 |j	                         }| j                  t        dd	d
d|j                   dd             	 t        j                  t         dt        dddd      }	 |j	                         }| j                  t        ddd
d|j                   dd             	 t        j                  t         dt        dddd      }|j                  d v r,| j                  t        d!d"d
d|j                   d             n+| j                  t        d!d"dd|j                   d             	 t        j                  t         d#d$d%ddd      }	 |j	                         }|j                  d&      dur| j                  t        d'd(d
d)d             n| j                  t        d'd(dd*d             	 t        j                  t         d+d$d%gd,dd      }	 |j	                         }|j                  d&      dur| j                  t        d-d.d
d)d             n| j                  t        d-d.dd*d             	 t        j                  d0d$d%ddd1      }	 |j	                         }|j                  d&      dur| j                  t        d2d3d
d)d             n| j                  t        d2d3dd*d             yy# t        $ rl |j                  dk(  r-| j                  t        dd	dd|j                   dd             n,| j                  t        dd	dd|j                   dd             Y w xY w# t        j                  $ r5}| j                  t        dd	dt        |      dd d             Y d}~8d}~ww xY w# t        $ r0 | j                  t        dddd|j                   dd             Y w xY w# t        j                  $ r5}| j                  t        dddt        |      dd d             Y d}~Zd}~ww xY w# t        j                  $ r5}| j                  t        d!d"dt        |      dd d             Y d}~d}~ww xY w# t        $ r0 | j                  t        d'd(dd|j                   dd             Y w xY w# t        j                  $ r5}| j                  t        d'd(dt        |      dd d             Y d}~d}~ww xY w# t        $ r0 | j                  t        d-d.d
d|j                   d/d             Y w xY w# t        j                  $ r5}| j                  t        d-d.dt        |      dd d             Y d}~d}~ww xY w# t        $ r/ | j                  t        d2d3d
d|j                   d/d             Y yw xY w# t        j                  $ r4}| j                  t        d2d3dt        |      dd d             Y d}~yd}~ww xY w)4z4Checks 20-25: Photo API endpoints respond correctly.u$   Phase 5 — Remote Photo API (20-25)/upload.php
auth_tokeninvalid_testrz   Tr   r7   verifyr   zupload.php returns JSONr   HTTP z, JSON responsezRemote Photo API   r   z but not JSONr   
, not JSONNP   z/getimagelisting.phpr   )r
  job_id)datar7   r     zgetimagelisting.php respondsz, JSONz/fetch_image.phpznonexistent.jpg)r
  filename)paramsr7   r  )r  i     zfetch_image.php respondsz/delete.php	bad_tokenztest.jpgsuccess   zdelete.php rejects bad authzrejected invalid tokenzaccepted invalid token!z/delete_image.php)r
  	filenames   z!delete_image.php rejects bad authz, non-JSON rejectionz:https://upload.aeihawaii.com/upload/delete_local_photo.phpF   z'delete_local_photo.php rejects bad auth)rc   requestspostREMOTE_API_BASEr   r   r   status_coderr   RequestExceptionrD   
AUTH_TOKENrZ   )r   respr  rH   s       r   phase_remote_photo_apir%    ss   67g}}0<#/"@%'6
	a99;DNN6"&?$)$*:*:);?"KM_a bl}}00DE1;s"K%'6	]99;DNN6"&Df$)$*:*:);6"BDVX Yh||//?@2<J[#\$&t5 z)NN6"&@&$)$*:*:);"<>PR S NN6"&@&$)$*:*:);"<>PR Sk}}0<1<*"U%'6
	]99;Dxx	"$.vb*G&>@R T U vb*G&?AS U Vq}}00AB1<J<"X%'6	g99;Dxx	"$.vb*Mv&>@R T U vb*Mv&?AS U Vw||X2=:#V$&u6
	g99;Dxx	"$.vb*SU[&>@R T U vb*SU[&?AS U VUm  	a3&vb*CV(-d.>.>-?}&MOa c d vb*CV(-d.>.>-?z&JL^ ` a	a $$ gvb";VSVCR[Rdeffg  	]NN6"&Df$)$*:*:);:"FHZ\ ]	] $$ lvb"@&#a&QTRT+Wijkkl $$ hvb"<fc!fSbkSefggh   	]NN6"&CV$)$*:*:);:"FHZ\ ]	] $$ kvb"?QPSQSVhijjk   	gNN6"&I6$)$*:*:);;O"PRdf g	g $$ qvb"EvsSTvVYWY{\noppq   	gNN6"&OQW$)$*:*:);;O"PRdf g	g $$ wvb"KVUXYZU[\_]_U`btuvvws)  "M! <K) -'O( <N, BP3 #R: A Q> $$U 	A T *W A V )A1MM! MM! !N)4*N$$N),5O%!O( $O%%O( (P0;*P++P03Q;*Q66Q;>5R73R: 6R77R: :T*S==T5T>:U =T>>U V	*VV	5WW WW X*X		Xc                 <  
 t        d       dt         dt         dt         dt         dt         dt         dt         dt         dt         d}t	        |d	      \  }}}|j                  d
      

fd} |d      }|j                         dk(  r| j                  t        ddddd             n!| j                  t        ddd|dd d              |d      }	 t        |      }| j                  t        ddd|ddd              |d      }		 t        |	      }| j                  t        ddd|ddd             y# t        $ r$ | j                  t        ddd|dd d             Y cw xY w# t        $ r$ | j                  t        ddd|	dd d             Y yw xY w)z$Checks 26-28: Database connectivity.u#   Phase 6 — Remote Database (26-28)z'
    echo '===SELECT1==='
    mysql -u  -p'' zC -N -e "SELECT 1;" 2>&1

    echo '===METER_FILES==='
    mysql -u zT -N -e "SELECT COUNT(*) FROM meter_files;" 2>&1

    echo '===JOBS==='
    mysql -u z- -N -e "SELECT COUNT(*) FROM jobs;" 2>&1
    rz   r{   r   c                     t              D ]B  \  }}|j                         | k(  s|dz   t              k  s,|dz      j                         c S  yr   r   r   s      r   r   z&phase_remote_database.<locals>.section  r   r   SELECT11   zMySQL connection worksr   zSELECT 1 OKzRemote Databaser   Nr  METER_FILES   zmeter_files table accessibler    rowsJOBS   zjobs table accessible)rc   DB_USERDB_PASSDB_NAMErI   r   r?   r   r   r   rr   )r   rF   r   r   r   r   sel1mfr   jobsr   s             @r   phase_remote_databaser8  t  s   56Yd7)2gY / Yd7)2gY / Yd7)2gY /	C 3+LCbyyH 9Dzz|svb":FMSdefvb":FD"IO`ab 
	BgBvb"@&USTIUZJ[]nop
 6?DbD	vb"96eAYeCTVghi  gvb"@&"Sb'Sdefg  bvb"9649N_`abs$   -D> -E. >*E+*E+.*FFc           	         t        d       t        d      \  }}}|j                         }|j                         xr t	        |      dkD  }| j                  t        dd|rdnd|r| dnd	d
             t        d      \  }}}|j                         }|j                         xr t	        |      dkD  }| j                  t        dd|rdnd|r| dnd	d
             t        d      \  }}}|j                         }|j                         xr t	        |      dkD  }| j                  t        dd|rdnd|r| dnd	d
             t        d      \  }}}|j                         }|j                         xr t	        |      dkD  }| j                  t        dd|rdnd|r| dnd	d
             t        d      \  }}}|dk(  xr |j                         dk7  }| j                  t        dd|rdnd|rdndd
             y)z,Checks 29-33: Core services on local server.u"   Phase 7 — Local Services (29-33)zDpgrep -c apache2 2>/dev/null || pgrep -c httpd 2>/dev/null || echo 0r      zApache runningr   r   r   r   zLocal ServiceszFpgrep -c mariadbd 2>/dev/null || pgrep -c mysqld 2>/dev/null || echo 0r   zMariaDB runningz'pgrep -c fail2ban 2>/dev/null || echo 0   zfail2ban runningz#pgrep -c cron 2>/dev/null || echo 0    zcron runningz9[ -d /mnt/dropbox ] && ls /mnt/dropbox/ | head -3 | wc -lr   !   z/mnt/dropbox/ accessiblemounted and readablenot accessibleN)rc   rM   r?   r   r   r   r   )r   r   _r   r   r   s         r   phase_local_servicesrA    s   45 abJCBIIKE		+SZ!^BNN6"."&24UG:.-IY[ \ cdJCBIIKE		+SZ!^BNN6"/2624UG:.-IY[ \ DEJCBIIKE		+SZ!^BNN6"0B&F24UG:.-IY[ \ @AJCBIIKE		+SZ!^BNN6"nf24UG:.-IY[ \ VWJCB	q	'SYY[C'BNN6"8B&F460<LN^` ar   c                 \   t        d       t        dt         d      \  }}}d|v }| j                  t	        dd|rdnd|r	t         d	nt         d
d             t        d      \  }}}|j                  d      D cg c]#  }|j                         s|j                         % }}d}g }|dd D ]7  }	t        d|	 dt               \  }
}}t        |
v s%d}|j                  |	       9 |s,| j                  t	        ddddt        |       dd             n0| j                  t	        dddddj                  |       d             t        |      }|dk\  r"| j                  t	        ddd| dd             nH|dk\  r"| j                  t	        ddd| dd             n!| j                  t	        ddd| dd             	 t        j                  t        dd       }| j                  t	        d!d"dd#|j                   d             t        d%      \  }}}d&|v r	 |j                  d'd(      d(   j                         }t        j                   |d)      }|t        j"                  t$        j&                        j)                  d*      z
  j*                  }|dkD  r"| j                  t	        d+d,d| d-d             nG|d.kD  r"| j                  t	        d+d,d| d/d             n| j                  t	        d+d,dd0d             yyy| j                  t	        d+d,dd3d             yc c}w # t        j                  $ r5}| j                  t	        d!d"dt        |      dd$ d             Y d}~cd}~ww xY w# t,        t.        f$ r' | j                  t	        d+d,dd1|dd2  d             Y yw xY w)4z5Checks 34-38: Firewall, fail2ban, HTTP, SSL on local.u-   Phase 8 — Local Firewall & Security (34-38)z"sudo ipset test trusted_whitelist z 2>&1z	is in set"   zAWS IP in trusted_whitelistr   r   z whitelistedu!    NOT in ipset — sync will fail!zLocal FirewallzQsudo fail2ban-client status 2>/dev/null | grep 'Jail list' | sed 's/.*://;s/ //g'r   FNr   zsudo fail2ban-client status z 2>/dev/null | grep T#   zAWS IP not banned in fail2banzchecked z jailszBANNED in: r   r   $   zfail2ban jail countr   r   z jails (expected ~22)r7   r  %   zupload.aeihawaii.com respondsr  r  zecho | openssl s_client -servername upload.aeihawaii.com -connect localhost:443 2>/dev/null | openssl x509 -noout -enddate 2>/dev/nullr   r   r   r   r   &   zLocal SSL cert > 30 daysr   r   r   r   r   r   r   )rc   rM   AWS_IPr   r   r   r?   r   r   r  rZ   	LOCAL_URLr!  r"  rD   r   r   r   r   r   r   r   rr   r   )r   r   r@  r   r   jjailsbanned	banned_injailj_out
jail_countr$  rH   r   r  r  s                    r   phase_local_firewallrR    s   ?@ ?xuMNJCB		BNN6";rVv57VHL1xGh=i*, -
 npJCB #		#<1!'')QWWY<E<FIcr
 #">tfDXY_X` abq!U?FT"	#
 vb"A6 (UF;=MO 	P 	vb"A6 +DIIi,@+ABDTV 	W UJRvb"7J<vAVXhij	r	vb"7J<OdAegwxyvb"7J<OdAegwxy>||Ir%@vb"A6 %d&6&6%78:JL 	M 	5JCB c	yyya(+113H&&x1GHF(,,x||"<"D"DD"D"QQWWI2~vb*Df)2:&>@P R SQvb*Df)23C&DFV X Y vb*DfiYijkYS 	vb"<fFWYijkm =: $$ >vb"A6!!fSbk+;= 	> 	>>, J' 	yNN6"&@&MZ]^a_aZbYcJdfvwx	ys8   =L%L%AL* C%M5 *M2=*M--M253N+*N+c                    t        d       t        d      \  }}}|r<| j                  t        dddt        j
                  j                  |      d             nTt        d      \  }}}|r%| j                  t        dddd	|d
d  d             n| j                  t        ddddd             t        d      \  }}}|r"| j                  t        dddd| d             n:t        d      \  }}}| j                  t        dddd|rd|d
d  ndz   d             t        d      \  }}}|rZ|dk(  rUd|v rt        j
                  j                  |      n|j                         d
d }| j                  t        ddd|d             n| j                  t        ddddd             t        d      \  }}}|j                  dd      }	 t        |      }|dk\  r#| j                  t        ddd|d d!d             y
|d"k\  r#| j                  t        ddd|d d#d             y
| j                  t        ddd$|d d%d             y
# t        $ r$ | j                  t        dddd&| d             Y y
w xY w)'z2Checks 39-42: Local cron jobs, disk, and database.u%   Phase 9 — Local Cron Health (39-42)zfind /var/www/html/map_dropbox/logs/ -name 'sync*.log' -mtime -1 2>/dev/null | head -1 || find /var/log/ -name '*database*sync*' -mtime -1 2>/dev/null | head -1'   zdaily_database_sync ran <24hr   z
Local CronzJgrep -a 'map_dropbox\|database_sync' /var/log/syslog 2>/dev/null | tail -1r   z
last log: Nr   zno recent log foundz.pgrep -f 'process_queue' 2>/dev/null | head -1(   zprocess_queue.sh runningzPID z=grep -a 'process_queue' /var/log/syslog 2>/dev/null | tail -1znot currently runningu    — last: r+   zfind /var/www/SQL_backups/ -name '*.sql*' -mtime -1 2>/dev/null | head -1 || ls -lt /var/www/SQL_backups/ 2>/dev/null | head -2 | tail -1r   /)   zDB backup < 24h oldzno recent backup foundz df -BG / | awk 'NR==2{print $4}'ri   r   *   zLocal disk free > 50GBr   r   r   z
GB (<50GB)r   z
GB (<20GB)r   )rc   rM   r   r   ospathbasenamer?   r   ro   rr   )r   r   r@  r   out2r[  r   r   s           r   phase_local_cronr]    s   78 	QJCB vb"@&"''JZJZ[^J_amno lm
aNN6"&DfPZ[_`cac[dZeNfhtuvNN6"&DfNceqrs KLJCB
vb"<fSElT`ab ^_
avb"<f5VZ;tCRyk9R`bceqs 	t 	GJCB rQw,/3J277##C(CIIK<Lvb"7<XYvb"7AY[ghi =>JCBC$I	p98NN6"&>2c(RTWcde2XNN6"&>2c(R\H]_klmNN6"&>2c(R\H]_klm pvb":FmT]S^D_amnops   2I 8'I  "I *I0/I0c                    t        d       	 t        j                  t        dd      }|j                  dk(  r,| j                  t        dddd	|j                   d
             n+| j                  t        dddd	|j                   d
             	 t        j                  t         ddt        idd      }	 |j                         }| j                  t        dddd	|j                   d
             y# t        j                  $ r4}| j                  t        dddt        |      dd d
             Y d}~d}~ww xY w# t        $ r/ | j                  t        dddd	|j                   dd
             Y yw xY w# t        j                  $ r4}| j                  t        dddt        |      dd d
             Y d}~yd}~ww xY w)z(Checks 43-44: Cross-server connectivity.u.   Phase 10 — Cross-System Connectivity (43-44)rz   TrF  r  +   zScheduler login page respondsr   r  zCross-Systemr   r   Nr  r	  r
  r  ,   zPhoto API responds (JSON)r  )rc   r  rZ   REMOTE_SCHEDULER_URLr!  r   r   r"  rD   r  r   r#  r   rr   )r   r$  rH   r  s       r   phase_cross_systemrb  K  s   @A
<||0"TJs"NN6"&Ev$)$*:*:);"<nN O NN6"&Ev$)$*:*:);"<nN O<}}0<#/"<%'6	Y99;DNN6"&A6$)$*:*:);"<nN O $$ <vb"A6!!fSbk>; 	< 	<<  	YNN6"&A6$)$*:*:);:"FX Y	Y $$ <vb"=v!!fSbk>; 	< 	<<sN   BC3 &E8 7;D= 3D:*D55D:=5E52E8 4E55E8 8F?*F::F?c                 H   t        d       dt         dt         dt         dt         dt         dt         d}t        |d	
      \  }}}|dk7  rLd|v sd|v sd|v r@t        dd      D ]0  }| j                  t        |d| ddt         d|dd  d             2 y|j                  d      fd} |d      }|j                         xr t        |      dkD  }| j                  t        dd|rdnd|r| dndd              |d      }|j                         xr t        |      dkD  }| j                  t        d d!|rdnd|r| dndd              |d"      }	|	d#k(  }| j                  t        d$d%|rdnd|rd&nd'd              |d(      }	|	d#k(  }| j                  t        d)d*|rdnd|rd&nd'd              |d+      }	|	d#k(  }| j                  t        d,d-|rdnd|rd&nd'd              |d.      }
	 t        |
      }| j                  t        d/d0d|d1d2d              |d4      }	 t        |      }|dkD  }| j                  t        d6d7|rdnd|rd8nd9d             y# t        $ r$ | j                  t        d/d0d|
dd3 d             Y lw xY w# t        $ r d5}Y dw xY w):zDChecks 45-51: aei-webserv2 (upload.aeihawaii.com / 192.168.141.219).u(   Phase 11 — aei-webserv2 Health (45-51)zecho '===APACHE==='; pgrep -c apache2 2>/dev/null || echo 0; echo '===MYSQL==='; pgrep -c mysqld 2>/dev/null || pgrep -c mariadbd 2>/dev/null || echo 0; echo '===UPLOAD_PHP==='; [ -f zb/uploadlocallat_kuldeep.php ] && echo 'EXISTS' || echo 'MISSING'; echo '===DELETE_LOCAL==='; [ -f z^/delete_local_photo.php ] && echo 'EXISTS' || echo 'MISSING'; echo '===CHECK_PHOTOS==='; [ -f z\/check_photos.php ] && echo 'EXISTS' || echo 'MISSING'; echo '===LOCAL_PHOTOS==='; mysql -u r'  r(  zo -N -e 'SELECT COUNT(*) FROM local_photos;' 2>&1; echo '===DROPBOX==='; ls /mnt/dropbox/ 2>&1 | head -3 | wc -lr   r{   r   r|   zConnection refusedzNo route-   4   zwebserv2 check r   zSSH to z	 failed: Nr   zwebserv2 Healthr   c                     t              D ]B  \  }}|j                         | k(  s|dz   t              k  s,|dz      j                         c S  yr   r   r   s      r   r   z&phase_webserv2_health.<locals>.section  r   r   APACHEzApache running (webserv2)r   r   r   r   .   zMySQL running (webserv2)
UPLOAD_PHPEXISTS/   z!uploadlocallat_kuldeep.php existspresentMISSINGDELETE_LOCAL0   zdelete_local_photo.php existsCHECK_PHOTOS1   zcheck_photos.php existsLOCAL_PHOTOSr   zlocal_photos table accessibler   r/  r  DROPBOXF3   z"/mnt/dropbox accessible (webserv2)r>  r?  )rc   WEBSERV2_UPLOAD_DIRWEBSERV2_DB_USERWEBSERV2_DB_PASSWEBSERV2_DB_NAMErR   r   r   r   rQ   r   r   r   rr   )r   rF   r   r   r   r   r   r   r   r$   lpdropboxr   s               @r   phase_webserv2_healthr{  n  s?   :;	
 $$ %#$ %#$ %$%T*:);2>N=O P2	2   (R8LCb	QwK3&*>#*EWZIZr2 	eANN6!qc%:F$+N+;9S"XJ"OQbd e	e 	yyH HE		+SZ!^BNN6"9R6V24UG:.-IZ\ ] GE		+SZ!^BNN6"8B&F24UG:.-IZ\ ] \"F	8	BNN6"AR6U['))y:KM N ^$F	8	BNN6"=vQW'))y:KM N ^$F	8	BNN6"726'))y:KM N 
	 BhBvb"A6eTUYV[K\^opq
 i GGQY NN6"BbFV\460<LN_a b  hvb"A62cr7Tefgh  s$   6-I# ,J #*JJJ! J!c                     t        j                  d      } | j                  ddd       | j                  ddd       | j                  d	dd
       | j                  ddd       | j                         }|j                  ag }t        j                         }|j                  st        dt        j                   dt        j                          t        d        t        dt        j                         j                  d              |j                  rt        d       n#|j                  rt        d       nt        d       |j                  sJt!        |       |j                  s |dd  D ]  }t#        ||j$                          t'        |       |j                  s |dd  D ]  }t#        ||j$                          t)        |       |j                  s |dd  D ]  }t#        ||j$                          t+        |       |j                  s |dd  D ]  }t#        ||j$                          t-        |       |j                  s |dd  D ]  }t#        ||j$                          t/        |       |j                  s |dd  D ]  }t#        ||j$                          |j                  st1        |       |j                  s |dd  D ]  }t#        ||j$                          t3        |       |j                  s |dd  D ]  }t#        ||j$                          t5        |       |j                  s |dd  D ]  }t#        ||j$                          |j                  sC|j                  s7t7        |       |j                  s |dd  D ]  }t#        ||j$                          |j                  s7t9        |       |j                  s |dd  D ]  }t#        ||j$                          t        j                         |z
  }t;        d |D              }t;        d |D              }t;        d |D              }t;        d |D              }	t=        |      }
|j                  rut        j                         j?                         tA        |d      |
||||	d|D cg c]  }|jC                          c}d }t        t	        jD                  |d!"             nt        dd        t        d#t        j                   d$t        j                   d%|
 d&|d'd(	       t        d)t        jF                   d*| t        j                   d#t        jH                   d+| t        j                   d#t        jJ                   d,| t        j                   |	rd-|	 nd.z          |d/kD  rwt        d0t        jJ                   d1t        j                          |D ]E  }|jL                  d2k(  st        d)|jN                  d3d4|jP                   d5|jR                          G |d/kD  r|j$                  swt        d0t        jH                   d6t        j                          |D ]E  }|jL                  d7k(  st        d)|jN                  d3d4|jP                   d5|jR                          G t                tU        jV                  |d/kD  rd       y d/       y c c}w )8NzAEI System Operational Audit)descriptionz--remote-only
store_truez"Skip local checks (phases 7-9, 11))actionhelpz--local-onlyzSkip remote checks (phases 1-6)z--jsonzOutput JSON instead of textz--warn-onlyzOnly show WARN and FAIL resultsrb   u   ────────────────────────────────────────────────────────────z  Started: z%Y-%m-%d %H:%M:%Sz  Mode: Remote onlyz  Mode: Local onlyz!  Mode: Full audit (both servers)iic              3   @   K   | ]  }|j                   d k(  sd  yw)r   r   Nr$   .0r\   s     r   	<genexpr>zmain.<locals>.<genexpr>       >1188v+=Q>   c              3   @   K   | ]  }|j                   d k(  sd  yw)r   r   Nr  r  s     r   r  zmain.<locals>.<genexpr>   r  r  c              3   @   K   | ]  }|j                   d k(  sd  yw)r   r   Nr  r  s     r   r  zmain.<locals>.<genexpr>!  r  r  c              3   @   K   | ]  }|j                   d k(  sd  yw)rT   r   Nr  r  s     r   r  zmain.<locals>.<genexpr>"  r  r  r   )totalpasswarnfailskip)	timestampelapsed_secondssummaryr   re   )indentz  zSummary:r   z checks in z.1fr   z    zPASS: zWARN: zFAIL: z  SKIP: r+   r   z
  zFailed checks:r   rW   rX   rU   z	Warnings:r   ),argparseArgumentParseradd_argument
parse_argsr   rY   timer[   r   r   r   r   r   strftimeremote_only
local_onlyr   r`   r]   r   r   r  r%  r8  rA  rR  r]  rb  r{  sumr   	isoformatroundr*   dumpsr   r   r   r$   r"   r#   r%   sysexit)parserargsr   
start_timer\   elapsed
pass_count
warn_count
fail_count
skip_countr  outputs               r   mainr    s   $$1OPF
Cgh
|Bcd
<YZ
lAbcD ))KGJ996;;-;FLL>JKHLLN334GHIJK')__&(57 ??g&yyRS\ 0Q/0 	'"yyRS\ 0Q/0 	'"yyRS\ 0Q/0 	g&yyRS\ 0Q/0 	w'yyRS\ 0Q/0 	g&yyRS\ 0Q/0 W%yyRS\ 0Q/0 	W%yyRS\ 0Q/0 	!yyRS\ 0Q/0 DOO7#yyRS\ 0Q/0 g&yyRS\ 0Q/0 iikJ&G >>>J>>>J>>>J>>>JLEyy!113$Wa0"""" .55		5
 	djj*+:, 6;;-x~Qug[QTUVWXV[[M
|FLL>VJ<~RVJ<~? -78J<(B@ 	A
 >D^FLL>BC G88v%D2affXU188*EFG >$..DYv||n=> G88v%D2affXU188*EFG 	HH*q.Q(a(3 6s   [
__main__)r   )F);r,   r  r   rY  rl   r9   r  r  warningsr   r   r   filterwarningsr  ImportErrorr[   r  r;   r<   r=   rI  r2  r3  r4  r   ra  rJ  REMOTE_PHOTOAPIREMOTE_SCHEDULERr#  rP   rQ   rO   ru  rv  rw  rx  r   r   rI   rM   rR   rY   r`   rc   rx   r   r   r   r  r%  r8  rA  rR  r]  rb  r{  r  r   r   r   r   <module>r     s  @   	 	  
   2 2   *D E *	


!2B *	CE 
 ",   #   
 
$
  W2(<L@rYlAiJGBVlw`,b`$aPClN4pp<FRblE)N zF u   	
()CHHQKs   C C.-C.