In [3]:
#TO DO
##########
##########
#Create as a django site allowing users to add their own sites for monitoring
#chart available cpu and memory over time

#Then we know:
#-- The page is available
#-- The time taken to load
#-- The server resource usage
In [51]:
import multiprocessing.dummy as mp 
import requests
import pandas as pd
import datetime
import time
import plotly.express as px

urls = [
  'http://netshock.co.uk',
  'http://netshock.co.uk/code-repo-2/',
  'http://netshock.co.uk/spark-repo-2/',
  'http://netshock.co.uk/strategy-learning-path',
  'https://keene.gq',
  'http://netshock.co.uk/strategy-learning-path/'
]

def do_print(url):

    now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")

    try:
        response = requests.get(url, timeout=6)
    except:
        response = 'null'

    try:
        response_code = response.status_code
    except:
        response_code = 'null'
    
    try:
        response_content = response.content
    except:
        response_content = 'null'
        
    try:        
        response_text = response.text
    except:
        response_text = 'null'
    
    try:
        response_content_type = response.headers['Content-Type']
    except:
        response_content_type = 'null'
    
    try:
        response_server = response.headers['Server']
    except:
        response_server = 'null'
        
    try:
        response_elapsed_time =  str(round(response.elapsed.total_seconds(),2))
    except:
        response_elapsed_time = 'null'
    
    try:
        response_last_modified = response.headers['Last-Modified']
    except:
        response_last_modified = 'null'
        
    try:
        response_content_encoding = response.headers['Content-Encoding']
    except:
        response_content_encoding = 'null'
    
    try:
        response_content_length = response.headers['Content-Length']
    except:
        response_content_length = 'null'    
    
    try:
        response_url = response.url
    except:
        response_url = 'null'
    
    if int(response_code) <400:
        availability = 'OK'
    elif int(response_code) >399 and int(response_code) < 500:
        availability = 'Client Error'
    elif int(response_code) >499:
        availability = 'Server Error'

    if int(response_code) <400:
        availability_score = 1
    elif int(response_code) >399 and int(response_code) < 500:
        availability_score = 0
    elif int(response_code) >499:
        availability_score = 0
    
    d.append([now, response_code,  response_content_type, response_url, response_server, response_last_modified, response_content_encoding, availability, availability_score, response_elapsed_time])
    return d

if __name__=="__main__":
    d = []
    #1 thread = 1.5s
    #2 threads = 0.9s
    #4 threads = 0.8s
    #10 threads = 0.7s
    p=mp.Pool(10)
    out = p.map(do_print, urls)
    p.close()
    p.join()

    output = pd.DataFrame(d, columns = ['time_checked', 'response_code', 'response_content_type', 'response_url', 'response_server', 'last_modified', 'encoding', 'availability', 'availability_score', 'response_elapsed_time'])
    #write output to csv
    with open('output.csv', 'a') as f:
        output.to_csv(f, header=False)

#read the CSV in to get historical data
output = pd.read_csv('output.csv')
output.columns = ['something', 'time_checked', 'response_code', 'response_content_type', 'response_url', 'response_server', 'last_modified', 'encoding', 'availability', 'availability_score', 'response_elapsed_time']
output = output.sort_values(by=['time_checked'])

#Group by response cde
output2 = output.groupby(['time_checked', 'response_code'])['availability_score'].agg('count').reset_index()

#Group by time checked
loadTimeGraph = output.groupby(['time_checked'])['response_elapsed_time'].agg('sum').reset_index()

#response code counts
fig = px.bar(output2, x='time_checked', y='availability_score', color='response_code')
fig.show()

#load times
fig2 = px.line(loadTimeGraph, x='time_checked', y='response_elapsed_time')
fig2.show()
In [76]:
import paramiko

hosts = [
    ['IP ADDRESS', 'USER', 'PASS']
]

ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 
ssh.load_system_host_keys()

i=0
for host in hosts:
    ssh.connect(hosts[i][0], username=hosts[i][1], password=hosts[i][2])
    ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command('echo $[100-$(vmstat 1 2|tail -1|awk \'{print $15}\')]')
    ssh_stdin2, ssh_stdout2, ssh_stderr2 = ssh.exec_command('free -m')

    for line in ssh_stdout:        
        print(line)
        
    for line in ssh_stdout2:        
        print(line)
              
    ssh.close()
    i=i+1
2

              total        used        free      shared  buff/cache   available

Mem:            985         334         122           2         527         496

Swap:             0           0           0

In [ ]: