1. pandas-highcharts
pandas-highcharts is very easy to use. It has some demos showing how to plot. The only con I found is I did not get how to draw dynamic graph through it.
The following example is a small modification of the demo code.
%load_ext autoreload
%autoreload 2
import pandas as pd
import datetime
import os
import numpy as np
from pandas.compat import StringIO
from pandas.io.common import urlopen
from IPython.display import display, display_pretty, Javascript, HTML
from pandas_highcharts.core import serialize
from pandas_highcharts.display import display_charts
import matplotlib.pyplot as plt
1.1 spline plot of two series
import time
import random
s = []
i = time.time()
while i < time.time() + 100:
i += 1
s.append([i, random.randint(0, 10), random.random()])
df = pd.DataFrame(s , columns = ["dt", "val1", "val2"])
df.dt = df.dt.astype("datetime64[s]")
print df.head()
df = df.set_index("dt")
dt val1 val2
0 2016-08-18 02:05:47 5 0.139195
1 2016-08-18 02:05:48 9 0.419186
2 2016-08-18 02:05:49 1 0.381455
3 2016-08-18 02:05:50 6 0.275681
4 2016-08-18 02:05:51 9 0.140637
display_charts(df, title="Plot with pandas-highcharts")
Graph 3. 打印出数据(string)
1.2 cut to count frequency
pd.cut(range(100), 10, right = False, retbins = True)
([[0, 9.9), [0, 9.9), [0, 9.9), [0, 9.9), [0, 9.9), ..., [89.1, 99.099), [89.1, 99.099), [89.1, 99.099), [89.1, 99.099), [89.1, 99.099)]
Length: 100
Categories (10, object): [[0, 9.9) < [9.9, 19.8) < [19.8, 29.7) < [29.7, 39.6) ... [59.4, 69.3) < [69.3, 79.2) < [79.2, 89.1) < [89.1, 99.099)],
array([ 0. , 9.9 , 19.8 , 29.7 , 39.6 , 49.5 , 59.4 ,
69.3 , 79.2 , 89.1 , 99.099]))
import time
import random
s = []
i = time.time()
while i < time.time() + 100:
i += 1
s.append([i + random.normalvariate(0, 1) * 2000, random.randint(0, 10), random.random()])
df = pd.DataFrame(s , columns = ["dt", "val1", "val2"])
print df.head()
s = pd.cut(df.dt, 10, right = False, retbins = True)
freq = s[0].value_counts().values
dt = s[1].tolist()[:10]
print freq
print dt
df_freq = pd.DataFrame({"dt" : dt, "freq" : freq})
df_freq.dt = df_freq.dt.astype("datetime64[s]")
df_freq = df_freq.set_index("dt")
print df_freq.head().to_string()
dt val1 val2
0 1.471485e+09 6 0.231597
1 1.471489e+09 4 0.111525
2 1.471485e+09 10 0.458811
3 1.471483e+09 10 0.027168
4 1.471487e+09 9 0.275005
[22 16 14 13 11 8 8 4 3 2]
[1471481582.4727995, 1471482543.2894208, 1471483504.1060421, 1471484464.9226635, 1471485425.7392848, 1471486386.555906, 1471487347.3725274, 1471488308.1891487, 1471489269.00577, 1471490229.8223913]
freq
dt
2016-08-18 00:53:02 22
2016-08-18 01:09:03 16
2016-08-18 01:25:04 14
2016-08-18 01:41:04 13
2016-08-18 01:57:05 11
display_charts(df_freq, title="my first test")
2. static plot in flask
To show the static highcharts plot through flask, a simple way is to prepare the data in python code, and render to the html page. The following is an example.
The structure of the dir will be like /app.py /static/main.css /static/js/main.js /templates/index.html
2.1 app.py
from flask import Flask, render_template
app = Flask(__name__)
@app.route('/')
@app.route('/index')
def index(chartID = 'chart_ID', chart_type = 'bar', chart_height = 350):
chart = {"renderTo": chartID, "type": chart_type, "height": chart_height,}
series = [{"name": 'Label1', "data": [1,2,3]}, {"name": 'Label2', "data": [4, 5, 6]}]
title = {"text": 'My Title'}
xAxis = {"categories": ['xAxis Data1', 'xAxis Data2', 'xAxis Data3']}
yAxis = {"title": {"text": 'yAxis Label'}}
return render_template('index.html', chartID=chartID, chart=chart, series=series, title=title, xAxis=xAxis, yAxis=yAxis)
if __name__ == "__main__":
app.run(debug = True, host='0.0.0.0', port=8080, passthrough_errors=True)
2.2 index.html
<!DOCTYPE html>
<html lang="en">
<head>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="">
<meta name="keywords" content="">
<meta name="author" content="">
<meta charset="utf-8">
<!-- <link href="../static/css/main.css" rel="stylesheet" type="text/css" /> -->
<!-- SUPPORT FOR IE6-8 OF HTML5 ELEMENTS -->
<!--[if lt IE 9]>
<script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->
<!-- <link rel="shortcut icon" href="{{ url_for('static', filename='ico/favicon.ico') }}"> -->
{% block head %}
<title>{% block title %} Title!{% endblock %}</title>
{% endblock %}
</head>
<body>
<div id={{ chartID|safe }} class="chart" style="height: 100px; width: 500px"></div>
<script>
var chart_id = {{ chartID|safe }}
var series = {{ series|safe }}
var title = {{ title|safe }}
var xAxis = {{ xAxis|safe }}
var yAxis = {{ yAxis|safe }}
var chart = {{ chart|safe }}
</script>
<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.8.2/jquery.min.js"></script>
<script src="http://code.highcharts.com/highcharts.js"></script>
<script src="../static/js/main.js"></script>
</body>
</html>
2.3 main.css
* {
margin: 20;
padding: 100;
}
header {
background-color:rgba(33, 33, 33, 0.9);
color:#ffffff;
display:block;
font: 14px/1.3 Arial,sans-serif;
height:50px;
position:relative;
}
header h2{
font-size: 22px;
margin: 0px auto;
padding: 10px 0;
width: 80%;
text-align: center;
}
header a, a:visited {
text-decoration:none;
color:#fcfcfc;
}
body {
background:url("background.jpg") no-repeat scroll top center transparent;
}
.actions, .chart {
margin: 15px auto;
width: 34px;
}
button {
background: none repeat scroll 0 0 #E3E3E3;
border: 1px solid #BBBBBB;
border-radius: 3px 3px 3px 3px;
box-shadow: 0 0 1px 1px #F6F6F6 inset;
color: #333333;
font: bold 12px;
margin: 0 5px;
padding: 8px 0 9px;
text-align: center;
text-shadow: 0 1px 0 #FFFFFF;
width: 150px;
}
button:hover {
background: none repeat scroll 0 0 #D9D9D9;
box-shadow: 0 0 1px 1px #EAEAEA inset;
color: #222222;
cursor: pointer;
}
button:active {
background: none repeat scroll 0 0 #D0D0D0;
box-shadow: 0 0 1px 1px #E3E3E3 inset;
color: #000000;
}
2.4 main.js
$(document).ready(function() {
$(chart_id).highcharts({
chart: chart,
title: title,
xAxis: xAxis,
yAxis: yAxis,
series: series
});
});
3. An example of system monitor: Memory Usage Plot
The folder structure is like /app.py /python_monitor.py * /templates/mon.html
The process is to catch the memory info in /proc/meminfo and save the data in mysql db. So first, we need to create the mysql db and tables.
3.0 create mysql db and table
mysql> create database falcon character set utf8;
Query OK, 1 row affected (0.00 sec)
CREATE TABLE `stat` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`host` varchar(256) DEFAULT NULL,
`mem_free` int(11) DEFAULT NULL,
`mem_usage` int(11) DEFAULT NULL,
`mem_total` int(11) DEFAULT NULL,
`load_avg` varchar(128) DEFAULT NULL,
`time` bigint(11) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `host` (`host`(255))
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8;
3.1 app.py
on the web server side, it will use flask.
Once there is data post the page, python will run to insert that data into mysql table stat.
Once the page is refreshed, GET method will be activated and the function getdata() will run to pull data from mysql db and data will be dumped to json file for highcharts to call.
import sys
sys.path.append('/usr/lib/python2.7/dist-packages')
import MySQLdb as mysql
import json
from flask import Flask, request, render_template
'''
run sql to create table
CREATE TABLE `stat` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`host` varchar(256) DEFAULT NULL,
`mem_free` int(11) DEFAULT NULL,
`mem_usage` int(11) DEFAULT NULL,
`mem_total` int(11) DEFAULT NULL,
`load_avg` varchar(128) DEFAULT NULL,
`time` bigint(11) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `host` (`host`(255))
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8;
'''
app = Flask(__name__)
db = mysql.connect(user="root", passwd="password", db="twitterdb", charset="utf8")
db.autocommit(True)
c = db.cursor()
@app.route("/", methods=["GET", "POST"])
def hello():
sql = ""
if request.method == "POST":
data = request.json
try:
sql = "INSERT INTO `stat` (`host`,`mem_free`,`mem_usage`,`mem_total`,`load_avg`,`time`) VALUES('%s', '%d', '%d', '%d', '%s', '%d')" % (data['Host'], data['MemFree'], data['MemUsage'], data['MemTotal'], data['LoadAvg'], int(data['Time']))
ret = c.execute(sql)
except mysql.IntegrityError:
pass
return "OK"
else:
return render_template("mon.html")
@app.route("/data", methods=["GET"])
def getdata():
c.execute("SELECT `time`,`mem_usage` FROM `stat`")
ones = [[i[0]*1000, i[1]] for i in c.fetchall()]
return "%s(%s);" % (request.args.get('callback'), json.dumps(ones))
if __name__ == "__main__":
app.run(host="localhost", port=8888, debug=True)
3.2 python_monitor.py
This will read the /proc/meminfo data and dump data to the web page.
#!/usr/bin/env python
#coding=utf-8
import inspect
import time
import urllib, urllib2
import json
import socket
class mon:
def __init__(self):
self.data = {}
def getTime(self):
return str(int(time.time()) + 8 * 3600)
def getHost(self):
return socket.gethostname()
def getLoadAvg(self):
with open('/proc/loadavg') as load_open:
a = load_open.read().split()[:3]
return ','.join(a)
def getMemTotal(self):
with open('/proc/meminfo') as mem_open:
a = int(mem_open.readline().split()[1])
return a / 1024
def getMemUsage(self, noBufferCache=True):
if noBufferCache:
with open('/proc/meminfo') as mem_open:
T = int(mem_open.readline().split()[1])
F = int(mem_open.readline().split()[1])
B = int(mem_open.readline().split()[1])
C = int(mem_open.readline().split()[1])
return (T-F-B-C)/1024
else:
with open('/proc/meminfo') as mem_open:
a = int(mem_open.readline().split()[1]) - int(mem_open.readline().split()[1])
return a / 1024
def getMemFree(self, noBufferCache=True):
if noBufferCache:
with open('/proc/meminfo') as mem_open:
T = int(mem_open.readline().split()[1])
F = int(mem_open.readline().split()[1])
B = int(mem_open.readline().split()[1])
C = int(mem_open.readline().split()[1])
return (F+B+C)/1024
else:
with open('/proc/meminfo') as mem_open:
mem_open.readline()
a = int(mem_open.readline().split()[1])
return a / 1024
def runAllGet(self):
for fun in inspect.getmembers(self, predicate=inspect.ismethod):
if fun[0][:3] == 'get':
self.data[fun[0][3:]] = fun[1]()
return self.data
if __name__ == "__main__":
while True:
m = mon()
data = m.runAllGet()
print "data is %s " %(data)
req = urllib2.Request("http://localhost:8888", json.dumps(data), {'Content-Type': 'application/json'})
f = urllib2.urlopen(req)
print f
response = f.read()
print "response is %s" %(response)
f.close()
time.sleep(5)
3.3 mon.html
This will use highcharts.
[root@91it templates]# cat mon.html
<title>memory monitor</title>
<!DOCTYPE HTML>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>Highstock Example</title>
<!-- <script type="text/javascript" src="{{ url_for('static', filename='jquery.min.js') }}"></script> -->
<script type="text/javascript" src="http://ajax.useso.com/ajax/libs/jquery/1.8.2/jquery.min.js"></script>
<style type="text/css">
${demo.css}
</style>
<script type="text/javascript">
$(function () {
$.getJSON('/data?callback=?', function (data) {
// Create the chart
$('#container').highcharts('StockChart', {
rangeSelector: {
inputEnabled: $('#container').width() > 480,
selected: 1
},
title: {
text: 'memory monitor'
},
series: [{
name: 'memory monitor',
data: data,
type: 'spline',
tooltip: {
valueDecimals: 2
}
}]
});
});
});
</script>
</head>
<body>
<!-- <script src="{{ url_for('static', filename='highstock.js') }}"></script> -->
<script src="http://cdnjs.cloudflare.com/ajax/libs/highstock/2.0.4/highstock.js"></script>
<!-- <script src="{{ url_for('static', filename='exporting.js') }}"></script> -->
<script src="http://code.highcharts.com/modules/exporting.js"></script>
<div id="container" style="height: 400px"></div>
</body>
</html>
first run python app.py and then run nohup python python_monitor.py > null 2>&1 &. After it is run, input http://localhost:8888/ to check the result and http://localhost:8888/data to view data.
Reference