Connection Statistics

In many mobile use cases customers want to have additional information on network coverage, signal and connections quality, roaming and so on. This data can be generated using a SDK script such as logger.are. It queries cli status periodically and logs the items you are interested in into a comma separated values file e.g./home/sdk/sandbox/logger/current.csv. When the script restarts or when a configurable timeout is reached, a new file is written. A second script will upload the log files in the working directory from time to time to a FTP server. Once having the data on the server, it can be evaluated, e.g. by drawing it onto a map or analyse it with Excel or NI DIAdem.

logger.are

/* DESC: A script that can be used for a longtime logging of a NetModule Router. 
 * The upper part of the script is the logger template. The lower part is the real programm, where you describe the settings and workflow.
 * Copyright (C) 2014 NetModule AG, Switzerland
 */
 
template logger {
    workfile="current.csv";
    serialNumber=nb_status('system').SERIAL_NUMBER;
    currentFilename="";
    currentLogStart=0; // the current logfile was startet at this time
    intervall=5; // new logentry every # seconds
    logfileIntervall=3600; // new logfile every # seconds
    // due to the fact that we cannot see the amount of free space in the sdk
    // we need to improvise and add up the current filesizes   
    currentLogSize=0; // in Bytes
    maxLogSize=15000000;  // in Bytes
    freespace=1000000; // in Bytes
    # give the constructor the absolute path to store the Loggs into
    void logger(string path) {
        dh=opendir(path);    
        if(dh == -1) {
            if ( mkdir(path,0666) == -1 ); {
                nb_syslog("Could not create dir: %s. Exiting",path);
                exit(1);
            }
        } else {
        nb_syslog("directory already existent: %s", path);
        closedir(dh);
        }
        this.path = path;
        this.fields = mkarray();
        namepart=strftime("%Y%m%d_%H%M%S_start.csv",localtime(time()));
        this.currentFilename=sprintf("%s_%s",this.serialNumber, namepart);
    }
 
    int jiffy() {
        sys = sysinfo();
        u = struct_get(sys, "uptime");
 
        if (is_void(u) || u < 1) {
            return 0;
        } else {
            return u;
        }
    } 
 
    int addField(string fieldname) {
        if (strlen(fieldname) > 0 ) {
            this.fields = array_merge(this.fields, fieldname);
            return 0;
        } else {
            nb_syslog("Fieldname to short, not adding");
            return -1;
        }
    }
 
    int addFields(array localFields) {
       for (i=0; i < length(localFields); i++) {
            if (this.addField(localFields[i]) == -1 ) return -1;
       }   
    return 0;
 
    }
 
    int newLogfile() {
        // check if file if we have a work file an rename it to currentFile
        fd=fopen(sprintf("%s%s", this.path,this.workfile),"r" );
        if(fd) {
            // we have a current workfile lets rename it
            fclose(fd);
            if(!rename(sprintf("%s%s",this.path,this.workfile), sprintf("%s%s",this.path,this.currentFilename))){
                nb_syslog("Could not rename %s to %s, %s will be deleted",this.workfile, this.currentFilename,this.workfile);
                remove(sprintf("%s%s",this.path,this.workfile));  
            }
        } 
        //set new current Filename as we stored the old one away
        namepart=strftime("%Y%m%d_%H%M%S.csv",localtime(time()));
        this.currentFilename=sprintf("%s_%s",this.serialNumber,namepart);
        this.currentLogStart=this.jiffy();
        //creating Header   
        header="";
        for (i=0; i < length(this.fields); i++) {
            header=sprintf('%s%s;',header,this.fields[i]);   
        }   
 
        if(this.addLineToFile(header) == -1 ) {
            nb_syslog("writing of header to file: %s%s:", this.path,this.workfile);
            return -1;
        } else {
            return 0;
        }
    }
 
    int addLineToFile(string line) {
        fd=fopen(sprintf("%s%s", this.path,this.workfile),"a" );
        if (fd < 0 ) {
            nb_syslog("could not open file %s%s", this.path, this.workfile);
        }
 
        if(!fwrite(fd,sprintf("%s\n",line))) {
            nb_syslog("writing of line to file: %s%s:", this.path,this.workfile);
            fclose(fd);
            return -1;
        } else {
            fclose(fd);
            return 0;
        }
    } 
 
    int filesize(string file) {
        fd=fopen(file,"r");
        if (fd) {
            fseek(fd,-1);
            size=ftell(fd);
            fclose(fd);
            return size;
        } else {
            return -1;
        }
    }
 
 
    int logNow()  {
        //get current status output
        sections=mkarray("system","wan","wwan","wlan","gnss","lan","openvpn","ipsec","dio","license", "hotspot");
        status=mkstruct();
        for (i=0;i<length(sections);i++) {
            status=struct_merge(status,nb_status(sections[i]));
        }
 
        line="";
        for (i=0; i < length(this.fields); i++) {
            value=struct_get(status,this.fields[i]);
            if (strlen(value) == 0) {   
                value=sprintf("n/a");
            }
            line=sprintf('%s%s;',line,value);   
        }
 
        if(this.addLineToFile(line) == -1 ) {
            nb_syslog("writing of line to file failed: %s%s:", this.path,this.workfile);
            return -1;
        } else {
            return 0;
        }
    }
 
    int logRotate() {
        // get all files in path
        nb_syslog("Starting Logrotate");
        files=mkarray();
        handle = opendir(this.path);
        if (handle != -1) {
            while ((entry = readdir(handle))) {
                if (entry == ".") continue;
                if (entry == "..") continue;
                if (entry == this.workfile) continue;
                files=array_merge(files,entry);
            }
            closedir(handle);
        } else {
            nb_syslog("cannot open directory %s", this.path);
        }
        // sort old to new
        qsort(files);
        // files=array_reverse(files);
        // get current filesizes
        this.currenLogSize=0;
        sizes=mkarray();
        for(i=0;i<length(files);i++){
            size[i]=this.filesize(sprintf("%s%s",this.path,files[i]));
            this.currentLogSize=this.currentLogSize+size[i]; 
        }
        nb_syslog("Current Logsize: %i byte",this.currentLogSize);
        //if we don't have enough space delete enough old ones to have enough space
    return 0;
    }
} 
// ######################################################
// ######################################################
// ######################################################
 
l = new logger("/logger/");
// Timestamp of the Logentry
l.addField("SYSTEM_TIME");
 
 
// GPS Fields
l.addField("GNSS1_LONGITUDE");
l.addField("GNSS1_LATITUDE");
l.addField("GNSS1_SATELLITES_INVIEW");
 
 
// LTE Connection Fields
// Can be cloneed with MOBILE2 in case more LTE Connections are used
// Arbitrary Strength Unit (ASU) https://en.wikipedia.org/wiki/Mobile_phone_signal#ASU
l.addField("MOBILE1_SIGNAL_LEVEL");
// received signal code power  https://en.wikipedia.org/wiki/Received_signal_code_power
l.addField("MOBILE1_SIGNAL_RSCP");
// Received signal strength indication https://en.wikipedia.org/wiki/Received_signal_strength_indication           
l.addField("MOBILE1_SIGNAL_RSSI");          
// Service Type: LTE/HSPA/3G/2G/EDGE        
l.addField("MOBILE1_SERVICE_TYPE");
// NETWORK ID of the LTE Network we are connected to
l.addField("MOBILE1_LAI");
// Area Code in  LTE Network
l.addField("MOBILE1_LAC");
// Cell ID we are connected to
l.addField("MOBILE1_CID");
// Is the SIM registered in Home or Roaming
l.addField("MOBILE1_REGISTRATION_STATE");
 
 
// WAN Data Connection field
// Connection that is current default Gateway
l.addField("WAN_HOTLINK");
// Per WANLINK/LTE Connection:
// Current State: up/down/dialin
l.addField("WANLINK1_STATE");
// Current upload and download rates 
l.addField("WANLINK1_DOWNLOAD_RATE");
l.addField("WANLINK1_UPLOAD_RATE");
// Bytes that are already down- or uploaded.
l.addField("WANLINK1_DATA_DOWNLOADED");
l.addField("WANLINK1_DATA_UPLOADED");
 
// WLAN Fields
// WLAN Clients Attached to the Accesspoint
l.addField("WLAN1_STATION_COUNT");
 
l.newLogfile();
l.intervall=5; // log every 5sec
l.logfileIntervall=3600; // create a new file every 5 min
 
// ######################################################
// ######################################################
// ######################################################
// ######################################################
 
 
while (true) {
    l.logNow();
    sleep(l.intervall);
    // lets see if we need a new logfile
    if (l.currentLogStart+l.logfileIntervall<l.jiffy()) {
       l.newLogfile();
       l.logRotate();
    }
}
exit(0);

log-data.csv

SYSTEM_TIME;GNSS1_LONGITUDE;GNSS1_LATITUDE;GNSS1_SATELLITES_INVIEW;MOBILE1_SIGNAL;MOBILE1_SERVICE_TYPE;MOBILE1_LAI;MOBILE1_LAC;MOBILE1_CID;WAN_HOTLINK;WANLINK1_STATE;WANLINK1_DOWNLOAD_RATE;WANLINK1_UPLOAD_RATE;WANLINK1_DATA_DOWNLOADED;WANLINK1_DATA_UPLOADED;WLAN1_STATION_COUNT;
2015-03-17 11:50:12;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:50:19;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:50:25;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:50:31;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:50:36;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:50:43;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:50:49;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:50:55;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:51:01;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:51:11;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:51:17;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:51:23;n/a;n/a;n/a;-87;UMTS;22803;1B5F;2B306;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:51:29;n/a;n/a;n/a;-87;UMTS;22803;1B5F;2B306;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:51:35;n/a;n/a;n/a;-87;UMTS;22803;1B5F;2B306;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:51:41;n/a;n/a;n/a;-87;UMTS;22803;1B5F;2B306;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:51:47;n/a;n/a;n/a;-87;UMTS;22803;1B5F;2B306;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:51:53;n/a;n/a;n/a;-87;UMTS;22803;1B5F;2B306;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:51:59;n/a;n/a;n/a;-87;UMTS;22803;1B5F;2B306;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:52:05;n/a;n/a;n/a;-87;UMTS;22803;1B5F;2B306;WANLINK1;up;0;0;28562684;29225261;0;

uploader.are

/* desc: This script uploaed all files in a directory (exception can be specified) to a ftp server. 
 * copyright (c) 2014 netmodule ag, switzerland
 */
 
 
template uploader {   
    workfile="current.csv";
    path="";
    removefilesafterupload=false;
    server="";
    serverpath="/";
    user="";
    password="";
 
    void uploader(string path) {
        dh=opendir(path);    
        if(dh == -1) {
           nb_syslog("path %s not existent, exiting",path);     
           exit(1);           
        } else {
            nb_syslog("uploader path: %s", path);
            closedir(dh);
        }
        this.path = path;
    }
 
    int uploadfiles() {
        nb_syslog("starting uploading");
        files=mkarray();
        handle = opendir(this.path);
        if (handle != -1) {
            while ((entry = readdir(handle))) {
                if (entry == ".") continue;
                if (entry == "..") continue;
                if (entry == this.workfile) continue;
                files=array_merge(files,entry);
            }
            closedir(handle);
        } else {
            nb_syslog("cannot open directory %s", this.path);
        }
        // sort old to new
        qsort(files);
        // get current filesizes
        dump(files);
        for(i=0;i<length(files);i++){
            // lets upload every file
            if (nb_transfer_put(this.user, this.password, sprintf("%s/%s/%s", this.server, this.serverpath,files[i]), sprintf("%s%s",this.path,files[i])) == 0) {
                nb_syslog("stored '%s%s' on remote site", this.path, files[i]);
                if(this.removefilesafterupload) {
                    if(remove(sprintf("%s%s",this.path,files[i]))) {
                        nb_syslog("file %s%s removed after upload", this.path, files[i]);
                    } else {
                        nb_syslog("file %s%s can't be removed after upload", this.path, files[i]);
                    }
                }
            } else {
                nb_syslog("upload of '%s%s' was not successfull ", this.path, files[i]);
            }
        }
    return 0;
    }
} // of template uploader
 
 
########### The real program ###############
 
u = new uploader("/logger/");
u.server="https://share.netmodule.com/router/public";
u.serverpath="/logger/";
u.user="*******";
u.password="*******";
u.removefilesafterupload=true;
u.uploadfiles();

Example Python script to visualize the Data:

scatter.py

from  mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
import matplotlib.pylab as pylab
import numpy as np
import string
import matplotlib.cm as cm
import csv
from datetime import datetime as dt
 
 
#2014-11-13 06:18:09
tf="%Y-%m-%d %H:%M:%S"
startTime=dt.strptime("2015-02-05 06:00:00", tf)
endTime=dt.strptime("2015-02-21 22:00:00", tf)
 
twogx=[] #longitudes
twogy=[] #latitudes
threegx=[] #longitudes
threegy=[] #latitudes
fourgx=[] #longitudes
fourgy=[] #latitudes
nagx=[] #longitudes
nagy=[] #latitudes
d1x=[]
d1y=[]
d2x=[]
d2y=[]
cy=[]
cx=[]
cities=[]        
 
 
 
errorlist=[]
maxlon=-180.0
minlon=180.0
maxlat=-85.0
minlat=85.0
 
fi=open(r'data/<logfile>.log','r')
csvlog=csv.DictReader(fi, delimiter=',') 
for row in csvlog:
        try:    
                timestamp=dt.strptime(row["system_time"],tf)
 
                if timestamp > startTime and timestamp < endTime:
                        try: 
 
                            stype=row["mobile1_service_type"] 
                            row_x=float(row["gnss1_longitude"])
                            row_y=float(row["gnss1_latidude"])
                            provider=row["mobile1_network"]     
                            if row_x == 0.0 or row_y == 0.0:   
                                raise NameError('NoGPsFix')
                            if stype=="EDGE":
                                    twogx.append(row_x)
                                    twogy.append(row_y)
                            elif stype=="HSPA+" or stype=="HSDPA" or stype=="UMTS": 
                                    threegx.append(row_x)
                                    threegy.append(row_y)
                            elif stype=="LTE":
                                    fourgx.append(row_x)
                                    fourgy.append(row_y)
                            else:
                                    nagx.append(row_x)
                                    nagy.append(row_y)
 
                            if provider=="Vodafone.de":
                                    d2x.append(row_x)
                                    d2y.append(row_y)
                            elif provider=="Telekom.de":
                                    d1x.append(row_x)
                                    d1y.append(row_y)
 
 
                            if row_x > maxlon : maxlon=row_x
                            if row_x < minlon : minlon=row_x
                            if row_y > maxlat : maxlat=row_y
                            if row_y < minlat : minlat=row_y
                        except:
                            errorlist.append(row["system_time"])
        except:
             errorlist.append(row["system_time"])
fi.close()
 
 
fi=open(r'data/cities.csv','r')
citycsv=csv.DictReader(fi, delimiter=',') 
for row in citycsv:
        try:    
            name=row["city"]      
            row_x=float(row["lon"])
            row_y=float(row["lat"])
            cy.append(row_y)
            cx.append(row_x)
            cities.append(name)
        except: 
            print "Error Cities"
fi.close
 
middlelat=maxlat-minlat
middlelon=maxlon-minlon
ma=middlelat
mo=middlelon
print "lon"
print maxlon
print minlon
print "lat"
print maxlat
print minlat
print min(threegx)
print max(threegy)
print mo
print ma
print middlelat
print middlelon
 
 
m= Basemap(llcrnrlon=(minlon-(mo*0.1)),llcrnrlat=(minlat-(ma*0.1)),urcrnrlon=(maxlon+(mo*0.1)),urcrnrlat=(maxlat+(ma*0.1)),lat_ts=20,
resolution='h',projection='merc',lon_0=-7.95,lat_0=48.48,epsg=3857) 
m.drawmapboundary(fill_color='white') # fill to edge
#m.drawmapscale(lon=(minlon+(middlelon*0.05)), lat=(minlat+(middlelat*0.05)),length=100,lat0=middlelat,lon0=middlelon,barstyle="fancy" )
print("getting wms image")
m.wmsimage("http://irs.gis-lab.info/?layers=osm&",layers=["osm"],xpixels=3200,format="png" )
print("received wma image")
twogx1,twogy1=m(twogx,twogy)
threegx1,threegy1=m(threegx,threegy)
fourgx1,fourgy1=m(fourgx,fourgy)
nagx1,nagy1=m(nagx,nagy)
d1x1,d1y1=m(d1x,d1y)
d2x1,d2y1=m(d2x,d2y)
cx1,cy1=m(cx,cy)
 
#m.scatter(d1x1,d1y1,s=20,c='m',marker="D",  alpha=1, edgecolors='none',label="Telekom.de ")
#m.scatter(d2x1,d2y1,s=20,c='r',marker="D",  alpha=1, edgecolors='none',label="Vodafone.de")
 
m.scatter(nagx1,nagy1,s=20,c='r',marker="D",   alpha=0.5, edgecolors='none' ,label="No Signal")
m.scatter(twogx1,twogy1,s=20,c='b',marker="D",  alpha=1, edgecolors='none',label="2G GSM/EDGE")
m.scatter(threegx1,threegy1,s=20,c='y',marker="D",  alpha=1, edgecolors='none',label="3G UMTS/HSPA+" )
m.scatter(fourgx1,fourgy1,s=20,c='g',marker="D",   alpha=1, edgecolors='none' ,label="4G LTE")
# some cities
m.scatter(cx1,cy1,s=80,c='w',marker="o", alpha=1, edgecolors='k' )
 
 
# print city name
for city, xc, yc in zip(cities, cx1, cy1):
    #draw the city name in a yellow (shaded) box
    plt.text(xc+200, yc, city, bbox=dict(facecolor='white', alpha=1))
 
 
plt.legend()
plt.title("NetModule test drive from " + startTime.strftime(tf) + " until " + endTime.strftime(tf) ) # might want to change this!
plt.savefig(dt.now().strftime("%Y%m%d_%H%M%S.png"),dpi=200)
plt.show()