Table of Contents

Connection Statistics

In many mobile use cases customers want to have additional information on network coverage, signal and connections quality, roaming and so on. This data can be generated using a SDK script such as logger.are. It queries cli status periodically and logs the items you are interested in into a comma separated values file e.g./home/sdk/sandbox/logger/current.csv. When the script restarts or when a configurable timeout is reached, a new file is written. A second script will upload the log files in the working directory from time to time to a FTP server. Once having the data on the server, it can be evaluated, e.g. by drawing it onto a map or analyse it with Excel or NI DIAdem.

Logging Skript

logger.are
/* DESC: A script that can be used for a longtime logging of a NetModule Router. 
 * The upper part of the script is the logger template. The lower part is the real programm, where you describe the settings and workflow.
 * Copyright (C) 2014 NetModule AG, Switzerland
 */
 
template logger {
    workfile="current.csv";
    serialNumber=nb_status('system').SERIAL_NUMBER;
    currentFilename="";
    currentLogStart=0; // the current logfile was startet at this time
    intervall=5; // new logentry every # seconds
    logfileIntervall=3600; // new logfile every # seconds
    // due to the fact that we cannot see the amount of free space in the sdk
    // we need to improvise and add up the current filesizes   
    currentLogSize=0; // in Bytes
    maxLogSize=15000000;  // in Bytes
    freespace=1000000; // in Bytes
    # give the constructor the absolute path to store the Loggs into
    void logger(string path) {
        dh=opendir(path);    
        if(dh == -1) {
            if ( mkdir(path,0666) == -1 ); {
                nb_syslog("Could not create dir: %s. Exiting",path);
                exit(1);
            }
        } else {
        nb_syslog("directory already existent: %s", path);
        closedir(dh);
        }
        this.path = path;
        this.fields = mkarray();
        namepart=strftime("%Y%m%d_%H%M%S_start.csv",localtime(time()));
        this.currentFilename=sprintf("%s_%s",this.serialNumber, namepart);
    }
 
    int jiffy() {
        sys = sysinfo();
        u = struct_get(sys, "uptime");
 
        if (is_void(u) || u < 1) {
            return 0;
        } else {
            return u;
        }
    } 
 
    int addField(string fieldname) {
        if (strlen(fieldname) > 0 ) {
            this.fields = array_merge(this.fields, fieldname);
            return 0;
        } else {
            nb_syslog("Fieldname to short, not adding");
            return -1;
        }
    }
 
    int addFields(array localFields) {
       for (i=0; i < length(localFields); i++) {
            if (this.addField(localFields[i]) == -1 ) return -1;
       }   
    return 0;
 
    }
 
    int newLogfile() {
        // check if file if we have a work file an rename it to currentFile
        fd=fopen(sprintf("%s%s", this.path,this.workfile),"r" );
        if(fd) {
            // we have a current workfile lets rename it
            fclose(fd);
            if(!rename(sprintf("%s%s",this.path,this.workfile), sprintf("%s%s",this.path,this.currentFilename))){
                nb_syslog("Could not rename %s to %s, %s will be deleted",this.workfile, this.currentFilename,this.workfile);
                remove(sprintf("%s%s",this.path,this.workfile));  
            }
        } 
        //set new current Filename as we stored the old one away
        namepart=strftime("%Y%m%d_%H%M%S.csv",localtime(time()));
        this.currentFilename=sprintf("%s_%s",this.serialNumber,namepart);
        this.currentLogStart=this.jiffy();
        //creating Header   
        header="";
        for (i=0; i < length(this.fields); i++) {
            header=sprintf('%s%s;',header,this.fields[i]);   
        }   
 
        if(this.addLineToFile(header) == -1 ) {
            nb_syslog("writing of header to file: %s%s:", this.path,this.workfile);
            return -1;
        } else {
            return 0;
        }
    }
 
    int addLineToFile(string line) {
        fd=fopen(sprintf("%s%s", this.path,this.workfile),"a" );
        if (fd < 0 ) {
            nb_syslog("could not open file %s%s", this.path, this.workfile);
        }
 
        if(!fwrite(fd,sprintf("%s\n",line))) {
            nb_syslog("writing of line to file: %s%s:", this.path,this.workfile);
            fclose(fd);
            return -1;
        } else {
            fclose(fd);
            return 0;
        }
    } 
 
    int filesize(string file) {
        fd=fopen(file,"r");
        if (fd) {
            fseek(fd,-1);
            size=ftell(fd);
            fclose(fd);
            return size;
        } else {
            return -1;
        }
    }
 
 
    int logNow()  {
        //get current status output
        sections=mkarray("system","wan","wwan","wlan","gnss","lan","openvpn","ipsec","dio","license", "hotspot");
        status=mkstruct();
        for (i=0;i<length(sections);i++) {
            status=struct_merge(status,nb_status(sections[i]));
        }
 
        line="";
        for (i=0; i < length(this.fields); i++) {
            value=struct_get(status,this.fields[i]);
            if (strlen(value) == 0) {   
                value=sprintf("n/a");
            }
            line=sprintf('%s%s;',line,value);   
        }
 
        if(this.addLineToFile(line) == -1 ) {
            nb_syslog("writing of line to file failed: %s%s:", this.path,this.workfile);
            return -1;
        } else {
            return 0;
        }
    }
 
    int logRotate() {
        // get all files in path
        nb_syslog("Starting Logrotate");
        files=mkarray();
        handle = opendir(this.path);
        if (handle != -1) {
            while ((entry = readdir(handle))) {
                if (entry == ".") continue;
                if (entry == "..") continue;
                if (entry == this.workfile) continue;
                files=array_merge(files,entry);
            }
            closedir(handle);
        } else {
            nb_syslog("cannot open directory %s", this.path);
        }
        // sort old to new
        qsort(files);
        // files=array_reverse(files);
        // get current filesizes
        this.currenLogSize=0;
        sizes=mkarray();
        for(i=0;i<length(files);i++){
            size[i]=this.filesize(sprintf("%s%s",this.path,files[i]));
            this.currentLogSize=this.currentLogSize+size[i]; 
        }
        nb_syslog("Current Logsize: %i byte",this.currentLogSize);
        //if we don't have enough space delete enough old ones to have enough space
    return 0;
    }
} 
// ######################################################
// ######################################################
// ######################################################
 
l = new logger("/logger/");
// Timestamp of the Logentry
l.addField("SYSTEM_TIME");
 
 
// GPS Fields
l.addField("GNSS1_LONGITUDE");
l.addField("GNSS1_LATITUDE");
l.addField("GNSS1_SATELLITES_INVIEW");
 
 
// LTE Connection Fields
// Can be cloneed with MOBILE2 in case more LTE Connections are used
// Arbitrary Strength Unit (ASU) https://en.wikipedia.org/wiki/Mobile_phone_signal#ASU
l.addField("MOBILE1_SIGNAL_LEVEL");
// received signal code power  https://en.wikipedia.org/wiki/Received_signal_code_power
l.addField("MOBILE1_SIGNAL_RSCP");
// Received signal strength indication https://en.wikipedia.org/wiki/Received_signal_strength_indication           
l.addField("MOBILE1_SIGNAL_RSSI");          
// Service Type: LTE/HSPA/3G/2G/EDGE        
l.addField("MOBILE1_SERVICE_TYPE");
// NETWORK ID of the LTE Network we are connected to
l.addField("MOBILE1_LAI");
// Area Code in  LTE Network
l.addField("MOBILE1_LAC");
// Cell ID we are connected to
l.addField("MOBILE1_CID");
// Is the SIM registered in Home or Roaming
l.addField("MOBILE1_REGISTRATION_STATE");
 
 
// WAN Data Connection field
// Connection that is current default Gateway
l.addField("WAN_HOTLINK");
// Per WANLINK/LTE Connection:
// Current State: up/down/dialin
l.addField("WANLINK1_STATE");
// Current upload and download rates 
l.addField("WANLINK1_DOWNLOAD_RATE");
l.addField("WANLINK1_UPLOAD_RATE");
// Bytes that are already down- or uploaded.
l.addField("WANLINK1_DATA_DOWNLOADED");
l.addField("WANLINK1_DATA_UPLOADED");
 
// WLAN Fields
// WLAN Clients Attached to the Accesspoint
l.addField("WLAN1_STATION_COUNT");
 
l.newLogfile();
l.intervall=5; // log every 5sec
l.logfileIntervall=3600; // create a new file every 5 min
 
// ######################################################
// ######################################################
// ######################################################
// ######################################################
 
 
while (true) {
    l.logNow();
    sleep(l.intervall);
    // lets see if we need a new logfile
    if (l.currentLogStart+l.logfileIntervall<l.jiffy()) {
       l.newLogfile();
       l.logRotate();
    }
}
exit(0);
log-data.csv
SYSTEM_TIME;GNSS1_LONGITUDE;GNSS1_LATITUDE;GNSS1_SATELLITES_INVIEW;MOBILE1_SIGNAL;MOBILE1_SERVICE_TYPE;MOBILE1_LAI;MOBILE1_LAC;MOBILE1_CID;WAN_HOTLINK;WANLINK1_STATE;WANLINK1_DOWNLOAD_RATE;WANLINK1_UPLOAD_RATE;WANLINK1_DATA_DOWNLOADED;WANLINK1_DATA_UPLOADED;WLAN1_STATION_COUNT;
2015-03-17 11:50:12;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:50:19;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:50:25;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:50:31;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:50:36;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:50:43;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:50:49;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:50:55;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:51:01;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;168;219;28560577;29223265;0;
2015-03-17 11:51:11;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:51:17;n/a;n/a;n/a;-87;UMTS;22803;1B5F;216C6;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:51:23;n/a;n/a;n/a;-87;UMTS;22803;1B5F;2B306;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:51:29;n/a;n/a;n/a;-87;UMTS;22803;1B5F;2B306;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:51:35;n/a;n/a;n/a;-87;UMTS;22803;1B5F;2B306;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:51:41;n/a;n/a;n/a;-87;UMTS;22803;1B5F;2B306;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:51:47;n/a;n/a;n/a;-87;UMTS;22803;1B5F;2B306;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:51:53;n/a;n/a;n/a;-87;UMTS;22803;1B5F;2B306;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:51:59;n/a;n/a;n/a;-87;UMTS;22803;1B5F;2B306;WANLINK1;up;0;0;28562684;29225261;0;
2015-03-17 11:52:05;n/a;n/a;n/a;-87;UMTS;22803;1B5F;2B306;WANLINK1;up;0;0;28562684;29225261;0;

Skript to periodcly upload the csv Files

uploader.are
/* desc: This script uploaed all files in a directory (exception can be specified) to a ftp server. 
 * copyright (c) 2014 netmodule ag, switzerland
 */
 
 
template uploader {   
    workfile="current.csv";
    path="";
    removefilesafterupload=false;
    server="";
    serverpath="/";
    user="";
    password="";
 
    void uploader(string path) {
        dh=opendir(path);    
        if(dh == -1) {
           nb_syslog("path %s not existent, exiting",path);     
           exit(1);           
        } else {
            nb_syslog("uploader path: %s", path);
            closedir(dh);
        }
        this.path = path;
    }
 
    int uploadfiles() {
        nb_syslog("starting uploading");
        files=mkarray();
        handle = opendir(this.path);
        if (handle != -1) {
            while ((entry = readdir(handle))) {
                if (entry == ".") continue;
                if (entry == "..") continue;
                if (entry == this.workfile) continue;
                files=array_merge(files,entry);
            }
            closedir(handle);
        } else {
            nb_syslog("cannot open directory %s", this.path);
        }
        // sort old to new
        qsort(files);
        // get current filesizes
        dump(files);
        for(i=0;i<length(files);i++){
            // lets upload every file
            if (nb_transfer_put(this.user, this.password, sprintf("%s/%s/%s", this.server, this.serverpath,files[i]), sprintf("%s%s",this.path,files[i])) == 0) {
                nb_syslog("stored '%s%s' on remote site", this.path, files[i]);
                if(this.removefilesafterupload) {
                    if(remove(sprintf("%s%s",this.path,files[i]))) {
                        nb_syslog("file %s%s removed after upload", this.path, files[i]);
                    } else {
                        nb_syslog("file %s%s can't be removed after upload", this.path, files[i]);
                    }
                }
            } else {
                nb_syslog("upload of '%s%s' was not successfull ", this.path, files[i]);
            }
        }
    return 0;
    }
} // of template uploader
 
 
########### The real program ###############
 
u = new uploader("/logger/");
u.server="https://share.netmodule.com/router/public";
u.serverpath="/logger/";
u.user="*******";
u.password="*******";
u.removefilesafterupload=true;
u.uploadfiles();

Example Python script to visualize the Data:

scatter.py
from  mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
import matplotlib.pylab as pylab
import numpy as np
import string
import matplotlib.cm as cm
import csv
from datetime import datetime as dt
 
 
#2014-11-13 06:18:09
tf="%Y-%m-%d %H:%M:%S"
startTime=dt.strptime("2015-02-05 06:00:00", tf)
endTime=dt.strptime("2015-02-21 22:00:00", tf)
 
twogx=[] #longitudes
twogy=[] #latitudes
threegx=[] #longitudes
threegy=[] #latitudes
fourgx=[] #longitudes
fourgy=[] #latitudes
nagx=[] #longitudes
nagy=[] #latitudes
d1x=[]
d1y=[]
d2x=[]
d2y=[]
cy=[]
cx=[]
cities=[]        
 
 
 
errorlist=[]
maxlon=-180.0
minlon=180.0
maxlat=-85.0
minlat=85.0
 
fi=open(r'data/<logfile>.log','r')
csvlog=csv.DictReader(fi, delimiter=',') 
for row in csvlog:
        try:    
                timestamp=dt.strptime(row["system_time"],tf)
 
                if timestamp > startTime and timestamp < endTime:
                        try: 
 
                            stype=row["mobile1_service_type"] 
                            row_x=float(row["gnss1_longitude"])
                            row_y=float(row["gnss1_latidude"])
                            provider=row["mobile1_network"]     
                            if row_x == 0.0 or row_y == 0.0:   
                                raise NameError('NoGPsFix')
                            if stype=="EDGE":
                                    twogx.append(row_x)
                                    twogy.append(row_y)
                            elif stype=="HSPA+" or stype=="HSDPA" or stype=="UMTS": 
                                    threegx.append(row_x)
                                    threegy.append(row_y)
                            elif stype=="LTE":
                                    fourgx.append(row_x)
                                    fourgy.append(row_y)
                            else:
                                    nagx.append(row_x)
                                    nagy.append(row_y)
 
                            if provider=="Vodafone.de":
                                    d2x.append(row_x)
                                    d2y.append(row_y)
                            elif provider=="Telekom.de":
                                    d1x.append(row_x)
                                    d1y.append(row_y)
 
 
                            if row_x > maxlon : maxlon=row_x
                            if row_x < minlon : minlon=row_x
                            if row_y > maxlat : maxlat=row_y
                            if row_y < minlat : minlat=row_y
                        except:
                            errorlist.append(row["system_time"])
        except:
             errorlist.append(row["system_time"])
fi.close()
 
 
fi=open(r'data/cities.csv','r')
citycsv=csv.DictReader(fi, delimiter=',') 
for row in citycsv:
        try:    
            name=row["city"]      
            row_x=float(row["lon"])
            row_y=float(row["lat"])
            cy.append(row_y)
            cx.append(row_x)
            cities.append(name)
        except: 
            print "Error Cities"
fi.close
 
middlelat=maxlat-minlat
middlelon=maxlon-minlon
ma=middlelat
mo=middlelon
print "lon"
print maxlon
print minlon
print "lat"
print maxlat
print minlat
print min(threegx)
print max(threegy)
print mo
print ma
print middlelat
print middlelon
 
 
m= Basemap(llcrnrlon=(minlon-(mo*0.1)),llcrnrlat=(minlat-(ma*0.1)),urcrnrlon=(maxlon+(mo*0.1)),urcrnrlat=(maxlat+(ma*0.1)),lat_ts=20,
resolution='h',projection='merc',lon_0=-7.95,lat_0=48.48,epsg=3857) 
m.drawmapboundary(fill_color='white') # fill to edge
#m.drawmapscale(lon=(minlon+(middlelon*0.05)), lat=(minlat+(middlelat*0.05)),length=100,lat0=middlelat,lon0=middlelon,barstyle="fancy" )
print("getting wms image")
m.wmsimage("http://irs.gis-lab.info/?layers=osm&",layers=["osm"],xpixels=3200,format="png" )
print("received wma image")
twogx1,twogy1=m(twogx,twogy)
threegx1,threegy1=m(threegx,threegy)
fourgx1,fourgy1=m(fourgx,fourgy)
nagx1,nagy1=m(nagx,nagy)
d1x1,d1y1=m(d1x,d1y)
d2x1,d2y1=m(d2x,d2y)
cx1,cy1=m(cx,cy)
 
#m.scatter(d1x1,d1y1,s=20,c='m',marker="D",  alpha=1, edgecolors='none',label="Telekom.de ")
#m.scatter(d2x1,d2y1,s=20,c='r',marker="D",  alpha=1, edgecolors='none',label="Vodafone.de")
 
m.scatter(nagx1,nagy1,s=20,c='r',marker="D",   alpha=0.5, edgecolors='none' ,label="No Signal")
m.scatter(twogx1,twogy1,s=20,c='b',marker="D",  alpha=1, edgecolors='none',label="2G GSM/EDGE")
m.scatter(threegx1,threegy1,s=20,c='y',marker="D",  alpha=1, edgecolors='none',label="3G UMTS/HSPA+" )
m.scatter(fourgx1,fourgy1,s=20,c='g',marker="D",   alpha=1, edgecolors='none' ,label="4G LTE")
# some cities
m.scatter(cx1,cy1,s=80,c='w',marker="o", alpha=1, edgecolors='k' )
 
 
# print city name
for city, xc, yc in zip(cities, cx1, cy1):
    #draw the city name in a yellow (shaded) box
    plt.text(xc+200, yc, city, bbox=dict(facecolor='white', alpha=1))
 
 
plt.legend()
plt.title("NetModule test drive from " + startTime.strftime(tf) + " until " + endTime.strftime(tf) ) # might want to change this!
plt.savefig(dt.now().strftime("%Y%m%d_%H%M%S.png"),dpi=200)
plt.show()