add db stats

3 files changed, 75 insertions, 2 deletions

diff --git a/cron.sh b/cron.sh
index 109c55f..9e44c04 100644
--- a/cron.sh
+++ b/cron.sh
@@ -17,9 +17,10 @@ find $LOGS -name "apigrabber-out*__????-??-??_??-??-??.log"\
           put '$date = sec2gmt(gmt2sec($date) // (60*60*3) * (60*60*3));' \
           then cut -f date,connection_end \
           then stats1 -a min,mean,max -f connection_end -g date \
+          then sort -f date \
     >> $PF.csv
 
-gnuplot -e "inputfile='$PF.csv'; outputfile='$PF.png';" connection_end_over_time.plt
+gnuplot -e "inputfile='$PF.csv'; outputfile='$PF.png';" $DATA/../connection_end_over_time.plt
 
 
 PF=$DATA/apigrabber_error_processed  # progress file
@@ -35,6 +36,47 @@ find $LOGS -name "apigrabber-error*__????-??-??_??-??-??.log"\
           put '$date = sec2gmt(gmt2sec($date) // (60*60*3) * (60*60*3));' \
           then cut -f date \
           then stats1 -a count -f date -g date \
+          then sort -f date \
     >> $PF.csv
 
-gnuplot -e "inputfile='$PF.csv'; outputfile='$PF.png';" errors_over_time.plt
+gnuplot -e "inputfile='$PF.csv'; outputfile='$PF.png';" $DATA/../errors_over_time.plt
+
+
+PF=$DATA/processor_processed  # progress file
+touch $PF $PF.csv
+find $LOGS -name "processor-out*__????-??-??_??-??-??.log"\
+    | sort\
+    | comm -13 $PF - \
+    | tee -a $PF \
+    | xargs grep -h " database transaction " \
+    | sed -E 's_^(.*?) - .*? database transaction _date=\1, _g' \
+    | mlr --fs ", " \
+          --ocsv \
+          put '$date = sec2gmt(gmt2sec($date) // (60*60*3) * (60*60*3));' \
+          then put '$duration = gsub($duration, "ms", "");' \
+          then cut -f date,duration \
+          then stats1 -a mean,count -f duration -g date \
+          then sort -f date \
+    | grep -v "1970" \
+    >> $PF.csv
+
+gnuplot -e "inputfile='$PF.csv'; outputfile='$PF.png';" $DATA/../transactions_over_time.plt
+
+
+PF=$DATA/processor_batch_processed  # progress file
+touch $PF $PF.csv
+find $LOGS -name "processor-out*__????-??-??_??-??-??.log"\
+    | sort\
+    | comm -13 $PF - \
+    | tee -a $PF \
+    | xargs grep -h " processing batch " \
+    | sed -E 's_^(.*?) - .*? processing batch _date=\1, _g' \
+    | mlr --fs ", " \
+          --ocsv \
+          put '$date = sec2gmt(gmt2sec($date) // (60*60*3) * (60*60*3));' \
+          then stats1 -a count -f players,matches,telemetries -g date \
+          then sort -f date \
+    | grep -v "1970" \
+    >> $PF.csv
+
+gnuplot -e "inputfile='$PF.csv'; outputfile='$PF.png';" $DATA/../objects_over_time.plt
diff --git a/objects_over_time.plt b/objects_over_time.plt
new file mode 100644
index 0000000..0e7b893
--- /dev/null
+++ b/objects_over_time.plt
@@ -0,0 +1,14 @@
+set title "processor data"
+set timefmt "%Y-%m-%dT%H:%M:%SZ";
+set xdata time;
+set ylabel "number of objects within 3h";
+set autoscale y;
+
+set datafile separator ", ";
+
+set terminal png noenhanced;
+set key autotitle columnhead;
+set output outputfile;
+plot inputfile using "date":"players_count" with lines, \
+     inputfile using "date":"matches_count" with lines, \
+     inputfile using "date":"telemetries_count" with lines;
diff --git a/transactions_over_time.plt b/transactions_over_time.plt
new file mode 100644
index 0000000..1517f85
--- /dev/null
+++ b/transactions_over_time.plt
@@ -0,0 +1,17 @@
+set title "processor performance"
+set timefmt "%Y-%m-%dT%H:%M:%SZ";
+set xdata time;
+set ytics nomirror;
+set ylabel "transaction duration in ms";
+set yrange [ 0:300 ];
+set y2tics;
+set y2label "number of transactions within 3h";
+set y2range [ 0:3000 ];
+
+set datafile separator ", ";
+
+set terminal png noenhanced;
+set key autotitle columnhead;
+set output outputfile;
+plot inputfile using "date":"duration_mean" with lines axes x1y1, \
+     inputfile using "date":"duration_count" with lines axes x1y2;