Event logging/capture (#3)

allow longer event type strings in db

add graf container w/ pre-built dashboard

remove elasticsearch and use db based event logging

add init command

Co-authored-by: lza_menace <lza_menace@protonmail.com>
Reviewed-on: #3

Dockerfile

@@ -1,9 +0,0 @@
-FROM ubuntu:19.10
-WORKDIR /srv
-COPY requirements.txt .
-RUN apt-get update && apt-get install python3-pip -y
-RUN python3 -m pip install -r requirements.txt
-COPY wowstash wowstash/
-COPY bin/ bin/
-EXPOSE 4001
-CMD ["/srv/bin/prod-container"]

docker-compose.es.yaml

@@ -1,22 +0,0 @@
-services:
-  kibana:
-    image: docker.elastic.co/kibana/kibana:7.1.0
-    ports:
-      - 5601:5601
-    environment:
-      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
-  elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.1.0
-    environment:
-      - discovery.type=single-node
-      - node.name=elasticsearch
-      - cluster.name=es-docker-cluster
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-    volumes:
-      - ./data/elasticsearch:/usr/share/elasticsearch/data
-    ports:
-      - 9200:9200

docker-compose.yaml

@@ -18,3 +18,22 @@ services:
     container_name: wowstash_cache
     ports:
       - 6379:6379
+  grafana:
+    image: grafana/grafana:6.5.0
+    container_name: grafana
+    restart: unless-stopped
+    ports:
+      - 127.0.0.1:3001:3000
+    environment:
+      HOSTNAME: grafana
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD}
+      GF_SERVER_ROOT_URL: ${GRAFANA_URL}
+      GF_ANALYTICS_REPORTING_ENABLED: "false"
+      GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
+      GF_USERS_ALLOW_SIGN_UP: "false"
+      GF_USERS_ALLOW_ORG_CREATE: "false"
+    volumes:
+      - ./files/dashboards.yaml:/etc/grafana/provisioning/dashboards/default.yaml:ro
+      - ./files/wowstash_ops.json:/var/lib/grafana/dashboards/wowstash_ops.json:ro
+      - grafana:/var/lib/grafana

files/dashboards.yaml

@@ -0,0 +1,13 @@
+apiVersion: 1
+
+providers:
+- name: 'default'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: true
+  editable: true
+  updateIntervalSeconds: 60
+  allowUiUpdates: true
+  options:
+    path: /var/lib/grafana/dashboards

files/wowstash_ops.json

@@ -0,0 +1,311 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 1,
+  "links": [],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 9,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 4,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 1,
+      "points": true,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "format": "time_series",
+          "group": [
+            {
+              "params": [
+                "$__interval",
+                "none"
+              ],
+              "type": "time"
+            }
+          ],
+          "metricColumn": "none",
+          "rawQuery": false,
+          "rawSql": "SELECT\n  $__timeGroupAlias(register_date,$__interval),\n  avg(id) AS \"id\"\nFROM users\nWHERE\n  $__timeFilter(register_date)\nGROUP BY 1\nORDER BY 1",
+          "refId": "A",
+          "select": [
+            [
+              {
+                "params": [
+                  "id"
+                ],
+                "type": "column"
+              },
+              {
+                "params": [
+                  "avg"
+                ],
+                "type": "aggregate"
+              },
+              {
+                "params": [
+                  "id"
+                ],
+                "type": "alias"
+              }
+            ]
+          ],
+          "table": "users",
+          "timeColumn": "register_date",
+          "timeColumnType": "timestamp",
+          "where": [
+            {
+              "name": "$__timeFilter",
+              "params": [],
+              "type": "macro"
+            }
+          ]
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "User Registrations",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 0,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 11,
+        "w": 24,
+        "x": 0,
+        "y": 9
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "format": "time_series",
+          "group": [
+            {
+              "params": [
+                "$__interval",
+                "0"
+              ],
+              "type": "time"
+            }
+          ],
+          "metricColumn": "type",
+          "rawQuery": false,
+          "rawSql": "SELECT\n  $__timeGroupAlias(date,$__interval,0),\n  type AS metric,\n  count(\"user\") AS \"id\"\nFROM events\nWHERE\n  $__timeFilter(date)\nGROUP BY 1,2\nORDER BY 1,2",
+          "refId": "A",
+          "select": [
+            [
+              {
+                "params": [
+                  "\"user\""
+                ],
+                "type": "column"
+              },
+              {
+                "params": [
+                  "count"
+                ],
+                "type": "aggregate"
+              },
+              {
+                "params": [
+                  "id"
+                ],
+                "type": "alias"
+              }
+            ]
+          ],
+          "table": "events",
+          "timeColumn": "date",
+          "timeColumnType": "timestamp",
+          "where": [
+            {
+              "name": "$__timeFilter",
+              "params": [],
+              "type": "macro"
+            }
+          ]
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Event Activity",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": true,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": false,
+  "schemaVersion": 21,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ]
+  },
+  "timezone": "",
+  "title": "Wowstash Ops",
+  "uid": "zvTlfCbGz",
+  "version": 1
+}

wowstash/blueprints/auth/routes.py

@@ -7,7 +7,7 @@ from wowstash.forms import Register, Login, Delete
 from wowstash.models import User
 from wowstash.factory import db, bcrypt
 from wowstash.library.docker import docker
-from wowstash.library.elasticsearch import send_es
+from wowstash.library.helpers import capture_event
 
 
 @auth_bp.route("/register", methods=["GET", "POST"])
@@ -33,7 +33,7 @@ def register():
         db.session.commit()
 
         # Capture event, login user and redirect to wallet page
-        send_es({'type': 'register', 'user': user.email})
+        capture_event(user.id, 'register')
         login_user(user)
         return redirect(url_for('wallet.setup'))
 
@@ -63,7 +63,7 @@ def login():
             return redirect(url_for('auth.login'))
 
         # Capture event, login user, and redirect to wallet page
-        send_es({'type': 'login', 'user': user.email})
+        capture_event(user.id, 'login')
         login_user(user)
         return redirect(url_for('wallet.dashboard'))
 
@@ -73,9 +73,9 @@ def login():
 def logout():
     if current_user.is_authenticated:
         docker.stop_container(current_user.wallet_container)
-        send_es({'type': 'stop_container', 'user': current_user.email})
+        capture_event(current_user.id, 'stop_container')
         current_user.clear_wallet_data()
-        send_es({'type': 'logout', 'user': current_user.email})
+        capture_event(current_user.id, 'logout')
         logout_user()
     return redirect(url_for('meta.index'))
 
@@ -85,10 +85,10 @@ def delete():
     form = Delete()
     if form.validate_on_submit():
         docker.stop_container(current_user.wallet_container)
-        send_es({'type': 'stop_container', 'user': current_user.email})
+        capture_event(current_user.id, 'stop_container')
         sleep(1)
         docker.delete_wallet_data(current_user.id)
-        send_es({'type': 'delete_wallet', 'user': current_user.email})
+        capture_event(current_user.id, 'delete_wallet')
         current_user.clear_wallet_data(reset_password=True, reset_wallet=True)
         flash('Successfully deleted wallet data')
         return redirect(url_for('wallet.setup'))

wowstash/blueprints/wallet/routes.py

@@ -10,7 +10,7 @@ from socket import socket
 from datetime import datetime
 from wowstash.blueprints.wallet import wallet_bp
 from wowstash.library.docker import docker
-from wowstash.library.elasticsearch import send_es
+from wowstash.library.helpers import capture_event
 from wowstash.library.jsonrpc import Wallet, to_atomic
 from wowstash.library.cache import cache
 from wowstash.forms import Send, Delete, Restore
@@ -29,6 +29,7 @@ def setup():
         if restore_form.validate_on_submit():
             c = docker.create_wallet(current_user.id, restore_form.seed.data)
             cache.store_data(f'init_wallet_{current_user.id}', 30, c)
+            capture_event(current_user.id, 'restore_wallet')
             current_user.wallet_created = True
             db.session.commit()
             return redirect(url_for('wallet.loading'))
@@ -81,7 +82,7 @@ def dashboard():
     seed = wallet.seed()
     spend_key = wallet.spend_key()
     view_key = wallet.view_key()
-    send_es({'type': 'load_dashboard', 'user': current_user.email})
+    capture_event(current_user.id, 'load_dashboard')
     return render_template(
         'wallet/dashboard.html',
         transfers=all_transfers,
@@ -115,6 +116,7 @@ def connect():
         current_user.wallet_container = wallet
         current_user.wallet_start = datetime.utcnow()
         db.session.commit()
+        capture_event(current_user.id, 'start_wallet')
         data = {
             'result': 'success',
             'message': 'Wallet has been connected'
@@ -133,6 +135,7 @@ def create():
     if current_user.wallet_created is False:
         c = docker.create_wallet(current_user.id)
         cache.store_data(f'init_wallet_{current_user.id}', 30, c)
+        capture_event(current_user.id, 'create_wallet')
         current_user.wallet_created = True
         db.session.commit()
         return redirect(url_for('wallet.loading'))
@@ -173,13 +176,13 @@ def send():
         # Check if Wownero wallet is available
         if wallet.connected is False:
             flash('Wallet RPC interface is unavailable at this time. Try again later.')
-            send_es({'type': 'tx_fail_rpc_unavailable', 'user': user.email})
+            capture_event(user.id, 'tx_fail_rpc_unavailable')
             return redirect(redirect_url)
 
         # Quick n dirty check to see if address is WOW
         if len(address) not in [97, 108]:
             flash('Invalid Wownero address provided.')
-            send_es({'type': 'tx_fail_address_invalid', 'user': user.email})
+            capture_event(user.id, 'tx_fail_address_invalid')
             return redirect(redirect_url)
 
         # Check if we're sweeping or not
@@ -191,7 +194,7 @@ def send():
                 amount = to_atomic(Decimal(send_form.amount.data))
             except:
                 flash('Invalid Wownero amount specified.')
-                send_es({'type': 'tx_fail_amount_invalid', 'user': user.email})
+                capture_event(user.id, 'tx_fail_amount_invalid')
                 return redirect(redirect_url)
 
             # Send transfer
@@ -202,10 +205,10 @@ def send():
             msg = tx['message'].capitalize()
             msg_lower = tx['message'].replace(' ', '_').lower()
             flash(f'There was a problem sending the transaction: {msg}')
-            send_es({'type': f'tx_fail_{msg_lower}', 'user': user.email})
+            capture_event(user.id, f'tx_fail_{msg_lower}')
         else:
             flash('Successfully sent transfer.')
-            send_es({'type': 'tx_success', 'user': user.email})
+            capture_event(user.id, 'tx_success')
 
         return redirect(redirect_url)
     else:

wowstash/config.example.py

@@ -37,7 +37,6 @@ DB_PASS = 'zzzzzzzzz'
 
 # Development
 TEMPLATES_AUTO_RELOAD = True
-ELASTICSEARCH_ENABLED = False
 
 # Social
 SOCIAL = {

wowstash/factory.py

@@ -77,6 +77,11 @@ def create_app():
             user.clear_wallet_data()
             print(f'Wallet data cleared for user {user.id}')
 
+        @app.cli.command('init')
+        def init():
+            import wowstash.models
+            db.create_all()
+
         # Routes/blueprints
         from wowstash.blueprints.auth import auth_bp
         from wowstash.blueprints.wallet import wallet_bp

wowstash/library/docker.py

@@ -9,7 +9,6 @@ from wowstash import config
 from wowstash.models import User
 from wowstash.factory import db
 from wowstash.library.jsonrpc import daemon
-from wowstash.library.elasticsearch import send_es
 
 
 class Docker(object):
@@ -66,7 +65,6 @@ class Docker(object):
                 }
             }
         )
-        send_es({'type': f'init_wallet', 'user': u.email})
         return container.short_id
 
     def start_wallet(self, user_id):
@@ -103,7 +101,6 @@ class Docker(object):
                     }
                 }
             )
-            send_es({'type': 'start_wallet', 'user': u.email})
             return container.short_id
         except APIError as e:
             if str(e).startswith('409'):

wowstash/library/elasticsearch.py

@@ -1,22 +0,0 @@
-from datetime import datetime
-from elasticsearch import Elasticsearch
-from wowstash import config
-
-
-def send_es(data):
-    if getattr(config, 'ELASTICSEARCH_ENABLED', False):
-        try:
-            es = Elasticsearch(
-                [getattr(config, 'ELASTICSEARCH_HOST', 'localhost')]
-            )
-            now = datetime.utcnow()
-            index_ts = now.strftime('%Y%m%d')
-            data['datetime'] = now
-            es.index(
-                index="{}-{}".format(
-                    getattr(config, 'ELASTICSEARCH_INDEX_NAME', 'wowstash'),
-                    index_ts
-            ), body=data)
-        except Exception as e:
-            print('Could not capture event in Elasticsearch: ', e)
-            pass # I don't really care if this logs...

wowstash/library/helpers.py

@@ -0,0 +1,12 @@
+from wowstash.models import Event
+from wowstash.factory import db
+
+
+def capture_event(user_id, event_type):
+    event = Event(
+        user=user_id,
+        type=event_type
+    )
+    db.session.add(event)
+    db.session.commit()
+    return

wowstash/models.py

@@ -53,3 +53,15 @@ class User(db.Model):
 
     def __repr__(self):
         return self.email
+
+
+class Event(db.Model):
+    __tablename__ = 'events'
+
+    id = db.Column(db.Integer, primary_key=True)
+    type = db.Column(db.String(60))
+    user = db.Column(db.Integer, db.ForeignKey(User.id))
+    date = db.Column(db.DateTime, server_default=func.now())
+
+    def __repr__(self):
+        return self.id