Browse Source

Event logging/capture (#3)

allow longer event type strings in db

add graf container w/ pre-built dashboard

remove elasticsearch and use db based event logging

add init command

Co-authored-by: lza_menace <lza_menace@protonmail.com>
Reviewed-on: #3
master
lza_menace 3 months ago
parent
commit
2567db144f
13 changed files with 389 additions and 71 deletions
  1. +0
    -9
      Dockerfile
  2. +0
    -22
      docker-compose.es.yaml
  3. +19
    -0
      docker-compose.yaml
  4. +13
    -0
      files/dashboards.yaml
  5. +311
    -0
      files/wowstash_ops.json
  6. +7
    -7
      wowstash/blueprints/auth/routes.py
  7. +10
    -7
      wowstash/blueprints/wallet/routes.py
  8. +0
    -1
      wowstash/config.example.py
  9. +5
    -0
      wowstash/factory.py
  10. +0
    -3
      wowstash/library/docker.py
  11. +0
    -22
      wowstash/library/elasticsearch.py
  12. +12
    -0
      wowstash/library/helpers.py
  13. +12
    -0
      wowstash/models.py

+ 0
- 9
Dockerfile View File

@ -1,9 +0,0 @@
FROM ubuntu:19.10
WORKDIR /srv
COPY requirements.txt .
RUN apt-get update && apt-get install python3-pip -y
RUN python3 -m pip install -r requirements.txt
COPY wowstash wowstash/
COPY bin/ bin/
EXPOSE 4001
CMD ["/srv/bin/prod-container"]

+ 0
- 22
docker-compose.es.yaml View File

@ -1,22 +0,0 @@
services:
kibana:
image: docker.elastic.co/kibana/kibana:7.1.0
ports:
- 5601:5601
environment:
ELASTICSEARCH_HOSTS: http://elasticsearch:9200
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.1.0
environment:
- discovery.type=single-node
- node.name=elasticsearch
- cluster.name=es-docker-cluster
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- ./data/elasticsearch:/usr/share/elasticsearch/data
ports:
- 9200:9200

+ 19
- 0
docker-compose.yaml View File

@ -18,3 +18,22 @@ services:
container_name: wowstash_cache
ports:
- 6379:6379
grafana:
image: grafana/grafana:6.5.0
container_name: grafana
restart: unless-stopped
ports:
- 127.0.0.1:3001:3000
environment:
HOSTNAME: grafana
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD}
GF_SERVER_ROOT_URL: ${GRAFANA_URL}
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_USERS_ALLOW_SIGN_UP: "false"
GF_USERS_ALLOW_ORG_CREATE: "false"
volumes:
- ./files/dashboards.yaml:/etc/grafana/provisioning/dashboards/default.yaml:ro
- ./files/wowstash_ops.json:/var/lib/grafana/dashboards/wowstash_ops.json:ro
- grafana:/var/lib/grafana

+ 13
- 0
files/dashboards.yaml View File

@ -0,0 +1,13 @@
apiVersion: 1
providers:
- name: 'default'
orgId: 1
folder: ''
type: file
disableDeletion: true
editable: true
updateIntervalSeconds: 60
allowUiUpdates: true
options:
path: /var/lib/grafana/dashboards

+ 311
- 0
files/wowstash_ops.json View File

@ -0,0 +1,311 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 1,
"links": [],
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 0
},
"hiddenSeries": false,
"id": 4,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"format": "time_series",
"group": [
{
"params": [
"$__interval",
"none"
],
"type": "time"
}
],
"metricColumn": "none",
"rawQuery": false,
"rawSql": "SELECT\n $__timeGroupAlias(register_date,$__interval),\n avg(id) AS \"id\"\nFROM users\nWHERE\n $__timeFilter(register_date)\nGROUP BY 1\nORDER BY 1",
"refId": "A",
"select": [
[
{
"params": [
"id"
],
"type": "column"
},
{
"params": [
"avg"
],
"type": "aggregate"
},
{
"params": [
"id"
],
"type": "alias"
}
]
],
"table": "users",
"timeColumn": "register_date",
"timeColumnType": "timestamp",
"where": [
{
"name": "$__timeFilter",
"params": [],
"type": "macro"
}
]
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "User Registrations",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 11,
"w": 24,
"x": 0,
"y": 9
},
"hiddenSeries": false,
"id": 2,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"format": "time_series",
"group": [
{
"params": [
"$__interval",
"0"
],
"type": "time"
}
],
"metricColumn": "type",
"rawQuery": false,
"rawSql": "SELECT\n $__timeGroupAlias(date,$__interval,0),\n type AS metric,\n count(\"user\") AS \"id\"\nFROM events\nWHERE\n $__timeFilter(date)\nGROUP BY 1,2\nORDER BY 1,2",
"refId": "A",
"select": [
[
{
"params": [
"\"user\""
],
"type": "column"
},
{
"params": [
"count"
],
"type": "aggregate"
},
{
"params": [
"id"
],
"type": "alias"
}
]
],
"table": "events",
"timeColumn": "date",
"timeColumnType": "timestamp",
"where": [
{
"name": "$__timeFilter",
"params": [],
"type": "macro"
}
]
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Event Activity",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": true,
"alignLevel": null
}
}
],
"refresh": false,
"schemaVersion": 21,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
]
},
"timezone": "",
"title": "Wowstash Ops",
"uid": "zvTlfCbGz",
"version": 1
}

+ 7
- 7
wowstash/blueprints/auth/routes.py View File

@ -7,7 +7,7 @@ from wowstash.forms import Register, Login, Delete
from wowstash.models import User
from wowstash.factory import db, bcrypt
from wowstash.library.docker import docker
from wowstash.library.elasticsearch import send_es
from wowstash.library.helpers import capture_event
@auth_bp.route("/register", methods=["GET", "POST"])
@ -33,7 +33,7 @@ def register():
db.session.commit()
# Capture event, login user and redirect to wallet page
send_es({'type': 'register', 'user': user.email})
capture_event(user.id, 'register')
login_user(user)
return redirect(url_for('wallet.setup'))
@ -63,7 +63,7 @@ def login():
return redirect(url_for('auth.login'))
# Capture event, login user, and redirect to wallet page
send_es({'type': 'login', 'user': user.email})
capture_event(user.id, 'login')
login_user(user)
return redirect(url_for('wallet.dashboard'))
@ -73,9 +73,9 @@ def login():
def logout():
if current_user.is_authenticated:
docker.stop_container(current_user.wallet_container)
send_es({'type': 'stop_container', 'user': current_user.email})
capture_event(current_user.id, 'stop_container')
current_user.clear_wallet_data()
send_es({'type': 'logout', 'user': current_user.email})
capture_event(current_user.id, 'logout')
logout_user()
return redirect(url_for('meta.index'))
@ -85,10 +85,10 @@ def delete():
form = Delete()
if form.validate_on_submit():
docker.stop_container(current_user.wallet_container)
send_es({'type': 'stop_container', 'user': current_user.email})
capture_event(current_user.id, 'stop_container')
sleep(1)
docker.delete_wallet_data(current_user.id)
send_es({'type': 'delete_wallet', 'user': current_user.email})
capture_event(current_user.id, 'delete_wallet')
current_user.clear_wallet_data(reset_password=True, reset_wallet=True)
flash('Successfully deleted wallet data')
return redirect(url_for('wallet.setup'))

+ 10
- 7
wowstash/blueprints/wallet/routes.py View File

@ -10,7 +10,7 @@ from socket import socket
from datetime import datetime
from wowstash.blueprints.wallet import wallet_bp
from wowstash.library.docker import docker
from wowstash.library.elasticsearch import send_es
from wowstash.library.helpers import capture_event
from wowstash.library.jsonrpc import Wallet, to_atomic
from wowstash.library.cache import cache
from wowstash.forms import Send, Delete, Restore
@ -29,6 +29,7 @@ def setup():
if restore_form.validate_on_submit():
c = docker.create_wallet(current_user.id, restore_form.seed.data)
cache.store_data(f'init_wallet_{current_user.id}', 30, c)
capture_event(current_user.id, 'restore_wallet')
current_user.wallet_created = True
db.session.commit()
return redirect(url_for('wallet.loading'))
@ -81,7 +82,7 @@ def dashboard():
seed = wallet.seed()
spend_key = wallet.spend_key()
view_key = wallet.view_key()
send_es({'type': 'load_dashboard', 'user': current_user.email})
capture_event(current_user.id, 'load_dashboard')
return render_template(
'wallet/dashboard.html',
transfers=all_transfers,
@ -115,6 +116,7 @@ def connect():
current_user.wallet_container = wallet
current_user.wallet_start = datetime.utcnow()
db.session.commit()
capture_event(current_user.id, 'start_wallet')
data = {
'result': 'success',
'message': 'Wallet has been connected'
@ -133,6 +135,7 @@ def create():
if current_user.wallet_created is False:
c = docker.create_wallet(current_user.id)
cache.store_data(f'init_wallet_{current_user.id}', 30, c)
capture_event(current_user.id, 'create_wallet')
current_user.wallet_created = True
db.session.commit()
return redirect(url_for('wallet.loading'))
@ -173,13 +176,13 @@ def send():
# Check if Wownero wallet is available
if wallet.connected is False:
flash('Wallet RPC interface is unavailable at this time. Try again later.')
send_es({'type': 'tx_fail_rpc_unavailable', 'user': user.email})
capture_event(user.id, 'tx_fail_rpc_unavailable')
return redirect(redirect_url)
# Quick n dirty check to see if address is WOW
if len(address) not in [97, 108]:
flash('Invalid Wownero address provided.')
send_es({'type': 'tx_fail_address_invalid', 'user': user.email})
capture_event(user.id, 'tx_fail_address_invalid')
return redirect(redirect_url)
# Check if we're sweeping or not
@ -191,7 +194,7 @@ def send():
amount = to_atomic(Decimal(send_form.amount.data))
except:
flash('Invalid Wownero amount specified.')
send_es({'type': 'tx_fail_amount_invalid', 'user': user.email})
capture_event(user.id, 'tx_fail_amount_invalid')
return redirect(redirect_url)
# Send transfer
@ -202,10 +205,10 @@ def send():
msg = tx['message'].capitalize()
msg_lower = tx['message'].replace(' ', '_').lower()
flash(f'There was a problem sending the transaction: {msg}')
send_es({'type': f'tx_fail_{msg_lower}', 'user': user.email})
capture_event(user.id, f'tx_fail_{msg_lower}')
else:
flash('Successfully sent transfer.')
send_es({'type': 'tx_success', 'user': user.email})
capture_event(user.id, 'tx_success')
return redirect(redirect_url)
else:

+ 0
- 1
wowstash/config.example.py View File

@ -37,7 +37,6 @@ DB_PASS = 'zzzzzzzzz'
# Development
TEMPLATES_AUTO_RELOAD = True
ELASTICSEARCH_ENABLED = False
# Social
SOCIAL = {

+ 5
- 0
wowstash/factory.py View File

@ -77,6 +77,11 @@ def create_app():
user.clear_wallet_data()
print(f'Wallet data cleared for user {user.id}')
@app.cli.command('init')
def init():
import wowstash.models
db.create_all()
# Routes/blueprints
from wowstash.blueprints.auth import auth_bp
from wowstash.blueprints.wallet import wallet_bp

+ 0
- 3
wowstash/library/docker.py View File

@ -9,7 +9,6 @@ from wowstash import config
from wowstash.models import User
from wowstash.factory import db
from wowstash.library.jsonrpc import daemon
from wowstash.library.elasticsearch import send_es
class Docker(object):
@ -66,7 +65,6 @@ class Docker(object):
}
}
)
send_es({'type': f'init_wallet', 'user': u.email})
return container.short_id
def start_wallet(self, user_id):
@ -103,7 +101,6 @@ class Docker(object):
}
}
)
send_es({'type': 'start_wallet', 'user': u.email})
return container.short_id
except APIError as e:
if str(e).startswith('409'):

+ 0
- 22
wowstash/library/elasticsearch.py View File

@ -1,22 +0,0 @@
from datetime import datetime
from elasticsearch import Elasticsearch
from wowstash import config
def send_es(data):
if getattr(config, 'ELASTICSEARCH_ENABLED', False):
try:
es = Elasticsearch(
[getattr(config, 'ELASTICSEARCH_HOST', 'localhost')]
)
now = datetime.utcnow()
index_ts = now.strftime('%Y%m%d')
data['datetime'] = now
es.index(
index="{}-{}".format(
getattr(config, 'ELASTICSEARCH_INDEX_NAME', 'wowstash'),
index_ts
), body=data)
except Exception as e:
print('Could not capture event in Elasticsearch: ', e)
pass # I don't really care if this logs...

+ 12
- 0
wowstash/library/helpers.py View File

@ -0,0 +1,12 @@
from wowstash.models import Event
from wowstash.factory import db
def capture_event(user_id, event_type):
event = Event(
user=user_id,
type=event_type
)
db.session.add(event)
db.session.commit()
return

+ 12
- 0
wowstash/models.py View File

@ -53,3 +53,15 @@ class User(db.Model):
def __repr__(self):
return self.email
class Event(db.Model):
__tablename__ = 'events'
id = db.Column(db.Integer, primary_key=True)
type = db.Column(db.String(60))
user = db.Column(db.Integer, db.ForeignKey(User.id))
date = db.Column(db.DateTime, server_default=func.now())
def __repr__(self):
return self.id

Loading…
Cancel
Save