make improvements to dashboards and logic of trading script

fix-graf
lza_menace 1 year ago
parent e121cc8920
commit 2c1d398465

29
db.py

@ -14,10 +14,6 @@ db = PostgresqlDatabase(
port=getenv('DB_PORT', 5432) port=getenv('DB_PORT', 5432)
) )
def get_time():
now = datetime.now()
now = now + timedelta(hours=7)
return now
class Ticker(Model): class Ticker(Model):
id = AutoField() id = AutoField()
@ -32,7 +28,7 @@ class Ticker(Model):
spread_btc = DoubleField() spread_btc = DoubleField()
spread_sats = IntegerField() spread_sats = IntegerField()
spread_perc = DoubleField() spread_perc = DoubleField()
date = DateTimeField(default=get_time) date = DateTimeField(default=datetime.utcnow)
class Meta: class Meta:
database = db database = db
@ -41,7 +37,22 @@ class Balance(Model):
total = DoubleField() total = DoubleField()
available = DoubleField() available = DoubleField()
currency = CharField() currency = CharField()
date = DateTimeField(default=get_time) date = DateTimeField(default=datetime.utcnow)
class Meta:
database = db
class Portfolio(Model):
usd = DoubleField()
btc = DoubleField()
date = DateTimeField(default=datetime.utcnow)
class Meta:
database = db
class BitcoinPrice(Model):
price_usd = DoubleField()
date = DateTimeField(default=datetime.utcnow)
class Meta: class Meta:
database = db database = db
@ -55,7 +66,7 @@ class Order(Model):
uuid = TextField(null=True) uuid = TextField(null=True)
active = BooleanField(default=True) active = BooleanField(default=True)
cancelled = BooleanField(default=False) cancelled = BooleanField(default=False)
date = DateTimeField(default=get_time) date = DateTimeField(default=datetime.utcnow)
class Meta: class Meta:
database = db database = db
@ -63,9 +74,9 @@ class Order(Model):
class Earning(Model): class Earning(Model):
trade_pair = CharField() trade_pair = CharField()
quantity = DoubleField() quantity = DoubleField()
date = DateTimeField(default=get_time) date = DateTimeField(default=datetime.utcnow)
class Meta: class Meta:
database = db database = db
db.create_tables([Ticker, Balance, Order, Earning]) db.create_tables([Ticker, Balance, Order, Earning, BitcoinPrice, Portfolio])

@ -20,11 +20,14 @@ services:
GF_LOG_LEVEL: "debug" GF_LOG_LEVEL: "debug"
volumes: volumes:
- grafana:/var/lib/grafana - grafana:/var/lib/grafana
- ./grafana/grafana.ini:/etc/grafana/grafana.ini:ro
- ./grafana/provisioning:/etc/grafana/provisioning:ro
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
postgres: postgres:
image: postgres:9.6.15-alpine image: postgres:9.6.15-alpine
container_name: trader_postgres container_name: trader_postgres
ports: ports:
- 5432:5432 - 127.0.0.1:5432:5432
environment: environment:
POSTGRES_PASSWORD: ${DB_PASS} POSTGRES_PASSWORD: ${DB_PASS}
POSTGRES_USER: ${DB_USER} POSTGRES_USER: ${DB_USER}

@ -1,8 +1,8 @@
GRAFANA_PASSWORD=xxxx GRAFANA_PASSWORD=xxxx
GRAFANA_URL=localhost:3000 GRAFANA_URL=localhost:3000
DB_PASS=yyyyy DB_PASS=trader
DB_USER=trader DB_USER=trader
DB_NAME=trader_xmr DB_NAME=trader
DB_HOST=127.0.0.1 DB_HOST=127.0.0.1
TO_USER=zzzzzzzzz TO_USER=zzzzzzzzz
TO_PASS=vvvvvvvvv TO_PASS=vvvvvvvvv

@ -1,13 +0,0 @@
apiVersion: 1
providers:
- name: 'default'
orgId: 1
folder: ''
type: file
disableDeletion: true
editable: true
updateIntervalSeconds: 60
allowUiUpdates: true
options:
path: /var/lib/grafana/dashboards

@ -21,7 +21,7 @@
"panels": [ "panels": [
{ {
"content": "\n# Orders\n\nOrders placed on TradeOgre\n\n\n\n", "content": "\n# Orders\n\nOrders placed on TradeOgre\n\n\n\n",
"datasource": null, "datasource": "postgres",
"gridPos": { "gridPos": {
"h": 3, "h": 3,
"w": 24, "w": 24,
@ -72,7 +72,7 @@
"cacheTimeout": null, "cacheTimeout": null,
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": null, "datasource": "postgres",
"decimals": 8, "decimals": 8,
"fill": 1, "fill": 1,
"fillGradient": 0, "fillGradient": 0,
@ -360,7 +360,7 @@
"rgba(237, 129, 40, 0.89)", "rgba(237, 129, 40, 0.89)",
"#d44a3a" "#d44a3a"
], ],
"datasource": null, "datasource": "postgres",
"format": "none", "format": "none",
"gauge": { "gauge": {
"maxValue": 100, "maxValue": 100,
@ -506,7 +506,7 @@
"rgba(237, 129, 40, 0.89)", "rgba(237, 129, 40, 0.89)",
"#d44a3a" "#d44a3a"
], ],
"datasource": null, "datasource": "postgres",
"format": "none", "format": "none",
"gauge": { "gauge": {
"maxValue": 100, "maxValue": 100,
@ -643,7 +643,7 @@
}, },
{ {
"content": "\n# Balances\n\nBalances and trade info of your account.\n\n\n\n", "content": "\n# Balances\n\nBalances and trade info of your account.\n\n\n\n",
"datasource": null, "datasource": "postgres",
"gridPos": { "gridPos": {
"h": 3, "h": 3,
"w": 24, "w": 24,
@ -663,7 +663,7 @@
"bars": false, "bars": false,
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": null, "datasource": "postgres",
"decimals": 8, "decimals": 8,
"fill": 1, "fill": 1,
"fillGradient": 0, "fillGradient": 0,
@ -884,7 +884,7 @@
"bars": false, "bars": false,
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": null, "datasource": "postgres",
"decimals": 8, "decimals": 8,
"fill": 1, "fill": 1,
"fillGradient": 0, "fillGradient": 0,
@ -1104,7 +1104,7 @@
}, },
{ {
"content": "\n# Markets\n\nGeneral stats and metrics of the $currency markets\n\n\n\n", "content": "\n# Markets\n\nGeneral stats and metrics of the $currency markets\n\n\n\n",
"datasource": null, "datasource": "postgres",
"gridPos": { "gridPos": {
"h": 3, "h": 3,
"w": 24, "w": 24,
@ -1124,7 +1124,7 @@
"bars": false, "bars": false,
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": null, "datasource": "postgres",
"decimals": 8, "decimals": 8,
"fill": 1, "fill": 1,
"fillGradient": 0, "fillGradient": 0,
@ -1260,7 +1260,7 @@
"bars": false, "bars": false,
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": null, "datasource": "postgres",
"decimals": 0, "decimals": 0,
"fill": 1, "fill": 1,
"fillGradient": 0, "fillGradient": 0,
@ -1394,7 +1394,7 @@
"bars": false, "bars": false,
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": null, "datasource": "postgres",
"decimals": 8, "decimals": 8,
"fill": 1, "fill": 1,
"fillGradient": 0, "fillGradient": 0,
@ -1547,7 +1547,7 @@
"bars": false, "bars": false,
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": null, "datasource": "postgres",
"fill": 1, "fill": 1,
"fillGradient": 0, "fillGradient": 0,
"gridPos": { "gridPos": {
@ -1678,7 +1678,7 @@
"bars": false, "bars": false,
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": null, "datasource": "postgres",
"decimals": null, "decimals": null,
"fill": 1, "fill": 1,
"fillGradient": 0, "fillGradient": 0,

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

@ -0,0 +1,29 @@
[analytics]
reporting_enabled = false
check_for_updates = false
[auth]
disable_login_form = true
[auth.anonymous]
enabled = true
org_role = Admin
[dashboards]
min_refresh_interval = 1m
default_home_dashboard_path = /var/lib/grafana/dashboards/xmr_trading_desk.json
[paths]
provisioning = /etc/grafana/provisioning
[server]
root_url = https://127.0.0.1
enable_gzip = true
read_timeout = 2m
[snapshots]
external_enabled = false
[security]
admin_user = admin
admin_password = admin

@ -1,9 +0,0 @@
apiVersion: 1
datasources:
- name: PostgreSQL
type: postgresql
url: http://prometheus:9090
access: proxy
isDefault: true
timeInterval: 10s

@ -0,0 +1,12 @@
apiVersion: 1
providers:
- name: 'fs'
orgId: 1
folder: ''
type: 'file'
updateIntervalSeconds: 30
allowUiUpdates: true
options:
path: '/var/lib/grafana/dashboards'
foldersFromFilesStructure: true

@ -0,0 +1,17 @@
apiVersion: 1
datasources:
- name: postgres
type: postgres
url: trader_postgres:5432
database: trader
user: trader
secureJsonData:
password: "trader"
jsonData:
sslmode: "disable"
maxOpenConns: 0
maxIdleConns: 2
connMaxLifetime: 14400
postgresVersion: 906
timescaledb: false

File diff suppressed because it is too large Load Diff

@ -8,7 +8,7 @@ from datetime import datetime
from decimal import Decimal from decimal import Decimal
from random import uniform from random import uniform
from time import sleep from time import sleep
from db import Ticker, Balance, Order, Earning from db import *
from tradeogre import TradeOgre from tradeogre import TradeOgre
@ -22,16 +22,17 @@ class Trader(TradeOgre):
load_dotenv('.env') load_dotenv('.env')
satoshi = .00000001 satoshi = .00000001
satoshi_multiplier = getenv('SATOSHI_MULTIPLIER', 10)
base_currency = getenv('BASE_CURRENCY', 'BTC') base_currency = getenv('BASE_CURRENCY', 'BTC')
trade_currency = getenv('TRADE_CURRENCY', 'WOW') trade_currency = getenv('TRADE_CURRENCY', 'XMR')
trade_pair = f'{base_currency}-{trade_currency}' trade_pair = f'{base_currency}-{trade_currency}'
trade_amount = float(getenv('TRADE_AMOUNT', 200)) trade_amount = float(getenv('TRADE_AMOUNT', .5))
spread_target = float(getenv('SPREAD_TARGET', 4)) spread_target = float(getenv('SPREAD_TARGET', 10))
amount_multiplier = float(getenv('AMOUNT_MULTIPLIER', 1.2)) amount_multiplier = float(getenv('AMOUNT_MULTIPLIER', 1.2))
active_order_limit = float(getenv('ACTIVE_ORDER_LIMIT', 10)) active_order_limit = float(getenv('ACTIVE_ORDER_LIMIT', 10))
def get_market_data(self): def get_market_data(self):
logging.info(f'[MARKET] Getting market data for trade pair {self.trade_pair}') logging.info(f'Getting market data for trade pair {self.trade_pair}')
res = self.get_trade_pair(self.trade_pair) res = self.get_trade_pair(self.trade_pair)
spread_btc = Decimal(res['ask']) - Decimal(res['bid']) spread_btc = Decimal(res['ask']) - Decimal(res['bid'])
spread_sats = float(spread_btc / Decimal(self.satoshi)) spread_sats = float(spread_btc / Decimal(self.satoshi))
@ -58,38 +59,40 @@ class Trader(TradeOgre):
spread_perc=res['spread_perc'] spread_perc=res['spread_perc']
) )
t.save() t.save()
logging.info(f'[MARKET] Stored market data as ID {t.id}') logging.info(f'Stored market data as ID {t.id}')
return t return t
def store_balance(self, currency): def store_balance(self, currency, dst):
logging.info(f'[BALANCE] Storing balance for currency {currency}') logging.info(f'Storing balance for currency {currency}')
res = self.get_balance(currency) res = self.get_balance(currency)
logging.debug(res) logging.debug(res)
b = Balance( b = Balance(
currency=currency, currency=currency,
total=res['balance'], total=res['balance'],
available=res['available'] available=res['available'],
date=dst
) )
b.save() b.save()
logging.info(f'[BALANCE] Stored market data as ID {b.id}') logging.info(f'Stored market data as ID {b.id}')
return b return b
def store_balances(self): def store_balances(self):
now = datetime.utcnow()
for cur in self.base_currency, self.trade_currency: for cur in self.base_currency, self.trade_currency:
self.store_balance(cur) self.store_balance(cur, now)
sleep(3) sleep(3)
def get_active_orders(self): def get_active_orders(self):
logging.info('[ORDERS] Getting active orders in local database') logging.info('Getting active orders in local database')
orders = Order.select().where(Order.active==True, Order.trade_pair==self.trade_pair) orders = Order.select().where(Order.active == True, Order.trade_pair == self.trade_pair)
logging.debug(f'Found {len(orders)} in database') logging.debug(f'Found {len(orders)} in database')
return orders return orders
def reconcile_orders(self): def reconcile_orders(self):
logging.info('[ORDERS] Reconciling orders on TradeOgre with local database') logging.info('Reconciling orders on TradeOgre with local database')
to_orders = self.get_orders(self.trade_pair) to_orders = self.get_orders(self.trade_pair)
for order in to_orders: for order in to_orders:
if not Order.filter(Order.uuid==order['uuid']): if not Order.filter(Order.uuid == order['uuid']):
o = Order( o = Order(
trade_pair=order['market'], trade_pair=order['market'],
trade_type='manual', trade_type='manual',
@ -100,10 +103,10 @@ class Trader(TradeOgre):
date=datetime.utcfromtimestamp(order['date']) date=datetime.utcfromtimestamp(order['date'])
) )
o.save() o.save()
logging.info(f'[ORDERS] Saved order {order["uuid"]} to the database') logging.info(f'Saved order {order["uuid"]} to the database')
def update_orders(self): def update_orders(self):
logging.info('[ORDERS] Updating orders in local database against TradeOgre') logging.info('Updating orders in local database against TradeOgre')
for order in self.get_active_orders(): for order in self.get_active_orders():
logging.info(f'Checking order {order.uuid}') logging.info(f'Checking order {order.uuid}')
o = self.get_order(order.uuid) o = self.get_order(order.uuid)
@ -116,7 +119,7 @@ class Trader(TradeOgre):
def calculate_earnings(self): def calculate_earnings(self):
orders = {'buy': [], 'sell': []} orders = {'buy': [], 'sell': []}
completed_orders = Order.select().where(Order.cancelled==False, Order.active==False) completed_orders = Order.select().where(Order.cancelled == False, Order.active == False)
for order in completed_orders: for order in completed_orders:
if order.buy: if order.buy:
type = 'buy' type = 'buy'
@ -133,23 +136,42 @@ class Trader(TradeOgre):
quantity=orders['total_earnings'] quantity=orders['total_earnings']
) )
e.save() e.save()
def update_portfolio(self):
logging.info('Updating portfolio')
base_balance = Balance.select().where(Balance.currency == self.base_currency).order_by(Balance.date.desc()).first()
trade_balance = Balance.select().where(Balance.currency == self.trade_currency).order_by(Balance.date.desc()).first()
trade_ticker = Ticker.select().where(Ticker.trade_pair == self.trade_pair).order_by(Ticker.date.desc()).first()
btc_price = BitcoinPrice.select().order_by(BitcoinPrice.date.desc()).first()
portfolio_btc = (trade_balance.total * trade_ticker.current_price) + base_balance.total
portfolio_usd = portfolio_btc * btc_price.price_usd
p = Portfolio(
usd=float(portfolio_usd),
btc=float(portfolio_btc)
)
p.save()
def update_bitcoin_price(self):
logging.info('Updating Bitcoin price')
btc = BitcoinPrice(price_usd=float(self.get_bitcoin_price()))
btc.save()
def start_market_maker(self): def start_market_maker(self):
logging.info('[MARKET MAKER] Starting market maker') logging.info('Starting market maker')
latest = Ticker.select().where(Ticker.trade_pair==self.trade_pair).order_by(Ticker.date.desc()).get() latest = Ticker.select().where(Ticker.trade_pair == self.trade_pair).order_by(Ticker.date.desc()).get()
trade_type = 'market_maker' trade_type = 'market_maker'
active_orders = len(self.get_active_orders()) active_orders = len(self.get_active_orders())
if active_orders > self.active_order_limit: if active_orders >= self.active_order_limit:
logging.info(f'[MARKET MAKER] Too many active orders in place ({active_orders}). Skipping.') logging.info(f'Too many active orders in place ({active_orders}). Skipping.')
return False return False
if latest.spread_sats >= self.spread_target: if latest.spread_sats >= self.spread_target:
bid_amount = uniform(self.trade_amount, self.trade_amount * self.amount_multiplier) bid_amount = uniform(self.trade_amount, self.trade_amount * self.amount_multiplier)
ask_amount = uniform(self.trade_amount, self.trade_amount * self.amount_multiplier) ask_amount = uniform(self.trade_amount, self.trade_amount * self.amount_multiplier)
bid_price = '{:.8f}'.format(latest.bid + self.satoshi) bid_price = '{:.8f}'.format(latest.bid + self.satoshi * self.satoshi_multiplier)
ask_price = '{:.8f}'.format(latest.ask - self.satoshi) ask_price = '{:.8f}'.format(latest.ask - self.satoshi * self.satoshi_multiplier)
logging.info(f'[MARKET MAKER] Submitting buy order for {bid_amount} at {bid_price} {self.trade_pair}') logging.info(f'Submitting buy order for {bid_amount} at {bid_price} {self.trade_pair}')
buy = self.submit_order('buy', self.trade_pair, bid_amount, bid_price) buy = self.submit_order('buy', self.trade_pair, bid_amount, bid_price)
logging.debug(buy) logging.debug(buy)
if 'uuid' in buy: if 'uuid' in buy:
@ -169,10 +191,10 @@ class Trader(TradeOgre):
active=active active=active
) )
_bo.save() _bo.save()
logging.info(f'[MARKET MAKER] Stored buy order as ID {_bo.id}') logging.info(f'Stored buy order as ID {_bo.id}')
sleep(3) sleep(3)
logging.info(f'[MARKET MAKER] Submitting sell order for {ask_amount} at {ask_price} {self.trade_pair}') logging.info(f'Submitting sell order for {ask_amount} at {ask_price} {self.trade_pair}')
sell = self.submit_order('sell', self.trade_pair, ask_amount, ask_price) sell = self.submit_order('sell', self.trade_pair, ask_amount, ask_price)
logging.debug(sell) logging.debug(sell)
if 'uuid' in sell: if 'uuid' in sell:
@ -192,13 +214,13 @@ class Trader(TradeOgre):
active=active active=active
) )
_so.save() _so.save()
logging.info(f'[MARKET MAKER] Stored sell order as ID {_so.id}') logging.info(f'Stored sell order as ID {_so.id}')
else: else:
logging.info(sell) logging.info(sell)
else: else:
logging.info(buy) logging.info(buy)
else: else:
logging.info(f'[MARKET MAKER] Not enough bid-ask spread ({latest.spread_sats} sats). Skipping market maker.') logging.info(f'Not enough bid-ask spread ({latest.spread_sats} sats). Skipping market maker.')
if __name__ == '__main__': if __name__ == '__main__':
@ -208,12 +230,14 @@ if __name__ == '__main__':
parser.add_argument('--update-orders', action='store_true', help='Update status of orders') parser.add_argument('--update-orders', action='store_true', help='Update status of orders')
parser.add_argument('--market-data', action='store_true', help='Update market data') parser.add_argument('--market-data', action='store_true', help='Update market data')
parser.add_argument('--calculate-earnings', action='store_true', help='Calculate earnings from all trades') parser.add_argument('--calculate-earnings', action='store_true', help='Calculate earnings from all trades')
parser.add_argument('--update-bitcoin-price', action='store_true', help='Update Bitcoin price (USD)')
parser.add_argument('--run', action='store_true', help='Run continuously') parser.add_argument('--run', action='store_true', help='Run continuously')
args = parser.parse_args() args = parser.parse_args()
t = Trader() t = Trader()
orders_counter = 0 orders_counter = 0
balances_counter = 0 balances_counter = 0
bitcoin_counter = 0
if not args.run and args.update_orders: if not args.run and args.update_orders:
t.reconcile_orders() t.reconcile_orders()
@ -224,12 +248,16 @@ if __name__ == '__main__':
if not args.run and args.balances: if not args.run and args.balances:
t.store_balances() t.store_balances()
t.update_portfolio()
if not args.run and args.market_data: if not args.run and args.market_data:
t.store_market_data() t.store_market_data()
if not args.run and args.calculate_earnings: if not args.run and args.calculate_earnings:
t.calculate_earnings() t.calculate_earnings()
if not args.run and args.update_bitcoin_price:
t.update_bitcoin_price()
if args.run: if args.run:
while True: while True:
@ -237,6 +265,15 @@ if __name__ == '__main__':
t.store_market_data() t.store_market_data()
except Exception as e: except Exception as e:
logging.info('[ERROR] Unable to store market data!', e) logging.info('[ERROR] Unable to store market data!', e)
# update bitcoin price every 10 runs
if args.update_bitcoin_price:
if bitcoin_counter == 10:
try:
t.update_bitcoin_price()
bitcoin_counter = 0
except Exception as e:
logging.info('Unable to update Bitcoin price!', e)
# update orders every 4 runs # update orders every 4 runs
if args.update_orders: if args.update_orders:
@ -254,6 +291,7 @@ if __name__ == '__main__':
if balances_counter == 2: if balances_counter == 2:
try: try:
t.store_balances() t.store_balances()
t.update_portfolio()
logging.info('[BALANCE] Resetting balances counter') logging.info('[BALANCE] Resetting balances counter')
balances_counter = 0 balances_counter = 0
except Exception as e: except Exception as e:
@ -268,8 +306,9 @@ if __name__ == '__main__':
orders_counter += 1 orders_counter += 1
balances_counter += 1 balances_counter += 1
bitcoin_counter += 1
# sleep # sleep
sleep_time = int(getenv('SLEEP_TIME', 20)) sleep_time = int(getenv('SLEEP_TIME', 1))
logging.info(f'Sleeping for {sleep_time} seconds') logging.info(f'Sleeping for {sleep_time} seconds')
sleep(sleep_time) sleep(sleep_time)

@ -48,3 +48,18 @@ class TradeOgre(object):
route = f'/account/orders' route = f'/account/orders'
data = {'market': pair} data = {'market': pair}
return self.req(route, 'post', data) return self.req(route, 'post', data)
def get_bitcoin_price(self):
url = 'https://api.coingecko.com/api/v3/coins/bitcoin'
headers = {'accept': 'application/json'}
data = {
'localization': False,
'tickers': False,
'market_data': True,
'community_data': False,
'developer_data': False,
'sparkline': False
}
r = requests_get(url, headers=headers, data=data)
return r.json()['market_data']['current_price']['usd']