Compare commits

..

1 Commits

Author SHA1 Message Date
Sander ae11e3d884 temp
3 years ago

@ -1,10 +1,15 @@
# wowlet-backend # feather-ws
Back-end websocket server for wowlet. Back-end websocket server for Feather wallet.
- Quart web framework, Py3 asyncio - Quart web framework, Py3 asyncio
- Redis - Redis
## Coins supported
- Monero
- Wownero
See also the environment variables `WOWLET_COIN_NAME`, `WOWLET_COIN_SYMBOL`, etc. in `settings.py`. See also the environment variables `WOWLET_COIN_NAME`, `WOWLET_COIN_SYMBOL`, etc. in `settings.py`.
## Tasks ## Tasks
@ -17,10 +22,9 @@ This websocket server has several scheduled recurring tasks:
- Fetch funding proposals - Fetch funding proposals
- Check status of RPC nodes (`data/nodes.json`) - Check status of RPC nodes (`data/nodes.json`)
When Wowlet (the wallet application) starts up, it When Feather wallet starts up, it will connect to
will connect to this websocket server and receive this websocket server and receive the information
the information listed above which is necessary listed above which is necessary for normal operation.
for normal operation.
See `wowlet_backend.tasks.*` for the various tasks. See `wowlet_backend.tasks.*` for the various tasks.

@ -2,24 +2,17 @@
"wow": { "wow": {
"mainnet": { "mainnet": {
"tor": [ "tor": [
"v2admi6gbeprxnk6i2oscizhgy4v5ixu6iezkhj5udiwbfjjs2w7dnid.onion:34568", "wowbuxx535x4exuexja2xfezpwcyznxkofui4ndjiectj4yuh2xheiid.onion:34568"
"awbibkoaa67jhuaqes4n2243gd6vtidjzqj2djukrubp2eudrmxr5mid.onion:34568",
"7ftpbpp6rbgqi5kjmhyin46essnh3eqb3m3rhfi7r2fr33iwkeuer3yd.onion:34568",
"j7rf2jcccizcp47y5moehguyuqdpg4lusk642sw4nayuruitqaqbc7ad.onion:34568",
"aje53o5z5twne5q2ljw44zkahhsuhjtwaxuburxddbf7n4pfsj4rj6qd.onion:34568",
"nepc4lxndsooj2akn7ofrj3ooqc25242obchcag6tw3f2mxrms2uuvyd.onion:34568",
"666l2ajxqjgj5lskvbokvworjysgvqag4oitokjuy7wz6juisul4jqad.onion:34568",
"ty7ppqozzodz75audgvkprekiiqsovbyrkfdjwadrkbe3etyzloatxad.onion:34568",
"ewynwpnprbgqllv2syn3drjdrqkw7ehoeg73znelm6mevvmpddexsoqd.onion:34568",
"mqkiqwmhhpqtzlrf26stv7jvtaudbyzkbg3lttkmvvvauzgtxm62tgyd.onion:34568",
"zao3w6isidntdbnyee5ufs7fyzmv7wzchpw32i3uo5eldjwmo4bxg2qd.onion:34568"
], ],
"clearnet": [ "clearnet": [
"spippolatori.it:34568", "global.wownodes.com:34568",
"node2.monerodevs.org:34568", "super.fast.node.xmr.pm:34568",
"node3.monerodevs.org:34568", "node.wownero.club:34568",
"node.wowne.ro:34568", "eu-west-1.wow.xmr.pm:34568",
"node.suchwow.xyz:34568" "eu-west-2.wow.xmr.pm:34568",
"eu-west-3.wow.xmr.pm:34568",
"eu-west-4.wow.xmr.pm:34568",
"wow.pwned.systems:34568"
] ]
}, },
"stagenet": { "stagenet": {

@ -1,5 +1,5 @@
quart quart
aioredis>=2.0.0 aioredis
aiohttp aiohttp
aiofiles aiofiles
quart_session quart_session
@ -8,6 +8,5 @@ aiohttp_socks
python-dateutil python-dateutil
psutil psutil
psutil psutil
pillow pillow-simd
python-magic python-magic
asyncio-multisubscriber-queue==0.3.1

@ -14,21 +14,21 @@ DEBUG = bool_env(os.environ.get("WOWLET_DEBUG", False))
HOST = os.environ.get("WOWLET_HOST", "127.0.0.1") HOST = os.environ.get("WOWLET_HOST", "127.0.0.1")
PORT = int(os.environ.get("WOWLET_PORT", 1337)) PORT = int(os.environ.get("WOWLET_PORT", 1337))
REDIS_HOST = os.environ.get("WOWLET_REDIS_HOST", "localhost") REDIS_ADDRESS = os.environ.get("WOWLET_REDIS_ADDRESS", "redis://localhost")
REDIS_PORT = os.environ.get("WOWLET_REDIS_PORT", 6379)
REDIS_PASSWORD = os.environ.get("WOWLET_REDIS_PASSWORD") REDIS_PASSWORD = os.environ.get("WOWLET_REDIS_PASSWORD")
COIN_NAME = os.environ.get("WOWLET_COIN_NAME", "wownero").lower() # as per coingecko COIN_NAME = os.environ.get("WOWLET_COIN_NAME", "monero").lower() # as per coingecko
COIN_SYMBOL = os.environ.get("WOWLET_COIN_SYMBOL", "wow").lower() # as per coingecko COIN_SYMBOL = os.environ.get("WOWLET_COIN_SYMBOL", "xmr").lower() # as per coingecko
COIN_GENESIS_DATE = os.environ.get("WOWLET_COIN_GENESIS_DATE", "20180401") COIN_GENESIS_DATE = os.environ.get("WOWLET_COIN_GENESIS_DATE", "20140418")
COIN_MODE = os.environ.get("WOWLET_COIN_MODE", "mainnet").lower() COIN_MODE = os.environ.get("WOWLET_COIN_MODE", "mainnet").lower()
TOR_SOCKS_PROXY = os.environ.get("WOWLET_TOR_SOCKS_PROXY", "socks5://127.0.0.1:9050") TOR_SOCKS_PROXY = os.environ.get("WOWLET_TOR_SOCKS_PROXY", "socks5://127.0.0.1:9050")
RPC_WOWNERO_HOST = "127.0.0.1"
RPC_WOWNERO_PORT = 34568
# while fetching USD price from coingecko, also include these extra coins: # while fetching USD price from coingecko, also include these extra coins:
CRYPTO_RATES_COINS_EXTRA = { CRYPTO_RATES_COINS_EXTRA = {
"wownero": "wow" "wownero": "wow",
"aeon": "aeon",
"turtlecoin": "trtl",
"haven": "xhv",
"loki": "loki"
} }

@ -7,19 +7,19 @@ import asyncio
from typing import List, Set from typing import List, Set
from datetime import datetime from datetime import datetime
from asyncio_multisubscriber_queue import MultisubscriberQueue
from quart import Quart from quart import Quart
from quart_session import Session from quart_session import Session
import aioredis import aioredis
from wowlet_backend.utils import print_banner from wowlet_backend.utils import current_worker_thread_is_primary, print_banner
import settings import settings
now = datetime.now() now = datetime.now()
app: Quart = None app: Quart = None
cache = None cache = None
user_agents: List[str] = None user_agents: List[str] = None
broadcast = MultisubscriberQueue() connected_websockets: Set[asyncio.Queue] = set()
_is_primary_worker_thread = False
async def _setup_nodes(app: Quart): async def _setup_nodes(app: Quart):
@ -38,16 +38,14 @@ async def _setup_user_agents(app: Quart):
async def _setup_cache(app: Quart): async def _setup_cache(app: Quart):
global cache global cache
# Each coin has it's own Redis DB index; `redis-cli -n $INDEX` # Each coin has it's own Redis DB index; `redis-cli -n $INDEX`
db = {"wow": 0, "xmr": 1, "aeon": 2, "trtl": 3, "msr": 4, "xhv": 5, "loki": 6}[settings.COIN_SYMBOL] db = {"xmr": 0, "wow": 1, "aeon": 2, "trtl": 3, "msr": 4, "xhv": 5, "loki": 6}[settings.COIN_SYMBOL]
data = { data = {
"host": settings.REDIS_HOST, "address": settings.REDIS_ADDRESS,
"port": settings.REDIS_PORT,
"db": db, "db": db,
"password": settings.REDIS_PASSWORD if settings.REDIS_PASSWORD else None "password": settings.REDIS_PASSWORD if settings.REDIS_PASSWORD else None
} }
cache = await aioredis.Redis(**data) cache = await aioredis.create_redis_pool(**data)
app.config['SESSION_TYPE'] = 'redis' app.config['SESSION_TYPE'] = 'redis'
app.config['SESSION_REDIS'] = cache app.config['SESSION_REDIS'] = cache
Session(app) Session(app)
@ -55,11 +53,13 @@ async def _setup_cache(app: Quart):
async def _setup_tasks(app: Quart): async def _setup_tasks(app: Quart):
"""Schedules a series of tasks at an interval.""" """Schedules a series of tasks at an interval."""
if not _is_primary_worker_thread:
return
from wowlet_backend.tasks import ( from wowlet_backend.tasks import (
BlockheightTask, HistoricalPriceTask, FundingProposalsTask, BlockheightTask, HistoricalPriceTask, FundingProposalsTask,
CryptoRatesTask, FiatRatesTask, RedditTask, RPCNodeCheckTask, CryptoRatesTask, FiatRatesTask, RedditTask, RPCNodeCheckTask,
XmrigTask, SuchWowTask, WowletReleasesTask, ForumThreadsTask, XmrigTask, SuchWowTask)
YellWowTask, WownerodTask)
asyncio.create_task(BlockheightTask().start()) asyncio.create_task(BlockheightTask().start())
asyncio.create_task(HistoricalPriceTask().start()) asyncio.create_task(HistoricalPriceTask().start())
@ -69,10 +69,6 @@ async def _setup_tasks(app: Quart):
asyncio.create_task(RPCNodeCheckTask().start()) asyncio.create_task(RPCNodeCheckTask().start())
asyncio.create_task(XmrigTask().start()) asyncio.create_task(XmrigTask().start())
asyncio.create_task(SuchWowTask().start()) asyncio.create_task(SuchWowTask().start())
asyncio.create_task(WowletReleasesTask().start())
asyncio.create_task(ForumThreadsTask().start())
asyncio.create_task(YellWowTask().start())
asyncio.create_task(WownerodTask().start())
if settings.COIN_SYMBOL in ["xmr", "wow"]: if settings.COIN_SYMBOL in ["xmr", "wow"]:
asyncio.create_task(FundingProposalsTask().start()) asyncio.create_task(FundingProposalsTask().start())
@ -102,7 +98,11 @@ def create_app():
@app.before_serving @app.before_serving
async def startup(): async def startup():
print_banner() global _is_primary_worker_thread
_is_primary_worker_thread = current_worker_thread_is_primary()
if _is_primary_worker_thread:
print_banner()
await _setup_cache(app) await _setup_cache(app)
await _setup_nodes(app) await _setup_nodes(app)

@ -9,14 +9,14 @@ import json
from quart import websocket, jsonify, send_from_directory from quart import websocket, jsonify, send_from_directory
import settings import settings
from wowlet_backend.factory import app, broadcast from wowlet_backend.factory import app
from wowlet_backend.wsparse import WebsocketParse from wowlet_backend.wsparse import WebsocketParse
from wowlet_backend.utils import wowlet_data from wowlet_backend.utils import collect_websocket, feather_data
@app.route("/") @app.route("/")
async def root(): async def root():
data = await wowlet_data() data = await feather_data()
return jsonify(data) return jsonify(data)
@ -28,8 +28,9 @@ async def suchwow(name: str):
@app.websocket('/ws') @app.websocket('/ws')
async def ws(): @collect_websocket
data = await wowlet_data() async def ws(queue):
data = await feather_data()
# blast available data on connect # blast available data on connect
for task_key, task_value in data.items(): for task_key, task_value in data.items():
@ -54,8 +55,9 @@ async def ws():
continue continue
async def tx(): async def tx():
async for _data in broadcast.subscribe(): while True:
payload = json.dumps(_data).encode() data = await queue.get()
payload = json.dumps(data).encode()
await websocket.send(payload) await websocket.send(payload)
# bidirectional async rx and tx loops # bidirectional async rx and tx loops

@ -8,7 +8,7 @@ import random
from typing import Union from typing import Union
class WowletTask: class FeatherTask:
""" """
The base class of many recurring tasks for this The base class of many recurring tasks for this
project. This abstracts away some functionality: project. This abstracts away some functionality:
@ -39,7 +39,7 @@ class WowletTask:
self._running = False self._running = False
async def start(self, *args, **kwargs): async def start(self, *args, **kwargs):
from wowlet_backend.factory import app, broadcast from wowlet_backend.factory import app, connected_websockets
if not self._active: if not self._active:
# invalid task # invalid task
return return
@ -94,10 +94,11 @@ class WowletTask:
propagate = False propagate = False
if propagate: if propagate:
await broadcast.put({ for queue in connected_websockets:
"cmd": self._websocket_cmd, await queue.put({
"data": result "cmd": self._websocket_cmd,
}) "data": result
})
# optional: cache the result # optional: cache the result
if self._cache_key and result: if self._cache_key and result:
@ -165,7 +166,3 @@ from wowlet_backend.tasks.reddit import RedditTask
from wowlet_backend.tasks.rpc_nodes import RPCNodeCheckTask from wowlet_backend.tasks.rpc_nodes import RPCNodeCheckTask
from wowlet_backend.tasks.xmrig import XmrigTask from wowlet_backend.tasks.xmrig import XmrigTask
from wowlet_backend.tasks.suchwow import SuchWowTask from wowlet_backend.tasks.suchwow import SuchWowTask
from wowlet_backend.tasks.wowlet import WowletReleasesTask
from wowlet_backend.tasks.forum import ForumThreadsTask
from wowlet_backend.tasks.yellow import YellWowTask
from wowlet_backend.tasks.wownerod import WownerodTask

@ -7,14 +7,12 @@ from typing import Union
from collections import Counter from collections import Counter
from functools import partial from functools import partial
import aiohttp
import settings import settings
from wowlet_backend.utils import httpget, popularity_contest from wowlet_backend.utils import httpget, popularity_contest
from wowlet_backend.tasks import WowletTask from wowlet_backend.tasks import FeatherTask
class BlockheightTask(WowletTask): class BlockheightTask(FeatherTask):
""" """
Fetch latest blockheight using webcrawling. We pick the most popular Fetch latest blockheight using webcrawling. We pick the most popular
height from a list of websites. Arguably this approach has benefits height from a list of websites. Arguably this approach has benefits
@ -30,14 +28,134 @@ class BlockheightTask(WowletTask):
self._websocket_cmd = "blockheights" self._websocket_cmd = "blockheights"
self._fns = {
"xmr": {
"mainnet": [
self._blockchair,
partial(self._onion_explorer, url="https://xmrchain.net/"),
partial(self._onion_explorer, url="https://community.xmr.to/explorer/mainnet/"),
partial(self._onion_explorer, url="https://monero.exan.tech/")
],
"stagenet": [
partial(self._onion_explorer, url="https://stagenet.xmrchain.net/"),
partial(self._onion_explorer, url="https://community.xmr.to/explorer/stagenet/"),
partial(self._onion_explorer, url="https://monero-stagenet.exan.tech/")
]
},
"wow": {
"mainnet": [
partial(self._onion_explorer, url="https://explore.wownero.com/"),
]
},
"aeon": {
"mainnet": [
partial(self._onion_explorer, url="https://aeonblockexplorer.com/"),
],
"stagenet": [
partial(self._onion_explorer, url="http://162.210.173.151:8083/"),
]
},
"trtl": {
"mainnet": [
self._turtlenode,
self._turtlenetwork,
self._l33d4n
]
},
"xhv": {
"mainnet": [
partial(self._onion_explorer, url="https://explorer.havenprotocol.org/")
],
"stagenet": [
partial(self._onion_explorer, url="https://explorer.stagenet.havenprotocol.org/page/1")
]
},
"loki": {
"mainnet": [
partial(self._onion_explorer, url="https://lokiblocks.com/")
],
"testnet": [
partial(self._onion_explorer, url="https://lokitestnet.com/")
]
}
}
async def task(self) -> Union[dict, None]: async def task(self) -> Union[dict, None]:
from wowlet_backend.factory import app from wowlet_backend.factory import app
timeout = aiohttp.ClientTimeout(total=3) coin_network_types = ["mainnet", "stagenet", "testnet"]
data = {t: 0 for t in coin_network_types}
for coin_network_type in coin_network_types:
if coin_network_type not in self._fns[settings.COIN_SYMBOL]:
continue
heights = []
for fn in self._fns[settings.COIN_SYMBOL][coin_network_type]:
fn_name = fn.func.__name__ if isinstance(fn, partial) else fn.__name__
try:
result = await fn()
heights.append(result)
except Exception as ex:
app.logger.error(f"blockheight fetch failed from {fn_name}(): {ex}")
continue
if heights:
data[coin_network_type] = popularity_contest(heights)
if data["mainnet"] == 0: # only care about mainnet
app.logger.error(f"Failed to parse latest blockheight!")
return
return data
async def _blockchair(self) -> int:
re_blockheight = r"<a href=\".*\">(\d+)</a>"
url = "https://blockchair.com/monero"
content = await httpget(url, json=False, raise_for_status=True)
height = re.findall(re_blockheight, content)
height = max(map(int, height))
return height
async def _wownero(self) -> int:
url = "https://explore.wownero.com/"
return await BlockheightTask._onion_explorer(url)
async def _turtlenode(self) -> int:
url = "https://public.turtlenode.net/info"
blob = await httpget(url, json=True, raise_for_status=True)
height = int(blob.get("height", 0))
if height <= 0:
raise Exception("bad height")
return height
async def _turtlenetwork(self) -> int:
url = "https://tnnode2.turtlenetwork.eu/blocks/height"
blob = await httpget(url, json=True, raise_for_status=True)
height = int(blob.get("height", 0))
if height <= 0:
raise Exception("bad height")
return height
async def _l33d4n(self):
url = "https://blockapi.turtlepay.io/block/header/top"
blob = await httpget(url, json=True, raise_for_status=True)
height = int(blob.get("height", 0))
if height <= 0:
raise Exception("bad height")
return height
async with aiohttp.ClientSession(timeout=timeout) as session: @staticmethod
async with session.get(f'http://{settings.RPC_WOWNERO_HOST}:{settings.RPC_WOWNERO_PORT}/get_height') as resp: async def _onion_explorer(url):
blob = await resp.json() """
Pages that are based on:
https://github.com/moneroexamples/onion-monero-blockchain-explorer
"""
re_blockheight = r"block\/(\d+)\"\>"
content = await httpget(url, json=False)
return { height = re.findall(re_blockheight, content)
'mainnet': blob['height'] height = max(map(int, height))
} return height

@ -1,32 +0,0 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2020, The Monero Project.
# Copyright (c) 2020, dsc@xmr.pm
from bs4 import BeautifulSoup
from typing import List
from dateutil.parser import parse
import settings
from wowlet_backend.utils import httpget
from wowlet_backend.tasks import WowletTask
class ForumThreadsTask(WowletTask):
"""Fetch recent forum threads."""
def __init__(self, interval: int = 300):
from wowlet_backend.factory import app
super(ForumThreadsTask, self).__init__(interval)
self._cache_key = "forum"
self._cache_expiry = self.interval * 10
# url
self._http_endpoint = "https://forum.wownero.com/latest.json"
self._websocket_cmd = "forum"
async def task(self):
from wowlet_backend.factory import app
blob = await httpget(self._http_endpoint, json=True)
return blob

@ -9,14 +9,13 @@ from typing import List, Union
from datetime import datetime from datetime import datetime
import aiofiles import aiofiles
from quart import current_app as app
import settings import settings
from wowlet_backend.utils import httpget from wowlet_backend.utils import httpget
from wowlet_backend.tasks import WowletTask from wowlet_backend.tasks import FeatherTask
class HistoricalPriceTask(WowletTask): class HistoricalPriceTask(FeatherTask):
""" """
This class manages a historical price (USD) database, saved in a This class manages a historical price (USD) database, saved in a
textfile at `self._path`. A Feather wallet instance will ask textfile at `self._path`. A Feather wallet instance will ask
@ -94,15 +93,6 @@ class HistoricalPriceTask(WowletTask):
data filtered by the parameters.""" data filtered by the parameters."""
from wowlet_backend.factory import cache from wowlet_backend.factory import cache
# redis keys are always strings, convert input parameters
year = str(year)
if isinstance(month, int):
month = str(month)
if not year.isnumeric() or (month and not month.isnumeric()):
app.logger.error(f"get() called with year: {year}, month {month}, not a number")
return
blob = await cache.get("historical_fiat") blob = await cache.get("historical_fiat")
blob = json.loads(blob) blob = json.loads(blob)
if year not in blob: if year not in blob:
@ -112,13 +102,13 @@ class HistoricalPriceTask(WowletTask):
if not month: if not month:
for _m, days in blob[year].items(): for _m, days in blob[year].items():
for day, price in days.items(): for day, price in days.items():
rtn[datetime(int(year), int(_m), int(day)).strftime('%Y%m%d')] = price rtn[datetime(year, _m, day).strftime('%Y%m%d')] = price
return rtn return rtn
if month not in blob[year]: if month not in blob[year]:
return return
for day, price in blob[year][month].items(): for day, price in blob[year][month].items():
rtn[datetime(int(year), int(month), int(day)).strftime('%Y%m%d')] = price rtn[datetime(year, month, day).strftime('%Y%m%d')] = price
return rtn return rtn

@ -7,10 +7,10 @@ from typing import List
import settings import settings
from wowlet_backend.utils import httpget from wowlet_backend.utils import httpget
from wowlet_backend.tasks import WowletTask from wowlet_backend.tasks import FeatherTask
class FundingProposalsTask(WowletTask): class FundingProposalsTask(FeatherTask):
"""Fetch funding proposals made by the community.""" """Fetch funding proposals made by the community."""
def __init__(self, interval: int = 600): def __init__(self, interval: int = 600):
from wowlet_backend.factory import app from wowlet_backend.factory import app

@ -2,17 +2,15 @@
# Copyright (c) 2020, The Monero Project. # Copyright (c) 2020, The Monero Project.
# Copyright (c) 2020, dsc@xmr.pm # Copyright (c) 2020, dsc@xmr.pm
import json, asyncio
from typing import List, Union from typing import List, Union
import settings import settings
from wowlet_backend.utils import httpget from wowlet_backend.utils import httpget
from wowlet_backend.tasks import WowletTask from wowlet_backend.tasks import FeatherTask
from wowlet_backend.factory import cache
class CryptoRatesTask(WowletTask): class CryptoRatesTask(FeatherTask):
def __init__(self, interval: int = 600): def __init__(self, interval: int = 180):
super(CryptoRatesTask, self).__init__(interval) super(CryptoRatesTask, self).__init__(interval)
self._cache_key = "crypto_rates" self._cache_key = "crypto_rates"
@ -39,56 +37,23 @@ class CryptoRatesTask(WowletTask):
"price_change_percentage_24h": r["price_change_percentage_24h"] "price_change_percentage_24h": r["price_change_percentage_24h"]
} for r in rates] } for r in rates]
# limit the list, only include specific coins. see wowlet:src/utils/prices.cpp
whitelist = ["XMR", "ZEC", "BTC", "ETH", "BCH", "LTC", "EOS", "ADA", "XLM", "TRX", "DASH", "DCR", "VET", "DOGE", "XRP", "WOW"]
rates = [r for r in rates if r["symbol"].upper() in whitelist]
# additional coins as defined by `settings.CRYPTO_RATES_COINS_EXTRA` # additional coins as defined by `settings.CRYPTO_RATES_COINS_EXTRA`
for coin, symbol in settings.CRYPTO_RATES_COINS_EXTRA.items(): for coin, symbol in settings.CRYPTO_RATES_COINS_EXTRA.items():
obj = {} url = f"{self._http_api_gecko}/simple/price?ids={coin}&vs_currencies=usd"
try: try:
results = {} data = await httpget(url, json=True)
for vs_currency in ["usd", "btc"]: if coin not in data or "usd" not in data[coin]:
url = f"{self._http_api_gecko}/simple/price?ids={coin}&vs_currencies={vs_currency}" continue
data = await httpget(url, json=True, timeout=15)
results[vs_currency] = data[coin][vs_currency]
price_btc = "{:.8f}".format(results["btc"]) rates.append({
price_sat = int(price_btc.replace(".", "").lstrip("0")) # yolo
obj = {
"id": coin, "id": coin,
"symbol": symbol, "symbol": symbol,
"image": "", "image": "",
"name": coin.capitalize(), "name": coin.capitalize(),
"current_price": results["usd"], "current_price": data[coin]["usd"],
"current_price_btc": price_btc,
"current_price_satoshi": price_sat,
"price_change_percentage_24h": 0.0 "price_change_percentage_24h": 0.0
} })
except Exception as ex: except Exception as ex:
app.logger.error(f"extra coin: {coin}; {ex}") app.logger.error(f"extra coin: {coin}; {ex}")
# use cache if present
data = await cache.get(self._cache_key)
if data:
data = json.loads(data)
extra_coin = [e for e in data if e['symbol'] == symbol]
if extra_coin:
app.logger.warning(f"using cache for extra coin: {coin}")
rates.append(extra_coin[0])
continue
try:
# additional call to fetch 24h pct change
url = f"{self._http_api_gecko}/coins/{coin}?tickers=false&market_data=true&community_data=false&developer_data=false&sparkline=false"
blob = await httpget(url, json=True, timeout=15)
obj["price_change_percentage_24h"] = blob.get("market_data", {}).get("price_change_percentage_24h")
except:
pass
rates.append(obj)
await asyncio.sleep(10)
return rates return rates

@ -6,10 +6,10 @@ from datetime import datetime, timedelta
import re import re
from wowlet_backend.utils import httpget from wowlet_backend.utils import httpget
from wowlet_backend.tasks import WowletTask from wowlet_backend.tasks import FeatherTask
class FiatRatesTask(WowletTask): class FiatRatesTask(FeatherTask):
def __init__(self, interval: int = 43200): def __init__(self, interval: int = 43200):
super(FiatRatesTask, self).__init__(interval) super(FiatRatesTask, self).__init__(interval)

@ -5,10 +5,10 @@
import html import html
import settings import settings
from wowlet_backend.utils import httpget from wowlet_backend.utils import httpget
from wowlet_backend.tasks import WowletTask from wowlet_backend.tasks import FeatherTask
class RedditTask(WowletTask): class RedditTask(FeatherTask):
def __init__(self, interval: int = 900): def __init__(self, interval: int = 900):
from wowlet_backend.factory import app from wowlet_backend.factory import app
super(RedditTask, self).__init__(interval) super(RedditTask, self).__init__(interval)

@ -7,10 +7,10 @@ from typing import List
import settings import settings
from wowlet_backend.utils import httpget, popularity_contest from wowlet_backend.utils import httpget, popularity_contest
from wowlet_backend.tasks import WowletTask from wowlet_backend.tasks import FeatherTask
class RPCNodeCheckTask(WowletTask): class RPCNodeCheckTask(FeatherTask):
def __init__(self, interval: int = 60): def __init__(self, interval: int = 60):
super(RPCNodeCheckTask, self).__init__(interval) super(RPCNodeCheckTask, self).__init__(interval)
@ -41,26 +41,19 @@ class RPCNodeCheckTask(WowletTask):
for network_type, _nodes in _.items(): for network_type, _nodes in _.items():
for node in _nodes: for node in _nodes:
for scheme in ["https", "http"]: try:
try: blob = await self.node_check(node, network_type=network_type)
blob = await self.node_check(f"{scheme}://{node}", network_type=network_type) data.append(blob)
blob['tls'] = True if scheme == "https" else False except Exception as ex:
data.append(blob) app.logger.warning(f"node {node} not reachable; {ex}")
break
except Exception as ex:
continue
if not data:
app.logger.warning(f"node {node} not reachable")
data.append(self._bad_node({ data.append(self._bad_node({
"address": node, "address": node,
"nettype": network_type_coin, "nettype": network_type_coin,
"type": network_type, "type": network_type,
"height": 0, "height": 0
"tls": False
}, reason="unreachable")) }, reason="unreachable"))
# not necessary for stagenet/testnet nodes to be validated # not neccesary for stagenet/testnet nodes to be validated
if network_type_coin != "mainnet": if network_type_coin != "mainnet":
nodes += data nodes += data
continue continue
@ -72,7 +65,7 @@ class RPCNodeCheckTask(WowletTask):
# data = list(map(lambda _node: _node if _node['target_height'] <= _node['height'] # data = list(map(lambda _node: _node if _node['target_height'] <= _node['height']
# else self._bad_node(_node, reason="+2_attack"), data)) # else self._bad_node(_node, reason="+2_attack"), data))
allowed_offset = 3 allowed_offset = 5
valid_heights = [] valid_heights = []
# current_blockheight = heights.get(network_type_coin, 0) # current_blockheight = heights.get(network_type_coin, 0)
@ -89,15 +82,14 @@ class RPCNodeCheckTask(WowletTask):
"""Call /get_info on the RPC, return JSON""" """Call /get_info on the RPC, return JSON"""
opts = { opts = {
"timeout": self._http_timeout, "timeout": self._http_timeout,
"json": True, "json": True
"verify_tls": False
} }
if network_type == "tor": if network_type == "tor":
opts["socks5"] = settings.TOR_SOCKS_PROXY opts["socks5"] = settings.TOR_SOCKS_PROXY
opts["timeout"] = self._http_timeout_onion opts["timeout"] = self._http_timeout_onion
blob = await httpget(f"{node}/get_info", **opts) blob = await httpget(f"http://{node}/get_info", **opts)
for expect in ["nettype", "height", "target_height"]: for expect in ["nettype", "height", "target_height"]:
if expect not in blob: if expect not in blob:
raise Exception(f"Invalid JSON response from RPC; expected key '{expect}'") raise Exception(f"Invalid JSON response from RPC; expected key '{expect}'")

@ -14,11 +14,11 @@ from bs4 import BeautifulSoup
import settings import settings
from wowlet_backend.utils import httpget, image_resize from wowlet_backend.utils import httpget, image_resize
from wowlet_backend.tasks import WowletTask from wowlet_backend.tasks import FeatherTask
class SuchWowTask(WowletTask): class SuchWowTask(FeatherTask):
def __init__(self, interval: int = 300): def __init__(self, interval: int = 600):
""" """
This task is specifically for Wownero - fetching a listing This task is specifically for Wownero - fetching a listing
of recent SuchWow submissions. of recent SuchWow submissions.
@ -31,16 +31,15 @@ class SuchWowTask(WowletTask):
self._http_endpoint = "https://suchwow.xyz/" self._http_endpoint = "https://suchwow.xyz/"
self._tmp_dir = os.path.join(settings.cwd, "data", "suchwow") self._tmp_dir = os.path.join(settings.cwd, "data", "suchwow")
self._websocket_cmd = "suchwow"
if not os.path.exists(self._tmp_dir): if not os.path.exists(self._tmp_dir):
os.mkdir(self._tmp_dir) os.mkdir(self._tmp_dir)
async def task(self): async def task(self):
from wowlet_backend.factory import app from wowlet_backend.factory import app
result = await httpget(f"{self._http_endpoint}api/list?limit=15&offset=0", json=True) result = await httpget(f"{self._http_endpoint}api/list", json=True)
result = list(sorted(result, key=lambda k: k['id'], reverse=True)) result = list(sorted(result, key=lambda k: k['id'], reverse=True))
result = result[:15]
for post in result: for post in result:
post_id = int(post['id']) post_id = int(post['id'])

@ -1,62 +0,0 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2020, The Monero Project.
# Copyright (c) 2020, dsc@xmr.pm
from dateutil.parser import parse
import settings
from wowlet_backend.utils import httpget
from wowlet_backend.tasks import WowletTask
class WowletReleasesTask(WowletTask):
"""Fetches the latest Wowlet releases using gitea's API"""
def __init__(self, interval: int = 3600):
super(WowletReleasesTask, self).__init__(interval)
self._cache_key = "wowlet_releases"
self._cache_expiry = self.interval
self._websocket_cmd = "wowlet_releases"
self._http_endpoint = "https://git.wownero.com/api/v1/repos/wowlet/wowlet/releases?limit=1"
async def task(self) -> dict:
blob = await httpget(self._http_endpoint)
if not isinstance(blob, list) or not blob:
raise Exception(f"Invalid JSON response for {self._http_endpoint}")
blob = blob[0]
data = {}
for asset in blob['assets']:
operating_system = "linux"
if "msvc" in asset['name'] or "win64" in asset['name'] or "windows" in asset['name']:
operating_system = "windows"
elif "macos" in asset["name"]:
operating_system = "macos"
data[operating_system] = {
"name": asset["name"],
"created_at": parse(asset["created_at"]).strftime("%Y-%m-%d"),
"url": asset['browser_download_url'],
"download_count": asset["download_count"],
"size": asset['size']
}
tag = blob['tag_name']
if tag.startswith("v"):
tag = tag[1:]
try:
t = [int(z) for z in tag.split(".")]
if len(t) != 3:
raise Exception()
except:
raise Exception(f"invalid tag: {tag}")
return {
"assets": data,
"body": blob['body'],
"version": tag
}

@ -1,55 +0,0 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2022, The Monero Project.
# Copyright (c) 2022, dsc@xmr.pm
from dateutil.parser import parse
import settings
from wowlet_backend.utils import httpget
from wowlet_backend.tasks import WowletTask
class WownerodTask(WowletTask):
"""Fetches the latest Wownerod releases using gitea's API"""
def __init__(self, interval: int = 3600):
super(WownerodTask, self).__init__(interval)
self._cache_key = "wownerod_releases"
self._cache_expiry = self.interval
self._websocket_cmd = "wownerod_releases"
self._http_endpoint = "https://git.wownero.com/api/v1/repos/wownero/wownero/releases?limit=1"
async def task(self) -> dict:
blob = await httpget(self._http_endpoint)
if not isinstance(blob, list) or not blob:
raise Exception(f"Invalid JSON response for {self._http_endpoint}")
blob = blob[0]
data = {}
for asset in blob['assets']:
operating_system = "linux"
if "w64" in asset['name'] or "msvc" in asset['name'] or "win64" in asset['name'] or "windows" in asset['name']:
operating_system = "windows"
elif "macos" in asset["name"] or "apple" in asset["name"]:
operating_system = "macos"
data.setdefault(operating_system, [])
data[operating_system].append({
"name": asset["name"],
"created_at": parse(asset["created_at"]).strftime("%Y-%m-%d"),
"url": asset['browser_download_url'],
"download_count": asset["download_count"],
"size": asset['size']
})
tag = blob['tag_name']
if tag.startswith("v"):
tag = tag[1:]
return {
"version": tag,
"assets": data
}

@ -6,10 +6,10 @@ from dateutil.parser import parse
import settings import settings
from wowlet_backend.utils import httpget from wowlet_backend.utils import httpget
from wowlet_backend.tasks import WowletTask from wowlet_backend.tasks import FeatherTask
class XmrigTask(WowletTask): class XmrigTask(FeatherTask):
"""Fetches the latest XMRig releases using Github's API""" """Fetches the latest XMRig releases using Github's API"""
def __init__(self, interval: int = 43200): def __init__(self, interval: int = 43200):
super(XmrigTask, self).__init__(interval) super(XmrigTask, self).__init__(interval)

@ -1,29 +0,0 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2022, The Monero Project.
# Copyright (c) 2022, dsc@xmr.pm
from typing import List
from dateutil.parser import parse
import settings
from wowlet_backend.utils import httpget
from wowlet_backend.tasks import WowletTask
class YellWowTask(WowletTask):
"""Fetches yellwowpages usernames/addresses"""
def __init__(self, interval: int = 3600):
super(YellWowTask, self).__init__(interval)
self._cache_key = "yellwow"
self._cache_expiry = self.interval * 10
self._websocket_cmd = "yellwow"
self._http_endpoint = "https://yellow.wownero.com/api/user/"
async def task(self) -> List[dict]:
blob = await httpget(self._http_endpoint)
if not isinstance(blob, list) or not blob:
raise Exception(f"Invalid JSON response for {self._http_endpoint}")
return blob

@ -1,26 +0,0 @@
import re, sys, os
from datetime import datetime
import time
from wowlet_backend.factory import broadcast
HISTORY = []
async def add_chat(message: str, balance: float = 0):
global HISTORY
item = {
"message": message,
"balance": balance,
"date": int(time.time())
}
HISTORY.append(item)
if len(HISTORY) >= 25:
HISTORY = HISTORY[:25]
await broadcast.put({
"cmd": "trollEntry",
"data": item
})

@ -1,6 +1,6 @@
# SPDX-License-Identifier: BSD-3-Clause # SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2022, The Monero Project. # Copyright (c) 2020, The Monero Project.
# Copyright (c) 2022, dsc@xmr.pm # Copyright (c) 2020, dsc@xmr.pm
import re import re
import json import json
@ -41,14 +41,27 @@ def print_banner():
""".strip()) """.strip())
async def httpget(url: str, json=True, timeout: int = 5, socks5: str = None, raise_for_status=True, verify_tls=True): def collect_websocket(func):
@wraps(func)
async def wrapper(*args, **kwargs):
from wowlet_backend.factory import connected_websockets
queue = asyncio.Queue()
connected_websockets.add(queue)
try:
return await func(queue, *args, **kwargs)
finally:
connected_websockets.remove(queue)
return wrapper
async def httpget(url: str, json=True, timeout: int = 5, socks5: str = None, raise_for_status=True):
headers = {"User-Agent": random_agent()} headers = {"User-Agent": random_agent()}
opts = {"timeout": aiohttp.ClientTimeout(total=timeout)} opts = {"timeout": aiohttp.ClientTimeout(total=timeout)}
if socks5: if socks5:
opts['connector'] = ProxyConnector.from_url(socks5) opts['connector'] = ProxyConnector.from_url(socks5)
async with aiohttp.ClientSession(**opts) as session: async with aiohttp.ClientSession(**opts) as session:
async with session.get(url, headers=headers, ssl=verify_tls) as response: async with session.get(url, headers=headers) as response:
if raise_for_status: if raise_for_status:
response.raise_for_status() response.raise_for_status()
@ -63,31 +76,23 @@ def random_agent():
return random.choice(user_agents) return random.choice(user_agents)
async def wowlet_data(): async def feather_data():
"""A collection of data collected by the various wowlet tasks""" """A collection of data collected by
`FeatherTask`, for Feather wallet clients."""
from wowlet_backend.factory import cache, now from wowlet_backend.factory import cache, now
data = await cache.get("data") data = await cache.get("data")
if data: if data:
data = json.loads(data) data = json.loads(data)
return data return data
keys = [ keys = ["blockheights", "funding_proposals", "crypto_rates", "fiat_rates", "reddit", "rpc_nodes", "xmrig", "xmrto_rates", "suchwow"]
"blockheights",
"funding_proposals",
"crypto_rates",
"fiat_rates",
"reddit",
"rpc_nodes",
"xmrig",
"xmrto_rates",
"suchwow",
"forum",
"wowlet_releases",
"yellwow",
"wownerod_releases"
]
data = {keys[i]: json.loads(val) if val else None for i, val in enumerate(await cache.mget(*keys))} data = {keys[i]: json.loads(val) if val else None for i, val in enumerate(await cache.mget(*keys))}
# @TODO: for backward-compat reasons we're including some legacy keys which can be removed after 1.0 release
data['nodes'] = data['rpc_nodes']
data['ccs'] = data['funding_proposals']
data['wfs'] = data['funding_proposals']
# start caching when application lifetime is more than 20 seconds # start caching when application lifetime is more than 20 seconds
if (datetime.now() - now).total_seconds() > 20: if (datetime.now() - now).total_seconds() > 20:
await cache.setex("data", 30, json.dumps(data)) await cache.setex("data", 30, json.dumps(data))
@ -105,6 +110,34 @@ def popularity_contest(lst: List[int]) -> Union[int, None]:
return Counter(lst).most_common(1)[0][0] return Counter(lst).most_common(1)[0][0]
def current_worker_thread_is_primary() -> bool:
"""
ASGI server (Hypercorn) may start multiple
worker threads, but we only want one feather-ws
instance to schedule `FeatherTask` tasks at an
interval. Therefor this function determines if the
current instance is responsible for the
recurring Feather tasks.
"""
from wowlet_backend.factory import app
current_pid = os.getpid()
parent_pid = os.getppid()
app.logger.debug(f"current_pid: {current_pid}, "
f"parent_pid: {parent_pid}")
if parent_pid == 0:
return True
parent = psutil.Process(parent_pid)
if parent.name() != "hypercorn":
return True
lowest_pid = min(c.pid for c in parent.children(recursive=True) if c.name() == "hypercorn")
if current_pid == lowest_pid:
return True
async def image_resize(buffer: bytes, max_bounding_box: int = 512, quality: int = 70) -> bytes: async def image_resize(buffer: bytes, max_bounding_box: int = 512, quality: int = 70) -> bytes:
""" """
- Resize if the image is too large - Resize if the image is too large
@ -128,54 +161,3 @@ async def image_resize(buffer: bytes, max_bounding_box: int = 512, quality: int
buffer.seek(0) buffer.seek(0)
return buffer.read() return buffer.read()
async def whaleornot(amount: float):
if amount <= 0:
fish_str = "amoeba"
elif amount < 100:
fish_str = "plankton"
elif amount >= 100 and amount < 200:
fish_str = "Paedocypris"
elif amount >= 200 and amount < 500:
fish_str = "Dwarf Goby"
elif amount >= 500 and amount < 1000:
fish_str = "European Pilchard"
elif amount >= 1000 and amount < 2000:
fish_str = "Goldfish"
elif amount >= 2000 and amount < 4000:
fish_str = "Herring"
elif amount >= 4000 and amount < 7000:
fish_str = "Atlantic Macerel"
elif amount >= 7000 and amount < 9000:
fish_str = "Gilt-head Bream"
elif amount >= 9000 and amount < 12000:
fish_str = "Salmonidae"
elif amount >= 12000 and amount < 20000:
fish_str = "Gadidae"
elif amount >= 20000 and amount < 40000:
fish_str = "Norwegian Delicious Salmon"
elif amount >= 40000 and amount < 60000:
fish_str = "Electric eel"
elif amount >= 60000 and amount < 80000:
fish_str = "Tuna"
elif amount >= 80000 and amount < 100000:
fish_str = "Wels catfish"
elif amount >= 100000 and amount < 120000:
fish_str = "Black marlin"
elif amount >= 120000 and amount < 160000:
fish_str = "Shark"
elif amount >= 160000 and amount < 220000:
fish_str = "Dolphin"
elif amount >= 220000 and amount < 320000:
fish_str = "Narwhal"
elif amount >= 320000 and amount < 500000:
fish_str = "Orca"
elif amount >= 500000 and amount < 700000:
fish_str = "Blue Whale"
elif amount >= 700000 and amount < 1000000:
fish_str = "Leviathan"
else:
fish_str = "Cthulu"
return fish_str

@ -10,7 +10,6 @@ from typing import Dict, Union, Optional
from copy import deepcopy from copy import deepcopy
import asyncio import asyncio
import re import re
from wowlet_backend.trollbox import add_chat
from wowlet_backend.utils import RE_ADDRESS from wowlet_backend.utils import RE_ADDRESS
@ -28,21 +27,6 @@ class WebsocketParse:
return await WebsocketParse.requestPIN(data) return await WebsocketParse.requestPIN(data)
elif cmd == "lookupPIN": elif cmd == "lookupPIN":
return await WebsocketParse.lookupPIN(data) return await WebsocketParse.lookupPIN(data)
elif cmd == "trollbox":
pass
#return await WebsocketParse.addTrollChat(data)
@staticmethod
async def addTrollChat(data):
if not data or not isinstance(data, dict):
return
for needle in ['message', 'balance']:
if needle not in data:
return
message = data['message']
balance = data['balance']
await add_chat(message, balance)
@staticmethod @staticmethod
async def txFiatHistory(data=None): async def txFiatHistory(data=None):

Loading…
Cancel
Save