YellWOWpages API - distribute contacts to WS clients

yellwow
dsc 2 years ago
parent c3138a6fb3
commit 7e64dcd133

@ -1,5 +1,5 @@
quart
aioredis
aioredis>=2.0.0
aiohttp
aiofiles
quart_session

@ -14,21 +14,18 @@ DEBUG = bool_env(os.environ.get("WOWLET_DEBUG", False))
HOST = os.environ.get("WOWLET_HOST", "127.0.0.1")
PORT = int(os.environ.get("WOWLET_PORT", 1337))
REDIS_ADDRESS = os.environ.get("WOWLET_REDIS_ADDRESS", "redis://localhost")
REDIS_HOST = os.environ.get("WOWLET_REDIS_HOST", "localhost")
REDIS_PORT = os.environ.get("WOWLET_REDIS_PORT", 6379)
REDIS_PASSWORD = os.environ.get("WOWLET_REDIS_PASSWORD")
COIN_NAME = os.environ.get("WOWLET_COIN_NAME", "monero").lower() # as per coingecko
COIN_SYMBOL = os.environ.get("WOWLET_COIN_SYMBOL", "xmr").lower() # as per coingecko
COIN_GENESIS_DATE = os.environ.get("WOWLET_COIN_GENESIS_DATE", "20140418")
COIN_NAME = os.environ.get("WOWLET_COIN_NAME", "wownero").lower() # as per coingecko
COIN_SYMBOL = os.environ.get("WOWLET_COIN_SYMBOL", "wow").lower() # as per coingecko
COIN_GENESIS_DATE = os.environ.get("WOWLET_COIN_GENESIS_DATE", "20180401")
COIN_MODE = os.environ.get("WOWLET_COIN_MODE", "mainnet").lower()
TOR_SOCKS_PROXY = os.environ.get("WOWLET_TOR_SOCKS_PROXY", "socks5://127.0.0.1:9050")
# while fetching USD price from coingecko, also include these extra coins:
CRYPTO_RATES_COINS_EXTRA = {
"wownero": "wow",
"aeon": "aeon",
"turtlecoin": "trtl",
"haven": "xhv",
"loki": "loki"
"wownero": "wow"
}

@ -12,7 +12,7 @@ from quart import Quart
from quart_session import Session
import aioredis
from wowlet_backend.utils import current_worker_thread_is_primary, print_banner
from wowlet_backend.utils import print_banner
import settings
now = datetime.now()
@ -20,7 +20,6 @@ app: Quart = None
cache = None
user_agents: List[str] = None
broadcast = MultisubscriberQueue()
_is_primary_worker_thread = False
async def _setup_nodes(app: Quart):
@ -39,14 +38,16 @@ async def _setup_user_agents(app: Quart):
async def _setup_cache(app: Quart):
global cache
# Each coin has it's own Redis DB index; `redis-cli -n $INDEX`
db = {"xmr": 0, "wow": 1, "aeon": 2, "trtl": 3, "msr": 4, "xhv": 5, "loki": 6}[settings.COIN_SYMBOL]
db = {"wow": 0, "xmr": 1, "aeon": 2, "trtl": 3, "msr": 4, "xhv": 5, "loki": 6}[settings.COIN_SYMBOL]
data = {
"address": settings.REDIS_ADDRESS,
"host": settings.REDIS_HOST,
"port": settings.REDIS_PORT,
"db": db,
"password": settings.REDIS_PASSWORD if settings.REDIS_PASSWORD else None
}
cache = await aioredis.create_redis_pool(**data)
cache = await aioredis.Redis(**data)
app.config['SESSION_TYPE'] = 'redis'
app.config['SESSION_REDIS'] = cache
Session(app)
@ -54,13 +55,11 @@ async def _setup_cache(app: Quart):
async def _setup_tasks(app: Quart):
"""Schedules a series of tasks at an interval."""
if not _is_primary_worker_thread:
return
from wowlet_backend.tasks import (
BlockheightTask, HistoricalPriceTask, FundingProposalsTask,
CryptoRatesTask, FiatRatesTask, RedditTask, RPCNodeCheckTask,
XmrigTask, SuchWowTask, WowletReleasesTask, ForumThreadsTask)
XmrigTask, SuchWowTask, WowletReleasesTask, ForumThreadsTask,
YellWowTask)
asyncio.create_task(BlockheightTask().start())
asyncio.create_task(HistoricalPriceTask().start())
@ -72,6 +71,7 @@ async def _setup_tasks(app: Quart):
asyncio.create_task(SuchWowTask().start())
asyncio.create_task(WowletReleasesTask().start())
asyncio.create_task(ForumThreadsTask().start())
asyncio.create_task(YellWowTask().start())
if settings.COIN_SYMBOL in ["xmr", "wow"]:
asyncio.create_task(FundingProposalsTask().start())
@ -101,11 +101,7 @@ def create_app():
@app.before_serving
async def startup():
global _is_primary_worker_thread
_is_primary_worker_thread = current_worker_thread_is_primary()
if _is_primary_worker_thread:
print_banner()
print_banner()
await _setup_cache(app)
await _setup_nodes(app)

@ -9,14 +9,14 @@ import json
from quart import websocket, jsonify, send_from_directory
import settings
from wowlet_backend.factory import app
from wowlet_backend.factory import app, broadcast
from wowlet_backend.wsparse import WebsocketParse
from wowlet_backend.utils import broadcast, feather_data
from wowlet_backend.utils import wowlet_data
@app.route("/")
async def root():
data = await feather_data()
data = await wowlet_data()
return jsonify(data)
@ -29,7 +29,7 @@ async def suchwow(name: str):
@app.websocket('/ws')
async def ws():
data = await feather_data()
data = await wowlet_data()
# blast available data on connect
for task_key, task_value in data.items():
@ -54,9 +54,8 @@ async def ws():
continue
async def tx():
while True:
data = await broadcast.get()
payload = json.dumps(data).encode()
async for _data in broadcast.subscribe():
payload = json.dumps(_data).encode()
await websocket.send(payload)
# bidirectional async rx and tx loops

@ -167,3 +167,4 @@ from wowlet_backend.tasks.xmrig import XmrigTask
from wowlet_backend.tasks.suchwow import SuchWowTask
from wowlet_backend.tasks.wowlet import WowletReleasesTask
from wowlet_backend.tasks.forum import ForumThreadsTask
from wowlet_backend.tasks.yellow import YellWowTask

@ -0,0 +1,28 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2022, The Monero Project.
# Copyright (c) 2022, dsc@xmr.pm
from dateutil.parser import parse
import settings
from wowlet_backend.utils import httpget
from wowlet_backend.tasks import WowletTask
class YellWowTask(WowletTask):
"""Fetches yellwowpages usernames/addresses"""
def __init__(self, interval: int = 3600):
super(YellWowTask, self).__init__(interval)
self._cache_key = "yellwow"
self._cache_expiry = self.interval * 10
self._websocket_cmd = "yellwow"
self._http_endpoint = "https://yellow.wownero.com/api/user/"
async def task(self) -> list[dict]:
blob = await httpget(self._http_endpoint)
if not isinstance(blob, list) or not blob:
raise Exception(f"Invalid JSON response for {self._http_endpoint}")
return blob

@ -1,6 +1,6 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2020, The Monero Project.
# Copyright (c) 2020, dsc@xmr.pm
# Copyright (c) 2022, The Monero Project.
# Copyright (c) 2022, dsc@xmr.pm
import re
import json
@ -41,19 +41,6 @@ def print_banner():
""".strip())
def collect_websocket(func):
@wraps(func)
async def wrapper(*args, **kwargs):
from wowlet_backend.factory import connected_websockets
queue = asyncio.Queue()
connected_websockets.add(queue)
try:
return await func(queue, *args, **kwargs)
finally:
connected_websockets.remove(queue)
return wrapper
async def httpget(url: str, json=True, timeout: int = 5, socks5: str = None, raise_for_status=True, verify_tls=True):
headers = {"User-Agent": random_agent()}
opts = {"timeout": aiohttp.ClientTimeout(total=timeout)}
@ -76,23 +63,30 @@ def random_agent():
return random.choice(user_agents)
async def feather_data():
"""A collection of data collected by
`FeatherTask`, for Feather wallet clients."""
async def wowlet_data():
"""A collection of data collected by the various wowlet tasks"""
from wowlet_backend.factory import cache, now
data = await cache.get("data")
if data:
data = json.loads(data)
return data
keys = ["blockheights", "funding_proposals", "crypto_rates", "fiat_rates", "reddit", "rpc_nodes", "xmrig", "xmrto_rates", "suchwow", "forum", "wowlet_releases"]
keys = [
"blockheights",
"funding_proposals",
"crypto_rates",
"fiat_rates",
"reddit",
"rpc_nodes",
"xmrig",
"xmrto_rates",
"suchwow",
"forum",
"wowlet_releases",
"yellwow"
]
data = {keys[i]: json.loads(val) if val else None for i, val in enumerate(await cache.mget(*keys))}
# @TODO: for backward-compat reasons we're including some legacy keys which can be removed after 1.0 release
data['nodes'] = data['rpc_nodes']
data['ccs'] = data['funding_proposals']
data['wfs'] = data['funding_proposals']
# start caching when application lifetime is more than 20 seconds
if (datetime.now() - now).total_seconds() > 20:
await cache.setex("data", 30, json.dumps(data))
@ -110,34 +104,6 @@ def popularity_contest(lst: List[int]) -> Union[int, None]:
return Counter(lst).most_common(1)[0][0]
def current_worker_thread_is_primary() -> bool:
"""
ASGI server (Hypercorn) may start multiple
worker threads, but we only want one feather-ws
instance to schedule `FeatherTask` tasks at an
interval. Therefor this function determines if the
current instance is responsible for the
recurring Feather tasks.
"""
from wowlet_backend.factory import app
current_pid = os.getpid()
parent_pid = os.getppid()
app.logger.debug(f"current_pid: {current_pid}, "
f"parent_pid: {parent_pid}")
if parent_pid == 0:
return True
parent = psutil.Process(parent_pid)
if parent.name() != "hypercorn":
return True
lowest_pid = min(c.pid for c in parent.children(recursive=True) if c.name() == "hypercorn")
if current_pid == lowest_pid:
return True
async def image_resize(buffer: bytes, max_bounding_box: int = 512, quality: int = 70) -> bytes:
"""
- Resize if the image is too large

Loading…
Cancel
Save