add a node

Reviewed-on: #20

README.md

@@ -1,15 +1,10 @@
-# feather-ws
+# wowlet-backend
 
-Back-end websocket server for Feather wallet.
+Back-end websocket server for wowlet.
 
 - Quart web framework, Py3 asyncio
 - Redis
 
-## Coins supported
-
-- Monero
-- Wownero
-
 See also the environment variables `WOWLET_COIN_NAME`, `WOWLET_COIN_SYMBOL`, etc. in `settings.py`.
 
 ## Tasks
@@ -22,9 +17,10 @@ This websocket server has several scheduled recurring tasks:
 - Fetch funding proposals
 - Check status of RPC nodes (`data/nodes.json`)
 
-When Feather wallet starts up, it will connect to
-this websocket server and receive the information
-listed above which is necessary for normal operation.
+When Wowlet (the wallet application) starts up, it 
+will connect to this websocket server and receive 
+the information listed above which is necessary 
+for normal operation.
 
 See `wowlet_backend.tasks.*` for the various tasks.
 

data/nodes.json

@@ -2,14 +2,24 @@
   "wow": {
     "mainnet": {
       "tor": [
-        "wowbuxx535x4exuexja2xfezpwcyznxkofui4ndjiectj4yuh2xheiid.onion:34568"
+        "v2admi6gbeprxnk6i2oscizhgy4v5ixu6iezkhj5udiwbfjjs2w7dnid.onion:34568",
+        "awbibkoaa67jhuaqes4n2243gd6vtidjzqj2djukrubp2eudrmxr5mid.onion:34568",
+        "7ftpbpp6rbgqi5kjmhyin46essnh3eqb3m3rhfi7r2fr33iwkeuer3yd.onion:34568",
+        "j7rf2jcccizcp47y5moehguyuqdpg4lusk642sw4nayuruitqaqbc7ad.onion:34568",
+        "aje53o5z5twne5q2ljw44zkahhsuhjtwaxuburxddbf7n4pfsj4rj6qd.onion:34568",
+        "nepc4lxndsooj2akn7ofrj3ooqc25242obchcag6tw3f2mxrms2uuvyd.onion:34568",
+        "666l2ajxqjgj5lskvbokvworjysgvqag4oitokjuy7wz6juisul4jqad.onion:34568",
+        "ty7ppqozzodz75audgvkprekiiqsovbyrkfdjwadrkbe3etyzloatxad.onion:34568",
+        "ewynwpnprbgqllv2syn3drjdrqkw7ehoeg73znelm6mevvmpddexsoqd.onion:34568",
+        "mqkiqwmhhpqtzlrf26stv7jvtaudbyzkbg3lttkmvvvauzgtxm62tgyd.onion:34568",
+        "zao3w6isidntdbnyee5ufs7fyzmv7wzchpw32i3uo5eldjwmo4bxg2qd.onion:34568"
       ],
       "clearnet": [
-        "wow.pwned.systems:34568",
-        "global.wownodes.com:34568",
-        "node.suchwow.xyz:34568",
-        "super.fast.node.xmr.pm:34568",
-        "wowbux.org:34568"
+          "spippolatori.it:34568",
+          "node2.monerodevs.org:34568",
+          "node3.monerodevs.org:34568",
+          "node.wowne.ro:34568",
+          "node.suchwow.xyz:34568"
       ]
     },
     "stagenet": {

requirements.txt

@@ -1,5 +1,5 @@
 quart
-aioredis
+aioredis>=2.0.0
 aiohttp
 aiofiles
 quart_session
@@ -8,5 +8,6 @@ aiohttp_socks
 python-dateutil
 psutil
 psutil
-pillow-simd
-python-magic
+pillow
+python-magic
+asyncio-multisubscriber-queue==0.3.1

settings.py_example

@@ -14,21 +14,21 @@ DEBUG = bool_env(os.environ.get("WOWLET_DEBUG", False))
 HOST = os.environ.get("WOWLET_HOST", "127.0.0.1")
 PORT = int(os.environ.get("WOWLET_PORT", 1337))
 
-REDIS_ADDRESS = os.environ.get("WOWLET_REDIS_ADDRESS", "redis://localhost")
+REDIS_HOST = os.environ.get("WOWLET_REDIS_HOST", "localhost")
+REDIS_PORT = os.environ.get("WOWLET_REDIS_PORT", 6379)
 REDIS_PASSWORD = os.environ.get("WOWLET_REDIS_PASSWORD")
 
-COIN_NAME = os.environ.get("WOWLET_COIN_NAME", "monero").lower()  # as per coingecko
-COIN_SYMBOL = os.environ.get("WOWLET_COIN_SYMBOL", "xmr").lower()  # as per coingecko
-COIN_GENESIS_DATE = os.environ.get("WOWLET_COIN_GENESIS_DATE", "20140418")
+COIN_NAME = os.environ.get("WOWLET_COIN_NAME", "wownero").lower()  # as per coingecko
+COIN_SYMBOL = os.environ.get("WOWLET_COIN_SYMBOL", "wow").lower()  # as per coingecko
+COIN_GENESIS_DATE = os.environ.get("WOWLET_COIN_GENESIS_DATE", "20180401")
 COIN_MODE = os.environ.get("WOWLET_COIN_MODE", "mainnet").lower()
 
 TOR_SOCKS_PROXY = os.environ.get("WOWLET_TOR_SOCKS_PROXY", "socks5://127.0.0.1:9050")
 
+RPC_WOWNERO_HOST = "127.0.0.1"
+RPC_WOWNERO_PORT = 34568
+
 # while fetching USD price from coingecko, also include these extra coins:
 CRYPTO_RATES_COINS_EXTRA = {
-    "wownero": "wow",
-    "aeon": "aeon",
-    "turtlecoin": "trtl",
-    "haven": "xhv",
-    "loki": "loki"
+    "wownero": "wow"
 }

wowlet_backend/factory.py

@@ -7,19 +7,19 @@ import asyncio
 from typing import List, Set
 from datetime import datetime
 
+from asyncio_multisubscriber_queue import MultisubscriberQueue
 from quart import Quart
 from quart_session import Session
 import aioredis
 
-from wowlet_backend.utils import current_worker_thread_is_primary, print_banner
+from wowlet_backend.utils import print_banner
 import settings
 
 now = datetime.now()
 app: Quart = None
 cache = None
 user_agents: List[str] = None
-connected_websockets: Set[asyncio.Queue] = set()
-_is_primary_worker_thread = False
+broadcast = MultisubscriberQueue() 
 
 
 async def _setup_nodes(app: Quart):
@@ -38,14 +38,16 @@ async def _setup_user_agents(app: Quart):
 async def _setup_cache(app: Quart):
     global cache
     # Each coin has it's own Redis DB index; `redis-cli -n $INDEX`
-    db = {"xmr": 0, "wow": 1, "aeon": 2, "trtl": 3, "msr": 4, "xhv": 5, "loki": 6}[settings.COIN_SYMBOL]
+    db = {"wow": 0, "xmr": 1, "aeon": 2, "trtl": 3, "msr": 4, "xhv": 5, "loki": 6}[settings.COIN_SYMBOL]
+
     data = {
-        "address": settings.REDIS_ADDRESS,
+        "host": settings.REDIS_HOST,
+        "port": settings.REDIS_PORT,
         "db": db,
         "password": settings.REDIS_PASSWORD if settings.REDIS_PASSWORD else None
     }
 
-    cache = await aioredis.create_redis_pool(**data)
+    cache = await aioredis.Redis(**data)
     app.config['SESSION_TYPE'] = 'redis'
     app.config['SESSION_REDIS'] = cache
     Session(app)
@@ -53,13 +55,11 @@ async def _setup_cache(app: Quart):
 
 async def _setup_tasks(app: Quart):
     """Schedules a series of tasks at an interval."""
-    if not _is_primary_worker_thread:
-        return
-
     from wowlet_backend.tasks import (
         BlockheightTask, HistoricalPriceTask, FundingProposalsTask,
         CryptoRatesTask, FiatRatesTask, RedditTask, RPCNodeCheckTask,
-        XmrigTask, SuchWowTask)
+        XmrigTask, SuchWowTask, WowletReleasesTask, ForumThreadsTask,
+        YellWowTask, WownerodTask)
 
     asyncio.create_task(BlockheightTask().start())
     asyncio.create_task(HistoricalPriceTask().start())
@@ -69,6 +69,10 @@ async def _setup_tasks(app: Quart):
     asyncio.create_task(RPCNodeCheckTask().start())
     asyncio.create_task(XmrigTask().start())
     asyncio.create_task(SuchWowTask().start())
+    asyncio.create_task(WowletReleasesTask().start())
+    asyncio.create_task(ForumThreadsTask().start())
+    asyncio.create_task(YellWowTask().start())
+    asyncio.create_task(WownerodTask().start())
 
     if settings.COIN_SYMBOL in ["xmr", "wow"]:
         asyncio.create_task(FundingProposalsTask().start())
@@ -98,11 +102,7 @@ def create_app():
 
     @app.before_serving
     async def startup():
-        global _is_primary_worker_thread
-        _is_primary_worker_thread = current_worker_thread_is_primary()
-
-        if _is_primary_worker_thread:
-            print_banner()
+        print_banner()
 
         await _setup_cache(app)
         await _setup_nodes(app)

wowlet_backend/routes.py

@@ -9,14 +9,14 @@ import json
 from quart import websocket, jsonify, send_from_directory
 
 import settings
-from wowlet_backend.factory import app
+from wowlet_backend.factory import app, broadcast
 from wowlet_backend.wsparse import WebsocketParse
-from wowlet_backend.utils import collect_websocket, feather_data
+from wowlet_backend.utils import wowlet_data
 
 
 @app.route("/")
 async def root():
-    data = await feather_data()
+    data = await wowlet_data()
     return jsonify(data)
 
 
@@ -28,9 +28,8 @@ async def suchwow(name: str):
 
 
 @app.websocket('/ws')
-@collect_websocket
-async def ws(queue):
-    data = await feather_data()
+async def ws():
+    data = await wowlet_data()
 
     # blast available data on connect
     for task_key, task_value in data.items():
@@ -55,9 +54,8 @@ async def ws(queue):
                 continue
 
     async def tx():
-        while True:
-            data = await queue.get()
-            payload = json.dumps(data).encode()
+        async for _data in broadcast.subscribe():
+            payload = json.dumps(_data).encode()
             await websocket.send(payload)
 
     # bidirectional async rx and tx loops

wowlet_backend/tasks/__init__.py

@@ -8,7 +8,7 @@ import random
 from typing import Union
 
 
-class FeatherTask:
+class WowletTask:
     """
     The base class of many recurring tasks for this
     project. This abstracts away some functionality:
@@ -39,7 +39,7 @@ class FeatherTask:
         self._running = False
 
     async def start(self, *args, **kwargs):
-        from wowlet_backend.factory import app, connected_websockets
+        from wowlet_backend.factory import app, broadcast
         if not self._active:
             # invalid task
             return
@@ -94,11 +94,10 @@ class FeatherTask:
                         propagate = False
 
                 if propagate:
-                    for queue in connected_websockets:
-                        await queue.put({
-                            "cmd": self._websocket_cmd,
-                            "data": result
-                        })
+                    await broadcast.put({
+                        "cmd": self._websocket_cmd,
+                        "data": result
+                    })
 
             # optional: cache the result
             if self._cache_key and result:
@@ -166,3 +165,7 @@ from wowlet_backend.tasks.reddit import RedditTask
 from wowlet_backend.tasks.rpc_nodes import RPCNodeCheckTask
 from wowlet_backend.tasks.xmrig import XmrigTask
 from wowlet_backend.tasks.suchwow import SuchWowTask
+from wowlet_backend.tasks.wowlet import WowletReleasesTask
+from wowlet_backend.tasks.forum import ForumThreadsTask
+from wowlet_backend.tasks.yellow import YellWowTask
+from wowlet_backend.tasks.wownerod import WownerodTask

wowlet_backend/tasks/blockheight.py

@@ -7,12 +7,14 @@ from typing import Union
 from collections import Counter
 from functools import partial
 
+import aiohttp
+
 import settings
 from wowlet_backend.utils import httpget, popularity_contest
-from wowlet_backend.tasks import FeatherTask
+from wowlet_backend.tasks import WowletTask
 
 
-class BlockheightTask(FeatherTask):
+class BlockheightTask(WowletTask):
     """
     Fetch latest blockheight using webcrawling. We pick the most popular
     height from a list of websites. Arguably this approach has benefits
@@ -28,134 +30,14 @@ class BlockheightTask(FeatherTask):
 
         self._websocket_cmd = "blockheights"
 
-        self._fns = {
-            "xmr": {
-                "mainnet": [
-                    self._blockchair,
-                    partial(self._onion_explorer, url="https://xmrchain.net/"),
-                    partial(self._onion_explorer, url="https://community.xmr.to/explorer/mainnet/"),
-                    partial(self._onion_explorer, url="https://monero.exan.tech/")
-                ],
-                "stagenet": [
-                    partial(self._onion_explorer, url="https://stagenet.xmrchain.net/"),
-                    partial(self._onion_explorer, url="https://community.xmr.to/explorer/stagenet/"),
-                    partial(self._onion_explorer, url="https://monero-stagenet.exan.tech/")
-                ]
-            },
-            "wow": {
-                "mainnet": [
-                    partial(self._onion_explorer, url="https://explore.wownero.com/"),
-                ]
-            },
-            "aeon": {
-                "mainnet": [
-                    partial(self._onion_explorer, url="https://aeonblockexplorer.com/"),
-                ],
-                "stagenet": [
-                    partial(self._onion_explorer, url="http://162.210.173.151:8083/"),
-                ]
-            },
-            "trtl": {
-                "mainnet": [
-                    self._turtlenode,
-                    self._turtlenetwork,
-                    self._l33d4n
-                ]
-            },
-            "xhv": {
-                "mainnet": [
-                    partial(self._onion_explorer, url="https://explorer.havenprotocol.org/")
-                ],
-                "stagenet": [
-                    partial(self._onion_explorer, url="https://explorer.stagenet.havenprotocol.org/page/1")
-                ]
-            },
-            "loki": {
-                "mainnet": [
-                    partial(self._onion_explorer, url="https://lokiblocks.com/")
-                ],
-                "testnet": [
-                    partial(self._onion_explorer, url="https://lokitestnet.com/")
-                ]
-            }
-        }
-
     async def task(self) -> Union[dict, None]:
         from wowlet_backend.factory import app
-        coin_network_types = ["mainnet", "stagenet", "testnet"]
-        data = {t: 0 for t in coin_network_types}
-
-        for coin_network_type in coin_network_types:
-            if coin_network_type not in self._fns[settings.COIN_SYMBOL]:
-                continue
-
-            heights = []
-            for fn in self._fns[settings.COIN_SYMBOL][coin_network_type]:
-                fn_name = fn.func.__name__ if isinstance(fn, partial) else fn.__name__
-
-                try:
-                    result = await fn()
-                    heights.append(result)
-                except Exception as ex:
-                    app.logger.error(f"blockheight fetch failed from {fn_name}(): {ex}")
-                    continue
-
-            if heights:
-                data[coin_network_type] = popularity_contest(heights)
-
-        if data["mainnet"] == 0:  # only care about mainnet
-            app.logger.error(f"Failed to parse latest blockheight!")
-            return
-
-        return data
-
-    async def _blockchair(self) -> int:
-        re_blockheight = r"<a href=\".*\">(\d+)</a>"
-
-        url = "https://blockchair.com/monero"
-        content = await httpget(url, json=False, raise_for_status=True)
-
-        height = re.findall(re_blockheight, content)
-        height = max(map(int, height))
-        return height
-
-    async def _wownero(self) -> int:
-        url = "https://explore.wownero.com/"
-        return await BlockheightTask._onion_explorer(url)
-
-    async def _turtlenode(self) -> int:
-        url = "https://public.turtlenode.net/info"
-        blob = await httpget(url, json=True, raise_for_status=True)
-        height = int(blob.get("height", 0))
-        if height <= 0:
-            raise Exception("bad height")
-        return height
-
-    async def _turtlenetwork(self) -> int:
-        url = "https://tnnode2.turtlenetwork.eu/blocks/height"
-        blob = await httpget(url, json=True, raise_for_status=True)
-        height = int(blob.get("height", 0))
-        if height <= 0:
-            raise Exception("bad height")
-        return height
-
-    async def _l33d4n(self):
-        url = "https://blockapi.turtlepay.io/block/header/top"
-        blob = await httpget(url, json=True, raise_for_status=True)
-        height = int(blob.get("height", 0))
-        if height <= 0:
-            raise Exception("bad height")
-        return height
+        timeout = aiohttp.ClientTimeout(total=3)
 
-    @staticmethod
-    async def _onion_explorer(url):
-        """
-        Pages that are based on:
-        https://github.com/moneroexamples/onion-monero-blockchain-explorer
-        """
-        re_blockheight = r"block\/(\d+)\"\>"
-        content = await httpget(url, json=False)
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+            async with session.get(f'http://{settings.RPC_WOWNERO_HOST}:{settings.RPC_WOWNERO_PORT}/get_height') as resp:
+                blob = await resp.json()
 
-        height = re.findall(re_blockheight, content)
-        height = max(map(int, height))
-        return height
+                return {
+                    'mainnet': blob['height']
+                }

wowlet_backend/tasks/forum.py

@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+from bs4 import BeautifulSoup
+from typing import List
+from dateutil.parser import parse
+
+import settings
+from wowlet_backend.utils import httpget
+from wowlet_backend.tasks import WowletTask
+
+
+class ForumThreadsTask(WowletTask):
+    """Fetch recent forum threads."""
+    def __init__(self, interval: int = 300):
+        from wowlet_backend.factory import app
+        super(ForumThreadsTask, self).__init__(interval)
+
+        self._cache_key = "forum"
+        self._cache_expiry = self.interval * 10
+
+        # url
+        self._http_endpoint = "https://forum.wownero.com/latest.json"
+
+        self._websocket_cmd = "forum"
+
+    async def task(self):
+        from wowlet_backend.factory import app
+
+        blob = await httpget(self._http_endpoint, json=True)
+        return blob

wowlet_backend/tasks/historical_prices.py

@@ -9,13 +9,14 @@ from typing import List, Union
 from datetime import datetime
 
 import aiofiles
+from quart import current_app as app
 
 import settings
 from wowlet_backend.utils import httpget
-from wowlet_backend.tasks import FeatherTask
+from wowlet_backend.tasks import WowletTask
 
 
-class HistoricalPriceTask(FeatherTask):
+class HistoricalPriceTask(WowletTask):
     """
     This class manages a historical price (USD) database, saved in a
     textfile at `self._path`. A Feather wallet instance will ask
@@ -93,6 +94,15 @@ class HistoricalPriceTask(FeatherTask):
         data filtered by the parameters."""
         from wowlet_backend.factory import cache
 
+        # redis keys are always strings, convert input parameters
+        year = str(year)
+        if isinstance(month, int):
+            month = str(month)
+
+        if not year.isnumeric() or (month and not month.isnumeric()):
+            app.logger.error(f"get() called with year: {year}, month {month}, not a number")
+            return
+
         blob = await cache.get("historical_fiat")
         blob = json.loads(blob)
         if year not in blob:
@@ -102,13 +112,13 @@ class HistoricalPriceTask(FeatherTask):
         if not month:
             for _m, days in blob[year].items():
                 for day, price in days.items():
-                    rtn[datetime(year, _m, day).strftime('%Y%m%d')] = price
+                    rtn[datetime(int(year), int(_m), int(day)).strftime('%Y%m%d')] = price
             return rtn
 
         if month not in blob[year]:
             return
 
         for day, price in blob[year][month].items():
-            rtn[datetime(year, month, day).strftime('%Y%m%d')] = price
+            rtn[datetime(int(year), int(month), int(day)).strftime('%Y%m%d')] = price
 
         return rtn

wowlet_backend/tasks/proposals.py

@@ -7,10 +7,10 @@ from typing import List
 
 import settings
 from wowlet_backend.utils import httpget
-from wowlet_backend.tasks import FeatherTask
+from wowlet_backend.tasks import WowletTask
 
 
-class FundingProposalsTask(FeatherTask):
+class FundingProposalsTask(WowletTask):
     """Fetch funding proposals made by the community."""
     def __init__(self, interval: int = 600):
         from wowlet_backend.factory import app

wowlet_backend/tasks/rates_crypto.py

@@ -2,15 +2,17 @@
 # Copyright (c) 2020, The Monero Project.
 # Copyright (c) 2020, dsc@xmr.pm
 
+import json, asyncio
 from typing import List, Union
 
 import settings
 from wowlet_backend.utils import httpget
-from wowlet_backend.tasks import FeatherTask
+from wowlet_backend.tasks import WowletTask
+from wowlet_backend.factory import cache
 
 
-class CryptoRatesTask(FeatherTask):
-    def __init__(self, interval: int = 180):
+class CryptoRatesTask(WowletTask):
+    def __init__(self, interval: int = 600):
         super(CryptoRatesTask, self).__init__(interval)
 
         self._cache_key = "crypto_rates"
@@ -37,23 +39,56 @@ class CryptoRatesTask(FeatherTask):
             "price_change_percentage_24h": r["price_change_percentage_24h"]
         } for r in rates]
 
+        # limit the list, only include specific coins. see wowlet:src/utils/prices.cpp
+        whitelist = ["XMR", "ZEC", "BTC", "ETH", "BCH", "LTC", "EOS", "ADA", "XLM", "TRX", "DASH", "DCR", "VET", "DOGE", "XRP", "WOW"]
+        rates = [r for r in rates if r["symbol"].upper() in whitelist]
+
         # additional coins as defined by `settings.CRYPTO_RATES_COINS_EXTRA`
         for coin, symbol in settings.CRYPTO_RATES_COINS_EXTRA.items():
-            url = f"{self._http_api_gecko}/simple/price?ids={coin}&vs_currencies=usd"
+            obj = {}
+
             try:
-                data = await httpget(url, json=True)
-                if coin not in data or "usd" not in data[coin]:
-                    continue
+                results = {}
+                for vs_currency in ["usd", "btc"]:
+                    url = f"{self._http_api_gecko}/simple/price?ids={coin}&vs_currencies={vs_currency}"
+                    data = await httpget(url, json=True, timeout=15)
+                    results[vs_currency] = data[coin][vs_currency]
 
-                rates.append({
+                price_btc = "{:.8f}".format(results["btc"])
+                price_sat = int(price_btc.replace(".", "").lstrip("0"))  # yolo
+
+                obj = {
                     "id": coin,
                     "symbol": symbol,
                     "image": "",
                     "name": coin.capitalize(),
-                    "current_price": data[coin]["usd"],
+                    "current_price": results["usd"],
+                    "current_price_btc": price_btc,
+                    "current_price_satoshi": price_sat,
                     "price_change_percentage_24h": 0.0
-                })
+                }
+
             except Exception as ex:
                 app.logger.error(f"extra coin: {coin}; {ex}")
 
+                # use cache if present
+                data = await cache.get(self._cache_key)
+                if data:
+                    data = json.loads(data)
+                    extra_coin = [e for e in data if e['symbol'] == symbol]
+                    if extra_coin:
+                        app.logger.warning(f"using cache for extra coin: {coin}")
+                        rates.append(extra_coin[0])
+                        continue
+
+            try:
+                # additional call to fetch 24h pct change
+                url = f"{self._http_api_gecko}/coins/{coin}?tickers=false&market_data=true&community_data=false&developer_data=false&sparkline=false"
+                blob = await httpget(url, json=True, timeout=15)
+                obj["price_change_percentage_24h"] = blob.get("market_data", {}).get("price_change_percentage_24h")
+            except:
+                pass
+
+            rates.append(obj)
+            await asyncio.sleep(10)
         return rates

wowlet_backend/tasks/rates_fiat.py

@@ -6,10 +6,10 @@ from datetime import datetime, timedelta
 import re
 
 from wowlet_backend.utils import httpget
-from wowlet_backend.tasks import FeatherTask
+from wowlet_backend.tasks import WowletTask
 
 
-class FiatRatesTask(FeatherTask):
+class FiatRatesTask(WowletTask):
     def __init__(self, interval: int = 43200):
         super(FiatRatesTask, self).__init__(interval)
 

wowlet_backend/tasks/reddit.py

@@ -5,10 +5,10 @@
 import html
 import settings
 from wowlet_backend.utils import httpget
-from wowlet_backend.tasks import FeatherTask
+from wowlet_backend.tasks import WowletTask
 
 
-class RedditTask(FeatherTask):
+class RedditTask(WowletTask):
     def __init__(self, interval: int = 900):
         from wowlet_backend.factory import app
         super(RedditTask, self).__init__(interval)

wowlet_backend/tasks/rpc_nodes.py

@@ -7,10 +7,10 @@ from typing import List
 
 import settings
 from wowlet_backend.utils import httpget, popularity_contest
-from wowlet_backend.tasks import FeatherTask
+from wowlet_backend.tasks import WowletTask
 
 
-class RPCNodeCheckTask(FeatherTask):
+class RPCNodeCheckTask(WowletTask):
     def __init__(self, interval: int = 60):
         super(RPCNodeCheckTask, self).__init__(interval)
 
@@ -41,19 +41,26 @@ class RPCNodeCheckTask(FeatherTask):
 
             for network_type, _nodes in _.items():
                 for node in _nodes:
-                    try:
-                        blob = await self.node_check(node, network_type=network_type)
-                        data.append(blob)
-                    except Exception as ex:
-                        app.logger.warning(f"node {node} not reachable; {ex}")
+                    for scheme in ["https", "http"]:
+                        try:
+                            blob = await self.node_check(f"{scheme}://{node}", network_type=network_type)
+                            blob['tls'] = True if scheme == "https" else False
+                            data.append(blob)
+                            break
+                        except Exception as ex:
+                            continue
+
+                    if not data:
+                        app.logger.warning(f"node {node} not reachable")
                         data.append(self._bad_node({
                             "address": node,
                             "nettype": network_type_coin,
                             "type": network_type,
-                            "height": 0
+                            "height": 0,
+                            "tls": False
                         }, reason="unreachable"))
 
-            # not neccesary for stagenet/testnet nodes to be validated
+            # not necessary for stagenet/testnet nodes to be validated
             if network_type_coin != "mainnet":
                 nodes += data
                 continue
@@ -62,8 +69,8 @@ class RPCNodeCheckTask(FeatherTask):
                 continue
 
             # Filter out nodes affected by < v0.17.1.3 sybil attack
-            data = list(map(lambda _node: _node if _node['target_height'] <= _node['height']
-                            else self._bad_node(_node, reason="+2_attack"), data))
+#            data = list(map(lambda _node: _node if _node['target_height'] <= _node['height']
+#                            else self._bad_node(_node, reason="+2_attack"), data))
 
             allowed_offset = 3
             valid_heights = []
@@ -82,14 +89,15 @@ class RPCNodeCheckTask(FeatherTask):
         """Call /get_info on the RPC, return JSON"""
         opts = {
             "timeout": self._http_timeout,
-            "json": True
+            "json": True,
+            "verify_tls": False
         }
 
         if network_type == "tor":
             opts["socks5"] = settings.TOR_SOCKS_PROXY
             opts["timeout"] = self._http_timeout_onion
 
-        blob = await httpget(f"http://{node}/get_info", **opts)
+        blob = await httpget(f"{node}/get_info", **opts)
         for expect in ["nettype", "height", "target_height"]:
             if expect not in blob:
                 raise Exception(f"Invalid JSON response from RPC; expected key '{expect}'")

wowlet_backend/tasks/suchwow.py

@@ -14,11 +14,11 @@ from bs4 import BeautifulSoup
 
 import settings
 from wowlet_backend.utils import httpget, image_resize
-from wowlet_backend.tasks import FeatherTask
+from wowlet_backend.tasks import WowletTask
 
 
-class SuchWowTask(FeatherTask):
-    def __init__(self, interval: int = 600):
+class SuchWowTask(WowletTask):
+    def __init__(self, interval: int = 300):
         """
         This task is specifically for Wownero - fetching a listing
         of recent SuchWow submissions.
@@ -31,15 +31,16 @@ class SuchWowTask(FeatherTask):
         self._http_endpoint = "https://suchwow.xyz/"
         self._tmp_dir = os.path.join(settings.cwd, "data", "suchwow")
 
+        self._websocket_cmd = "suchwow"
+
         if not os.path.exists(self._tmp_dir):
             os.mkdir(self._tmp_dir)
 
     async def task(self):
         from wowlet_backend.factory import app
-        result = await httpget(f"{self._http_endpoint}api/list", json=True)
+        result = await httpget(f"{self._http_endpoint}api/list?limit=15&offset=0", json=True)
 
         result = list(sorted(result, key=lambda k: k['id'], reverse=True))
-        result = result[:15]
 
         for post in result:
             post_id = int(post['id'])

wowlet_backend/tasks/wowlet.py

@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+from dateutil.parser import parse
+
+import settings
+from wowlet_backend.utils import httpget
+from wowlet_backend.tasks import WowletTask
+
+
+class WowletReleasesTask(WowletTask):
+    """Fetches the latest Wowlet releases using gitea's API"""
+    def __init__(self, interval: int = 3600):
+        super(WowletReleasesTask, self).__init__(interval)
+
+        self._cache_key = "wowlet_releases"
+        self._cache_expiry = self.interval
+
+        self._websocket_cmd = "wowlet_releases"
+
+        self._http_endpoint = "https://git.wownero.com/api/v1/repos/wowlet/wowlet/releases?limit=1"
+
+    async def task(self) -> dict:
+        blob = await httpget(self._http_endpoint)
+        if not isinstance(blob, list) or not blob:
+            raise Exception(f"Invalid JSON response for {self._http_endpoint}")
+
+        blob = blob[0]
+
+        data = {}
+        for asset in blob['assets']:
+            operating_system = "linux"
+            if "msvc" in asset['name'] or "win64" in asset['name'] or "windows" in asset['name']:
+                operating_system = "windows"
+            elif "macos" in asset["name"]:
+                operating_system = "macos"
+
+            data[operating_system] = {
+                "name": asset["name"],
+                "created_at": parse(asset["created_at"]).strftime("%Y-%m-%d"),
+                "url": asset['browser_download_url'],
+                "download_count": asset["download_count"],
+                "size": asset['size']
+            }
+
+        tag = blob['tag_name']
+        if tag.startswith("v"):
+            tag = tag[1:]
+
+        try:
+            t = [int(z) for z in tag.split(".")]
+            if len(t) != 3:
+                raise Exception()
+        except:
+            raise Exception(f"invalid tag: {tag}")
+
+        return {
+            "assets": data,
+            "body": blob['body'],
+            "version": tag
+        }

wowlet_backend/tasks/wownerod.py

@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2022, The Monero Project.
+# Copyright (c) 2022, dsc@xmr.pm
+
+from dateutil.parser import parse
+
+import settings
+from wowlet_backend.utils import httpget
+from wowlet_backend.tasks import WowletTask
+
+
+class WownerodTask(WowletTask):
+    """Fetches the latest Wownerod releases using gitea's API"""
+    def __init__(self, interval: int = 3600):
+        super(WownerodTask, self).__init__(interval)
+
+        self._cache_key = "wownerod_releases"
+        self._cache_expiry = self.interval
+
+        self._websocket_cmd = "wownerod_releases"
+
+        self._http_endpoint = "https://git.wownero.com/api/v1/repos/wownero/wownero/releases?limit=1"
+
+    async def task(self) -> dict:
+        blob = await httpget(self._http_endpoint)
+        if not isinstance(blob, list) or not blob:
+            raise Exception(f"Invalid JSON response for {self._http_endpoint}")
+
+        blob = blob[0]
+
+        data = {}
+        for asset in blob['assets']:
+            operating_system = "linux"
+            if "w64" in asset['name'] or "msvc" in asset['name'] or "win64" in asset['name'] or "windows" in asset['name']:
+                operating_system = "windows"
+            elif "macos" in asset["name"] or "apple" in asset["name"]:
+                operating_system = "macos"
+
+            data.setdefault(operating_system, [])
+            data[operating_system].append({
+                "name": asset["name"],
+                "created_at": parse(asset["created_at"]).strftime("%Y-%m-%d"),
+                "url": asset['browser_download_url'],
+                "download_count": asset["download_count"],
+                "size": asset['size']
+            })
+
+        tag = blob['tag_name']
+        if tag.startswith("v"):
+            tag = tag[1:]
+
+        return {
+            "version": tag,
+            "assets": data
+        }

wowlet_backend/tasks/xmrig.py

@@ -6,10 +6,10 @@ from dateutil.parser import parse
 
 import settings
 from wowlet_backend.utils import httpget
-from wowlet_backend.tasks import FeatherTask
+from wowlet_backend.tasks import WowletTask
 
 
-class XmrigTask(FeatherTask):
+class XmrigTask(WowletTask):
     """Fetches the latest XMRig releases using Github's API"""
     def __init__(self, interval: int = 43200):
         super(XmrigTask, self).__init__(interval)

wowlet_backend/tasks/yellow.py

@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2022, The Monero Project.
+# Copyright (c) 2022, dsc@xmr.pm
+
+from typing import List
+from dateutil.parser import parse
+
+import settings
+from wowlet_backend.utils import httpget
+from wowlet_backend.tasks import WowletTask
+
+
+class YellWowTask(WowletTask):
+    """Fetches yellwowpages usernames/addresses"""
+    def __init__(self, interval: int = 3600):
+        super(YellWowTask, self).__init__(interval)
+
+        self._cache_key = "yellwow"
+        self._cache_expiry = self.interval * 10
+
+        self._websocket_cmd = "yellwow"
+
+        self._http_endpoint = "https://yellow.wownero.com/api/user/"
+
+    async def task(self) -> List[dict]:
+        blob = await httpget(self._http_endpoint)
+        if not isinstance(blob, list) or not blob:
+            raise Exception(f"Invalid JSON response for {self._http_endpoint}")
+        return blob

wowlet_backend/trollbox.py

@@ -0,0 +1,26 @@
+import re, sys, os
+from datetime import datetime
+import time
+
+from wowlet_backend.factory import broadcast
+
+HISTORY = []
+
+
+async def add_chat(message: str, balance: float = 0):
+    global HISTORY
+    item = {
+        "message": message,
+        "balance": balance,
+        "date": int(time.time())
+    }
+
+    HISTORY.append(item)
+    if len(HISTORY) >= 25:
+        HISTORY = HISTORY[:25]
+
+    await broadcast.put({
+        "cmd": "trollEntry",
+        "data": item
+    })
+

wowlet_backend/utils.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright (c) 2020, The Monero Project.
-# Copyright (c) 2020, dsc@xmr.pm
+# Copyright (c) 2022, The Monero Project.
+# Copyright (c) 2022, dsc@xmr.pm
 
 import re
 import json
@@ -41,27 +41,14 @@ def print_banner():
     """.strip())
 
 
-def collect_websocket(func):
-    @wraps(func)
-    async def wrapper(*args, **kwargs):
-        from wowlet_backend.factory import connected_websockets
-        queue = asyncio.Queue()
-        connected_websockets.add(queue)
-        try:
-            return await func(queue, *args, **kwargs)
-        finally:
-            connected_websockets.remove(queue)
-    return wrapper
-
-
-async def httpget(url: str, json=True, timeout: int = 5, socks5: str = None, raise_for_status=True):
+async def httpget(url: str, json=True, timeout: int = 5, socks5: str = None, raise_for_status=True, verify_tls=True):
     headers = {"User-Agent": random_agent()}
     opts = {"timeout": aiohttp.ClientTimeout(total=timeout)}
     if socks5:
         opts['connector'] = ProxyConnector.from_url(socks5)
 
     async with aiohttp.ClientSession(**opts) as session:
-        async with session.get(url, headers=headers) as response:
+        async with session.get(url, headers=headers, ssl=verify_tls) as response:
             if raise_for_status:
                 response.raise_for_status()
 
@@ -76,23 +63,31 @@ def random_agent():
     return random.choice(user_agents)
 
 
-async def feather_data():
-    """A collection of data collected by
-    `FeatherTask`, for Feather wallet clients."""
+async def wowlet_data():
+    """A collection of data collected by the various wowlet tasks"""
     from wowlet_backend.factory import cache, now
     data = await cache.get("data")
     if data:
         data = json.loads(data)
         return data
 
-    keys = ["blockheights", "funding_proposals", "crypto_rates", "fiat_rates", "reddit", "rpc_nodes", "xmrig", "xmrto_rates", "suchwow"]
+    keys = [
+        "blockheights",
+        "funding_proposals",
+        "crypto_rates",
+        "fiat_rates",
+        "reddit",
+        "rpc_nodes",
+        "xmrig",
+        "xmrto_rates",
+        "suchwow",
+        "forum",
+        "wowlet_releases",
+        "yellwow",
+        "wownerod_releases"
+    ]
     data = {keys[i]: json.loads(val) if val else None for i, val in enumerate(await cache.mget(*keys))}
 
-    # @TODO: for backward-compat reasons we're including some legacy keys which can be removed after 1.0 release
-    data['nodes'] = data['rpc_nodes']
-    data['ccs'] = data['funding_proposals']
-    data['wfs'] = data['funding_proposals']
-
     # start caching when application lifetime is more than 20 seconds
     if (datetime.now() - now).total_seconds() > 20:
         await cache.setex("data", 30, json.dumps(data))
@@ -110,34 +105,6 @@ def popularity_contest(lst: List[int]) -> Union[int, None]:
     return Counter(lst).most_common(1)[0][0]
 
 
-def current_worker_thread_is_primary() -> bool:
-    """
-    ASGI server (Hypercorn) may start multiple
-    worker threads, but we only want one feather-ws
-    instance to schedule `FeatherTask` tasks at an
-    interval. Therefor this function determines if the
-    current instance is responsible for the
-    recurring Feather tasks.
-    """
-    from wowlet_backend.factory import app
-
-    current_pid = os.getpid()
-    parent_pid = os.getppid()
-    app.logger.debug(f"current_pid: {current_pid}, "
-                     f"parent_pid: {parent_pid}")
-
-    if parent_pid == 0:
-        return True
-
-    parent = psutil.Process(parent_pid)
-    if parent.name() != "hypercorn":
-        return True
-
-    lowest_pid = min(c.pid for c in parent.children(recursive=True) if c.name() == "hypercorn")
-    if current_pid == lowest_pid:
-        return True
-
-
 async def image_resize(buffer: bytes, max_bounding_box: int = 512, quality: int = 70) -> bytes:
     """
     - Resize if the image is too large
@@ -161,3 +128,54 @@ async def image_resize(buffer: bytes, max_bounding_box: int = 512, quality: int
     buffer.seek(0)
 
     return buffer.read()
+
+
+async def whaleornot(amount: float):
+    if amount <= 0:
+        fish_str = "amoeba"
+    elif amount < 100:
+        fish_str = "plankton"
+    elif amount >= 100 and amount < 200:
+        fish_str = "Paedocypris"
+    elif amount >= 200 and amount < 500:
+        fish_str = "Dwarf Goby"
+    elif amount >= 500 and amount < 1000:
+        fish_str = "European Pilchard"
+    elif amount >= 1000 and amount < 2000:
+        fish_str = "Goldfish"
+    elif amount >= 2000 and amount < 4000:
+        fish_str = "Herring"
+    elif amount >= 4000 and amount < 7000:
+        fish_str = "Atlantic Macerel"
+    elif amount >= 7000 and amount < 9000:
+        fish_str = "Gilt-head Bream"
+    elif amount >= 9000 and amount < 12000:
+        fish_str = "Salmonidae"
+    elif amount >= 12000 and amount < 20000:
+        fish_str = "Gadidae"
+    elif amount >= 20000 and amount < 40000:
+        fish_str = "Norwegian Delicious Salmon"
+    elif amount >= 40000 and amount < 60000:
+        fish_str = "Electric eel"
+    elif amount >= 60000 and amount < 80000:
+        fish_str = "Tuna"
+    elif amount >= 80000 and amount < 100000:
+        fish_str = "Wels catfish"
+    elif amount >= 100000 and amount < 120000:
+        fish_str = "Black marlin"
+    elif amount >= 120000 and amount < 160000:
+        fish_str = "Shark"
+    elif amount >= 160000 and amount < 220000:
+        fish_str = "Dolphin"
+    elif amount >= 220000 and amount < 320000:
+        fish_str = "Narwhal"
+    elif amount >= 320000 and amount < 500000:
+        fish_str = "Orca"
+    elif amount >= 500000 and amount < 700000:
+        fish_str = "Blue Whale"
+    elif amount >= 700000 and amount < 1000000:
+        fish_str = "Leviathan"
+    else:
+        fish_str = "Cthulu"
+    return fish_str
+

wowlet_backend/wsparse.py

@@ -10,6 +10,7 @@ from typing import Dict, Union, Optional
 from copy import deepcopy
 import asyncio
 import re
+from wowlet_backend.trollbox import add_chat
 
 from wowlet_backend.utils import RE_ADDRESS
 
@@ -27,6 +28,21 @@ class WebsocketParse:
             return await WebsocketParse.requestPIN(data)
         elif cmd == "lookupPIN":
             return await WebsocketParse.lookupPIN(data)
+        elif cmd == "trollbox":
+            pass
+            #return await WebsocketParse.addTrollChat(data)
+
+    @staticmethod
+    async def addTrollChat(data):
+        if not data or not isinstance(data, dict):
+            return
+        for needle in ['message', 'balance']:
+            if needle not in data:
+                return
+        
+        message = data['message']
+        balance = data['balance']
+        await add_chat(message, balance)
 
     @staticmethod
     async def txFiatHistory(data=None):