# SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2020, The Monero Project. # Copyright (c) 2020, dsc@xmr.pm import json import aiohttp from bs4 import BeautifulSoup from aiohttp_socks import ProxyType, ProxyConnector, ChainProxyConnector from fapi.utils import broadcast_blockheight, broadcast_nodes, httpget, BlockHeight import settings class FeatherApi: @staticmethod async def redis_get(key): from fapi.factory import app, cache try: data = await cache.get(key) if data: return json.loads(data) except Exception as ex: app.logger.error(f"Redis error: {ex}") @staticmethod async def xmrto_rates(): from fapi.factory import app, cache xmrto_rates = await FeatherApi.redis_get("xmrto_rates") if xmrto_rates and app.config["DEBUG"]: return xmrto_rates try: result = await httpget(settings.urls["xmrto_rates"]) if not result: raise Exception("empty response") if "error" in result: raise Exception(f"${result['error']} ${result['error_msg']}") return result except Exception as ex: app.logger.error(f"error parsing xmrto_rates blob: {ex}") return xmrto_rates @staticmethod async def after_xmrto(data): from fapi.factory import app, cache, api_data, connected_websockets if not data: return _data = api_data.get("xmrto_rates", {}) _data = json.dumps(_data, sort_keys=True, indent=4) if json.dumps(data, sort_keys=True, indent=4) == _data: return api_data["xmrto_rates"] = data @staticmethod async def crypto_rates(): from fapi.factory import app, cache crypto_rates = await FeatherApi.redis_get("crypto_rates") if crypto_rates and app.config["DEBUG"]: return crypto_rates result = None try: result = await httpget(settings.urls["crypto_rates"]) if not result: raise Exception("empty response") crypto_rates = result except Exception as ex: app.logger.error(f"error parsing crypto_rates blob: {ex}") if not result and crypto_rates: app.logger.warning("USING OLD CACHE FOR CRYPTO RATES") return crypto_rates # grab WOW price while we're at it... try: _result = await httpget(settings.urls["crypto_wow_rates"]) if not _result: raise Exception("empty response") except Exception as ex: _result = {} if "wownero" in _result and "usd" in _result["wownero"]: crypto_rates.append({ "id": "wownero", "symbol": "wow", "image": "", "name": "Wownero", "current_price": _result["wownero"]["usd"], "price_change_percentage_24h": 0.0 }) await cache.set("crypto_rates", json.dumps(crypto_rates)) return crypto_rates @staticmethod async def after_crypto(data): from fapi.factory import app, cache, api_data, connected_websockets if not data: return _data = api_data.get("crypto_rates", {}) _data = json.dumps(_data, sort_keys=True, indent=4) if json.dumps(data, sort_keys=True, indent=4) == _data: return _data = [] for obj in data: _data.append({ "id": obj['id'], "symbol": obj['symbol'], "image": obj['image'], "name": obj['name'], "current_price": obj['current_price'], "price_change_percentage_24h": obj['price_change_percentage_24h'] }) api_data["crypto_rates"] = data for queue in connected_websockets: await queue.put({ "cmd": "crypto_rates", "data": { "crypto_rates": api_data["crypto_rates"] } }) @staticmethod async def fiat_rates(): from fapi.factory import app, cache fiat_rates = await FeatherApi.redis_get("fiat_rates") if fiat_rates and app.config["DEBUG"]: return fiat_rates try: result = await httpget(settings.urls["fiat_rates"], json=True) if not result: raise Exception("empty response") await cache.set("fiat_rates", json.dumps(result)) return result except Exception as ex: app.logger.error(f"error parsing fiat_rates blob: {ex}") # old cache app.logger.warning("USING OLD CACHE FOR FIAT RATES") return fiat_rates @staticmethod async def after_fiat(data): from fapi.factory import app, cache, api_data, connected_websockets if not data: return _data = api_data.get("fiat_rates", {}) _data = json.dumps(_data, sort_keys=True, indent=4) if json.dumps(data, sort_keys=True, indent=4) == _data: return api_data["fiat_rates"] = data for queue in connected_websockets: await queue.put({ "cmd": "fiat_rates", "data": { "fiat_rates": api_data["fiat_rates"] } }) @staticmethod async def ccs(): # CCS JSON api is broken ;x https://hackerone.com/reports/934231 from fapi.factory import app, cache ccs = await FeatherApi.redis_get("ccs") if ccs and app.config["DEBUG"]: return ccs try: content = await httpget(f"{settings.urls['ccs']}/funding-required/", json=False) except Exception as ex: app.logger.error(f"error fetching ccs HTML: {ex}") return ccs try: soup = BeautifulSoup(content, "html.parser") except Exception as ex: app.logger.error(f"error parsing ccs HTML page: {ex}") return ccs data = [] for x in soup.findAll("a", {"class": "ffs-idea"}): try: item = { "state": "FUNDING-REQUIRED", "author": x.find("p", {"class": "author-list"}).text, "date": x.find("p", {"class": "date-list"}).text, "title": x.find("h3").text, "raised_amount": float(x.find("span", {"class": "progress-number-funded"}).text), "target_amount": float(x.find("span", {"class": "progress-number-goal"}).text), "contributors": 0, "url": f"https://ccs.getmonero.org{x.attrs['href']}" } item["percentage_funded"] = item["raised_amount"] * (100 / item["target_amount"]) if item["percentage_funded"] >= 100: item["percentage_funded"] = 100.0 try: item["contributors"] = int(x.find("p", {"class": "contributor"}).text.split(" ")[0]) except: pass href = x.attrs['href'] try: content = await httpget(f"{settings.urls['ccs']}{href}", json=False) try: soup2 = BeautifulSoup(content, "html.parser") except Exception as ex: app.logger.error(f"error parsing ccs HTML page: {ex}") continue try: instructions = soup2.find("div", {"class": "instructions"}) if not instructions: raise Exception("could not parse div.instructions, page probably broken") address = instructions.find("p", {"class": "string"}).text if not address.strip(): raise Exception(f"error fetching ccs HTML: could not parse address") item["address"] = address.strip() except Exception as ex: app.logger.error(f"error parsing ccs address from HTML: {ex}") continue except Exception as ex: app.logger.error(f"error fetching ccs HTML: {ex}") continue data.append(item) except Exception as ex: app.logger.error(f"error parsing a ccs item: {ex}") await cache.set("ccs", json.dumps(data)) return data @staticmethod async def after_ccs(data): from fapi.factory import app, cache, api_data, connected_websockets if not data: return _data = api_data.get("ccs", {}) _data = json.dumps(_data, sort_keys=True, indent=4) if json.dumps(data, sort_keys=True, indent=4) == _data: return api_data["ccs"] = data for queue in connected_websockets: await queue.put({ "cmd": "ccs", "data": api_data["ccs"] }) @staticmethod async def reddit(): from fapi.factory import app, cache reddit = await FeatherApi.redis_get("reddit") if reddit and app.config["DEBUG"]: return reddit try: blob = await httpget(settings.urls["reddit"]) if not blob: raise Exception("no data from url") blob = [{ 'title': z['data']['title'], 'author': z['data']['author'], 'url': "https://old.reddit.com/" + z['data']['permalink'], 'comments': z['data']['num_comments'] } for z in blob['data']['children']] # success if blob: await cache.set("reddit", json.dumps(blob)) return blob except Exception as ex: app.logger.error(f"error parsing reddit blob: {ex}") # old cache return reddit @staticmethod async def after_reddit(data): from fapi.factory import app, cache, api_data, connected_websockets if not data: return _data = api_data.get("reddit", {}) _data = json.dumps(_data, sort_keys=True, indent=4) if json.dumps(data, sort_keys=True, indent=4) == _data: return api_data["reddit"] = data for queue in connected_websockets: await queue.put({ "cmd": "reddit", "data": api_data["reddit"] }) @staticmethod async def blockheight(): from fapi.factory import app, cache data = {"mainnet": 0, "stagenet": 0} for stagenet in [False, True]: try: data["mainnet" if stagenet is False else "stagenet"] = \ await BlockHeight.xmrchain(stagenet) except Exception as ex: app.logger.error(f"Could not fetch blockheight from xmrchain") try: data["mainnet" if stagenet is False else "stagenet"] = \ await BlockHeight.xmrto(stagenet) except: app.logger.error(f"Could not fetch blockheight from xmr.to") return data @staticmethod async def after_blockheight(data): from fapi.factory import app, cache, api_data changed = False api_data.setdefault("blockheights", {}) if data["mainnet"] > 1 and data["mainnet"] > api_data["blockheights"].get("mainnet", 1): api_data["blockheights"]["mainnet"] = data["mainnet"] changed = True if data["stagenet"] > 1 and data["stagenet"] > api_data["blockheights"].get("stagenet", 1): api_data["blockheights"]["stagenet"] = data["stagenet"] changed = True if changed: await broadcast_blockheight() @staticmethod async def check_nodes(): from fapi.factory import nodes, app data = [] for network_type, network_name in nodes.items(): for k, _nodes in nodes[network_type].items(): for node in _nodes: timeout = aiohttp.ClientTimeout(total=5) d = {'timeout': timeout} if ".onion" in node: d['connector'] = ProxyConnector.from_url('socks5://127.0.0.1:9050') d['timeout'] = aiohttp.ClientTimeout(total=12) try: async with aiohttp.ClientSession(**d) as session: async with session.get(f"http://{node}/get_info") as response: blob = await response.json() for expect in ["nettype", "height"]: assert expect in blob _node = { "address": node, "height": int(blob["height"]), "online": True, "nettype": blob["nettype"], "type": k } except Exception as ex: app.logger.warning(f"node {node} not reachable") _node = { "address": node, "height": 0, "online": False, "nettype": network_type, "type": k } data.append(_node) return data @staticmethod async def after_check_nodes(data): from fapi.factory import api_data api_data["nodes"] = data await broadcast_nodes()