diff --git a/requirements.txt b/requirements.txt index 7038f33..1490c6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,7 @@ quart_session beautifulsoup4 aiohttp_socks python-dateutil -psutil \ No newline at end of file +psutil +psutil +pillow-simd +python-magic \ No newline at end of file diff --git a/wowlet_backend/factory.py b/wowlet_backend/factory.py index 12ffdb0..188d47f 100644 --- a/wowlet_backend/factory.py +++ b/wowlet_backend/factory.py @@ -59,7 +59,7 @@ async def _setup_tasks(app: Quart): from wowlet_backend.tasks import ( BlockheightTask, HistoricalPriceTask, FundingProposalsTask, CryptoRatesTask, FiatRatesTask, RedditTask, RPCNodeCheckTask, - XmrigTask, XmrToTask) + XmrigTask, SuchWowTask) asyncio.create_task(BlockheightTask().start()) asyncio.create_task(HistoricalPriceTask().start()) @@ -68,6 +68,7 @@ async def _setup_tasks(app: Quart): asyncio.create_task(RedditTask().start()) asyncio.create_task(RPCNodeCheckTask().start()) asyncio.create_task(XmrigTask().start()) + asyncio.create_task(SuchWowTask().start()) if settings.COIN_SYMBOL in ["xmr", "wow"]: asyncio.create_task(FundingProposalsTask().start()) diff --git a/wowlet_backend/routes.py b/wowlet_backend/routes.py index f9a9f92..7d4b659 100644 --- a/wowlet_backend/routes.py +++ b/wowlet_backend/routes.py @@ -2,11 +2,13 @@ # Copyright (c) 2020, The Monero Project. # Copyright (c) 2020, dsc@xmr.pm +import os import asyncio import json -from quart import websocket, jsonify +from quart import websocket, jsonify, send_from_directory +import settings from wowlet_backend.factory import app from wowlet_backend.wsparse import WebsocketParse from wowlet_backend.utils import collect_websocket, feather_data @@ -18,6 +20,13 @@ async def root(): return jsonify(data) +@app.route("/suchwow/") +async def suchwow(name: str): + """Download a SuchWow.xyz image""" + base = os.path.join(settings.cwd, "data", "suchwow") + return await send_from_directory(base, name) + + @app.websocket('/ws') @collect_websocket async def ws(queue): diff --git a/wowlet_backend/tasks/__init__.py b/wowlet_backend/tasks/__init__.py index ff70e9e..2bff2c4 100644 --- a/wowlet_backend/tasks/__init__.py +++ b/wowlet_backend/tasks/__init__.py @@ -127,7 +127,7 @@ class FeatherTask: from wowlet_backend.factory import app, cache try: - data = await cache.execute('JSON.GET', key, path) + data = await cache.get(key) if data: return json.loads(data) except Exception as ex: @@ -165,4 +165,4 @@ from wowlet_backend.tasks.rates_crypto import CryptoRatesTask from wowlet_backend.tasks.reddit import RedditTask from wowlet_backend.tasks.rpc_nodes import RPCNodeCheckTask from wowlet_backend.tasks.xmrig import XmrigTask -from wowlet_backend.tasks.xmrto import XmrToTask +from wowlet_backend.tasks.suchwow import SuchWowTask diff --git a/wowlet_backend/tasks/rates_fiat.py b/wowlet_backend/tasks/rates_fiat.py index acf59ec..260fb7c 100644 --- a/wowlet_backend/tasks/rates_fiat.py +++ b/wowlet_backend/tasks/rates_fiat.py @@ -10,7 +10,7 @@ from wowlet_backend.tasks import FeatherTask class FiatRatesTask(FeatherTask): - def __init__(self, interval: int = 600): + def __init__(self, interval: int = 43200): super(FiatRatesTask, self).__init__(interval) self._cache_key = "fiat_rates" diff --git a/wowlet_backend/tasks/suchwow.py b/wowlet_backend/tasks/suchwow.py new file mode 100644 index 0000000..b09a511 --- /dev/null +++ b/wowlet_backend/tasks/suchwow.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2020, The Monero Project. +# Copyright (c) 2020, dsc@xmr.pm + +import json +import os +import re +import glob + +import magic +import aiohttp +import aiofiles +from bs4 import BeautifulSoup + +import settings +from wowlet_backend.utils import httpget, image_resize +from wowlet_backend.tasks import FeatherTask + + +class SuchWowTask(FeatherTask): + def __init__(self, interval: int = 600): + """ + This task is specifically for Wownero - fetching a listing + of recent SuchWow submissions. + """ + super(SuchWowTask, self).__init__(interval) + + self._cache_key = "suchwow" + self._cache_expiry = self.interval * 10 + + self._http_endpoint = "https://suchwow.xyz/" + self._tmp_dir = os.path.join(settings.cwd, "data", "suchwow") + + if not os.path.exists(self._tmp_dir): + os.mkdir(self._tmp_dir) + + async def task(self): + from wowlet_backend.factory import app + result = await httpget(f"{self._http_endpoint}api/list", json=True) + + result = list(sorted(result, key=lambda k: k['id'], reverse=True)) + result = result[:15] + + for post in result: + post_id = int(post['id']) + path_img = os.path.join(self._tmp_dir, f"{post_id}.jpg") + path_img_thumb = os.path.join(self._tmp_dir, f"{post_id}.thumb.jpg") + path_img_tmp = os.path.join(self._tmp_dir, f"{post_id}.tmp.jpg") + path_metadata = os.path.join(self._tmp_dir, f"{post_id}.json") + if os.path.exists(path_metadata): + continue + + try: + url = post['image'] + await self.download_and_write(url, path_img_tmp) + + async with aiofiles.open(path_img_tmp, mode="rb") as f: + image = await f.read() + + # security: only images + if not await self.is_image(image): + app.logger.error(f"skipping {post_id} because of invalid mimetype") + + resized = await image_resize(image, max_bounding_box=800, quality=80) + thumbnail = await image_resize(image, max_bounding_box=400, quality=80) + + async with aiofiles.open(path_img, mode="wb") as f: + await f.write(resized) + + async with aiofiles.open(path_img_thumb, mode="wb") as f: + await f.write(thumbnail) + + except Exception as ex: + app.logger.error(f"Failed to download or resize {post_id}, cleaning up leftover files. {ex}") + for path in [path_img, path_img_tmp, path_img_thumb]: + if os.path.exists(path): + os.unlink(path) + continue + + f = open(path_metadata, "w") + f.write(json.dumps({ + "img": os.path.basename(path_img), + "thumb": os.path.basename(path_img_thumb), + "added_by": post['submitter'].replace("<", ""), + "addy": post['address'], + "title": post['title'].replace("<", ""), + "href": post['href'], + "id": post['id'] + })) + f.close() + + images = [] + try: + for fn in glob.glob(f"{self._tmp_dir}/*.json", recursive=False): + async with aiofiles.open(fn, mode="rb") as f: + blob = json.loads(await f.read()) + images.append(blob) + except Exception as ex: + pass + + # sort on id, limit + images = list(sorted(images, key=lambda k: k['id'], reverse=True)) + images = images[:15] + + return images + + async def download_and_write(self, url: str, destination: str): + async with aiohttp.ClientSession() as session: + async with session.get(url) as resp: + if resp.status != 200: + raise Exception(f"Failed to download image from {url}; status non 200") + f = await aiofiles.open(destination, mode='wb') + await f.write(await resp.read()) + await f.close() + + async def is_image(self, buffer: bytes): + mime = magic.from_buffer(buffer, mime=True) + if mime in ["image/jpeg", "image/jpg", "image/png"]: + return True diff --git a/wowlet_backend/tasks/xmrto.py b/wowlet_backend/tasks/xmrto.py deleted file mode 100644 index bb4e280..0000000 --- a/wowlet_backend/tasks/xmrto.py +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2020, The Monero Project. -# Copyright (c) 2020, dsc@xmr.pm - -import settings -from wowlet_backend.utils import httpget -from wowlet_backend.tasks import FeatherTask - - -class XmrToTask(FeatherTask): - def __init__(self, interval: int = 30): - super(XmrToTask, self).__init__(interval) - - self._cache_key = "xmrto_rates" - self._cache_expiry = self.interval * 10 - - if settings.COIN_MODE == 'stagenet': - self._http_endpoint = "https://test.xmr.to/api/v3/xmr2btc/order_parameter_query/" - else: - self._http_endpoint = "https://xmr.to/api/v3/xmr2btc/order_parameter_query/" - - async def task(self): - result = await httpget(self._http_endpoint) - if "error" in result: - raise Exception(f"${result['error']} ${result['error_msg']}") - return result diff --git a/wowlet_backend/utils.py b/wowlet_backend/utils.py index 806b0af..ba48afc 100644 --- a/wowlet_backend/utils.py +++ b/wowlet_backend/utils.py @@ -10,10 +10,12 @@ from datetime import datetime from collections import Counter from functools import wraps from typing import List, Union +from io import BytesIO import psutil import aiohttp from aiohttp_socks import ProxyConnector +from PIL import Image import settings @@ -79,7 +81,7 @@ async def feather_data(): data = json.loads(data) return data - keys = ["blockheights", "funding_proposals", "crypto_rates", "fiat_rates", "reddit", "rpc_nodes", "xmrig", "xmrto_rates"] + keys = ["blockheights", "funding_proposals", "crypto_rates", "fiat_rates", "reddit", "rpc_nodes", "xmrig", "xmrto_rates", "suchwow"] data = {keys[i]: json.loads(val) if val else None for i, val in enumerate(await cache.mget(*keys))} # @TODO: for backward-compat reasons we're including some legacy keys which can be removed after 1.0 release @@ -131,3 +133,27 @@ def current_worker_thread_is_primary() -> bool: if current_pid == lowest_pid: return True + +async def image_resize(buffer: bytes, max_bounding_box: int = 512, quality: int = 70) -> bytes: + """ + - Resize if the image is too large + - PNG -> JPEG + - Removes EXIF + """ + buffer = BytesIO(buffer) + buffer.seek(0) + image = Image.open(buffer) + image = image.convert('RGB') + + if max([image.height, image.width]) > max_bounding_box: + image.thumbnail((max_bounding_box, max_bounding_box), Image.BICUBIC) + + data = list(image.getdata()) + image_without_exif = Image.new(image.mode, image.size) + image_without_exif.putdata(data) + + buffer = BytesIO() + image_without_exif.save(buffer, "JPEG", quality=quality) + buffer.seek(0) + + return buffer.read()