Suchwow.xyz task

requirements.txt

@@ -6,4 +6,7 @@ quart_session
 beautifulsoup4
 aiohttp_socks
 python-dateutil
-psutil
+psutil
+psutil
+pillow-simd
+python-magic

wowlet_backend/factory.py

@@ -59,7 +59,7 @@ async def _setup_tasks(app: Quart):
     from wowlet_backend.tasks import (
         BlockheightTask, HistoricalPriceTask, FundingProposalsTask,
         CryptoRatesTask, FiatRatesTask, RedditTask, RPCNodeCheckTask,
-        XmrigTask, XmrToTask)
+        XmrigTask, SuchWowTask)
 
     asyncio.create_task(BlockheightTask().start())
     asyncio.create_task(HistoricalPriceTask().start())
@@ -68,6 +68,7 @@ async def _setup_tasks(app: Quart):
     asyncio.create_task(RedditTask().start())
     asyncio.create_task(RPCNodeCheckTask().start())
     asyncio.create_task(XmrigTask().start())
+    asyncio.create_task(SuchWowTask().start())
 
     if settings.COIN_SYMBOL in ["xmr", "wow"]:
         asyncio.create_task(FundingProposalsTask().start())

wowlet_backend/routes.py

@@ -2,11 +2,13 @@
 # Copyright (c) 2020, The Monero Project.
 # Copyright (c) 2020, dsc@xmr.pm
 
+import os
 import asyncio
 import json
 
-from quart import websocket, jsonify
+from quart import websocket, jsonify, send_from_directory
 
+import settings
 from wowlet_backend.factory import app
 from wowlet_backend.wsparse import WebsocketParse
 from wowlet_backend.utils import collect_websocket, feather_data
@@ -18,6 +20,13 @@ async def root():
     return jsonify(data)
 
 
+@app.route("/suchwow/<path:name>")
+async def suchwow(name: str):
+    """Download a SuchWow.xyz image"""
+    base = os.path.join(settings.cwd, "data", "suchwow")
+    return await send_from_directory(base, name)
+
+
 @app.websocket('/ws')
 @collect_websocket
 async def ws(queue):

wowlet_backend/tasks/__init__.py

@@ -127,7 +127,7 @@ class FeatherTask:
         from wowlet_backend.factory import app, cache
 
         try:
-            data = await cache.execute('JSON.GET', key, path)
+            data = await cache.get(key)
             if data:
                 return json.loads(data)
         except Exception as ex:
@@ -165,4 +165,4 @@ from wowlet_backend.tasks.rates_crypto import CryptoRatesTask
 from wowlet_backend.tasks.reddit import RedditTask
 from wowlet_backend.tasks.rpc_nodes import RPCNodeCheckTask
 from wowlet_backend.tasks.xmrig import XmrigTask
-from wowlet_backend.tasks.xmrto import XmrToTask
+from wowlet_backend.tasks.suchwow import SuchWowTask

wowlet_backend/tasks/rates_fiat.py

@@ -10,7 +10,7 @@ from wowlet_backend.tasks import FeatherTask
 
 
 class FiatRatesTask(FeatherTask):
-    def __init__(self, interval: int = 600):
+    def __init__(self, interval: int = 43200):
         super(FiatRatesTask, self).__init__(interval)
 
         self._cache_key = "fiat_rates"

wowlet_backend/tasks/suchwow.py

@@ -0,0 +1,119 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+import json
+import os
+import re
+import glob
+
+import magic
+import aiohttp
+import aiofiles
+from bs4 import BeautifulSoup
+
+import settings
+from wowlet_backend.utils import httpget, image_resize
+from wowlet_backend.tasks import FeatherTask
+
+
+class SuchWowTask(FeatherTask):
+    def __init__(self, interval: int = 600):
+        """
+        This task is specifically for Wownero - fetching a listing
+        of recent SuchWow submissions.
+        """
+        super(SuchWowTask, self).__init__(interval)
+
+        self._cache_key = "suchwow"
+        self._cache_expiry = self.interval * 10
+
+        self._http_endpoint = "https://suchwow.xyz/"
+        self._tmp_dir = os.path.join(settings.cwd, "data", "suchwow")
+
+        if not os.path.exists(self._tmp_dir):
+            os.mkdir(self._tmp_dir)
+
+    async def task(self):
+        from wowlet_backend.factory import app
+        result = await httpget(f"{self._http_endpoint}api/list", json=True)
+
+        result = list(sorted(result, key=lambda k: k['id'], reverse=True))
+        result = result[:15]
+
+        for post in result:
+            post_id = int(post['id'])
+            path_img = os.path.join(self._tmp_dir, f"{post_id}.jpg")
+            path_img_thumb = os.path.join(self._tmp_dir, f"{post_id}.thumb.jpg")
+            path_img_tmp = os.path.join(self._tmp_dir, f"{post_id}.tmp.jpg")
+            path_metadata = os.path.join(self._tmp_dir, f"{post_id}.json")
+            if os.path.exists(path_metadata):
+                continue
+
+            try:
+                url = post['image']
+                await self.download_and_write(url, path_img_tmp)
+
+                async with aiofiles.open(path_img_tmp, mode="rb") as f:
+                    image = await f.read()
+
+                # security: only images
+                if not await self.is_image(image):
+                    app.logger.error(f"skipping {post_id} because of invalid mimetype")
+
+                resized = await image_resize(image, max_bounding_box=800, quality=80)
+                thumbnail = await image_resize(image, max_bounding_box=400, quality=80)
+
+                async with aiofiles.open(path_img, mode="wb") as f:
+                    await f.write(resized)
+
+                async with aiofiles.open(path_img_thumb, mode="wb") as f:
+                    await f.write(thumbnail)
+
+            except Exception as ex:
+                app.logger.error(f"Failed to download or resize {post_id}, cleaning up leftover files. {ex}")
+                for path in [path_img, path_img_tmp, path_img_thumb]:
+                    if os.path.exists(path):
+                        os.unlink(path)
+                continue
+
+            f = open(path_metadata, "w")
+            f.write(json.dumps({
+                "img": os.path.basename(path_img),
+                "thumb": os.path.basename(path_img_thumb),
+                "added_by": post['submitter'].replace("<", ""),
+                "addy": post['address'],
+                "title": post['title'].replace("<", ""),
+                "href": post['href'],
+                "id": post['id']
+            }))
+            f.close()
+
+        images = []
+        try:
+            for fn in glob.glob(f"{self._tmp_dir}/*.json", recursive=False):
+                async with aiofiles.open(fn, mode="rb") as f:
+                    blob = json.loads(await f.read())
+                    images.append(blob)
+        except Exception as ex:
+            pass
+
+        # sort on id, limit
+        images = list(sorted(images, key=lambda k: k['id'], reverse=True))
+        images = images[:15]
+
+        return images
+
+    async def download_and_write(self, url: str, destination: str):
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as resp:
+                if resp.status != 200:
+                    raise Exception(f"Failed to download image from {url}; status non 200")
+                f = await aiofiles.open(destination, mode='wb')
+                await f.write(await resp.read())
+                await f.close()
+
+    async def is_image(self, buffer: bytes):
+        mime = magic.from_buffer(buffer, mime=True)
+        if mime in ["image/jpeg", "image/jpg", "image/png"]:
+            return True

wowlet_backend/tasks/xmrto.py

@@ -1,26 +0,0 @@
-# SPDX-License-Identifier: BSD-3-Clause
-# Copyright (c) 2020, The Monero Project.
-# Copyright (c) 2020, dsc@xmr.pm
-
-import settings
-from wowlet_backend.utils import httpget
-from wowlet_backend.tasks import FeatherTask
-
-
-class XmrToTask(FeatherTask):
-    def __init__(self, interval: int = 30):
-        super(XmrToTask, self).__init__(interval)
-
-        self._cache_key = "xmrto_rates"
-        self._cache_expiry = self.interval * 10
-
-        if settings.COIN_MODE == 'stagenet':
-            self._http_endpoint = "https://test.xmr.to/api/v3/xmr2btc/order_parameter_query/"
-        else:
-            self._http_endpoint = "https://xmr.to/api/v3/xmr2btc/order_parameter_query/"
-
-    async def task(self):
-        result = await httpget(self._http_endpoint)
-        if "error" in result:
-            raise Exception(f"${result['error']} ${result['error_msg']}")
-        return result

wowlet_backend/utils.py

@@ -10,10 +10,12 @@ from datetime import datetime
 from collections import Counter
 from functools import wraps
 from typing import List, Union
+from io import BytesIO
 
 import psutil
 import aiohttp
 from aiohttp_socks import ProxyConnector
+from PIL import Image
 
 import settings
 
@@ -79,7 +81,7 @@ async def feather_data():
         data = json.loads(data)
         return data
 
-    keys = ["blockheights", "funding_proposals", "crypto_rates", "fiat_rates", "reddit", "rpc_nodes", "xmrig", "xmrto_rates"]
+    keys = ["blockheights", "funding_proposals", "crypto_rates", "fiat_rates", "reddit", "rpc_nodes", "xmrig", "xmrto_rates", "suchwow"]
     data = {keys[i]: json.loads(val) if val else None for i, val in enumerate(await cache.mget(*keys))}
 
     # @TODO: for backward-compat reasons we're including some legacy keys which can be removed after 1.0 release
@@ -131,3 +133,27 @@ def current_worker_thread_is_primary() -> bool:
     if current_pid == lowest_pid:
         return True
 
+
+async def image_resize(buffer: bytes, max_bounding_box: int = 512, quality: int = 70) -> bytes:
+    """
+    - Resize if the image is too large
+    - PNG -> JPEG
+    - Removes EXIF
+    """
+    buffer = BytesIO(buffer)
+    buffer.seek(0)
+    image = Image.open(buffer)
+    image = image.convert('RGB')
+
+    if max([image.height, image.width]) > max_bounding_box:
+        image.thumbnail((max_bounding_box, max_bounding_box), Image.BICUBIC)
+
+    data = list(image.getdata())
+    image_without_exif = Image.new(image.mode, image.size)
+    image_without_exif.putdata(data)
+
+    buffer = BytesIO()
+    image_without_exif.save(buffer, "JPEG", quality=quality)
+    buffer.seek(0)
+
+    return buffer.read()