Browse Source

Suchwow.xyz task

tempfix
dsc 3 months ago
parent
commit
4e7338a4ea
  1. 5
      requirements.txt
  2. 3
      wowlet_backend/factory.py
  3. 11
      wowlet_backend/routes.py
  4. 4
      wowlet_backend/tasks/__init__.py
  5. 2
      wowlet_backend/tasks/rates_fiat.py
  6. 119
      wowlet_backend/tasks/suchwow.py
  7. 26
      wowlet_backend/tasks/xmrto.py
  8. 28
      wowlet_backend/utils.py

5
requirements.txt

@ -6,4 +6,7 @@ quart_session
beautifulsoup4
aiohttp_socks
python-dateutil
psutil
psutil
psutil
pillow-simd
python-magic

3
wowlet_backend/factory.py

@ -59,7 +59,7 @@ async def _setup_tasks(app: Quart):
from wowlet_backend.tasks import (
BlockheightTask, HistoricalPriceTask, FundingProposalsTask,
CryptoRatesTask, FiatRatesTask, RedditTask, RPCNodeCheckTask,
XmrigTask, XmrToTask)
XmrigTask, SuchWowTask)
asyncio.create_task(BlockheightTask().start())
asyncio.create_task(HistoricalPriceTask().start())
@ -68,6 +68,7 @@ async def _setup_tasks(app: Quart):
asyncio.create_task(RedditTask().start())
asyncio.create_task(RPCNodeCheckTask().start())
asyncio.create_task(XmrigTask().start())
asyncio.create_task(SuchWowTask().start())
if settings.COIN_SYMBOL in ["xmr", "wow"]:
asyncio.create_task(FundingProposalsTask().start())

11
wowlet_backend/routes.py

@ -2,11 +2,13 @@
# Copyright (c) 2020, The Monero Project.
# Copyright (c) 2020, dsc@xmr.pm
import os
import asyncio
import json
from quart import websocket, jsonify
from quart import websocket, jsonify, send_from_directory
import settings
from wowlet_backend.factory import app
from wowlet_backend.wsparse import WebsocketParse
from wowlet_backend.utils import collect_websocket, feather_data
@ -18,6 +20,13 @@ async def root():
return jsonify(data)
@app.route("/suchwow/<path:name>")
async def suchwow(name: str):
"""Download a SuchWow.xyz image"""
base = os.path.join(settings.cwd, "data", "suchwow")
return await send_from_directory(base, name)
@app.websocket('/ws')
@collect_websocket
async def ws(queue):

4
wowlet_backend/tasks/__init__.py

@ -127,7 +127,7 @@ class FeatherTask:
from wowlet_backend.factory import app, cache
try:
data = await cache.execute('JSON.GET', key, path)
data = await cache.get(key)
if data:
return json.loads(data)
except Exception as ex:
@ -165,4 +165,4 @@ from wowlet_backend.tasks.rates_crypto import CryptoRatesTask
from wowlet_backend.tasks.reddit import RedditTask
from wowlet_backend.tasks.rpc_nodes import RPCNodeCheckTask
from wowlet_backend.tasks.xmrig import XmrigTask
from wowlet_backend.tasks.xmrto import XmrToTask
from wowlet_backend.tasks.suchwow import SuchWowTask

2
wowlet_backend/tasks/rates_fiat.py

@ -10,7 +10,7 @@ from wowlet_backend.tasks import FeatherTask
class FiatRatesTask(FeatherTask):
def __init__(self, interval: int = 600):
def __init__(self, interval: int = 43200):
super(FiatRatesTask, self).__init__(interval)
self._cache_key = "fiat_rates"

119
wowlet_backend/tasks/suchwow.py

@ -0,0 +1,119 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2020, The Monero Project.
# Copyright (c) 2020, dsc@xmr.pm
import json
import os
import re
import glob
import magic
import aiohttp
import aiofiles
from bs4 import BeautifulSoup
import settings
from wowlet_backend.utils import httpget, image_resize
from wowlet_backend.tasks import FeatherTask
class SuchWowTask(FeatherTask):
def __init__(self, interval: int = 600):
"""
This task is specifically for Wownero - fetching a listing
of recent SuchWow submissions.
"""
super(SuchWowTask, self).__init__(interval)
self._cache_key = "suchwow"
self._cache_expiry = self.interval * 10
self._http_endpoint = "https://suchwow.xyz/"
self._tmp_dir = os.path.join(settings.cwd, "data", "suchwow")
if not os.path.exists(self._tmp_dir):
os.mkdir(self._tmp_dir)
async def task(self):
from wowlet_backend.factory import app
result = await httpget(f"{self._http_endpoint}api/list", json=True)
result = list(sorted(result, key=lambda k: k['id'], reverse=True))
result = result[:15]
for post in result:
post_id = int(post['id'])
path_img = os.path.join(self._tmp_dir, f"{post_id}.jpg")
path_img_thumb = os.path.join(self._tmp_dir, f"{post_id}.thumb.jpg")
path_img_tmp = os.path.join(self._tmp_dir, f"{post_id}.tmp.jpg")
path_metadata = os.path.join(self._tmp_dir, f"{post_id}.json")
if os.path.exists(path_metadata):
continue
try:
url = post['image']
await self.download_and_write(url, path_img_tmp)
async with aiofiles.open(path_img_tmp, mode="rb") as f:
image = await f.read()
# security: only images
if not await self.is_image(image):
app.logger.error(f"skipping {post_id} because of invalid mimetype")
resized = await image_resize(image, max_bounding_box=800, quality=80)
thumbnail = await image_resize(image, max_bounding_box=400, quality=80)
async with aiofiles.open(path_img, mode="wb") as f:
await f.write(resized)
async with aiofiles.open(path_img_thumb, mode="wb") as f:
await f.write(thumbnail)
except Exception as ex:
app.logger.error(f"Failed to download or resize {post_id}, cleaning up leftover files. {ex}")
for path in [path_img, path_img_tmp, path_img_thumb]:
if os.path.exists(path):
os.unlink(path)
continue
f = open(path_metadata, "w")
f.write(json.dumps({
"img": os.path.basename(path_img),
"thumb": os.path.basename(path_img_thumb),
"added_by": post['submitter'].replace("<", ""),
"addy": post['address'],
"title": post['title'].replace("<", ""),
"href": post['href'],
"id": post['id']
}))
f.close()
images = []
try:
for fn in glob.glob(f"{self._tmp_dir}/*.json", recursive=False):
async with aiofiles.open(fn, mode="rb") as f:
blob = json.loads(await f.read())
images.append(blob)
except Exception as ex:
pass
# sort on id, limit
images = list(sorted(images, key=lambda k: k['id'], reverse=True))
images = images[:15]
return images
async def download_and_write(self, url: str, destination: str):
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp:
if resp.status != 200:
raise Exception(f"Failed to download image from {url}; status non 200")
f = await aiofiles.open(destination, mode='wb')
await f.write(await resp.read())
await f.close()
async def is_image(self, buffer: bytes):
mime = magic.from_buffer(buffer, mime=True)
if mime in ["image/jpeg", "image/jpg", "image/png"]:
return True

26
wowlet_backend/tasks/xmrto.py

@ -1,26 +0,0 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2020, The Monero Project.
# Copyright (c) 2020, dsc@xmr.pm
import settings
from wowlet_backend.utils import httpget
from wowlet_backend.tasks import FeatherTask
class XmrToTask(FeatherTask):
def __init__(self, interval: int = 30):
super(XmrToTask, self).__init__(interval)
self._cache_key = "xmrto_rates"
self._cache_expiry = self.interval * 10
if settings.COIN_MODE == 'stagenet':
self._http_endpoint = "https://test.xmr.to/api/v3/xmr2btc/order_parameter_query/"
else:
self._http_endpoint = "https://xmr.to/api/v3/xmr2btc/order_parameter_query/"
async def task(self):
result = await httpget(self._http_endpoint)
if "error" in result:
raise Exception(f"${result['error']} ${result['error_msg']}")
return result

28
wowlet_backend/utils.py

@ -10,10 +10,12 @@ from datetime import datetime
from collections import Counter
from functools import wraps
from typing import List, Union
from io import BytesIO
import psutil
import aiohttp
from aiohttp_socks import ProxyConnector
from PIL import Image
import settings
@ -79,7 +81,7 @@ async def feather_data():
data = json.loads(data)
return data
keys = ["blockheights", "funding_proposals", "crypto_rates", "fiat_rates", "reddit", "rpc_nodes", "xmrig", "xmrto_rates"]
keys = ["blockheights", "funding_proposals", "crypto_rates", "fiat_rates", "reddit", "rpc_nodes", "xmrig", "xmrto_rates", "suchwow"]
data = {keys[i]: json.loads(val) if val else None for i, val in enumerate(await cache.mget(*keys))}
# @TODO: for backward-compat reasons we're including some legacy keys which can be removed after 1.0 release
@ -131,3 +133,27 @@ def current_worker_thread_is_primary() -> bool:
if current_pid == lowest_pid:
return True
async def image_resize(buffer: bytes, max_bounding_box: int = 512, quality: int = 70) -> bytes:
"""
- Resize if the image is too large
- PNG -> JPEG
- Removes EXIF
"""
buffer = BytesIO(buffer)
buffer.seek(0)
image = Image.open(buffer)
image = image.convert('RGB')
if max([image.height, image.width]) > max_bounding_box:
image.thumbnail((max_bounding_box, max_bounding_box), Image.BICUBIC)
data = list(image.getdata())
image_without_exif = Image.new(image.mode, image.size)
image_without_exif.putdata(data)
buffer = BytesIO()
image_without_exif.save(buffer, "JPEG", quality=quality)
buffer.seek(0)
return buffer.read()
Loading…
Cancel
Save