more compact data collection scheme, update wownero data

getdata.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2018 The Monero Project
+# 
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+# 
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+# 
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Test blockchain RPC calls
+
+Test the following RPCs:
+    - get_info
+    - generateblocks
+    - [TODO: many tests still need to be written]
+
+"""
+
+from test_framework.daemon import Daemon
+
+import sys
+import datetime
+import json
+
+class GetData():
+    def run(self, port, height_window_index):
+        HEIGHT_WINDOW_SIZE = 100000
+        start_height = height_window_index * HEIGHT_WINDOW_SIZE
+        end_height = start_height + HEIGHT_WINDOW_SIZE
+        daemon = Daemon(port=port)
+        res_info = daemon.get_info()
+        if res_info['height'] < end_height:
+            end_height = res_info['height']
+        assert start_height < end_height
+        print('var chartData_%d = [' % height_window_index)
+        for i in range(start_height, end_height):
+            res_blk = daemon.getblock(i)
+            block_header = res_blk['block_header']
+
+            timestamp  = block_header['timestamp']
+            difficulty = block_header['difficulty']
+            reward     = block_header['reward']
+            block_size = block_header['block_size']
+            num_txes   = block_header['num_txes']
+            # print(res_blk)
+            # print('====================================================')
+
+            res_txs = daemon.gettransactions(res_blk['tx_hashes']) if num_txes > 0 else None
+            txs_str = ''
+            for j in range(num_txes):
+                tx_json = res_txs['txs'][j]['as_json']
+                tx = json.loads(tx_json)
+                unlock_time = tx['unlock_time']
+                ins = len(tx['vin'])
+                outs = len(tx['vout'])
+                ring_size = len(tx['vin'][0]['key']['key_offsets'])
+                fee = 0
+                if tx['version'] > 1:
+                    fee = tx['rct_signatures']['txnFee']
+                else:
+                    ins_total = 0
+                    for k in range(ins):
+                        ins_total += tx['vin'][k]['key']['amount']
+                    outs_total = 0
+                    for k in range(outs):
+                        outs_total += tx['vout'][k]['amount']
+                    fee = ins_total - outs_total
+                extra_size = len(tx['extra'])
+                if j > 0:
+                    txs_str += ','
+                txs_str += "[%d,%d,%d,%d,%d,%d]" % (unlock_time, ins, outs, ring_size, fee, extra_size)
+            print('[%d,%d,%d,%d,[%s]],' % (timestamp, difficulty, reward, block_size, txs_str))
+        print(']')
+
+if __name__ == '__main__':
+    GetData().run(int(sys.argv[1]), int(sys.argv[2]))

test_framework/daemon.py

@@ -32,8 +32,8 @@ from .rpc import JSONRPC
 
 class Daemon(object):
 
-    def __init__(self, protocol='http', host='127.0.0.1', port=18081, path='/json_rpc'):
-        self.rpc = JSONRPC('{protocol}://{host}:{port}{path}'.format(protocol=protocol, host=host, port=port, path=path))
+    def __init__(self, protocol='http', host='127.0.0.1', port=18081):
+        self.rpc = JSONRPC(protocol=protocol, host=host, port=port)
 
     def getblocktemplate(self, address):
         getblocktemplate = {
@@ -45,7 +45,7 @@ class Daemon(object):
             'jsonrpc': '2.0', 
             'id': '0'
         }
-        return self.rpc.send_request(getblocktemplate)
+        return self.rpc.send_request('/json_rpc', getblocktemplate)
 
     def get_block_headers_range(self, start_height, end_height):
         get_block_headers_range = {
@@ -57,7 +57,7 @@ class Daemon(object):
             'jsonrpc': '2.0',
             'id': '0'
         }
-        return self.rpc.send_request(get_block_headers_range)
+        return self.rpc.send_request('/json_rpc', get_block_headers_range)
 
     def submitblock(self, block):
         submitblock = {
@@ -66,7 +66,7 @@ class Daemon(object):
             'jsonrpc': '2.0', 
             'id': '0'
         }    
-        return self.rpc.send_request(submitblock)
+        return self.rpc.send_request('/json_rpc', submitblock)
 
     def getblock(self, height=0):
         getblock = {
@@ -77,7 +77,15 @@ class Daemon(object):
             'jsonrpc': '2.0', 
             'id': '0'
         }
-        return self.rpc.send_request(getblock)
+        return self.rpc.send_request('/json_rpc', getblock)
+
+    def gettransactions(self, tx_hashes):
+        gettransactions = {
+            'txs_hashes': tx_hashes,
+            'decode_as_json': True,
+            'prune': True
+        }
+        return self.rpc.send_request('/gettransactions', gettransactions)
 
     def get_connections(self):
         get_connections = {
@@ -85,7 +93,7 @@ class Daemon(object):
             'jsonrpc': '2.0', 
             'id': '0'
         }
-        return self.rpc.send_request(get_connections)
+        return self.rpc.send_request('/json_rpc', get_connections)
 
     def get_info(self):
         get_info = {
@@ -93,7 +101,7 @@ class Daemon(object):
                 'jsonrpc': '2.0', 
                 'id': '0'
         }    
-        return self.rpc.send_request(get_info)    
+        return self.rpc.send_request('/json_rpc', get_info)    
 
     def hard_fork_info(self):
         hard_fork_info = {
@@ -101,7 +109,7 @@ class Daemon(object):
                 'jsonrpc': '2.0', 
                 'id': '0'
         }    
-        return self.rpc.send_request(hard_fork_info)    
+        return self.rpc.send_request('/json_rpc', hard_fork_info)    
 
     def generateblocks(self, address, blocks=1):
         generateblocks = {
@@ -114,4 +122,4 @@ class Daemon(object):
             'jsonrpc': '2.0', 
             'id': '0'
         }
-        return self.rpc.send_request(generateblocks)
+        return self.rpc.send_request('/json_rpc', generateblocks)

test_framework/rpc.py

@@ -30,19 +30,22 @@ import requests
 import json
 
 class JSONRPC(object):
-    def __init__(self, url):
-        self.url = url
+    def __init__(self, protocol, host, port):
+        self.protocol = protocol
+        self.host = host
+        self.port = port
 
-    def send_request(self, inputs):
+    def send_request(self, path, inputs):
+        url = '{protocol}://{host}:{port}{path}'.format(protocol=self.protocol, host=self.host, port=self.port, path=path)
         res = requests.post(
-            self.url,
+            url,
             data=json.dumps(inputs),
             headers={'content-type': 'application/json'})
         res = res.json()
         
         assert 'error' not in res, res
 
-        return res['result']
+        return res['result'] if path == '/json_rpc' else res
 
 
 

wownero-bc-size.html

@@ -33,7 +33,9 @@ function formatBytes(bytes, decimal) {
 }
 
 for (var i = 0; i < chartData.length; ++i) {
-    chartData[i].bc_size = chartData[i].block_size;
+    chartData[i].date = new Date(1000 * chartData[i][0]);
+    chartData[i].height = i + 1;
+    chartData[i].bc_size = chartData[i][3];
     if (i > 0)
       chartData[i].bc_size += chartData[i - 1].bc_size;
     chartData[i].bc_size_str = formatBytes(chartData[i].bc_size, 3);

wownero-block-reward.html

@@ -21,6 +21,12 @@
 var chartData = chartData_0;
 chartData.shift();
 
+for (var i = 0; i < chartData.length; ++i) {
+    chartData[i].date = new Date(1000 * chartData[i][0]);
+    chartData[i].height = i + 1;
+    chartData[i].reward = chartData[i][2] / Math.pow(10, 11);
+}
+
 var chart = AmCharts.makeChart("chartdiv", {
     "type": "serial",
     "theme": "light",

wownero-block-size.html

@@ -33,6 +33,9 @@ function formatBytes(bytes, decimal) {
 }
 
 for (var i = 0; i < chartData.length; ++i) {
+    chartData[i].date = new Date(1000 * chartData[i][0]);
+    chartData[i].height = i + 1;
+    chartData[i].block_size = chartData[i][3];
     chartData[i].block_size_str = formatBytes(chartData[i].block_size, 3);
 }
 

wownero-coin-supply.html

@@ -23,7 +23,9 @@ chartData.shift();
 
 var coin_supply = 0;
 for (var i = 0; i < chartData.length; ++i) {
-    coin_supply += chartData[i].reward;
+    chartData[i].date = new Date(1000 * chartData[i][0]);
+    chartData[i].height = i + 1;
+    coin_supply += chartData[i][2];
     chartData[i].coin_supply = coin_supply;
 }
 

wownero-tx-per-day.html

@@ -22,15 +22,17 @@ var chartData = chartData_0;
 chartData.shift();
 
 var chartData_tpd = [];
-var last_date = "";
+var last_date = null;
 var num_txes = 0;
 for (var i = 0; i < chartData.length; ++i) {
-    var block = chartData[i];
-    var block_date = block.date.substr(0, 10);
-    if (last_date === "") {
+    var block_date = new Date(1000 * chartData[i][0]);
+    block_date.setHours(0);
+    block_date.setMinutes(0);
+    block_date.setSeconds(0);
+    if (last_date === null) {
         last_date = block_date;
     }
-    if (Date.parse(last_date) < Date.parse(block_date) || i == chartData.length - 1) {
+    if (last_date < block_date || i == chartData.length - 1) {
         chartData_tpd.push({
             "date": last_date,
             "num_txes": num_txes,
@@ -38,7 +40,7 @@ for (var i = 0; i < chartData.length; ++i) {
         last_date = block_date;
         num_txes = 0;
     } else {
-        num_txes += block.num_txes;
+        num_txes += chartData[i][4].length;
     }
 }
 

wownero.html

@@ -21,6 +21,12 @@
 var chartData = chartData_0;
 chartData.shift();
 
+for (var i = 0; i < chartData.length; ++i) {
+    chartData[i].date = new Date(1000 * chartData[i][0]);
+    chartData[i].height = i + 1;
+    chartData[i].difficulty = chartData[i][1];
+}
+
 var chart = AmCharts.makeChart("chartdiv", {
     "type": "serial",
     "theme": "light",