More code added to ModularIdentifier

src/MicroCore.h

@@ -67,8 +67,8 @@ public:
     get_device() const;
 
     virtual void
-    get_output_key(const uint64_t& amount,
-                   const vector<uint64_t>& absolute_offsets,
+    get_output_key(uint64_t amount,
+                   vector<uint64_t> const& absolute_offsets,
                    vector<cryptonote::output_data_t>& outputs)
     {
         core_storage.get_db()
@@ -149,6 +149,15 @@ public:
         return core_storage.get_current_blockchain_height();
     }
 
+    virtual void
+    get_output_tx_and_index(
+            const uint64_t& amount,
+            const std::vector<uint64_t> &offsets,
+            std::vector<tx_out_index> &indices) const
+    {
+        core_storage.get_db().get_output_tx_and_index(
+                    amount, offsets, indices);
+    }
 
     virtual bool
     get_output_histogram(

src/PaymentSearcher.hpp

@@ -95,7 +95,7 @@ public:
             identifier.template get<Output>()->identify(tx, tx_pub_key);
 
             auto const& outputs_found
-                    = identifier.template get<Output>()->get_outputs();
+                    = identifier.template get<Output>()->get();
 
             if (!outputs_found.empty())
             {   // nothing was found

src/UniversalIdentifier.cpp

@@ -5,22 +5,41 @@ namespace xmreg
 
 void
 Output::identify(transaction const& tx,
-                 public_key const& tx_pub_key)
-{
-    cout << "Outputs identificataion" << endl;
-
-
-    bool tx_is_coinbase = is_coinbase(tx);
+                 public_key const& tx_pub_key,
+                 vector<public_key> const& additional_tx_pub_keys)
+{   
+    auto tx_is_coinbase = is_coinbase(tx);
 
     key_derivation derivation;
 
     if (!generate_key_derivation(tx_pub_key,
                                  *get_viewkey(), derivation))
     {
-        throw std::runtime_error("Cant get derived key for a tx");
+        throw std::runtime_error("generate_key_derivation failed");
     }
 
-    for (uint64_t i = 0; i < tx.vout.size(); ++i)
+    // since introduction of subaddresses, a tx can
+    // have extra public keys, thus we need additional
+    // derivations
+
+    vector<key_derivation> additional_derivations;
+
+    if (!additional_tx_pub_keys.empty())
+    {
+        additional_derivations.reserve(additional_tx_pub_keys.size());
+
+        for (size_t i = 0; i < additional_tx_pub_keys.size(); ++i)
+        {
+            if (!generate_key_derivation(additional_tx_pub_keys[i],
+                                         *get_viewkey(),
+                                         additional_derivations[i]))
+            {
+                throw std::runtime_error("additional generate_key_derivation failed");
+            }
+        }
+    }
+
+    for (auto i = 0u; i < tx.vout.size(); ++i)
     {
         // i will act as output indxes in the tx
 
@@ -46,6 +65,20 @@ Output::identify(transaction const& tx,
         // check if generated public key matches the current output's key
         bool mine_output = (txout_key.key == generated_tx_pubkey);
 
+        auto with_additional = false;
+
+        if (!mine_output && !additional_tx_pub_keys.empty())
+        {
+            // check for output using additional tx public keys
+            derive_public_key(additional_derivations[i],
+                              i,
+                              get_address()->address.m_spend_public_key,
+                              generated_tx_pubkey);
+
+            mine_output = (txout_key.key == generated_tx_pubkey);
+
+            with_additional = true;
+        }
 
         // placeholder variable for ringct outputs info
         // that we need to save in database
@@ -54,6 +87,10 @@ Output::identify(transaction const& tx,
         rct::key rtc_mask {0};
         rct::key rtc_amount {0};
 
+        // use primary derivation or the one from additional public keys
+        derivation = !with_additional ? derivation
+                                      : additional_derivations[i];
+
         // if mine output has RingCT, i.e., tx version is 2
         // need to decode its amount. otherwise its zero.
         if (mine_output && tx.version == 2)
@@ -66,8 +103,6 @@ Output::identify(transaction const& tx,
             // so use amount from ringct, only for non-coinbase txs
             if (!tx_is_coinbase)
             {
-                bool r;
-
                 // for ringct non-coinbase txs, these values are given
                 // with txs.
                 // coinbase ringctx dont have this information. we will provide
@@ -85,12 +120,11 @@ Output::identify(transaction const& tx,
                 rct::key mask =  tx.rct_signatures
                         .ecdhInfo[i].mask;
 
-                r = decode_ringct(tx.rct_signatures,
-                                  tx_pub_key,
-                                  *get_viewkey(),
-                                  i,
-                                  mask,
-                                  rct_amount_val);
+                auto r = decode_ringct(tx.rct_signatures,
+                                 derivation,
+                                 i,
+                                 mask,
+                                 rct_amount_val);
 
                 if (!r)
                 {
@@ -110,20 +144,206 @@ Output::identify(transaction const& tx,
 
             identified_outputs.emplace_back(
                     info{
-                        txout_key.key, amount, i,
+                        txout_key.key, amount, i, derivation,
                         rtc_outpk, rtc_mask, rtc_amount
                     });
 
+            total_xmr += amount;
+
         } //  if (mine_output)
 
     } // for (uint64_t i = 0; i < tx.vout.size(); ++i)
 }
 
 void Input::identify(transaction const& tx,
-              public_key const& tx_pub_key)
+                     public_key const& tx_pub_key,
+                     vector<public_key> const& additional_tx_pub_keys)
 {
-     cout << "Inputs identificataion" << endl;
+     auto search_misses {0};
+
+     auto input_no = tx.vin.size();
+
+     for (auto i = 0u; i < input_no; ++i)
+     {
+         if(tx.vin[i].type() != typeid(txin_to_key))
+             continue;
+
+         // get tx input key
+         txin_to_key const& in_key
+                 = boost::get<txin_to_key>(tx.vin[i]);
+
+         // get absolute offsets of mixins
+         vector<uint64_t> absolute_offsets
+                 = relative_output_offsets_to_absolute(
+                         in_key.key_offsets);
+
+         // get public keys of outputs used in the mixins that
+         // match to the offests
+         vector<output_data_t> mixin_outputs;
+
+         // this can THROW if no outputs are found
+         mcore->get_output_key(in_key.amount,
+                               absolute_offsets,
+                               mixin_outputs);
+
+         // indicates whether we found any matching mixin in the current input
+         auto found_a_match {false};
+
+         // for each found output public key check if its ours or not
+         for (auto count = 0u; count < absolute_offsets.size(); ++count)
+         {
+             // get basic information about mixn's output
+             output_data_t const& output_data
+                     = mixin_outputs[count];
+
+             // before going to the mysql, check our known outputs cash
+             // if the key exists. Its much faster than going to mysql
+             // for this.
+
+             auto it = known_outputs->find(output_data.pubkey);
+
+             if (it != known_outputs->end())
+             {
+                 // this seems to be our mixin.
+                 // save it into identified_inputs vector
+
+                 identified_inputs.push_back(info {
+                         in_key.k_image,
+                         it->second, // amount
+                         output_data.pubkey});
+
+                 total_xmr += it->second;
+
+                 found_a_match = true;
+             }
+
+         } // for (const cryptonote::output_data_t& output_data: outputs)
+
+         if (found_a_match == false)
+         {
+             // if we didnt find any match, break of the look.
+             // there is no reason to check remaining key images
+             // as when we spent something, our outputs should be
+             // in all inputs in a given txs. Thus, if a single input
+             // is without our output, we can assume this tx does
+             // not contain any of our spendings.
+
+             // just to be sure before we break out of this loop,
+             // do it only after two misses
+
+             if (++search_misses > 2)
+                 break;
+         }
+
+     } //  for (auto i = 0u; i < input_no; ++i)
 }
 
 
+void RealInput::identify(transaction const& tx,
+                         public_key const& tx_pub_key,
+                         vector<public_key> const& additional_tx_pub_keys)
+{
+     auto search_misses {0};
+
+     auto input_no = tx.vin.size();
+
+     for (auto i = 0u; i < input_no; ++i)
+     {
+         if(tx.vin[i].type() != typeid(txin_to_key))
+             continue;
+
+         // get tx input key
+         txin_to_key const& in_key
+                 = boost::get<cryptonote::txin_to_key>(tx.vin[i]);
+
+         // get absolute offsets of mixins
+         auto absolute_offsets
+                 = relative_output_offsets_to_absolute(
+                         in_key.key_offsets);
+
+
+         //tx_out_index is pair::<transaction hash, output index>
+         vector<tx_out_index> indices;
+
+         // get tx hashes and indices in the txs for the
+         // given outputs of mixins
+         //  this cant THROW DB_EXCEPTION
+         mcore->get_output_tx_and_index(
+                     in_key.amount, absolute_offsets, indices);
+
+         // placeholder for information about key image that
+         // we will find as ours
+         unique_ptr<info> key_image_info {nullptr};
+
+         // for each found mixin tx, check if any key image
+         // generated using our outputs in the mixin tx
+         // matches the given key image in the current tx
+         for (auto const& txi : indices)
+         {
+            auto const& mixin_tx_hash = txi.first;
+            auto const& output_index_in_tx = txi.second;
+
+            transaction mixin_tx;
+
+            if (!mcore->get_tx(mixin_tx_hash, mixin_tx))
+            {
+                throw std::runtime_error("Cant get tx: "
+                                         + pod_to_hex(mixin_tx_hash));
+            }
+
+            // use Output universal identifier to identify our outputs
+            // in a mixin tx
+            auto identifier = make_identifier(
+                        mixin_tx,
+                        make_unique<Output>(get_address(), get_viewkey()));
+
+            identifier.identify();
+
+            for (auto const& found_output: identifier.get<Output>()->get())
+            {
+                // generate key_image using this output
+                // to check for sure if the given key image is ours
+                // or not
+                crypto::key_image key_img_generated;
+
+                if (!xmreg::generate_key_image(found_output.derivation,
+                                               found_output.idx_in_tx, /* position in the tx */
+                                               *spendkey,
+                                               get_address()->address.m_spend_public_key,
+                                               key_img_generated))
+                {
+                    throw std::runtime_error("Cant generate key image for output: "
+                                                        + pod_to_hex(found_output.pub_key));
+                }
+
+                // now check if current key image in the tx which we
+                // analyze matches generated key image
+                if (in_key.k_image == key_img_generated)
+                {
+                    // this is our key image if they are equal!
+                    key_image_info.reset(new info {key_img_generated,
+                                                   found_output.amount,
+                                                   found_output.pub_key});
+
+                    break;
+                }
+
+            } // auto const& found_output: identifier.get<Output>()->get_outputs()
+
+            // if key_image_info has been populated, there is no
+            // reason to keep check remaning outputs in the mixin tx
+            // instead add its info to identified_inputs and move on
+            // to the next key image
+            if (!key_image_info)
+            {
+                identified_inputs.push_back(*key_image_info);
+                total_xmr += key_image_info->amount;
+                break;
+            }
+
+         } // for (auto const& txi : indices)
+
+     } //  for (auto i = 0u; i < input_no; ++i)
+}
+
 }

src/UniversalIdentifier.hpp

@@ -2,6 +2,7 @@
 
 #include "om_log.h"
 #include "tools.h"
+#include "MicroCore.h"
 
 #include <tuple>
 #include <utility>
@@ -15,7 +16,9 @@ class AbstractIdentifier
 {
 public:
     virtual void identify(transaction const& tx,
-                          public_key const& tx_pub_key) = 0;
+                          public_key const& tx_pub_key,
+                          vector<public_key> const& additional_tx_pub_keys
+                                = vector<public_key>{}) = 0;
 };
 
 
@@ -29,33 +32,28 @@ public:
    {}
 
     virtual void identify(transaction const& tx,
-                          public_key const& tx_pub_key) = 0;
+                          public_key const& tx_pub_key,
+                          vector<public_key> const& additional_tx_pub_keys
+                                = vector<public_key>{}) = 0;
 
     inline auto get_address() const {return address_info;}
     inline auto get_viewkey() const {return viewkey;}
+    inline auto get_total() const {return total_xmr;}
 
     virtual ~BaseIdentifier() = default;
 
-private:
-
+protected:
     address_parse_info const* address_info {nullptr};
     secret_key const* viewkey {nullptr};
+    uint64_t total_xmr {0};
 };
 
-
-
+/**
+ * @brief The Output class identifies our
+ * outputs in a given tx
+ */
 class Output : public BaseIdentifier
 {
-    struct info
-    {
-        public_key pub_key;
-        uint64_t   amount;
-        uint64_t   idx_in_tx;
-        rct::key   rtc_outpk;
-        rct::key   rtc_mask;
-        rct::key   rtc_amount;
-    };
-
 public:
 
     Output(address_parse_info const* _address,
@@ -64,21 +62,63 @@ public:
     {}
 
     void identify(transaction const& tx,
-                  public_key const& tx_pub_key) override;
+                  public_key const& tx_pub_key,
+                  vector<public_key> const& additional_tx_pub_keys
+                        = vector<public_key>{}) override;
 
-    inline auto get_outputs() const
+    inline auto get() const
     {
         return identified_outputs;
     }
 
-private:
+protected:
+
+    struct info
+    {
+        public_key pub_key;
+        uint64_t   amount;
+        uint64_t   idx_in_tx;
+        key_derivation derivation;
+        rct::key   rtc_outpk;
+        rct::key   rtc_mask;
+        rct::key   rtc_amount;
+    };
 
     uint64_t total_received {0};
     vector<info> identified_outputs;
 };
 
+/**
+ * @brief The Input class identifies our possible
+ * inputs (key images) in a given tx
+ */
 class Input : public BaseIdentifier
 {
+public:
+
+    using key_imgs_map_t = unordered_map<public_key, uint64_t>;
+
+    Input(address_parse_info const* _a,
+           secret_key const* _viewkey,
+           key_imgs_map_t const* _known_outputs,
+           MicroCore* _mcore)
+        : BaseIdentifier(_a, _viewkey),          
+          known_outputs {_known_outputs},
+          mcore {_mcore}
+    {}
+
+    void identify(transaction const& tx,
+                  public_key const& tx_pub_key,
+                  vector<public_key> const& additional_tx_pub_keys
+                        = vector<public_key>{}) override;
+
+    inline auto get() const
+    {
+        return identified_inputs;
+    }
+
+protected:
+
     struct info
     {
         key_image key_img;
@@ -86,40 +126,40 @@ class Input : public BaseIdentifier
         public_key out_pub_key;
     };
 
-public:
-
-    using key_imgs_map_t = unordered_map<public_key, uint64_t>;
+    secret_key const* viewkey {nullptr};   
+    key_imgs_map_t const* known_outputs {nullptr};    
+    MicroCore* mcore {nullptr};
+    vector<info> identified_inputs;
+};
 
-    Input(address_parse_info const* _a,
-           secret_key const* _viewkey,
-           secret_key const* _spendkey,
-           key_imgs_map_t const* _known_outputs)
-        : BaseIdentifier(_a, _viewkey),
-          spendkey {_spendkey},
-          known_outputs {_known_outputs}
-    {}
+/**
+ * Spendkey is optional. But if we have it,
+ * we can for sure determine which key images
+ * are ours or not. This is especially useful
+ * in unit testing, since we can compare wether
+ * guessed key images do contain all our key images
+ */
+class RealInput : public Input
+{
 
-    Input(address_parse_info const* _a,
-          secret_key const* _viewkey,
-          secret_key const* _spendkey)
-        : Input(_a, _viewkey, _spendkey, nullptr)
-    {}
+public:
 
-    Input(address_parse_info const* _a,
-          secret_key const* _viewkey,
-          key_imgs_map_t* _known_outputs)
-        : Input(_a, _viewkey, nullptr, _known_outputs)
+    RealInput(address_parse_info const* _a,
+              secret_key const* _viewkey,
+              secret_key const* _spendkey,
+              MicroCore* _mcore)
+        : Input(_a, _viewkey, nullptr, _mcore),
+          spendkey {_spendkey}
     {}
 
-
     void identify(transaction const& tx,
-                  public_key const& tx_pub_key) override;
+                  public_key const& tx_pub_key,
+                  vector<public_key> const& additional_tx_pub_keys
+                        = vector<public_key>{}) override;
 
-private:
-    secret_key const* viewkey;
-    secret_key const* spendkey;
-    key_imgs_map_t const* known_outputs {nullptr};
 
+protected:
+    secret_key const* spendkey {nullptr};
 };
 
 
@@ -135,7 +175,9 @@ public:
     {}
 
     void identify(transaction const& tx,
-                  public_key const& tx_pub_key) override
+                  public_key const& tx_pub_key,
+                  vector<public_key> const& additional_tx_pub_keys
+                        = vector<public_key>{}) override
     {   
         cout << "PaymentID decryption: "
                 + pod_to_hex(payment_id) << endl;
@@ -146,6 +188,7 @@ public:
 
         payment_id = std::get<HashT>(payment_id_tuple);
 
+        // if no payment_id found, return
         if (payment_id == null_hash)
             return;
 
@@ -180,8 +223,7 @@ public:
 
 private:
     HashT payment_id {};
-    HashT null_hash {0};
-
+    HashT null_hash {};
 };
 
 using LegacyPaymentID = PaymentID<crypto::hash>;
@@ -199,13 +241,21 @@ public:
         : identifiers {move(args)...},
           tx {_tx}
     {
+        // having tx public key is very common for all identifiers
+        // so we can get it here, instead of just obtaining it
+        // for each identifier seprately
         tx_pub_key = get_tx_pub_key_from_received_outs(tx);
+
+        // multi-output txs can have some additional public keys
+        // in the extra field. So we also get them, just in case
+        additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(tx);
     }
 
     void identify()
     {
          auto b = {(std::get<unique_ptr<T>>(
-                        identifiers)->identify(tx, tx_pub_key),
+                        identifiers)->identify(
+                            tx, tx_pub_key, additional_tx_pub_keys),
                    true)...};
          (void) b;
     }
@@ -221,10 +271,12 @@ public:
 private:
     transaction const& tx;
     public_key tx_pub_key;
-
+    vector<public_key> additional_tx_pub_keys;
 };
 
-
+/**
+ * A helper function to create ModularIdentifier object
+ */
 template<typename... T>
 auto make_identifier(transaction const& tx, T&&... identifiers)
 {
@@ -232,4 +284,16 @@ auto make_identifier(transaction const& tx, T&&... identifiers)
                 tx, std::forward<T>(identifiers)...);
 }
 
+template <typename T>
+auto
+calc_total_xmr(T&& infos)
+{
+    uint64_t total_xmr {0};
+
+    for (auto const& info: infos)
+        total_xmr += info.amount;
+
+    return total_xmr;
+}
+
 }

src/tools.cpp

@@ -838,12 +838,23 @@ decode_ringct(const rct::rctSig& rv,
         return false;
     }
 
-    crypto::secret_key scalar1;
+    return decode_ringct(rv, derivation, i, mask, amount);
+}
 
-    crypto::derivation_to_scalar(derivation, i, scalar1);
 
+bool
+decode_ringct(rct::rctSig const& rv,
+              crypto::key_derivation const& derivation,
+              unsigned int i,
+              rct::key& mask,
+              uint64_t& amount)
+{
     try
     {
+        crypto::secret_key scalar1;
+
+        crypto::derivation_to_scalar(derivation, i, scalar1);
+
         switch (rv.type)
         {
             case rct::RCTTypeSimple:
@@ -862,19 +873,20 @@ decode_ringct(const rct::rctSig& rv,
                                         hw::get_device("default"));
                 break;
             default:
-                cerr << "Unsupported rct type: " << rv.type << endl;
+                cerr << "Unsupported rct type: " << rv.type << '\n';
                 return false;
         }
     }
-    catch (const std::exception &e)
+    catch (...)
     {
-        cerr << "Failed to decode input " << i << endl;
+        cerr << "Failed to decode input " << i << '\n';
         return false;
     }
 
     return true;
 }
 
+
 bool
 url_decode(const std::string& in, std::string& out)
 {

src/tools.h

@@ -215,6 +215,12 @@ decode_ringct(const rct::rctSig & rv,
               uint64_t & amount);
 
 bool
+decode_ringct(rct::rctSig const& rv,
+              crypto::key_derivation const& derivation,
+              unsigned int i,
+              rct::key& mask,
+              uint64_t& amount);
+bool
 url_decode(const std::string& in, std::string& out);
 
 map<std::string, std::string>

tests/mocks.h

@@ -74,9 +74,9 @@ public:
                             transaction& tx));
 
     MOCK_METHOD3(get_output_key,
-                    void(const uint64_t& amount,
-                         const vector<uint64_t>& absolute_offsets,
-                         vector<cryptonote::output_data_t>& outputs));
+                    void(uint64_t amount,
+                         vector<uint64_t> const& absolute_offsets,
+                         vector<output_data_t>& outputs));
 
     MOCK_METHOD2(get_output_key,
                     output_data_t(uint64_t amount,
@@ -225,6 +225,19 @@ struct MockGettingOutputs
         return true;
     }
 
+    /*
+     * MicroCore uses get_output_key, so provide this for convinience
+     * which is just same as  get_output_keys above
+    */
+    virtual void
+    get_output_key(
+            const uint64_t& amount,
+            vector<uint64_t> absolute_offsets,
+            vector<cryptonote::output_data_t>& outputs)
+    {
+        get_output_keys(amount, absolute_offsets, outputs);
+    }
+
 };
 
 bool