Commit 847932c37d192f38d980e3b38f4b46927646ad74

Authored by Marius Hanne
1 parent 250a15d370
Exists in spv

watch addresses by block height

Showing 7 changed files with 82 additions and 165 deletions Side-by-side Diff

... ... @@ -536,6 +536,7 @@
536 536 # 542 contains invalid transaction
537 537 542 => "0000000083c1f82cf72c6724f7a317325806384b06408bce7a4327f418dfd5ad",
538 538 71018 => "000000000010dd93dc55541116b2744eb8f4c3b706df6e8512d231a03fb9e435",
  539 + 180606 => "00000000a132e751c0ad9ce04ee85ab98b344844b761f14236acbfe3c389a8e7"
539 540 }
540 541 },
541 542  
lib/bitcoin/network/connection_handler.rb
... ... @@ -106,12 +106,12 @@
106 106 @node.push_notification(:connection, [:connected, info])
107 107 @node.addrs << addr
108 108 filterload
109   - send_getdata_block(Bitcoin.network[:genesis_hash])
  109 + send_getdata_block(Bitcoin.network[:genesis_hash].htb)
110 110 end
111 111 end
112 112  
113 113 def filterload
114   - addrs = @node.store.watched_addrs
  114 + addrs = @node.store.watched_addrs @node.store.get_depth
115 115 return unless addrs.any?
116 116 filter = Bitcoin::BloomFilter.new(addrs.size * 22, 0.01, 1234, :update_all)
117 117 addrs.each {|a| filter.insert(a.htb) }
lib/bitcoin/protocol/block.rb
... ... @@ -95,7 +95,7 @@
95 95 @tx << t
96 96 }
97 97  
98   -# @payload = to_payload
  98 + @payload = to_payload
99 99 buf
100 100 end
101 101  
lib/bitcoin/protocol/merkle_block.rb
... ... @@ -39,9 +39,9 @@
39 39 b
40 40 end
41 41  
42   - def self.from_block blk
  42 + def self.from_block blk, header_only = true
43 43 # blk.tx = []
44   - b = new blk.to_payload
  44 + b = new blk.to_payload, header_only
45 45 b.tx = blk.tx
46 46 # b.hashes = blk.tx.map(&:hash).map(&:htb)
47 47 # TODO: flags
lib/bitcoin/storage/spv/spv_store.rb
... ... @@ -38,43 +38,32 @@
38 38 @head = nil
39 39 end
40 40  
41   - def add_watched_address addr
42   - @watched_addrs = watched_addrs
43   - @watched_addrs << addr
  41 + def add_watched_address addr, depth = 0
  42 + @watched_addrs = (JSON.load(@db["watched_addrs"]) || [])
  43 + @watched_addrs << [addr, depth]
44 44 @db["watched_addrs"] = @watched_addrs.to_json
45   - log.info { "Added watched address #{addr}, now watching #{addrs.count}." }
  45 + log.info { "Added watched address #{addr}, now watching #{@watched_addrs.count}." }
46 46 end
47 47  
48   - def watched_addrs
49   - # @watched_addrs ||= (JSON.load(@db["watched_addrs"]) || [])
50   - []
  48 + def watched_addrs depth
  49 + @watched_addrs ||= (JSON.load(@db["watched_addrs"]) || [])
  50 + @watched_addrs.map {|a, d| a if d <= depth }.compact
51 51 end
52 52  
  53 + def watch_addrs? depth = 0
  54 + watched_addrs(depth).any? || @config[:watch_all_addrs]
  55 + end
  56 +
53 57 # persist given block +blk+ to storage.
54 58 def persist_block blk, chain, depth, prev_work = 0
55 59  
56   - blk = Bitcoin::P::MerkleBlock.from_block(blk)
  60 +# blk = Bitcoin::P::MerkleBlock.from_block(blk, false)
57 61  
58 62 blk.chain, blk.depth, blk.work = chain, depth, prev_work + blk.block_work
59 63  
60   - # attrs = {
61   - # :depth => depth,
62   - # :chain => chain,
  64 + # make sure blk.to_payload is only the header and no tx data
  65 + tx = blk.tx; blk.tx = []
63 66  
64   - # :version => blk.ver,
65   - # :prev_hash => blk.prev_block.hth,
66   - # :mrkl_root => blk.mrkl_root.hth,
67   - # :time => blk.time,
68   - # :bits => blk.bits,
69   - # :nonce => blk.nonce,
70   - # :work => (prev_work + blk.block_work).to_s,
71   -
72   - # :hashes => blk.hashes,
73   - # :flags => blk.flags,
74   - # }
75   -
76   - # attrs[:aux_pow] = blk.aux_pow.to_payload.blob if blk.aux_pow
77   -
78 67 key = chain == 2 ? "o#{blk.hash.htb}" : "b#{blk.hash.htb}"
79 68 @db[key] = blk.to_disk
80 69 if chain == MAIN
81 70  
82 71  
... ... @@ -83,21 +72,21 @@
83 72 @db["d#{depth}"] = blk.hash.htb
84 73 end
85 74  
86   - blk.tx.each {|tx| store_tx(tx) } if watched_addrs.any?
  75 + tx.each.with_index {|tx, idx| store_tx(tx, true, blk, idx) } if watch_addrs?(depth)
87 76  
88   -# if !@last_block || @last_block.to_i < Time.now.to_i - 10
89   - # connect orphans
90   - @db.range("o", "p") do |hash, data|
91   - orphan = wrap_block(Bitcoin::P::MerkleBlock.from_disk(data))
92   - if orphan.prev_block.reverse.hth == blk.hash
93   - begin
94   - store_block(orphan)
95   - rescue SystemStackError
96   - EM.defer { store_block(orphan) } if EM.reactor_running?
  77 + if !@last_block || @last_block.to_i < Time.now.to_i - 10
  78 + # connect orphans
  79 + @db.range("o", "p") do |hash, data|
  80 + orphan = wrap_block(Bitcoin::P::MerkleBlock.from_disk(data))
  81 + if orphan.prev_block.reverse.hth == blk.hash
  82 + begin
  83 + store_block(orphan)
  84 + rescue SystemStackError
  85 + EM.defer { store_block(orphan) } if EM.reactor_running?
  86 + end
97 87 end
98 88 end
99 89 end
100   -# end
101 90 @last_block = Time.now
102 91  
103 92 return depth, chain
104 93  
105 94  
106 95  
107 96  
108 97  
109 98  
... ... @@ -114,93 +103,32 @@
114 103 blk = Bitcoin::P::MerkleBlock.from_disk(@db["b#{hash.htb}"])
115 104 blk.chain = 0
116 105 @db["b#{hash.htb}"] = blk.to_disk
117   - @db["d#{blk.depth}"] = hash
  106 + @db["d#{blk.depth}"] = hash.htb
118 107 end
119 108 end
120 109  
121   - # parse script and collect address/txout mappings to index
122   - def parse_script txout, i, tx_hash = "", tx_idx
123   - addrs, names = [], []
124   -
125   - script = Bitcoin::Script.new(txout.pk_script) rescue nil
126   - if script
127   - if script.is_hash160? || script.is_pubkey?
128   - addrs << [i, script.get_hash160]
129   - elsif script.is_multisig?
130   - script.get_multisig_pubkeys.map do |pubkey|
131   - addrs << [i, Bitcoin.hash160(pubkey.unpack("H*")[0])]
132   - end
133   - elsif Bitcoin.namecoin? && script.is_namecoin?
134   - addrs << [i, script.get_hash160]
135   - names << [i, script]
136   - else
137   - log.info { "Unknown script type in #{tx_hash}:#{tx_idx}" }
138   - log.debug { script.to_string }
139   - end
140   - script_type = SCRIPT_TYPES.index(script.type)
141   - else
142   - log.error { "Error parsing script #{tx_hash}:#{tx_idx}" }
143   - script_type = SCRIPT_TYPES.index(:unknown)
144   - end
145   - [script_type, addrs, names]
146   - end
147   -
148   - # # bulk-store addresses and txout mappings
149   - # def persist_addrs addrs
150   - # addr_txouts, new_addrs = [], []
151   - # addrs.group_by {|_, a| a }.each do |hash160, txouts|
152   - # if existing = @db[:addr][:hash160 => hash160]
153   - # txouts.each {|id, _| addr_txouts << [existing[:id], id] }
154   - # else
155   - # new_addrs << [hash160, txouts.map {|id, _| id }]
156   - # end
157   - # end
158   - # new_addr_ids = @db[:addr].insert_multiple(new_addrs.map {|hash160, txout_id|
159   - # { hash160: hash160 } })
160   - # new_addr_ids.each.with_index do |addr_id, idx|
161   - # new_addrs[idx][1].each do |txout_id|
162   - # addr_txouts << [addr_id, txout_id]
163   - # end
164   - # end
165   - # @db[:addr_txout].insert_multiple(addr_txouts.map {|addr_id, txout_id|
166   - # { addr_id: addr_id, txout_id: txout_id }})
167   - # end
168   -
169   - # prepare transaction data for storage
170   - def tx_data tx
171   - { hash: tx.hash.htb.blob,
172   - version: tx.ver, lock_time: tx.lock_time,
173   - coinbase: tx.in.size == 1 && tx.in[0].coinbase?,
174   - tx_size: tx.payload.bytesize }
175   - end
176   -
177 110 # store transaction +tx+
178   - def store_tx(tx, validate = true)
179   - if watched_addrs.any?
  111 + def store_tx(tx, validate = true, block = nil, idx = nil)
  112 + if !block || watch_addrs?(block.depth)
  113 + watched_addrs = watched_addrs(block ? block.depth : @head.depth)
180 114 relevant = false
181 115 # TODO optimize
182   - # maybe just matching the hash160 against the raw pk_script is faster
183 116 tx.out.each.with_index {|o,i|
184   - next unless o.pk_script =~ /[#{watched_addrs.join("|")}]/
185   - relevant = true
186   - break
187   -
188   - # script = Bitcoin::Script.new(o.pk_script)
189   - # address = script.get_hash160
190   - # relevant = true if watched_addrs.include?(address)
  117 + script = Bitcoin::Script.new(o.pk_script)
  118 + address = script.get_hash160
  119 + relevant = true if @config[:watch_all_addrs] || watched_addrs.include?(address)
191 120 }
192 121 tx.in.each {|i|
193   - next unless prev_out = get_tx(i.prev_out.reverse_hth).out[i.prev_out_index]
194   - next unless prev_out.pk_script =~ /[#{watched_addrs.join("|")}]/
195   - relevant = true
196   - break
197   - # relevant = true if (Bitcoin::Script.new(prev_out.pk_script).get_addresses.map {|a| Bitcoin.hash160_from_address(a) } & watched_addrs).any?
  122 + next unless prev_tx = get_tx(i.prev_out.reverse_hth)
  123 + next unless prev_out = prev_tx.out[i.prev_out_index]
  124 + relevant = true if @config[:watch_all_addrs] ||
  125 + (Bitcoin::Script.new(prev_out.pk_script).get_addresses.map {|a|
  126 + Bitcoin.hash160_from_address(a) } & watched_addrs).any?
198 127 }
199 128 return unless relevant
200 129 else
201 130 return
202 131 end
203   -
204 132 @log.debug { "Storing tx #{tx.hash} (#{tx.to_payload.bytesize} bytes)" }
205 133 @db["t#{tx.hash.htb}"] = tx.payload
206 134 end
207 135  
208 136  
... ... @@ -247,32 +175,12 @@
247 175 get_block(@db["d#{get_block(prev_hash).depth + 1}"])
248 176 end
249 177  
250   - # # get block by given +tx_hash+
251   - # def get_block_by_tx(tx_hash)
252   - # TODO
253   - # end
254   -
255 178 # get transaction for given +tx_hash+
256 179 def get_tx(tx_hash)
257   - wrap_tx(Bitcoin::P::Tx.new(@db["t#{tx_hash.htb}"]))
  180 + return nil unless data = @db["t#{tx_hash.htb}"]
  181 + wrap_tx(Bitcoin::P::Tx.new(data))
258 182 end
259 183  
260   - # # get corresponding Models::TxIn for the txout in transaction
261   - # # +tx_hash+ with index +txout_idx+
262   - # def get_txin_for_txout(tx_hash, txout_idx)
263   - # TODO
264   - # end
265   -
266   - # # get corresponding Models::TxOut for +txin+
267   - # def get_txout_for_txin(txin)
268   - # TODO
269   - # end
270   -
271   - # # get all Models::TxOut matching given +script+
272   - # def get_txouts_for_pk_script(script)
273   - # TODO
274   - # end
275   -
276 184 # get all Models::TxOut matching given +hash160+
277 185 def get_txouts_for_hash160(hash160, unconfirmed = false)
278 186 txouts = []
... ... @@ -289,11 +197,6 @@
289 197 txouts
290 198 end
291 199  
292   - # # Grab the position of a tx in a given block
293   - # def get_idx_from_tx_hash(tx_hash)
294   - # TODO
295   - # end
296   -
297 200 # wrap given +block+ into Models::Block
298 201 def wrap_block(block)
299 202 return nil unless block
... ... @@ -355,9 +258,6 @@
355 258 txout.value = output.value
356 259 txout.pk_script = output.pk_script
357 260 txout
358   - end
359   -
360   - def check_consistency count = 1000
361 261 end
362 262  
363 263 end
lib/bitcoin/storage/storage.rb
... ... @@ -422,6 +422,10 @@
422 422 raise "Not implemented"
423 423 end
424 424  
  425 + def check_consistency *a
  426 + log.warn { "Consistency check not implemented" }
  427 + end
  428 +
425 429 # import satoshi bitcoind blk0001.dat blockchain file
426 430 def import filename, max_depth = nil
427 431 if File.file?(filename)
... ... @@ -452,7 +456,9 @@
452 456 end
453 457  
454 458 def in_sync?
455   - (get_head && (Time.now - get_head.time).to_i < 3600) ? true : false
  459 + in_sync = (get_head && (Time.now - get_head.time).to_i < 3600) ? true : false
  460 + log.info { "Storage in sync with blockchain." } if in_sync && !@in_sync
  461 + @in_sync = in_sync
456 462 end
457 463  
458 464 end
spec/bitcoin/storage/storage_spec.rb
... ... @@ -142,6 +142,14 @@
142 142 @store.get_block("00000007199508e34a9ff81e6ec0c477a4cccff2a4767a8eee39c11db367b008")
143 143 end
144 144  
  145 + it "should filter watched addrs" do
  146 + @key = Bitcoin::Key.generate
  147 + @store.add_watched_address @key.hash160, @store.get_depth + 2
  148 + block = create_block(@store.get_head.hash, true, [], @key)
  149 + @store.get_tx(block.tx[0].hash).should == nil
  150 + block = create_block(block.hash, true, [], @key)
  151 + @store.get_tx(block.tx[0].hash).hash.should == block.tx[0].hash
  152 + end
145 153  
146 154  
147 155  
148 156  
149 157  
... ... @@ -163,20 +171,21 @@
163 171 # if @store.class.name =~ /SequelStore/
164 172 # describe :transactions do
165 173  
166   - # it "should store tx" do
167   - # @store.store_tx(@tx, false).should != false
168   - # end
  174 + it "should store tx" do
  175 + @store.store_tx(@tx, false).should != false
  176 + end
169 177  
170   - # it "should not store tx if already stored and return existing id" do
171   - # id = @store.store_tx(@tx, false)
172   - # @store.store_tx(@tx, false).should == id
173   - # end
  178 + it "should not store tx if already stored and return existing id" do
  179 + id = @store.store_tx(@tx, false)
  180 + @store.store_tx(@tx, false).should == id
  181 + end
174 182  
175   - # it "should check if tx is already stored" do
176   - # @store.has_tx(@tx.hash).should == false
177   - # @store.store_tx(@tx, false)
178   - # @store.has_tx(@tx.hash).should == true
179   - # end
  183 + it "should check if tx is already stored" do
  184 + @store.add_watched_address Script.new(@tx.out[0].pk_script).get_hash160
  185 + @store.has_tx(@tx.hash).should == false
  186 + @store.store_tx(@tx, false)
  187 + @store.has_tx(@tx.hash).should == true
  188 + end
180 189  
181 190 # it "should store hash160 for txout" do
182 191 # @store.store_tx(@tx, false)
183 192  
... ... @@ -184,14 +193,15 @@
184 193 # .should == "3129d7051d509424d23d533fa2d5258977e822e3"
185 194 # end
186 195  
187   - # it "should get tx" do
188   - # @store.store_tx(@tx, false)
189   - # @store.get_tx(@tx.hash).should == @tx
190   - # end
  196 + it "should get tx" do
  197 + @store.add_watched_address Script.new(@tx.out[0].pk_script).get_hash160
  198 + @store.store_tx(@tx, false)
  199 + @store.get_tx(@tx.hash).should == @tx
  200 + end
191 201  
192   - # it "should not get tx" do
193   - # @store.get_tx("nonexistant").should == nil
194   - # end
  202 + it "should not get tx" do
  203 + @store.get_tx("nonexistant").should == nil
  204 + end
195 205  
196 206 # it "should get the position for a given tx" do
197 207 # @store.store_block(@blk)