Commit 1f9fad64326821d68daa0d7a4db7a99a1a2a674f

Authored by Marius Hanne
Exists in master and in 1 other branch benchmark

Merge remote-tracking branch 'refs/remotes/comboy/fake_chain'

Showing 11 changed files Side-by-side Diff

... ... @@ -9,6 +9,7 @@
9 9 log/
10 10 rdoc/
11 11 coverage/
  12 +spec/bitcoin/fixtures/fake_chain
12 13 *.conf
13 14 *.db
14 15 .rbx/
... ... @@ -552,6 +552,7 @@
552 552 :target_spacing => 600, # block interval
553 553 :max_money => 21_000_000 * COIN,
554 554 :min_tx_fee => 10_000,
  555 + :no_difficulty => true,
555 556 :min_relay_tx_fee => 10_000,
556 557 :dns_seeds => [
557 558 "testnet-seed.bitcoin.petertodd.org",
lib/bitcoin/builder.rb
... ... @@ -71,6 +71,7 @@
71 71 def tx tx = nil
72 72 tx ||= ( c = TxBuilder.new; yield c; c.tx )
73 73 @block.tx << tx
  74 + tx
74 75 end
75 76  
76 77 # create the block according to values specified via DSL.
lib/bitcoin/validation.rb
... ... @@ -109,7 +109,7 @@
109 109  
110 110 # check that bits satisfy required difficulty
111 111 def difficulty
112   - return true if Bitcoin.network_name == :testnet3
  112 + return true if Bitcoin.network[:no_difficulty] == true
113 113 block.bits == next_bits_required || [block.bits, next_bits_required]
114 114 end
115 115  
spec/bitcoin/helpers/fake_blockchain.rb
  1 +require_relative '../spec_helper'
  2 +require 'fileutils'
  3 +
  4 +Bitcoin::NETWORKS[:fake] = {
  5 + :project => :bitcoin,
  6 + :no_difficulty => true,
  7 + :magic_head => "fake",
  8 + :address_version => "00",
  9 + :p2sh_version => "05",
  10 + :privkey_version => "80",
  11 + :default_port => 78333,
  12 + :coinbase_maturity => 0,
  13 + :protocol_version => 70001,
  14 + :max_money => 21_000_000 * 100_000_000,
  15 + :dns_seeds => [],
  16 + :genesis_hash => "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943",
  17 + :proof_of_work_limit => 553713663,
  18 + :alert_pubkeys => [],
  19 + :known_nodes => [],
  20 + :checkpoints => {},
  21 + :min_tx_fee => 10_000,
  22 + :min_relay_tx_fee => 10_000,
  23 +}
  24 +
  25 +
  26 +
  27 +# Small utility to generate fake blocks mostly to be able to test performance
  28 +# They are full from the start, so that we don't have to import 100K blocks to check
  29 +# how performance looks when storing or validating 1K transactions
  30 +class FakeBlockchain
  31 +
  32 + # Initialize fake blockchain and generate +num_blocks+ starting blocks with given
  33 + # +opts+ (see #generate).
  34 + def initialize(num = 50, opts = {})
  35 + Bitcoin.network = :fake
  36 + if File.exist? block_path(0)
  37 + genesis = Bitcoin::P::Block.new File.read block_path 0
  38 + Bitcoin.network[:genesis_hash] = genesis.hash
  39 + else
  40 + STDERR.puts "\nFake blockchain not present, generating (go take a nap)..."
  41 + depth = 0
  42 + FileUtils.mkdir_p fixtures_path "fake_chain"
  43 + generate(num, opts) do |blk|
  44 + File.open(block_path(depth),'w') {|f| f.write blk.to_payload }
  45 + depth += 1
  46 + end
  47 + end
  48 + end
  49 +
  50 + # Generate fake blockchain with +num+ number of blocks
  51 + # Blocks are provided as an argument to the block given to the method
  52 + # fake_chain.generate(5) {|b| save_block(b) }
  53 + def generate(num = 50, opts = {})
  54 + srand 1337
  55 +
  56 + default_opts = {
  57 + block_size: 950_000, # minimum block size
  58 + num_keys: 1000, # number of different keys being used
  59 + genesis_timestamp: Time.new(2009).to_i,
  60 + verbose: true, # print out debug information
  61 + }
  62 +
  63 + opts = default_opts.merge(opts)
  64 +
  65 + to_spend = [] # table of outputs that we can spend
  66 + lost_count = 0 # keeping track of lost coins
  67 + keys = Array.new(opts[:num_keys]) { Bitcoin::Key.generate }
  68 + timestamp = opts[:genesis_timestamp]
  69 +
  70 + genesis = Bitcoin::Builder.build_block do |blk|
  71 + blk.time timestamp
  72 + blk.prev_block "00"*32
  73 + blk.tx do |t|
  74 + t.input {|i| i.coinbase }
  75 + t.output {|o| o.value 50*Bitcoin::COIN; o.script {|s| s.recipient keys[0].addr } }
  76 + end
  77 + end
  78 + Bitcoin.network[:genesis_hash] = genesis.hash
  79 + yield(genesis)
  80 +
  81 + to_spend << {tx: genesis.tx[0], tx_idx: 0, key: keys[0], value: 50e8}
  82 +
  83 + prev_block = genesis
  84 +
  85 +
  86 + num.times do |blk_i|
  87 +
  88 + timestamp += 600
  89 + t0 = Time.now
  90 +
  91 + block = Bitcoin::Builder.build_block do |blk|
  92 + blk.time timestamp
  93 + blk.prev_block prev_block.hash
  94 + key0 = keys.sample
  95 + tx0 = blk.tx do |t|
  96 + t.input {|i| i.coinbase }
  97 + t.output {|o| o.value 50e8; o.script {|s| s.recipient key0.addr } }
  98 + end
  99 +
  100 + # We "lose" some coins, that is we decide never to spend some outputs
  101 + # It's to keep utxo growing without making block generation time growing
  102 + lost_count += to_spend.size
  103 + to_spend = to_spend.reject.with_index {|x,i| i==0 ? false : (((to_spend.size - i) / to_spend.size.to_f)**2 * rand > rand*0.2) }
  104 + lost_count -= to_spend.size
  105 +
  106 + # many txs vs many tx outputs in given block
  107 + many_outputs_prob = 0.5 * (rand ** 3)
  108 +
  109 + total_tx_size = 0
  110 +
  111 + # generate tranasctions
  112 + loop do
  113 + # we want utxo to keep growing so we use many inputs only with some small probability
  114 + ins = to_spend[(rand(to_spend.size)).to_i..-1].sample(rand < 0.01 ? (rand(50) + 1) : 1)
  115 + total = ins.map{|i| i[:value]}.inject(:+)
  116 + next if total < 20_000
  117 +
  118 + new_outs = []
  119 +
  120 + tx = blk.tx do |t|
  121 +
  122 + # generate inputs
  123 + ins.map do |input|
  124 + t.input do |i|
  125 + i.prev_out input[:tx]
  126 + i.prev_out_index input[:tx_idx]
  127 + i.signature_key input[:key]
  128 + end
  129 + end
  130 + # remove outputs that we just used
  131 + ins.each {|i| to_spend.delete i}
  132 +
  133 + fee = 10_000
  134 + # helper to split value randomly in a half
  135 + half_split = ->(v) { split = [rand, 0.1].max; [v*split, v*(1-split)] }
  136 + # helper to split value randomly to many pieces
  137 + value_split = ->(v, depth=0) {(depth < 10 && rand > 0.2) ? half_split[v].map{|x| value_split[x, depth+1]} : [v] }
  138 +
  139 + if rand < many_outputs_prob
  140 + # every now and then there are many outptus
  141 + out_values = value_split[total-fee].flatten.map {|x| x.round(8)}
  142 + out_values.each.with_index do |v,i|
  143 + key = keys.sample
  144 + t.output {|o| o.value v; o.script {|s| s.recipient key.addr }}
  145 + new_outs << {tx_idx: i, key: key, value: v}
  146 + end
  147 + else
  148 + # most txs seem to have 2 outputs
  149 + k1 = keys.sample
  150 + k2 = keys.sample
  151 + v1, v2 = half_split[total-fee]
  152 + t.output {|o| o.value v1; o.script {|s| s.recipient k1.addr }}
  153 + t.output {|o| o.value v2; o.script {|s| s.recipient k2.addr }}
  154 + new_outs << {tx_idx: 0, key: k1, value: v2}
  155 + new_outs << {tx_idx: 1, key: k2, value: v2}
  156 + end
  157 + end
  158 +
  159 + new_outs.each {|o| to_spend << {tx: tx}.merge(o) } # fun fact: the opposite merge is way slower
  160 +
  161 + total_tx_size += tx.to_payload.size
  162 + break if total_tx_size > opts[:block_size]
  163 + end
  164 +
  165 + # coinbase
  166 + to_spend << {tx: tx0, tx_idx: 0, key: key0, value: 50e8}
  167 + end
  168 + puts "depth #{blk_i+1}/#{num} \t txcount: #{block.tx.size} \t size: #{block.to_payload.size} \t utxo count: #{to_spend.size + lost_count} (#{to_spend.size}) \t ttg: #{'%.2f' % (Time.now - t0)}s" if opts[:verbose]
  169 + yield(block)
  170 + prev_block = block
  171 + end
  172 + true
  173 + end
  174 +
  175 + def block(depth)
  176 + Bitcoin::Protocol::Block.new File.read block_path depth
  177 + end
  178 +
  179 + def block_path(depth)
  180 + fixtures_path "fake_chain/#{depth}.blk"
  181 + end
  182 +
  183 +end
spec/bitcoin/namecoin_spec.rb
... ... @@ -89,7 +89,7 @@
89 89  
90 90 before do
91 91 Bitcoin.network = :namecoin
92   - class Bitcoin::Validation::Block; def difficulty; true; end; end
  92 + Bitcoin.network[:no_difficulty] = true
93 93 class Bitcoin::Validation::Block; def min_timestamp; true; end; end
94 94 Bitcoin.network[:proof_of_work_limit] = Bitcoin.encode_compact_bits("ff"*32)
95 95 [:name_new, :name_firstupdate, :name_update].each {|type|
spec/bitcoin/performance/storage_spec.rb
  1 +# encoding: ascii-8bit
  2 +
  3 +require_relative '../spec_helper'
  4 +require_relative '../helpers/fake_blockchain'
  5 +require 'benchmark'
  6 +
  7 +[
  8 + [:sequel, :postgres]
  9 +].compact.each do |options|
  10 +
  11 + next unless storage = setup_db(*options)
  12 +
  13 + describe "#{storage.backend_name} block storage" do
  14 +
  15 + before do
  16 + @store = storage
  17 + @store.reset
  18 + @store.log.level = :error
  19 + @fake_chain = FakeBlockchain.new 10
  20 + end
  21 +
  22 + it "block storage" do
  23 + blocks = (0..10).to_a.map{|i| @fake_chain.block(i) }
  24 +
  25 + bm = Benchmark.measure do
  26 + bm = Benchmark.bm do |b|
  27 + blocks.each.with_index do |blk,i|
  28 + b.report("storing fake block ##{i}") do
  29 + depth, chain = @store.new_block blk
  30 + chain.should == 0
  31 + end
  32 + end
  33 + end
  34 + end
  35 + puts '-'*80
  36 + puts "TOTAL #{bm.format}"
  37 + end
  38 +
  39 +
  40 + end
  41 +end
spec/bitcoin/spec_helper.rb
... ... @@ -20,10 +20,14 @@
20 20  
21 21 require 'bitcoin'
22 22  
  23 +def fixtures_path(relative_path)
  24 + File.join(File.dirname(__FILE__), 'fixtures', relative_path)
  25 +end
  26 +
23 27 def fixtures_file(relative_path)
24   - basedir = File.join(File.dirname(__FILE__), 'fixtures')
25   - Bitcoin::Protocol.read_binary_file( File.join(basedir, relative_path) )
  28 + Bitcoin::Protocol.read_binary_file( fixtures_path(relative_path) )
26 29 end
  30 +
27 31  
28 32 include Bitcoin::Builder
29 33  
spec/bitcoin/storage/models_spec.rb
... ... @@ -22,7 +22,7 @@
22 22 describe "Storage::Models (#{options[0].to_s.capitalize}Store, #{options[1]})" do
23 23  
24 24 before do
25   - class Bitcoin::Validation::Block; def difficulty; true; end; end
  25 + Bitcoin.network[:no_difficulty] = true
26 26 Bitcoin.network[:proof_of_work_limit] = Bitcoin.encode_compact_bits("ff"*32)
27 27  
28 28 @store = storage
... ... @@ -44,12 +44,7 @@
44 44 end
45 45  
46 46 after do
47   - class Bitcoin::Validation::Block
48   - def difficulty
49   - return true if Bitcoin.network_name == :testnet3
50   - block.bits == next_bits_required || [block.bits, next_bits_required]
51   - end
52   - end
  47 + Bitcoin.network.delete :no_difficulty
53 48 end
54 49  
55 50 describe "Block" do
spec/bitcoin/storage/reorg_spec.rb
... ... @@ -208,9 +208,9 @@
208 208  
209 209 # see https://bitcointalk.org/index.php?topic=46370.0
210 210 it "should pass reorg unit tests" do
211   - # Disable difficulty checks. Hackish, should be replaced with some sane API.**
212   - class Bitcoin::Validation::Block; def difficulty; true; end; end
213 211 Bitcoin.network = :bitcoin
  212 + # Disable difficulty check
  213 + Bitcoin.network[:no_difficulty] = true
214 214 @store.import "./spec/bitcoin/fixtures/reorg/blk_0_to_4.dat"
215 215 @store.get_depth.should == 4
216 216 @store.get_head.hash.should =~ /000000002f264d65040/
217 217  
... ... @@ -228,14 +228,8 @@
228 228 balance("1NiEGXeURREqqMjCvjCeZn6SwEBZ9AdVet").should == 1000000000
229 229 balance("1KXFNhNtrRMfgbdiQeuJqnfD7dR4PhniyJ").should == 0
230 230 balance("1JyMKvPHkrCQd8jQrqTR1rBsAd1VpRhTiE").should == 14000000000
  231 + Bitcoin.network.delete :no_difficulty
231 232 Bitcoin.network = :testnet
232   - # Re-enable difficulty checks. Hackish, should be replaced with some sane API.
233   - class Bitcoin::Validation::Block
234   - def difficulty
235   - return true if Bitcoin.network_name == :testnet3
236   - block.bits == next_bits_required || [block.bits, next_bits_required]
237   - end
238   - end
239 233 end
240 234  
241 235 end
spec/bitcoin/storage/storage_spec.rb
... ... @@ -24,7 +24,7 @@
24 24 describe "Storage::Backends::#{options[0].to_s.capitalize}Store (#{options[1]})" do
25 25  
26 26 before do
27   - class Bitcoin::Validation::Block; def difficulty; true; end; end
  27 + Bitcoin.network[:no_difficulty] = true
28 28 Bitcoin.network[:proof_of_work_limit] = Bitcoin.encode_compact_bits("ff"*32)
29 29  
30 30 @store = storage
... ... @@ -46,12 +46,7 @@
46 46 end
47 47  
48 48 after do
49   - class Bitcoin::Validation::Block
50   - def difficulty
51   - return true if Bitcoin.network_name == :testnet3
52   - block.bits == next_bits_required || [block.bits, next_bits_required]
53   - end
54   - end
  49 + Bitcoin.network.delete :no_difficulty
55 50 end
56 51  
57 52 it "should get backend name" do