performance report

.travis.yml

@@ -269,6 +269,14 @@ matrix:
       env: MATRIX_EVAL="CC=gcc-4.9 && CXX=g++-4.9 && CMAKE_BUILD_TYPE=Debug && COVERAGE=1"
       after_success: cd ${TRAVIS_BUILD_DIR}; ./coverage.sh
 
+    # Linux x GCC4.9 x Debug x Performance
+    - os: linux
+      addons:
+        apt:
+          sources: ['ubuntu-toolchain-r-test']
+          packages: ['g++-4.9']
+      env: MATRIX_EVAL="CC=gcc-4.9 && CXX=g++-4.9 && CMAKE_BUILD_TYPE=Release && PERFORMANCE=1"
+
   # Allow quality control jobs to fail
   allow_failures:
     - os: osx
@@ -292,6 +300,12 @@ matrix:
           sources: ['ubuntu-toolchain-r-test']
           packages: ['g++-4.9']
       env: MATRIX_EVAL="CC=gcc-4.9 && CXX=g++-4.9 && CMAKE_BUILD_TYPE=Debug && COVERAGE=1"
+    - os: linux
+      addons:
+        apt:
+          sources: ['ubuntu-toolchain-r-test']
+          packages: ['g++-4.9']
+      env: MATRIX_EVAL="CC=gcc-4.9 && CXX=g++-4.9 && CMAKE_BUILD_TYPE=Release && PERFORMANCE=1"
 
   # Allow fast finish, even if the optional quality control jobs are not done yet
   fast_finish: true
@@ -345,4 +359,7 @@ script:
   - cd build
   - cmake $CMAKE_PARAMS ..
   - eval make $REDIR
-  - GTEST_COLOR=1 ctest -VV
+  - if [ "$PERFORMANCE" != "1" ]; then GTEST_COLOR=1 ctest -VV; fi
+
+after_script:
+  - if [ "$PERFORMANCE" == "1" ] && [ "$TRAVIS_PULL_REQUEST" != "false" ]; then cd ../test/performance && ruby test_perf.rb; fi

test/performance/test_perf.rb

@@ -0,0 +1,69 @@
+# Check we have the required environment variables
+if not ENV['TRAVIS_PULL_REQUEST'] or not ENV['TRAVIS_REPO_SLUG'] or not ENV['GH_TOKEN'] or not ENV['TRAVIS_PULL_REQUEST_BRANCH']
+  puts "Environment misconfiguration"
+  exit
+end
+
+# Basic init
+result      = ""
+decrease    = ""
+improvement = ""
+notif       = ""
+
+# Get report from master branch
+`git checkout master -- test_perf_result`
+
+# For each test
+#  > get execution time
+#  > check if there is major regression or improvement
+#  > update the performance report
+Dir['../../build/bin/*_test'].each do |f|
+  start_time  = Time.now
+  `$(#{f} 1> /dev/null 2> /dev/null)`
+  end_time    = Time.now
+  total_time  = end_time - start_time
+
+  prev_perf    = `cat test_perf_result | grep #{f} | cut -d ' ' -f 2`.to_f
+  higher_bound = prev_perf + (total_time * 25 / 100)
+  lower_bound  = prev_perf - (total_time * 25 / 100)
+  diff         = (total_time - prev_perf) * 100 / prev_perf
+
+  if total_time > higher_bound
+    decrease += "#{f}: was #{prev_perf}, now #{total_time} (+#{'%0.02f' % diff}%)\\n"
+  elsif total_time < lower_bound
+    improvement += "#{f}: was #{prev_perf}, now #{total_time} (#{'%0.02f' % diff}%)\\n"
+  end
+
+  result += "#{f} #{total_time}\n"
+end
+
+# Update the performance report
+`echo '#{result}' > test_perf_result`
+
+# Setup regression report
+if not decrease.empty?
+  notif += '# Performance degradation report\n'
+  notif += decrease
+end
+
+# Setup improvement report
+if not improvement.empty?
+  notif += '# Performance improvement report\n'
+  notif += improvement
+end
+
+# Setup no change report
+if notif.empty?
+  notif += 'No major performance regression or improvement\n'
+end
+
+# Notify
+puts `curl -H "Authorization: token ${GH_TOKEN}" -X POST -d '{"body": "#{notif}"}' "https://api.github.com/repos/${TRAVIS_REPO_SLUG}/issues/${TRAVIS_PULL_REQUEST}/comments"`
+
+# Push the updated perf report
+`git remote rm origin && git remote add origin https://github.com/$TRAVIS_REPO_SLUG`
+`git fetch origin`
+`git checkout -b $TRAVIS_PULL_REQUEST_BRANCH origin/$TRAVIS_PULL_REQUEST_BRANCH`
+`git add test_perf_result`
+`git commit -m 'performance report [CI SKIP]'`
+`git push https://${GH_TOKEN}@github.com/$TRAVIS_REPO_SLUG $TRAVIS_PULL_REQUEST_BRANCH`

test/performance/test_perf_result

@@ -0,0 +1,99 @@
+../../build/bin/responses_were_addresses_spent_from_test 0.002673413
+../../build/bin/core_get_trytes_test 0.074619049
+../../build/bin/core_get_transactions_to_approve_test 0.052522042
+../../build/bin/core_get_balances_test 0.030051051
+../../build/bin/extended_send_transfer_test 15.595613762
+../../build/bin/models_transfer_test 0.00297322
+../../build/bin/responses_get_node_info_test 0.00253447
+../../build/bin/utils_stop_watch_test 14.003987851
+../../build/bin/extended_initiate_transfer_test 0.147724112
+../../build/bin/responses_get_tips_test 0.002230168
+../../build/bin/requests_get_inclusion_states_test 0.002160395
+../../build/bin/responses_add_neighbors_test 0.002024402
+../../build/bin/responses_send_transfer_test 0.00203437
+../../build/bin/core_get_node_info_test 0.017016791
+../../build/bin/core_were_addresses_spent_from_test 0.05861448
+../../build/bin/extended_is_reattachable_test 3.434427288
+../../build/bin/errors_illegal_state_test 0.002237674
+../../build/bin/extended_get_transactions_objects_test 0.036048461
+../../build/bin/core_interrupt_attaching_to_tangle_test 4.023164518
+../../build/bin/models_tag_test 0.002586472
+../../build/bin/extended_get_latest_inclusion_test 0.051134751
+../../build/bin/responses_get_transactions_to_approve_test 0.002579033
+../../build/bin/extended_is_promotable_test 0.045838763
+../../build/bin/models_seed_test 0.002818129
+../../build/bin/requests_get_neighbors_test 0.002317921
+../../build/bin/models_neighbor_test 0.002360679
+../../build/bin/errors_generic_test 0.002276989
+../../build/bin/responses_get_balances_test 0.00232869
+../../build/bin/responses_get_balances_and_format_test 0.002480821
+../../build/bin/requests_check_consistency_test 0.002302962
+../../build/bin/requests_get_trytes_test 0.002287192
+../../build/bin/requests_base_test 0.002208943
+../../build/bin/responses_get_trytes_test 0.002103742
+../../build/bin/responses_get_transfers_test 0.002068923
+../../build/bin/extended_send_trytes_test 4.248804643
+../../build/bin/extended_bundles_from_addresses_test 0.308622726
+../../build/bin/responses_check_consistency_test 0.002088344
+../../build/bin/extended_find_transaction_objects_by_bundle_test 0.061152426
+../../build/bin/crypto_pow_test 8.125748669
+../../build/bin/responses_get_account_data_test 0.002368888
+../../build/bin/requests_get_balances_test 0.002037807
+../../build/bin/requests_add_neighbors_test 0.001962252
+../../build/bin/core_store_transactions_test 2.063053968
+../../build/bin/extended_get_transfers_test 0.500654961
+../../build/bin/core_broadcast_transactions_test 2.064208122
+../../build/bin/models_address_test 0.002783341
+../../build/bin/core_neighbors_test 0.05454215
+../../build/bin/requests_get_tips_test 0.002321864
+../../build/bin/responses_get_new_addresses_test 0.00234684
+../../build/bin/crypto_curl_test 0.009824984
+../../build/bin/core_get_inclusions_states_test 0.058152284
+../../build/bin/responses_find_transactions_test 0.002415209
+../../build/bin/core_get_tips_test 0.016515135
+../../build/bin/crypto_signing_test 0.00270949
+../../build/bin/models_transaction_test 0.007840742
+../../build/bin/responses_remove_neighbors_test 0.00428531
+../../build/bin/extended_get_new_addresses_test 0.30295796
+../../build/bin/extended_get_account_data_test 0.799443023
+../../build/bin/responses_base_test 0.002369885
+../../build/bin/crypto_kerl_test 0.002272641
+../../build/bin/requests_find_transactions_test 0.002468121
+../../build/bin/extended_prepare_transfers_test 0.559005646
+../../build/bin/responses_get_bundle_test 0.002204882
+../../build/bin/crypto_multi_signing_test 0.028989643
+../../build/bin/responses_get_neighbors_test 0.00224885
+../../build/bin/extended_get_balances_and_format_test 0.032527705
+../../build/bin/requests_interrupt_attaching_to_tangle_test 0.002199682
+../../build/bin/requests_store_transactions_test 0.001986044
+../../build/bin/models_signature_test 0.002157005
+../../build/bin/core_attach_to_tangle_test 6.174935276
+../../build/bin/extended_get_bundle_test 0.13890563
+../../build/bin/extended_replay_bundle_test 2.10854185
+../../build/bin/errors_bad_request_test 0.002255862
+../../build/bin/requests_remove_neighbors_test 0.00194235
+../../build/bin/responses_replay_bundle_test 0.001901126
+../../build/bin/errors_network_test 0.001898647
+../../build/bin/requests_get_transactions_to_approve_test 0.001977679
+../../build/bin/errors_crypto_test 0.0018472
+../../build/bin/core_check_consistency_test 0.043053724
+../../build/bin/requests_get_node_info_test 0.002219457
+../../build/bin/requests_were_addresses_spent_from_test 0.002148877
+../../build/bin/core_find_transactions_test 0.050835434
+../../build/bin/extended_find_transactions_test 0.04339799
+../../build/bin/crypto_keccak384_test 0.002274911
+../../build/bin/types_trinary_test 0.002403102
+../../build/bin/extended_get_inputs_test 0.16328698
+../../build/bin/extended_add_remainder_test 0.281219814
+../../build/bin/extended_find_transaction_objects_test 0.064035028
+../../build/bin/extended_broadcast_and_store_test 4.077830567
+../../build/bin/errors_internal_server_error_test 0.002134477
+../../build/bin/models_bundle_test 0.028806677
+../../build/bin/extended_traverse_bundle_test 0.085511965
+../../build/bin/errors_unrecognized_test 0.002120729
+../../build/bin/requests_attach_to_tangle_test 0.001955771
+../../build/bin/errors_unauthorized_test 0.002027241
+../../build/bin/responses_get_inclusion_states_test 0.001831769
+../../build/bin/responses_attach_to_tangle_test 0.001852293
+../../build/bin/requests_broadcast_transactions_test 0.001973562
+