diff --git a/.travis.yml b/.travis.yml index 9874979..be1ee50 100644 --- a/.travis.yml +++ b/.travis.yml @@ -269,6 +269,14 @@ matrix: env: MATRIX_EVAL="CC=gcc-4.9 && CXX=g++-4.9 && CMAKE_BUILD_TYPE=Debug && COVERAGE=1" after_success: cd ${TRAVIS_BUILD_DIR}; ./coverage.sh + # Linux x GCC4.9 x Debug x Performance + - os: linux + addons: + apt: + sources: ['ubuntu-toolchain-r-test'] + packages: ['g++-4.9'] + env: MATRIX_EVAL="CC=gcc-4.9 && CXX=g++-4.9 && CMAKE_BUILD_TYPE=Release && PERFORMANCE=1" + # Allow quality control jobs to fail allow_failures: - os: osx @@ -292,6 +300,12 @@ matrix: sources: ['ubuntu-toolchain-r-test'] packages: ['g++-4.9'] env: MATRIX_EVAL="CC=gcc-4.9 && CXX=g++-4.9 && CMAKE_BUILD_TYPE=Debug && COVERAGE=1" + - os: linux + addons: + apt: + sources: ['ubuntu-toolchain-r-test'] + packages: ['g++-4.9'] + env: MATRIX_EVAL="CC=gcc-4.9 && CXX=g++-4.9 && CMAKE_BUILD_TYPE=Release && PERFORMANCE=1" # Allow fast finish, even if the optional quality control jobs are not done yet fast_finish: true @@ -345,4 +359,7 @@ script: - cd build - cmake $CMAKE_PARAMS .. - eval make $REDIR - - GTEST_COLOR=1 ctest -VV + - if [ "$PERFORMANCE" != "1" ]; then GTEST_COLOR=1 ctest -VV; fi + +after_script: + - if [ "$PERFORMANCE" == "1" ] && [ "$TRAVIS_PULL_REQUEST" != "false" ]; then cd ../test/performance && ruby test_perf.rb; fi diff --git a/test/performance/test_perf.rb b/test/performance/test_perf.rb new file mode 100755 index 0000000..5f2d796 --- /dev/null +++ b/test/performance/test_perf.rb @@ -0,0 +1,69 @@ +# Check we have the required environment variables +if not ENV['TRAVIS_PULL_REQUEST'] or not ENV['TRAVIS_REPO_SLUG'] or not ENV['GH_TOKEN'] or not ENV['TRAVIS_PULL_REQUEST_BRANCH'] + puts "Environment misconfiguration" + exit +end + +# Basic init +result = "" +decrease = "" +improvement = "" +notif = "" + +# Get report from master branch +`git checkout master -- test_perf_result` + +# For each test +# > get execution time +# > check if there is major regression or improvement +# > update the performance report +Dir['../../build/bin/*_test'].each do |f| + start_time = Time.now + `$(#{f} 1> /dev/null 2> /dev/null)` + end_time = Time.now + total_time = end_time - start_time + + prev_perf = `cat test_perf_result | grep #{f} | cut -d ' ' -f 2`.to_f + higher_bound = prev_perf + (total_time * 25 / 100) + lower_bound = prev_perf - (total_time * 25 / 100) + diff = (total_time - prev_perf) * 100 / prev_perf + + if total_time > higher_bound + decrease += "#{f}: was #{prev_perf}, now #{total_time} (+#{'%0.02f' % diff}%)\\n" + elsif total_time < lower_bound + improvement += "#{f}: was #{prev_perf}, now #{total_time} (#{'%0.02f' % diff}%)\\n" + end + + result += "#{f} #{total_time}\n" +end + +# Update the performance report +`echo '#{result}' > test_perf_result` + +# Setup regression report +if not decrease.empty? + notif += '# Performance degradation report\n' + notif += decrease +end + +# Setup improvement report +if not improvement.empty? + notif += '# Performance improvement report\n' + notif += improvement +end + +# Setup no change report +if notif.empty? + notif += 'No major performance regression or improvement\n' +end + +# Notify +puts `curl -H "Authorization: token ${GH_TOKEN}" -X POST -d '{"body": "#{notif}"}' "https://api.github.com/repos/${TRAVIS_REPO_SLUG}/issues/${TRAVIS_PULL_REQUEST}/comments"` + +# Push the updated perf report +`git remote rm origin && git remote add origin https://github.com/$TRAVIS_REPO_SLUG` +`git fetch origin` +`git checkout -b $TRAVIS_PULL_REQUEST_BRANCH origin/$TRAVIS_PULL_REQUEST_BRANCH` +`git add test_perf_result` +`git commit -m 'performance report [CI SKIP]'` +`git push https://${GH_TOKEN}@github.com/$TRAVIS_REPO_SLUG $TRAVIS_PULL_REQUEST_BRANCH` diff --git a/test/performance/test_perf_result b/test/performance/test_perf_result new file mode 100644 index 0000000..08ea5d3 --- /dev/null +++ b/test/performance/test_perf_result @@ -0,0 +1,99 @@ +../../build/bin/responses_were_addresses_spent_from_test 0.002673413 +../../build/bin/core_get_trytes_test 0.074619049 +../../build/bin/core_get_transactions_to_approve_test 0.052522042 +../../build/bin/core_get_balances_test 0.030051051 +../../build/bin/extended_send_transfer_test 15.595613762 +../../build/bin/models_transfer_test 0.00297322 +../../build/bin/responses_get_node_info_test 0.00253447 +../../build/bin/utils_stop_watch_test 14.003987851 +../../build/bin/extended_initiate_transfer_test 0.147724112 +../../build/bin/responses_get_tips_test 0.002230168 +../../build/bin/requests_get_inclusion_states_test 0.002160395 +../../build/bin/responses_add_neighbors_test 0.002024402 +../../build/bin/responses_send_transfer_test 0.00203437 +../../build/bin/core_get_node_info_test 0.017016791 +../../build/bin/core_were_addresses_spent_from_test 0.05861448 +../../build/bin/extended_is_reattachable_test 3.434427288 +../../build/bin/errors_illegal_state_test 0.002237674 +../../build/bin/extended_get_transactions_objects_test 0.036048461 +../../build/bin/core_interrupt_attaching_to_tangle_test 4.023164518 +../../build/bin/models_tag_test 0.002586472 +../../build/bin/extended_get_latest_inclusion_test 0.051134751 +../../build/bin/responses_get_transactions_to_approve_test 0.002579033 +../../build/bin/extended_is_promotable_test 0.045838763 +../../build/bin/models_seed_test 0.002818129 +../../build/bin/requests_get_neighbors_test 0.002317921 +../../build/bin/models_neighbor_test 0.002360679 +../../build/bin/errors_generic_test 0.002276989 +../../build/bin/responses_get_balances_test 0.00232869 +../../build/bin/responses_get_balances_and_format_test 0.002480821 +../../build/bin/requests_check_consistency_test 0.002302962 +../../build/bin/requests_get_trytes_test 0.002287192 +../../build/bin/requests_base_test 0.002208943 +../../build/bin/responses_get_trytes_test 0.002103742 +../../build/bin/responses_get_transfers_test 0.002068923 +../../build/bin/extended_send_trytes_test 4.248804643 +../../build/bin/extended_bundles_from_addresses_test 0.308622726 +../../build/bin/responses_check_consistency_test 0.002088344 +../../build/bin/extended_find_transaction_objects_by_bundle_test 0.061152426 +../../build/bin/crypto_pow_test 8.125748669 +../../build/bin/responses_get_account_data_test 0.002368888 +../../build/bin/requests_get_balances_test 0.002037807 +../../build/bin/requests_add_neighbors_test 0.001962252 +../../build/bin/core_store_transactions_test 2.063053968 +../../build/bin/extended_get_transfers_test 0.500654961 +../../build/bin/core_broadcast_transactions_test 2.064208122 +../../build/bin/models_address_test 0.002783341 +../../build/bin/core_neighbors_test 0.05454215 +../../build/bin/requests_get_tips_test 0.002321864 +../../build/bin/responses_get_new_addresses_test 0.00234684 +../../build/bin/crypto_curl_test 0.009824984 +../../build/bin/core_get_inclusions_states_test 0.058152284 +../../build/bin/responses_find_transactions_test 0.002415209 +../../build/bin/core_get_tips_test 0.016515135 +../../build/bin/crypto_signing_test 0.00270949 +../../build/bin/models_transaction_test 0.007840742 +../../build/bin/responses_remove_neighbors_test 0.00428531 +../../build/bin/extended_get_new_addresses_test 0.30295796 +../../build/bin/extended_get_account_data_test 0.799443023 +../../build/bin/responses_base_test 0.002369885 +../../build/bin/crypto_kerl_test 0.002272641 +../../build/bin/requests_find_transactions_test 0.002468121 +../../build/bin/extended_prepare_transfers_test 0.559005646 +../../build/bin/responses_get_bundle_test 0.002204882 +../../build/bin/crypto_multi_signing_test 0.028989643 +../../build/bin/responses_get_neighbors_test 0.00224885 +../../build/bin/extended_get_balances_and_format_test 0.032527705 +../../build/bin/requests_interrupt_attaching_to_tangle_test 0.002199682 +../../build/bin/requests_store_transactions_test 0.001986044 +../../build/bin/models_signature_test 0.002157005 +../../build/bin/core_attach_to_tangle_test 6.174935276 +../../build/bin/extended_get_bundle_test 0.13890563 +../../build/bin/extended_replay_bundle_test 2.10854185 +../../build/bin/errors_bad_request_test 0.002255862 +../../build/bin/requests_remove_neighbors_test 0.00194235 +../../build/bin/responses_replay_bundle_test 0.001901126 +../../build/bin/errors_network_test 0.001898647 +../../build/bin/requests_get_transactions_to_approve_test 0.001977679 +../../build/bin/errors_crypto_test 0.0018472 +../../build/bin/core_check_consistency_test 0.043053724 +../../build/bin/requests_get_node_info_test 0.002219457 +../../build/bin/requests_were_addresses_spent_from_test 0.002148877 +../../build/bin/core_find_transactions_test 0.050835434 +../../build/bin/extended_find_transactions_test 0.04339799 +../../build/bin/crypto_keccak384_test 0.002274911 +../../build/bin/types_trinary_test 0.002403102 +../../build/bin/extended_get_inputs_test 0.16328698 +../../build/bin/extended_add_remainder_test 0.281219814 +../../build/bin/extended_find_transaction_objects_test 0.064035028 +../../build/bin/extended_broadcast_and_store_test 4.077830567 +../../build/bin/errors_internal_server_error_test 0.002134477 +../../build/bin/models_bundle_test 0.028806677 +../../build/bin/extended_traverse_bundle_test 0.085511965 +../../build/bin/errors_unrecognized_test 0.002120729 +../../build/bin/requests_attach_to_tangle_test 0.001955771 +../../build/bin/errors_unauthorized_test 0.002027241 +../../build/bin/responses_get_inclusion_states_test 0.001831769 +../../build/bin/responses_attach_to_tangle_test 0.001852293 +../../build/bin/requests_broadcast_transactions_test 0.001973562 +