diff --git a/.cirrus.yml b/.cirrus.yml index b3ac6d06cbfd..5c319c08b783 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,4 +1,5 @@ env: # Global defaults + CIRRUS_CLONE_DEPTH: 1 PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y" MAKEJOBS: "-j10" TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache @@ -27,8 +28,9 @@ base_template: &BASE_TEMPLATE # Unconditionally install git (used in fingerprint_script). - bash -c "$PACKAGE_MANAGER_INSTALL git" - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi - - git fetch $CIRRUS_REPO_CLONE_URL "pull/${CIRRUS_PR}/merge" + - git fetch --depth=1 $CIRRUS_REPO_CLONE_URL "pull/${CIRRUS_PR}/merge" - git checkout FETCH_HEAD # Use merged changes to detect silent merge conflicts + # Also, the merge commit is used to lint COMMIT_RANGE="HEAD~..HEAD" main_template: &MAIN_TEMPLATE timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out @@ -37,7 +39,7 @@ main_template: &MAIN_TEMPLATE ci_script: - ./ci/test_run_all.sh -global_task_template: &GLOBAL_TASK_TEMPLATE +container_depends_template: &CONTAINER_DEPENDS_TEMPLATE << : *BASE_TEMPLATE container: # https://cirrus-ci.org/faq/#are-there-any-limits @@ -47,15 +49,10 @@ global_task_template: &GLOBAL_TASK_TEMPLATE memory: 8G # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers depends_built_cache: folder: "depends/built" - fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-list -1 HEAD ./depends) - << : *MAIN_TEMPLATE + fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-parse HEAD:depends) -macos_native_task_template: &MACOS_NATIVE_TASK_TEMPLATE - << : *BASE_TEMPLATE - check_clang_script: - - clang --version - brew_install_script: - - brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt +global_task_template: &GLOBAL_TASK_TEMPLATE + << : *CONTAINER_DEPENDS_TEMPLATE << : *MAIN_TEMPLATE compute_credits_template: &CREDITS_TEMPLATE @@ -64,24 +61,29 @@ compute_credits_template: &CREDITS_TEMPLATE use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'bitcoin/bitcoin' && $CIRRUS_PR != "" task: - name: 'lint [bionic]' + name: 'lint [bookworm]' << : *BASE_TEMPLATE container: - image: ubuntu:bionic # For python 3.6, oldest supported version according to doc/dependencies.md + image: debian:bookworm cpu: 1 memory: 1G # For faster CI feedback, immediately schedule the linters << : *CREDITS_TEMPLATE + python_cache: + folder: "/tmp/python" + fingerprint_script: cat .python-version /etc/os-release + unshallow_script: + - git fetch --unshallow --no-tags lint_script: - ./ci/lint_run_all.sh env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV task: - name: 'tidy [jammy]' + name: 'tidy [lunar]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:jammy + image: ubuntu:lunar cpu: 2 memory: 5G # For faster CI feedback, immediately schedule the linters @@ -101,7 +103,7 @@ task: env: PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\MSBuild\Current\Bin;%PATH%' PYTHONUTF8: 1 - CI_VCPKG_TAG: '2022.09.27' + CI_VCPKG_TAG: '2023.01.09' VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads' VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' CCACHE_DIR: 'C:\Users\ContainerAdministrator\AppData\Local\ccache' @@ -152,7 +154,7 @@ task: ccache_cache: folder: '%CCACHE_DIR%' install_tools_script: - - choco install --yes --no-progress ccache --version=4.6.1 + - choco install --yes --no-progress ccache --version=4.7.4 - choco install --yes --no-progress python3 --version=3.9.6 - pip install zmq - ccache --version @@ -175,16 +177,16 @@ task: - ccache --show-stats check_script: - src\test_bitcoin.exe -l test_suite - - src\bench_bitcoin.exe --sanity-check > NUL + - src\bench_bitcoin.exe --sanity-check - python test\util\test_runner.py - python test\util\rpcauth-test.py functional_tests_script: # Increase the dynamic port range to the maximum allowed value to mitigate "OSError: [WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted". - # See: https://docs.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance + # See: https://learn.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance - netsh int ipv4 set dynamicport tcp start=1025 num=64511 - netsh int ipv6 set dynamicport tcp start=1025 num=64511 # Exclude feature_dbcrash for now due to timeout - - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=6 --timeout-factor=8 --extended --exclude feature_dbcrash + - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=99999999 --jobs=6 --timeout-factor=8 --extended --exclude feature_dbcrash task: name: 'ARM [unit tests, no functional tests] [bullseye]' @@ -212,6 +214,8 @@ task: << : *GLOBAL_TASK_TEMPLATE container: image: quay.io/centos/centos:stream8 + # For faster CI feedback, immediately schedule one task that runs all tests + << : *CREDITS_TEMPLATE env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV PACKAGE_MANAGER_INSTALL: "yum install -y" @@ -228,10 +232,10 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh" task: - name: '[TSan, depends, gui] [jammy]' + name: '[TSan, depends, gui] [lunar]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:jammy + image: ubuntu:lunar cpu: 6 # Increase CPU and Memory to avoid timeout memory: 24G env: @@ -239,10 +243,10 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" task: - name: '[MSan, depends] [focal]' + name: '[MSan, depends] [lunar]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:focal + image: ubuntu:lunar env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" @@ -267,10 +271,10 @@ task: MAKEJOBS: "-j4" # Avoid excessive memory use task: - name: '[fuzzer,address,undefined,integer, no depends] [jammy]' + name: '[fuzzer,address,undefined,integer, no depends] [lunar]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:jammy + image: ubuntu:lunar cpu: 4 # Increase CPU and memory to avoid timeout memory: 16G env: @@ -289,23 +293,23 @@ task: FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" task: - name: '[no wallet, libbitcoinkernel] [bionic]' + name: '[no wallet, libbitcoinkernel] [buster]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:bionic + image: debian:buster env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh" task: name: 'macOS 10.15 [gui, no tests] [focal]' - << : *BASE_TEMPLATE + << : *CONTAINER_DEPENDS_TEMPLATE + container: + image: ubuntu:focal macos_sdk_cache: folder: "depends/SDKs/$MACOS_SDK" fingerprint_key: "$MACOS_SDK" << : *MAIN_TEMPLATE - container: - image: ubuntu:focal env: MACOS_SDK: "Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers" << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV @@ -316,7 +320,12 @@ task: macos_instance: # Use latest image, but hardcode version to avoid silent upgrades (and breaks) image: ghcr.io/cirruslabs/macos-ventura-xcode:14.1 # https://cirrus-ci.org/guide/macOS - << : *MACOS_NATIVE_TASK_TEMPLATE + << : *BASE_TEMPLATE + check_clang_script: + - clang --version + brew_install_script: + - brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt + << : *MAIN_TEMPLATE env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV CI_USE_APT_INSTALL: "no" @@ -324,17 +333,17 @@ task: FILE_ENV: "./ci/test/00_setup_env_mac_native_arm64.sh" task: - name: 'ARM64 Android APK [focal]' - << : *BASE_TEMPLATE + name: 'ARM64 Android APK [jammy]' + << : *CONTAINER_DEPENDS_TEMPLATE + container: + image: ubuntu:jammy android_sdk_cache: folder: "depends/SDKs/android" fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.2.8568313" depends_sources_cache: folder: "depends/sources" - fingerprint_script: git rev-list -1 HEAD ./depends + fingerprint_script: git rev-parse HEAD:depends/packages << : *MAIN_TEMPLATE - container: - image: ubuntu:focal env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_android.sh" diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index eedeeb4e54c5..000000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 000000000000..83922b54cbf4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,93 @@ +name: Bug report +description: Submit a new bug report. +labels: [bug] +body: + - type: markdown + attributes: + value: | + ## This issue tracker is only for technical issues related to Bitcoin Core. + + * General bitcoin questions and/or support requests should use Bitcoin StackExchange at https://bitcoin.stackexchange.com. + * For reporting security issues, please read instructions at https://bitcoincore.org/en/contact/. + * If the node is "stuck" during sync or giving "block checksum mismatch" errors, please ensure your hardware is stable by running `memtest` and observe CPU temperature with a load-test tool such as `linpack` before creating an issue. + + ---- + - type: checkboxes + attributes: + label: Is there an existing issue for this? + description: Please search to see if an issue already exists for the bug you encountered. + options: + - label: I have searched the existing issues + required: true + - type: textarea + id: current-behaviour + attributes: + label: Current behaviour + description: Tell us what went wrong + validations: + required: true + - type: textarea + id: expected-behaviour + attributes: + label: Expected behaviour + description: Tell us what you expected to happen + validations: + required: true + - type: textarea + id: reproduction-steps + attributes: + label: Steps to reproduce + description: | + Tell us how to reproduce your bug. Please attach related screenshots if necessary. + * Run-time or compile-time configuration options + * Actions taken + validations: + required: true + - type: textarea + id: logs + attributes: + label: Relevant log output + description: | + Please copy and paste any relevant log output or attach a debug log file. + + You can find the debug.log in your [data dir.](https://github.com/bitcoin/bitcoin/blob/master/doc/files.md#data-directory-location) + + Please be aware that the debug log might contain personally identifying information. + validations: + required: false + - type: dropdown + attributes: + label: How did you obtain Bitcoin Core + multiple: false + options: + - Compiled from source + - Pre-built binaries + - Package manager + - Other + validations: + required: true + - type: input + id: core-version + attributes: + label: What version of Bitcoin Core are you using? + description: Run `bitcoind --version` or in Bitcoin-QT use `Help > About Bitcoin Core` + placeholder: e.g. v24.0.1 or master@e1bf547 + validations: + required: true + - type: input + id: os + attributes: + label: Operating system and version + placeholder: e.g. "MacOS Ventura 13.2" or "Ubuntu 22.04 LTS" + validations: + required: true + - type: textarea + id: machine-specs + attributes: + label: Machine specifications + description: | + What are the specifications of the host machine? + e.g. OS/CPU and disk type, network connectivity + validations: + required: false + diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index fb91208954ea..000000000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve (use this for suspected bugs only, if not sure, open a regular issue below) -title: '' -labels: Bug -assignees: '' - ---- - - - - - -**Expected behavior** - - - -**Actual behavior** - - - -**To reproduce** - - - -**System information** - - - - - - - - - diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000000..40370284a6d8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: true +contact_links: + - name: Bitcoin Core Security Policy + url: https://github.com/bitcoin/bitcoin/blob/master/SECURITY.md + about: View security policy + - name: Bitcoin Core Developers + url: https://bitcoincore.org + about: Bitcoin Core homepage diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 2d5685185ea3..000000000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: '' -labels: Feature -assignees: '' - ---- - -**Is your feature request related to a problem? Please describe.** - - -**Describe the solution you'd like** - - -**Describe alternatives you've considered** - - -**Additional context** - diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 000000000000..4622fd981919 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,36 @@ +name: Feature Request +description: Suggest an idea for this project. +labels: [Feature] +body: + - type: textarea + id: feature + attributes: + label: Please describe the feature you'd like to see added. + description: Attach screenshots or logs if applicable. + validations: + required: true + - type: textarea + id: related-problem + attributes: + label: Is your feature related to a problem, if so please describe it. + description: Attach screenshots or logs if applicable. + validations: + required: false + - type: textarea + id: solution + attributes: + label: Describe the solution you'd like + validations: + required: false + - type: textarea + id: alternatives + attributes: + label: Describe any alternatives you've considered + validations: + required: false + - type: textarea + id: additional-context + attributes: + label: Please leave any additional context + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/good_first_issue.md b/.github/ISSUE_TEMPLATE/good_first_issue.md deleted file mode 100644 index d32e22d36079..000000000000 --- a/.github/ISSUE_TEMPLATE/good_first_issue.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -name: Good first issue -about: '(Regular devs only): Suggest a new good first issue' -title: '' -labels: '' -assignees: '' - ---- - - - - - - - -#### Useful skills: - - - -#### Want to work on this issue? - -For guidance on contributing, please read [CONTRIBUTING.md](https://github.com/bitcoin/bitcoin/blob/master/CONTRIBUTING.md) before opening your pull request. diff --git a/.github/ISSUE_TEMPLATE/good_first_issue.yml b/.github/ISSUE_TEMPLATE/good_first_issue.yml new file mode 100644 index 000000000000..c40dad9687a0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/good_first_issue.yml @@ -0,0 +1,42 @@ +name: Good First Issue +description: (Regular devs only) Suggest a new good first issue +labels: [good first issue] +body: + - type: markdown + attributes: + value: | + Please add the label "good first issue" manually before or after opening + + A good first issue is an uncontroversial issue, that has a relatively unique and obvious solution + + Motivate the issue and explain the solution briefly + - type: textarea + id: motivation + attributes: + label: Motivation + description: Motivate the issue + validations: + required: true + - type: textarea + id: solution + attributes: + label: Possible solution + description: Describe a possible solution + validations: + required: false + - type: textarea + id: useful-skills + attributes: + label: Useful Skills + description: For example, “`std::thread`”, “Qt5 GUI and async GUI design” or “basic understanding of Bitcoin mining and the Bitcoin Core RPC interface”. + validations: + required: false + - type: textarea + attributes: + label: Guidance for new contributors + description: Please leave this to automatically add the footer for new contributors + value: | + Want to work on this issue? + + For guidance on contributing, please read [CONTRIBUTING.md](https://github.com/bitcoin/bitcoin/blob/master/CONTRIBUTING.md) before opening your pull request. + diff --git a/.github/ISSUE_TEMPLATE/gui_issue.md b/.github/ISSUE_TEMPLATE/gui_issue.md deleted file mode 100644 index 37acc81e21e6..000000000000 --- a/.github/ISSUE_TEMPLATE/gui_issue.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -name: An issue or feature request related to the GUI -about: Any report, issue or feature request related to the GUI should be reported at https://github.com/bitcoin-core/gui/issues/ -title: Any report, issue or feature request related to the GUI should be reported at https://github.com/bitcoin-core/gui/issues/ -labels: GUI -assignees: '' - ---- - -Any report, issue or feature request related to the GUI should be reported at -https://github.com/bitcoin-core/gui/issues/ diff --git a/.github/ISSUE_TEMPLATE/gui_issue.yml b/.github/ISSUE_TEMPLATE/gui_issue.yml new file mode 100644 index 000000000000..4fe578e9b5b9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/gui_issue.yml @@ -0,0 +1,18 @@ +name: Issue or feature request related to the GUI +description: Any report, issue or feature request related to the GUI +labels: [GUI] +body: +- type: checkboxes + id: acknowledgement + attributes: + label: Issues, reports or feature requests related to the GUI should be opened directly on the GUI repo + description: https://github.com/bitcoin-core/gui/issues/ + options: + - label: I still think this issue should be opened here + required: true +- type: textarea + id: gui-request + attributes: + label: Report + validations: + required: true diff --git a/.python-version b/.python-version index cd337510bedc..36f601f10e93 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.6.15 +3.7.16 diff --git a/.tx/config b/.tx/config index 20eff98d28b7..ab1fb933d833 100644 --- a/.tx/config +++ b/.tx/config @@ -1,7 +1,7 @@ [main] host = https://www.transifex.com -[o:bitcoin:p:bitcoin:r:qt-translation-024x] +[o:bitcoin:p:bitcoin:r:qt-translation-025x] file_filter = src/qt/locale/bitcoin_.xlf source_file = src/qt/locale/bitcoin_en.xlf source_lang = en diff --git a/COPYING b/COPYING index b157c5fe4908..2f7add71aca6 100644 --- a/COPYING +++ b/COPYING @@ -1,7 +1,7 @@ The MIT License (MIT) -Copyright (c) 2009-2022 The Bitcoin Core developers -Copyright (c) 2009-2022 Bitcoin Developers +Copyright (c) 2009-2023 The Bitcoin Core developers +Copyright (c) 2009-2023 Bitcoin Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/REVIEWERS b/REVIEWERS deleted file mode 100644 index cb1bafa49605..000000000000 --- a/REVIEWERS +++ /dev/null @@ -1,17 +0,0 @@ -# ============================================================================== -# Bitcoin Core REVIEWERS -# ============================================================================== - -# Configuration of automated review requests for the bitcoin/bitcoin repo -# via DrahtBot. - -# Order is not important; if a modified file or directory matches a fnmatch, -# the reviewer will be mentioned in a PR comment requesting a review. - -# Regular contributors are free to add their names to specific directories or -# files provided that they are willing to provide a review. - -# Absence from this list should not be interpreted as a discouragement to -# review a pull request. Peer review is always welcome and is a critical -# component of the progress of the codebase. Information on peer review -# guidelines can be found in the CONTRIBUTING.md doc. diff --git a/build-aux/m4/ax_boost_base.m4 b/build-aux/m4/ax_boost_base.m4 index 6c944b160ff7..f6620882a21e 100644 --- a/build-aux/m4/ax_boost_base.m4 +++ b/build-aux/m4/ax_boost_base.m4 @@ -8,7 +8,7 @@ # # DESCRIPTION # -# Test for the Boost C++ libraries of a particular version (or newer) +# Test for the Boost C++ headers of a particular version (or newer) # # If no path to the installed boost library is given the macro searchs # under /usr, /usr/local, /opt, /opt/local and /opt/homebrew and evaluates @@ -17,12 +17,14 @@ # # This macro calls: # -# AC_SUBST(BOOST_CPPFLAGS) / AC_SUBST(BOOST_LDFLAGS) +# AC_SUBST(BOOST_CPPFLAGS) # # And sets: # # HAVE_BOOST # +# Note that this macro has been modified compared to upstream. +# # LICENSE # # Copyright (c) 2008 Thomas Porschberg @@ -59,26 +61,10 @@ AC_ARG_WITH([boost], ], [want_boost="yes"]) - -AC_ARG_WITH([boost-libdir], - [AS_HELP_STRING([--with-boost-libdir=LIB_DIR], - [Force given directory for boost libraries. - Note that this will override library path detection, - so use this parameter only if default library detection fails - and you know exactly where your boost libraries are located.])], - [ - AS_IF([test -d "$withval"], - [_AX_BOOST_BASE_boost_lib_path="$withval"], - [AC_MSG_ERROR([--with-boost-libdir expected directory name])]) - ], - [_AX_BOOST_BASE_boost_lib_path=""]) - -BOOST_LDFLAGS="" BOOST_CPPFLAGS="" AS_IF([test "x$want_boost" = "xyes"], [_AX_BOOST_BASE_RUNDETECT([$1],[$2],[$3])]) AC_SUBST(BOOST_CPPFLAGS) -AC_SUBST(BOOST_LDFLAGS) ]) @@ -139,7 +125,6 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ AC_MSG_CHECKING([for boostlib >= $1 ($WANT_BOOST_VERSION) lib path in "$_AX_BOOST_BASE_boost_path/$_AX_BOOST_BASE_boost_path_tmp"]) AS_IF([test -d "$_AX_BOOST_BASE_boost_path/$_AX_BOOST_BASE_boost_path_tmp" && test -r "$_AX_BOOST_BASE_boost_path/$_AX_BOOST_BASE_boost_path_tmp" ],[ AC_MSG_RESULT([yes]) - BOOST_LDFLAGS="-L$_AX_BOOST_BASE_boost_path/$_AX_BOOST_BASE_boost_path_tmp"; break; ], [AC_MSG_RESULT([no])]) @@ -156,27 +141,17 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ for libsubdir in $search_libsubdirs ; do if ls "$_AX_BOOST_BASE_boost_path_tmp/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi done - BOOST_LDFLAGS="-L$_AX_BOOST_BASE_boost_path_tmp/$libsubdir" BOOST_CPPFLAGS="-I$_AX_BOOST_BASE_boost_path_tmp/include" break; fi done ]) - dnl overwrite ld flags if we have required special directory with - dnl --with-boost-libdir parameter - AS_IF([test "x$_AX_BOOST_BASE_boost_lib_path" != "x"], - [BOOST_LDFLAGS="-L$_AX_BOOST_BASE_boost_lib_path"]) - - AC_MSG_CHECKING([for boostlib >= $1 ($WANT_BOOST_VERSION)]) + AC_MSG_CHECKING([for Boost headers >= $1 ($WANT_BOOST_VERSION)]) CPPFLAGS_SAVED="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" export CPPFLAGS - LDFLAGS_SAVED="$LDFLAGS" - LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" - export LDFLAGS - AC_REQUIRE([AC_PROG_CXX]) AC_LANG_PUSH(C++) AC_COMPILE_IFELSE([_AX_BOOST_BASE_PROGRAM($WANT_BOOST_VERSION)],[ @@ -193,11 +168,8 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ dnl built and installed without the --layout=system option or for a staged(not installed) version if test "x$succeeded" != "xyes" ; then CPPFLAGS="$CPPFLAGS_SAVED" - LDFLAGS="$LDFLAGS_SAVED" BOOST_CPPFLAGS= - if test -z "$_AX_BOOST_BASE_boost_lib_path" ; then - BOOST_LDFLAGS= - fi + _version=0 if test -n "$_AX_BOOST_BASE_boost_path" ; then if test -d "$_AX_BOOST_BASE_boost_path" && test -r "$_AX_BOOST_BASE_boost_path"; then @@ -216,14 +188,6 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ BOOST_CPPFLAGS="-I$_AX_BOOST_BASE_boost_path" fi fi - dnl if we found something and BOOST_LDFLAGS was unset before - dnl (because "$_AX_BOOST_BASE_boost_lib_path" = ""), set it here. - if test -n "$BOOST_CPPFLAGS" && test -z "$BOOST_LDFLAGS"; then - for libsubdir in $libsubdirs ; do - if ls "$_AX_BOOST_BASE_boost_path/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi - done - BOOST_LDFLAGS="-L$_AX_BOOST_BASE_boost_path/$libsubdir" - fi fi else if test "x$cross_compiling" != "xyes" ; then @@ -242,12 +206,6 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` BOOST_CPPFLAGS="-I$best_path/include/boost-$VERSION_UNDERSCORE" - if test -z "$_AX_BOOST_BASE_boost_lib_path" ; then - for libsubdir in $libsubdirs ; do - if ls "$best_path/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi - done - BOOST_LDFLAGS="-L$best_path/$libsubdir" - fi fi if test -n "$BOOST_ROOT" ; then @@ -259,10 +217,9 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ stage_version=`echo $version_dir | sed 's/boost_//' | sed 's/_/./g'` stage_version_shorten=`expr $stage_version : '\([[0-9]]*\.[[0-9]]*\)'` V_CHECK=`expr $stage_version_shorten \>\= $_version` - if test "x$V_CHECK" = "x1" && test -z "$_AX_BOOST_BASE_boost_lib_path" ; then + if test "x$V_CHECK" = "x1" ; then AC_MSG_NOTICE(We will use a staged boost library from $BOOST_ROOT) BOOST_CPPFLAGS="-I$BOOST_ROOT" - BOOST_LDFLAGS="-L$BOOST_ROOT/stage/$libsubdir" fi fi fi @@ -270,8 +227,6 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" export CPPFLAGS - LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" - export LDFLAGS AC_LANG_PUSH(C++) AC_COMPILE_IFELSE([_AX_BOOST_BASE_PROGRAM($WANT_BOOST_VERSION)],[ @@ -298,6 +253,4 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ fi CPPFLAGS="$CPPFLAGS_SAVED" - LDFLAGS="$LDFLAGS_SAVED" - ]) diff --git a/build_msvc/README.md b/build_msvc/README.md index 7520700a3449..ba6171fee7a3 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -17,7 +17,7 @@ Building with Visual Studio is an alternative to the Linux based [cross-compiler Prerequisites --------------------- To build [dependencies](../doc/dependencies.md) (except for [Qt](#qt)), -the default approach is to use the [vcpkg](https://docs.microsoft.com/en-us/cpp/vcpkg) package manager from Microsoft: +the default approach is to use the [vcpkg](https://vcpkg.io) package manager from Microsoft: 1. [Install](https://vcpkg.io/en/getting-started.html) vcpkg. @@ -69,7 +69,7 @@ Alternatively, open the `build_msvc/bitcoin.sln` file in Visual Studio. Security --------------------- -[Base address randomization](https://docs.microsoft.com/en-us/cpp/build/reference/dynamicbase-use-address-space-layout-randomization?view=msvc-160) is used to make Bitcoin Core more secure. When building Bitcoin using the `build_msvc` process base address randomization can be disabled by editing `common.init.vcproj` to change `RandomizedBaseAddress` from `true` to `false` and then rebuilding the project. +[Base address randomization](https://learn.microsoft.com/en-us/cpp/build/reference/dynamicbase-use-address-space-layout-randomization) is used to make Bitcoin Core more secure. When building Bitcoin using the `build_msvc` process base address randomization can be disabled by editing `common.init.vcproj` to change `RandomizedBaseAddress` from `true` to `false` and then rebuilding the project. To check if `bitcoind` has `RandomizedBaseAddress` enabled or disabled run diff --git a/build_msvc/libsecp256k1/libsecp256k1.vcxproj b/build_msvc/libsecp256k1/libsecp256k1.vcxproj index 16ee32d87e10..0b90f341a7b4 100644 --- a/build_msvc/libsecp256k1/libsecp256k1.vcxproj +++ b/build_msvc/libsecp256k1/libsecp256k1.vcxproj @@ -14,7 +14,7 @@ - ENABLE_MODULE_ECDH;ENABLE_MODULE_RECOVERY;ENABLE_MODULE_EXTRAKEYS;ENABLE_MODULE_SCHNORRSIG;%(PreprocessorDefinitions) + ENABLE_MODULE_RECOVERY;ENABLE_MODULE_EXTRAKEYS;ENABLE_MODULE_SCHNORRSIG;%(PreprocessorDefinitions) ..\..\src\secp256k1;%(AdditionalIncludeDirectories) 4146;4244;4267;4334 diff --git a/build_msvc/msvc-autogen.py b/build_msvc/msvc-autogen.py index ae48a52a2fe3..e02e3abdfa15 100755 --- a/build_msvc/msvc-autogen.py +++ b/build_msvc/msvc-autogen.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2021 The Bitcoin Core developers +# Copyright (c) 2016-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/build_msvc/vcpkg.json b/build_msvc/vcpkg.json index 86773d1fd3ce..3557269be0ca 100644 --- a/build_msvc/vcpkg.json +++ b/build_msvc/vcpkg.json @@ -13,5 +13,12 @@ "features": ["thread"] }, "zeromq" + ], + "builtin-baseline": "f14984af3738e69f197bf0e647a8dca12de92996", + "overrides": [ + { + "name": "libevent", + "version": "2.1.12#7" + } ] } diff --git a/ci/README.md b/ci/README.md index 3c5f04c39e34..de798607df93 100644 --- a/ci/README.md +++ b/ci/README.md @@ -8,8 +8,7 @@ Be aware that the tests will be built and run in-place, so please run at your ow If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first. The ci needs to perform various sysadmin tasks such as installing packages or writing to the user's home directory. -While most of the actions are done inside a docker container, this is not possible for all. Thus, cache directories, -such as the depends cache, previous release binaries, or ccache, are mounted as read-write into the docker container. While it should be fine to run +While it should be fine to run the ci system locally on you development box, the ci scripts can generally be assumed to have received less review and testing compared to other parts of the codebase. If you want to keep the work tree clean, you might want to run the ci system in a virtual machine with a Linux operating system of your choice. diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 6f9a4709dde8..f7147582dc1d 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -1,27 +1,45 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C ${CI_RETRY_EXE} apt-get update -${CI_RETRY_EXE} apt-get install -y python3-pip curl git gawk jq -( - # Temporary workaround for https://github.com/bitcoin/bitcoin/pull/26130#issuecomment-1260499544 - # Can be removed once the underlying image is bumped to something that includes git2.34 or later - sed -i -e 's/bionic/jammy/g' /etc/apt/sources.list - ${CI_RETRY_EXE} apt-get update - ${CI_RETRY_EXE} apt-get install -y --reinstall git -) +# Lint dependencies: +# - curl/xz-utils (to install shellcheck) +# - git (used in many lint scripts) +# - gpg (used by verify-commits) +${CI_RETRY_EXE} apt-get install -y curl xz-utils git gpg + +if [ -z "${SKIP_PYTHON_INSTALL}" ]; then + PYTHON_PATH=/tmp/python + if [ ! -d "${PYTHON_PATH}/bin" ]; then + ( + git clone https://github.com/pyenv/pyenv.git + cd pyenv/plugins/python-build || exit 1 + ./install.sh + ) + # For dependencies see https://github.com/pyenv/pyenv/wiki#suggested-build-environment + ${CI_RETRY_EXE} apt-get install -y build-essential libssl-dev zlib1g-dev \ + libbz2-dev libreadline-dev libsqlite3-dev curl llvm \ + libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \ + clang + env CC=clang python-build "$(cat "${BASE_ROOT_DIR}/.python-version")" "${PYTHON_PATH}" + fi + export PATH="${PYTHON_PATH}/bin:${PATH}" + command -v python3 + python3 --version +fi ${CI_RETRY_EXE} pip3 install codespell==2.2.1 -${CI_RETRY_EXE} pip3 install flake8==4.0.1 -${CI_RETRY_EXE} pip3 install mypy==0.942 -${CI_RETRY_EXE} pip3 install pyzmq==22.3.0 -${CI_RETRY_EXE} pip3 install vulture==2.3 +${CI_RETRY_EXE} pip3 install flake8==5.0.4 +${CI_RETRY_EXE} pip3 install mypy==0.971 +${CI_RETRY_EXE} pip3 install pyzmq==24.0.1 +${CI_RETRY_EXE} pip3 install vulture==2.6 SHELLCHECK_VERSION=v0.8.0 -curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ -export PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}" +curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | \ + tar --xz -xf - --directory /tmp/ +mv "/tmp/shellcheck-${SHELLCHECK_VERSION}/shellcheck" /usr/bin/ diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index 826888b6cc1b..fa28f6126c56 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -1,14 +1,17 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C -GIT_HEAD=$(git rev-parse HEAD) -if [ -n "$CIRRUS_PR" ]; then - COMMIT_RANGE="${CIRRUS_BASE_SHA}..$GIT_HEAD" +if [ -n "$LOCAL_BRANCH" ]; then + # To faithfully recreate CI linting locally, specify all commits on the current + # branch. + COMMIT_RANGE="$(git merge-base HEAD master)..HEAD" +elif [ -n "$CIRRUS_PR" ]; then + COMMIT_RANGE="HEAD~..HEAD" echo git log --no-merges --oneline "$COMMIT_RANGE" echo diff --git a/ci/lint/Dockerfile b/ci/lint/Dockerfile new file mode 100644 index 000000000000..03c20c728604 --- /dev/null +++ b/ci/lint/Dockerfile @@ -0,0 +1,29 @@ +# See test/lint/README.md for usage. +# +# This container basically has to live in this directory in order to pull in the CI +# install scripts. If it lived in the root directory, it would have to pull in the +# entire repo as docker context during build; if it lived elsewhere, it wouldn't be +# able to make back-references to pull in the install scripts. So here it lives. + +FROM python:3.7-buster + +ENV DEBIAN_FRONTEND=noninteractive +ENV LC_ALL=C.UTF-8 + +# This is used by the 04_install.sh script; we can't read the Python version from +# .python-version for the same reasons as above, and it's more efficient to pull a +# preexisting Python image than it is to build from source. +ENV SKIP_PYTHON_INSTALL=1 + +# Must be built from ./ci/lint/ for these paths to work. +COPY ./docker-entrypoint.sh /entrypoint.sh +COPY ./04_install.sh /install.sh + +RUN /install.sh && \ + echo 'alias lint="./ci/lint/06_script.sh"' >> ~/.bashrc && \ + chmod 755 /entrypoint.sh && \ + rm -rf /var/lib/apt/lists/* + + +WORKDIR /bitcoin +ENTRYPOINT ["/entrypoint.sh"] diff --git a/ci/lint/docker-entrypoint.sh b/ci/lint/docker-entrypoint.sh new file mode 100755 index 000000000000..3fdbbb0761c0 --- /dev/null +++ b/ci/lint/docker-entrypoint.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +export LC_ALL=C + +# Fixes permission issues when there is a container UID/GID mismatch with the owner +# of the mounted bitcoin src dir. +git config --global --add safe.directory /bitcoin + +if [ -z "$1" ]; then + LOCAL_BRANCH=1 bash -ic "./ci/lint/06_script.sh" +else + exec "$@" +fi diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 5a150d5f8031..4a54f47b03f5 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,11 +8,10 @@ export LC_ALL=C.UTF-8 # The root dir. # The ci system copies this folder. -# This is where the depends build is done. BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) export BASE_ROOT_DIR # The depends dir. -# This folder exists on the ci host and ci guest. Changes are propagated back and forth. +# This folder exists only on the ci guest, and on the ci host as a volume. export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...) # This folder only exists on the ci host. @@ -45,10 +44,9 @@ export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} export TEST_RUNNER_TIMEOUT_FACTOR=${TEST_RUNNER_TIMEOUT_FACTOR:-40} export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-} export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} -export EXPECTED_TESTS_DURATION_IN_SECONDS=${EXPECTED_TESTS_DURATION_IN_SECONDS:-1000} export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed} -export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:20.04} +export CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG:-ubuntu:20.04} # Randomize test order. # See https://www.boost.org/doc/libs/1_71_0/libs/test/doc/html/boost_test/utf_reference/rt_param_reference/random.html export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1} @@ -58,15 +56,18 @@ export CCACHE_SIZE=${CCACHE_SIZE:-100M} export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} # The cache dir. -# This folder exists on the ci host and ci guest. Changes are propagated back and forth. +# This folder exists only on the ci guest, and on the ci host as a volume. export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache} # Folder where the build result is put (bin and lib). export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} # Folder where the build is done (dist and out-of-tree build). export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build} +# The folder for previous release binaries. +# This folder exists only on the ci guest, and on the ci host as a volume. export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST} +export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu" export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} -export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison} +export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH diff --git a/ci/test/00_setup_env_android.sh b/ci/test/00_setup_env_android.sh index 6732db36adf7..1834bd0bc4ae 100755 --- a/ci/test/00_setup_env_android.sh +++ b/ci/test/00_setup_env_android.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export HOST=aarch64-linux-android export PACKAGES="unzip openjdk-8-jdk gradle" export CONTAINER_NAME=ci_android -export DOCKER_NAME_TAG="ubuntu:focal" +export CI_IMAGE_NAME_TAG="ubuntu:jammy" export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 932be4b43dc7..ac0c0be96a7b 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -18,7 +18,7 @@ if [ -n "$QEMU_USER_CMD" ]; then fi export CONTAINER_NAME=ci_arm_linux # Use debian to avoid 404 apt errors when cross compiling -export DOCKER_NAME_TAG="debian:bullseye" +export CI_IMAGE_NAME_TAG="debian:bullseye" export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index 1ce3261f44e8..8a931d44e5e2 100755 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,10 +8,11 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_centos -export DOCKER_NAME_TAG=quay.io/centos/centos:stream8 -export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison" +export CI_IMAGE_NAME_TAG=quay.io/centos/centos:stream8 +export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python38 python38-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison" export PIP_PACKAGES="pyzmq" export GOAL="install" +export NO_WERROR=1 # GCC 8 export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports" export CONFIG_SHELL="/bin/dash" export TEST_RUNNER_ENV="LC_ALL=en_US.UTF-8" diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index 76de87d955dd..9e3ea0d38390 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_multiprocess -export DOCKER_NAME_TAG=ubuntu:20.04 +export CI_IMAGE_NAME_TAG=ubuntu:20.04 export PACKAGES="cmake python3 llvm clang g++-multilib" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index c4f22c8f9ea0..fe42871c3160 100755 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross -export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos +export CI_IMAGE_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos export HOST=x86_64-apple-darwin export PACKAGES="cmake libz-dev libtinfo5 python3-setuptools xorriso" export XCODE_VERSION=12.2 diff --git a/ci/test/00_setup_env_mac_native_arm64.sh b/ci/test/00_setup_env_mac_native_arm64.sh index cb0e13e77c78..a6799d7b883f 100755 --- a/ci/test/00_setup_env_mac_native_arm64.sh +++ b/ci/test/00_setup_env_mac_native_arm64.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index d8ebbbcdc9f3..bb3f6997f368 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,7 +20,7 @@ fi export CONTAINER_NAME=ci_native_asan export PACKAGES="systemtap-sdt-dev clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" -export DOCKER_NAME_TAG=ubuntu:22.04 +export CI_IMAGE_NAME_TAG=ubuntu:22.04 export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index d7caec835906..05cb45c2d868 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -1,18 +1,18 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:22.04" +export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). export CONTAINER_NAME=ci_native_fuzz -export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev" +export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC='clang -ftrivial-auto-var-init=pattern' CXX='clang++ -ftrivial-auto-var-init=pattern'" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC='clang-16 -ftrivial-auto-var-init=pattern' CXX='clang++-16 -ftrivial-auto-var-init=pattern'" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 071bac8fb334..35a0de8034e6 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -1,23 +1,23 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:20.04" +export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" -export CONTAINER_NAME="ci_native_msan" -export PACKAGES="clang-12 llvm-12 cmake" +export CONTAINER_NAME="ci_native_fuzz_msan" +export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake" # BDB generates false-positives and will be removed in future -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_UNIT_TESTS="false" export RUN_FUNCTIONAL_TESTS="false" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index 97c530e19e64..5fee10e37e5c 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -1,20 +1,20 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:22.04" +export CI_IMAGE_NAME_TAG="debian:bookworm" export CONTAINER_NAME=ci_native_fuzz_valgrind -export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev valgrind" +export PACKAGES="clang llvm libclang-rt-dev python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev valgrind" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--valgrind" export GOAL="install" -# Temporarily pin dwarf 4, until valgrind can understand clang's dwarf 5 +# Temporarily pin dwarf 4, until using Valgrind 3.20 or later export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++ CFLAGS='-gdwarf-4' CXXFLAGS='-gdwarf-4'" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 34a792ec8ff2..bdb9bd7b5d87 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -1,23 +1,23 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:20.04" +export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" -export PACKAGES="clang-12 llvm-12 cmake" +export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake" # BDB generates false-positives and will be removed in future -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_FUNCTIONAL_TESTS="false" export CCACHE_SIZE=250M diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index 63560a5f5ccb..06bc2401c525 100755 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -1,14 +1,17 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel -export DOCKER_NAME_TAG=ubuntu:18.04 # Use bionic to have one config run the tests in python3.6, see doc/dependencies.md -export PACKAGES="python3-zmq clang-8 llvm-8 libc++abi-8-dev libc++-8-dev" # Use clang-8 to test C++17 compatibility, see doc/dependencies.md +export CI_IMAGE_NAME_TAG=debian:buster +# Use minimum supported python3.7 and clang-8, see doc/dependencies.md +export PACKAGES="-t buster-backports python3-zmq clang-8 llvm-8 libc++abi-8-dev libc++-8-dev" +export APPEND_APT_SOURCES_LIST="deb http://deb.debian.org/debian buster-backports main" export DEP_OPTS="NO_WALLET=1 CC=clang-8 CXX='clang++-8 -stdlib=libc++'" export GOAL="install" +export NO_WERROR=1 export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-8 CXX='clang++-8 -stdlib=libc++' --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 8a6ea62d5c54..5cc0addd33c6 100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -1,19 +1,21 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_qt5 -export DOCKER_NAME_TAG=debian:buster # Check that buster gcc-8 can compile our C++17 and run our functional tests in python3, see doc/dependencies.md +export CI_IMAGE_NAME_TAG=debian:buster +# Use minimum supported python3.7 and gcc-8, see doc/dependencies.md export PACKAGES="gcc-8 g++-8 python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1 CC=gcc-8 CXX=g++-8" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" +export NO_WERROR=1 export DOWNLOAD_PREVIOUS_RELEASES="true" export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports \ --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index 11a12e336a25..2fa61b84659e 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -6,14 +6,14 @@ export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:22.04" +export CI_IMAGE_NAME_TAG="ubuntu:lunar" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). export CONTAINER_NAME=ci_native_tidy -export PACKAGES="clang libclang-dev llvm-dev clang-tidy bear cmake libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" +export PACKAGES="clang-16 libclang-16-dev llvm-16-dev libomp-16-dev clang-tidy-16 bear cmake libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=false export RUN_TIDY=true export GOAL="install" -export BITCOIN_CONFIG="CC=clang CXX=clang++ --enable-c++20 --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0'" +export BITCOIN_CONFIG="CC=clang-16 CXX=clang++-16 --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0 -I/usr/lib/llvm-16/lib/clang/16/include'" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 6bf8391209fd..e3fa7ab7771f 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan -export DOCKER_NAME_TAG=ubuntu:22.04 -export PACKAGES="clang-13 llvm-13 libc++abi-13-dev libc++-13-dev python3-zmq" -export DEP_OPTS="CC=clang-13 CXX='clang++-13 -stdlib=libc++'" +export CI_IMAGE_NAME_TAG=ubuntu:23.04 # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). +export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev libc++abi-16-dev libc++-16-dev python3-zmq" +export DEP_OPTS="CC=clang-16 CXX='clang++-16 -stdlib=libc++'" export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION' CXXFLAGS='-g' --with-sanitizers=thread CC=clang-13 CXX='clang++-13 -stdlib=libc++'" +export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION' CXXFLAGS='-g' --with-sanitizers=thread" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index d8c08fca39c5..97b85755ef97 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -1,17 +1,17 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:22.04" +export CI_IMAGE_NAME_TAG="debian:bookworm" export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" +export PACKAGES="valgrind clang llvm libclang-rt-dev python3-zmq libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 export TEST_RUNNER_EXTRA="--nosandbox --exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" -# Temporarily pin dwarf 4, until valgrind can understand clang's dwarf 5 +# Temporarily pin dwarf 4, until using Valgrind 3.20 or later export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++ CFLAGS='-gdwarf-4' CXXFLAGS='-gdwarf-4'" # TODO enable GUI diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 136edb6662b7..af18703ce118 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,7 +18,7 @@ if [ -n "$QEMU_USER_CMD" ]; then fi # Use debian to avoid 404 apt errors export CONTAINER_NAME=ci_s390x -export DOCKER_NAME_TAG="debian:bookworm" +export CI_IMAGE_NAME_TAG="debian:bookworm" export TEST_RUNNER_ENV="LC_ALL=C" export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export RUN_FUNCTIONAL_TESTS=true diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 360011355184..3adfbf6e479b 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -1,16 +1,16 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 -export DOCKER_NAME_TAG=ubuntu:22.04 # Check that Jammy can cross-compile to win64 +export CI_IMAGE_NAME_TAG=ubuntu:22.04 # Check that Jammy can cross-compile to win64 export HOST=x86_64-w64-mingw32 export DPKG_ADD_ARCH="i386" export PACKAGES="python3 nsis g++-mingw-w64-x86-64-posix wine-binfmt wine64 wine32 file" export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="--enable-reduce-exports --disable-external-signer --disable-gui-tests" +export BITCOIN_CONFIG="--enable-reduce-exports --enable-external-signer --disable-gui-tests" diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh new file mode 100755 index 000000000000..67dd013f6f23 --- /dev/null +++ b/ci/test/01_base_install.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +CFG_DONE="ci.base-install-done" # Use a global git setting to remember whether this script ran to avoid running it twice + +if [ "$(git config --global ${CFG_DONE})" == "true" ]; then + echo "Skip base install" + exit 0 +fi + +if [ -n "$DPKG_ADD_ARCH" ]; then + dpkg --add-architecture "$DPKG_ADD_ARCH" +fi + +if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then + ${CI_RETRY_EXE} bash -c "dnf -y install epel-release" + ${CI_RETRY_EXE} bash -c "dnf -y --allowerasing install $CI_BASE_PACKAGES $PACKAGES" +elif [ "$CI_USE_APT_INSTALL" != "no" ]; then + if [[ "${ADD_UNTRUSTED_BPFCC_PPA}" == "true" ]]; then + # Ubuntu 22.04 LTS and Debian 11 both have an outdated bpfcc-tools packages. + # The iovisor PPA is outdated as well. The next Ubuntu and Debian releases will contain updated + # packages. Meanwhile, use an untrusted PPA to install an up-to-date version of the bpfcc-tools + # package. + # TODO: drop this once we can use newer images in GCE + add-apt-repository ppa:hadret/bpfcc + fi + if [[ -n "${APPEND_APT_SOURCES_LIST}" ]]; then + echo "${APPEND_APT_SOURCES_LIST}" >> /etc/apt/sources.list + fi + ${CI_RETRY_EXE} apt-get update + ${CI_RETRY_EXE} bash -c "apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $CI_BASE_PACKAGES" +fi + +if [ -n "$PIP_PACKAGES" ]; then + if [ "$CI_OS_NAME" == "macos" ]; then + sudo -H pip3 install --upgrade pip + # shellcheck disable=SC2086 + IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES + else + # shellcheck disable=SC2086 + ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES + fi +fi + +if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then + update-alternatives --install /usr/bin/clang++ clang++ "$(which clang++-16)" 100 + update-alternatives --install /usr/bin/clang clang "$(which clang-16)" 100 + git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.1 "${BASE_SCRATCH_DIR}"/msan/llvm-project + cmake -B "${BASE_SCRATCH_DIR}"/msan/build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes + make -C "${BASE_SCRATCH_DIR}"/msan/build/ "$MAKEJOBS" +fi + +if [[ "${RUN_TIDY}" == "true" ]]; then + git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_16 "${DIR_IWYU}"/include-what-you-use + cmake -B "${DIR_IWYU}"/build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-16 -S "${DIR_IWYU}"/include-what-you-use + make -C "${DIR_IWYU}"/build/ install "$MAKEJOBS" +fi + +mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" + +OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers" + +if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then + OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz" + OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}" + if [ ! -f "$OSX_SDK_PATH" ]; then + curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" + fi + tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" +fi + +if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then + ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip + if [ ! -f "$ANDROID_TOOLS_PATH" ]; then + curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" + fi + mkdir -p "$ANDROID_HOME" + unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME" + yes | "${ANDROID_HOME}"/cmdline-tools/bin/sdkmanager --sdk_root="${ANDROID_HOME}" --install "build-tools;${ANDROID_BUILD_TOOLS_VERSION}" "platform-tools" "platforms;android-${ANDROID_API_LEVEL}" "ndk;${ANDROID_NDK_VERSION}" +fi + +git config --global ${CFG_DONE} "true" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 4915797ca409..1024222e96a9 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,19 +20,23 @@ export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/t export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|PREVIOUS_RELEASES_DIR)' | tee /tmp/env if [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) - DOCKER_ADMIN="--cap-add SYS_PTRACE" + CI_CONTAINER_CAP="--cap-add SYS_PTRACE" fi export P_CI_DIR="$PWD" +export BINS_SCRATCH_DIR="${BASE_SCRATCH_DIR}/bins/" if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then - echo "Creating $DOCKER_NAME_TAG container to run in" - LOCAL_UID=$(id -u) - LOCAL_GID=$(id -g) - - # the name isn't important, so long as we use the same UID - LOCAL_USER=nonroot - ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" + echo "Creating $CI_IMAGE_NAME_TAG container to run in" + DOCKER_BUILDKIT=1 ${CI_RETRY_EXE} docker build \ + --file "${BASE_ROOT_DIR}/ci/test_imagefile" \ + --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \ + --build-arg "FILE_ENV=${FILE_ENV}" \ + --tag="${CONTAINER_NAME}" \ + "${BASE_ROOT_DIR}" + docker volume create "${CONTAINER_NAME}_ccache" || true + docker volume create "${CONTAINER_NAME}_depends" || true + docker volume create "${CONTAINER_NAME}_previous_releases" || true if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then echo "Restart docker before run to stop and clear all containers started with --rm" @@ -40,68 +44,34 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then fi # shellcheck disable=SC2086 - DOCKER_ID=$(docker run $DOCKER_ADMIN --rm --interactive --detach --tty \ + CI_CONTAINER_ID=$(docker run $CI_CONTAINER_CAP --rm --interactive --detach --tty \ --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ - --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \ - --mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \ - --mount type=bind,src=$PREVIOUS_RELEASES_DIR,dst=$PREVIOUS_RELEASES_DIR \ + --mount "type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" \ + --mount "type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR" \ + --mount "type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" \ -w $BASE_ROOT_DIR \ --env-file /tmp/env \ --name $CONTAINER_NAME \ - $DOCKER_NAME_TAG) - - # Create a non-root user inside the container which matches the local user. - # - # This prevents the root user in the container modifying the local file system permissions - # on the mounted directories - docker exec "$DOCKER_ID" useradd -u "$LOCAL_UID" -o -m "$LOCAL_USER" - docker exec "$DOCKER_ID" groupmod -o -g "$LOCAL_GID" "$LOCAL_USER" - docker exec "$DOCKER_ID" chown -R "$LOCAL_USER":"$LOCAL_USER" "${BASE_ROOT_DIR}" - export DOCKER_CI_CMD_PREFIX_ROOT="docker exec -u 0 $DOCKER_ID" - export DOCKER_CI_CMD_PREFIX="docker exec -u $LOCAL_UID $DOCKER_ID" + $CONTAINER_NAME) + export CI_CONTAINER_ID + export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}" else echo "Running on host system without docker wrapper" fi CI_EXEC () { - $DOCKER_CI_CMD_PREFIX bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd \"$P_CI_DIR\" && $*" -} -CI_EXEC_ROOT () { - $DOCKER_CI_CMD_PREFIX_ROOT bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd \"$P_CI_DIR\" && $*" + $CI_EXEC_CMD_PREFIX bash -c "export PATH=${BINS_SCRATCH_DIR}:\$PATH && cd \"$P_CI_DIR\" && $*" } export -f CI_EXEC -export -f CI_EXEC_ROOT -if [ -n "$DPKG_ADD_ARCH" ]; then - CI_EXEC_ROOT dpkg --add-architecture "$DPKG_ADD_ARCH" -fi +CI_EXEC rsync --archive --stats --human-readable /ci_base_install/ "${BASE_ROOT_DIR}" || echo "/ci_base_install/ missing" +CI_EXEC "${BASE_ROOT_DIR}/ci/test/01_base_install.sh" +CI_EXEC rsync --archive --stats --human-readable /ro_base/ "${BASE_ROOT_DIR}" || echo "Nothing to copy from ro_base" +# Fixes permission issues when there is a container UID/GID mismatch with the owner +# of the git source code directory. +CI_EXEC git config --global --add safe.directory \"*\" -if [[ $DOCKER_NAME_TAG == *centos* ]]; then - ${CI_RETRY_EXE} CI_EXEC_ROOT dnf -y install epel-release - ${CI_RETRY_EXE} CI_EXEC_ROOT dnf -y --allowerasing install "$DOCKER_PACKAGES" "$PACKAGES" -elif [ "$CI_USE_APT_INSTALL" != "no" ]; then - if [[ "${ADD_UNTRUSTED_BPFCC_PPA}" == "true" ]]; then - # Ubuntu 22.04 LTS and Debian 11 both have an outdated bpfcc-tools packages. - # The iovisor PPA is outdated as well. The next Ubuntu and Debian releases will contain updated - # packages. Meanwhile, use an untrusted PPA to install an up-to-date version of the bpfcc-tools - # package. - # TODO: drop this once we can use newer images in GCE - CI_EXEC_ROOT add-apt-repository ppa:hadret/bpfcc - fi - ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get update - ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$DOCKER_PACKAGES" -fi - -if [ -n "$PIP_PACKAGES" ]; then - if [ "$CI_OS_NAME" == "macos" ]; then - sudo -H pip3 install --upgrade pip - # shellcheck disable=SC2086 - IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES - else - # shellcheck disable=SC2086 - ${CI_RETRY_EXE} CI_EXEC pip3 install --user $PIP_PACKAGES - fi -fi +CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}" if [ "$CI_OS_NAME" == "macos" ]; then top -l 1 -s 0 | awk ' /PhysMem/ {print}' @@ -129,38 +99,12 @@ fi CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" -if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-12) 100" - CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang clang \$(which clang-12) 100" - CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/" - CI_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project" - CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" - CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx" -fi - -if [[ "${RUN_TIDY}" == "true" ]]; then - export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu" - if [ ! -d "${DIR_IWYU}" ]; then - CI_EXEC "mkdir -p ${DIR_IWYU}/build/" - CI_EXEC "git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_14 ${DIR_IWYU}/include-what-you-use" - CI_EXEC "cd ${DIR_IWYU}/build && cmake -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-14 ../include-what-you-use" - CI_EXEC_ROOT "cd ${DIR_IWYU}/build && make install $MAKEJOBS" - fi -fi - -if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then - echo "Create $BASE_ROOT_DIR" - CI_EXEC rsync -a /ro_base/ "$BASE_ROOT_DIR" -fi - if [ "$USE_BUSY_BOX" = "true" ]; then echo "Setup to use BusyBox utils" - CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/bins/" # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) - # find excluded for now because it does not recognize the -delete option in ./depends (fixed in later BusyBox version) # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed) # shellcheck disable=SC1010 - CI_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \| grep -v "^find$"\)\; do ln -s \$\(command -v busybox\) "${BASE_SCRATCH_DIR}/bins/\$util"\; done + CI_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \)\; do ln -s \$\(command -v busybox\) "${BINS_SCRATCH_DIR}/\$util"\; done # Print BusyBox version CI_EXEC patch --help fi diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index dd2b43d38bf9..199cdd64a7c6 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,34 +11,10 @@ if [ "$CI_OS_NAME" == "macos" ]; then echo > "${HOME}/Library/Application Support/Bitcoin" else CI_EXEC echo \> \$HOME/.bitcoin - CI_EXEC_ROOT echo \> \$HOME/.bitcoin -fi - -CI_EXEC mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" - -OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers" - -if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then - OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz" - OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}" - if [ ! -f "$OSX_SDK_PATH" ]; then - CI_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" - fi - CI_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" -fi - -if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then - ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip - if [ ! -f "$ANDROID_TOOLS_PATH" ]; then - CI_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" - fi - CI_EXEC mkdir -p "$ANDROID_HOME" - CI_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME" - CI_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/bin/sdkmanager --sdk_root=\"${ANDROID_HOME}\" --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\"" fi if [ -z "$NO_DEPENDS" ]; then - if [[ $DOCKER_NAME_TAG == *centos* ]]; then + if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then # CentOS has problems building the depends if the config shell is not explicitly set # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to # an error as the first command is executed) diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index 13693a2ecfb6..5856d33d2dbf 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -1,12 +1,15 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST" +BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking" +if [ -z "$NO_DEPENDS" ]; then + BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} CONFIG_SITE=$DEPENDS_DIR/$HOST/share/config.site" +fi if [ -z "$NO_WERROR" ]; then BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" fi @@ -23,7 +26,7 @@ if [ -n "$ANDROID_TOOLS_URL" ]; then exit 0 fi -BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" +BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --prefix=$BASE_OUTDIR" if [ -n "$CONFIG_SHELL" ]; then CI_EXEC "$CONFIG_SHELL" -c "./autogen.sh" diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 46312b50eb21..f7dcbcee5e85 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,12 +9,14 @@ export LC_ALL=C.UTF-8 if [[ $HOST = *-mingw32 ]]; then # Generate all binaries, so that they can be wrapped CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 + CI_EXEC make "$MAKEJOBS" -C src minisketch/test.exe VERBOSE=1 CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" fi if [ -n "$QEMU_USER_CMD" ]; then # Generate all binaries, so that they can be wrapped CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 + CI_EXEC make "$MAKEJOBS" -C src minisketch/test VERBOSE=1 CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" fi @@ -31,15 +33,16 @@ if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then - CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast + CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=99999999 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast fi if [ "${RUN_TIDY}" = "true" ]; then set -eo pipefail export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/src/" - ( CI_EXEC run-clang-tidy -quiet "${MAKEJOBS}" ) | grep -C5 "error" + ( CI_EXEC run-clang-tidy-16 -quiet "${MAKEJOBS}" ) | grep -C5 "error" export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/" CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/iwyu_tool.py"\ + " src/common/init.cpp"\ " src/common/url.cpp"\ " src/compat"\ " src/dbwrapper.cpp"\ @@ -48,19 +51,26 @@ if [ "${RUN_TIDY}" = "true" ]; then " src/node/chainstate.cpp"\ " src/node/chainstatemanager_args.cpp"\ " src/node/mempool_args.cpp"\ + " src/node/minisketchwrapper.cpp"\ + " src/node/utxo_snapshot.cpp"\ " src/node/validation_cache_args.cpp"\ " src/policy/feerate.cpp"\ " src/policy/packages.cpp"\ " src/policy/settings.cpp"\ " src/primitives/transaction.cpp"\ + " src/random.cpp"\ " src/rpc/fees.cpp"\ " src/rpc/signmessage.cpp"\ + " src/test/fuzz/string.cpp"\ " src/test/fuzz/txorphan.cpp"\ " src/test/fuzz/util/"\ + " src/test/util/coins.cpp"\ + " src/uint256.cpp"\ " src/util/bip32.cpp"\ " src/util/bytevectorhash.cpp"\ " src/util/check.cpp"\ " src/util/error.cpp"\ + " src/util/exception.cpp"\ " src/util/getuniquepath.cpp"\ " src/util/hasher.cpp"\ " src/util/message.cpp"\ @@ -72,7 +82,12 @@ if [ "${RUN_TIDY}" = "true" ]; then " src/util/syserror.cpp"\ " src/util/threadinterrupt.cpp"\ " src/zmq"\ - " -p . ${MAKEJOBS} -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" + " -p . ${MAKEJOBS}"\ + " -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp"\ + " |& tee /tmp/iwyu_ci.out" + export P_CI_DIR="${BASE_ROOT_DIR}/src" + CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/fix_includes.py --nosafe_headers < /tmp/iwyu_ci.out" + CI_EXEC "git --no-pager diff" fi if [ "$RUN_SECURITY_TESTS" = "true" ]; then @@ -82,3 +97,8 @@ fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/fuzz/test_runner.py "${FUZZ_TESTS_CONFIG}" "$MAKEJOBS" -l DEBUG "${DIR_FUZZ_IN}" fi + +if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then + echo "Stop and remove CI container by ID" + docker container kill "${CI_CONTAINER_ID}" +fi diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh index eb31edbce8c4..e028ede37888 100755 --- a/ci/test/wrap-qemu.sh +++ b/ci/test/wrap-qemu.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/wrap-wine.sh b/ci/test/wrap-wine.sh index 1662f8f6a335..90e53887bc4b 100755 --- a/ci/test/wrap-wine.sh +++ b/ci/test/wrap-wine.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test_imagefile b/ci/test_imagefile new file mode 100644 index 000000000000..4854708d1a9b --- /dev/null +++ b/ci/test_imagefile @@ -0,0 +1,10 @@ +ARG CI_IMAGE_NAME_TAG +FROM ${CI_IMAGE_NAME_TAG} + +ARG FILE_ENV +ENV FILE_ENV=${FILE_ENV} + +COPY ./ci/retry/retry /usr/bin/retry +COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh /ci_base_install/ci/test/ + +RUN ["bash", "-c", "cd /ci_base_install/ && set -o errexit && source ./ci/test/00_setup_env.sh && ./ci/test/01_base_install.sh"] diff --git a/configure.ac b/configure.ac index 541f31e3f0fc..5803876a1b1e 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ define(_CLIENT_VERSION_MINOR, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) define(_CLIENT_VERSION_IS_RELEASE, false) -define(_COPYRIGHT_YEAR, 2022) +define(_COPYRIGHT_YEAR, 2023) define(_COPYRIGHT_HOLDERS,[The %s developers]) define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[[Bitcoin Core]]) AC_INIT([Bitcoin Core],m4_join([.], _CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MINOR, _CLIENT_VERSION_BUILD)m4_if(_CLIENT_VERSION_RC, [0], [], [rc]_CLIENT_VERSION_RC),[https://github.com/bitcoin/bitcoin/issues],[bitcoin],[https://bitcoincore.org/]) @@ -76,6 +76,12 @@ dnl we have those under control, re-enable that functionality. case $host in *mingw*) lt_cv_deplibs_check_method="pass_all" + + dnl Remove unwanted -DDLL_EXPORT from these variables. + dnl We do not use this macro, but system headers may export unwanted symbols + dnl if it's set. + lt_cv_prog_compiler_pic="-DPIC" + lt_cv_prog_compiler_pic_CXX="-DPIC" ;; esac @@ -120,8 +126,8 @@ AC_PATH_TOOL([AR], [ar]) AC_PATH_TOOL([GCOV], [gcov]) AC_PATH_TOOL([LLVM_COV], [llvm-cov]) AC_PATH_PROG([LCOV], [lcov]) -dnl Python 3.6 is specified in .python-version and should be used if available, see doc/dependencies.md -AC_PATH_PROGS([PYTHON], [python3.6 python3.7 python3.8 python3.9 python3.10 python3.11 python3 python]) +dnl Python 3.7 is specified in .python-version and should be used if available, see doc/dependencies.md +AC_PATH_PROGS([PYTHON], [python3.7 python3.8 python3.9 python3.10 python3.11 python3.12 python3 python]) AC_PATH_PROG([GENHTML], [genhtml]) AC_PATH_PROG([GIT], [git]) AC_PATH_PROG([CCACHE], [ccache]) @@ -163,24 +169,12 @@ AC_ARG_WITH([miniupnpc], [use_upnp=$withval], [use_upnp=auto]) -AC_ARG_ENABLE([upnp-default], - [AS_HELP_STRING([--enable-upnp-default], - [if UPNP is enabled, turn it on at startup (default is no)])], - [use_upnp_default=$enableval], - [use_upnp_default=no]) - AC_ARG_WITH([natpmp], [AS_HELP_STRING([--with-natpmp], [enable NAT-PMP (default is yes if libnatpmp is found)])], [use_natpmp=$withval], [use_natpmp=auto]) -AC_ARG_ENABLE([natpmp-default], - [AS_HELP_STRING([--enable-natpmp-default], - [if NAT-PMP is enabled, turn it on at startup (default is no)])], - [use_natpmp_default=$enableval], - [use_natpmp_default=no]) - AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[do not compile tests (default is to compile)]), [use_tests=$enableval], @@ -448,7 +442,7 @@ if test "$CXXFLAGS_overridden" = "no"; then AX_CHECK_COMPILE_FLAG([-Wvla], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wshadow-field], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wshadow-field"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wthread-safety], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wloop-analysis], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wloop-analysis], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wloop-analysis"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wredundant-decls], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wunused-member-function], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-member-function"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wdate-time], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"], [], [$CXXFLAG_WERROR]) @@ -947,7 +941,9 @@ if test "$TARGET_OS" != "windows"; then AX_CHECK_COMPILE_FLAG([-fPIC], [PIC_FLAGS="-fPIC"]) fi -dnl All versions of gcc that we commonly use for building are subject to bug +dnl Versions of gcc prior to 12.1 (commit +dnl https://github.com/gcc-mirror/gcc/commit/551aa75778a4c5165d9533cd447c8fc822f583e1) +dnl are subject to a bug, see the gccbug_90348 test case and dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90348. To work around that, set dnl -fstack-reuse=none for all gcc builds. (Only gcc understands this flag) AX_CHECK_COMPILE_FLAG([-fstack-reuse=none], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-reuse=none"]) @@ -973,11 +969,11 @@ if test "$use_hardening" != "no"; then dnl However, FORTIFY_SOURCE requires that there is some level of optimization, otherwise it does nothing and just creates a compiler warning. dnl Since FORTIFY_SOURCE is a no-op without optimizations, do not enable it when enable_debug is yes. if test "$enable_debug" != "yes"; then - AX_CHECK_PREPROC_FLAG([-D_FORTIFY_SOURCE=2],[ + AX_CHECK_PREPROC_FLAG([-D_FORTIFY_SOURCE=3],[ AX_CHECK_PREPROC_FLAG([-U_FORTIFY_SOURCE],[ HARDENED_CPPFLAGS="$HARDENED_CPPFLAGS -U_FORTIFY_SOURCE" ]) - HARDENED_CPPFLAGS="$HARDENED_CPPFLAGS -D_FORTIFY_SOURCE=2" + HARDENED_CPPFLAGS="$HARDENED_CPPFLAGS -D_FORTIFY_SOURCE=3" ]) fi @@ -1006,7 +1002,7 @@ if test "$TARGET_OS" = "darwin"; then AX_CHECK_LINK_FLAG([-Wl,-bind_at_load], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"], [], [$LDFLAG_WERROR]) fi -AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h unistd.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) +AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) AC_CHECK_DECLS([getifaddrs, freeifaddrs],[CHECK_SOCKET],, [#include @@ -1428,14 +1424,15 @@ if test "$use_upnp" != "no"; then [AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS="$MINIUPNPC_LIBS -lminiupnpc"], [have_miniupnpc=no], [$MINIUPNPC_LIBS])], [have_miniupnpc=no] ) - dnl The minimum supported miniUPnPc API version is set to 10. This keeps compatibility - dnl with Ubuntu 16.04 LTS and Debian 8 libminiupnpc-dev packages. + + dnl The minimum supported miniUPnPc API version is set to 17. This excludes + dnl versions with known vulnerabilities. if test "$have_miniupnpc" != "no"; then AC_MSG_CHECKING([whether miniUPnPc API version is supported]) AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ @%:@include ]], [[ - #if MINIUPNPC_API_VERSION >= 10 + #if MINIUPNPC_API_VERSION >= 17 // Everything is okay #else # error miniUPnPc API version is too old @@ -1444,7 +1441,7 @@ if test "$use_upnp" != "no"; then AC_MSG_RESULT([yes]) ],[ AC_MSG_RESULT([no]) - AC_MSG_WARN([miniUPnPc API version < 10 is unsupported, disabling UPnP support.]) + AC_MSG_WARN([miniUPnPc API version < 17 is unsupported, disabling UPnP support.]) have_miniupnpc=no ]) fi @@ -1461,7 +1458,7 @@ if test "$use_natpmp" != "no"; then CPPFLAGS="$TEMP_CPPFLAGS" fi -if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench" = "nonononononono"; then +if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench$enable_fuzz_binary" = "nonononononononono"; then use_boost=no else use_boost=yes @@ -1479,9 +1476,11 @@ if test "$use_boost" = "yes"; then BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_MULTI_INDEX_DISABLE_SERIALIZATION" dnl Prevent use of std::unary_function, which was removed in C++17, - dnl and will generate warnings with newer compilers. - dnl See: https://github.com/boostorg/container_hash/issues/22. - BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_NO_CXX98_FUNCTION_BASE" + dnl and will generate warnings with newer compilers for Boost + dnl older than 1.80. + dnl See: https://github.com/boostorg/config/pull/430. + AX_CHECK_PREPROC_FLAG([-DBOOST_NO_CXX98_FUNCTION_BASE], [BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_NO_CXX98_FUNCTION_BASE"], [], [$CXXFLAG_WERROR], + [AC_LANG_PROGRAM([[#include ]])]) if test "$enable_debug" = "yes" || test "$enable_fuzz" = "yes"; then BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE" @@ -1493,45 +1492,45 @@ if test "$use_boost" = "yes"; then fi if test "$use_external_signer" != "no"; then - case $host in - *mingw*) - dnl Boost Process uses Boost Filesystem when targeting Windows. Also, - dnl since Boost 1.71.0, Process does not work with mingw-w64 without - dnl workarounds. See 67669ab425b52a2b6be3d2f3b3b7e3939b676a2c. - if test "$use_external_signer" = "yes"; then - AC_MSG_ERROR([External signing is not supported on Windows]) - fi - use_external_signer="no"; - ;; - *) - AC_MSG_CHECKING([whether Boost.Process can be used]) - TEMP_CXXFLAGS="$CXXFLAGS" - dnl Boost 1.78 requires the following workaround. - dnl See: https://github.com/boostorg/process/issues/235 - CXXFLAGS="$CXXFLAGS -Wno-error=narrowing" - TEMP_CPPFLAGS="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" - TEMP_LDFLAGS="$LDFLAGS" - dnl Boost 1.73 and older require the following workaround. - LDFLAGS="$LDFLAGS $PTHREAD_CFLAGS" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]])], - [have_boost_process="yes"], - [have_boost_process="no"]) - LDFLAGS="$TEMP_LDFLAGS" - CPPFLAGS="$TEMP_CPPFLAGS" - CXXFLAGS="$TEMP_CXXFLAGS" - AC_MSG_RESULT([$have_boost_process]) - if test "$have_boost_process" = "yes"; then - use_external_signer="yes" - AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [1], [Define if external signer support is enabled]) - else - if test "$use_external_signer" = "yes"; then - AC_MSG_ERROR([External signing is not supported for this Boost version]) - fi - use_external_signer="no"; - fi - ;; - esac + AC_MSG_CHECKING([whether Boost.Process can be used]) + TEMP_CXXFLAGS="$CXXFLAGS" + dnl Boost 1.78 requires the following workaround. + dnl See: https://github.com/boostorg/process/issues/235 + CXXFLAGS="$CXXFLAGS -Wno-error=narrowing" + TEMP_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + TEMP_LDFLAGS="$LDFLAGS" + dnl Boost 1.73 and older require the following workaround. + LDFLAGS="$LDFLAGS $PTHREAD_CFLAGS" + AC_LINK_IFELSE([AC_LANG_PROGRAM([[ + #define BOOST_PROCESS_USE_STD_FS + #include + ]],[[ + namespace bp = boost::process; + bp::opstream stdin_stream; + bp::ipstream stdout_stream; + bp::child c("dummy", bp::std_out > stdout_stream, bp::std_err > stdout_stream, bp::std_in < stdin_stream); + stdin_stream << std::string{"test"} << std::endl; + if (c.running()) c.terminate(); + c.wait(); + c.exit_code(); + ]])], + [have_boost_process="yes"], + [have_boost_process="no"]) + LDFLAGS="$TEMP_LDFLAGS" + CPPFLAGS="$TEMP_CPPFLAGS" + CXXFLAGS="$TEMP_CXXFLAGS" + AC_MSG_RESULT([$have_boost_process]) + if test "$have_boost_process" = "yes"; then + use_external_signer="yes" + AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [1], [Define if external signer support is enabled]) + AC_DEFINE([BOOST_PROCESS_USE_STD_FS], [1], [Defined to avoid Boost::Process trying to use Boost Filesystem]) + else + if test "$use_external_signer" = "yes"; then + AC_MSG_ERROR([External signing is not supported for this Boost version]) + fi + use_external_signer="no"; + fi fi AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "$use_external_signer" = "yes"]) @@ -1765,17 +1764,10 @@ if test "$have_miniupnpc" = "no"; then else if test "$use_upnp" != "no"; then AC_MSG_RESULT([yes]) - AC_MSG_CHECKING([whether to build with UPnP enabled by default]) use_upnp=yes - upnp_setting=0 - if test "$use_upnp_default" != "no"; then - use_upnp_default=yes - upnp_setting=1 - fi - AC_MSG_RESULT([$use_upnp_default]) - AC_DEFINE_UNQUOTED([USE_UPNP],[$upnp_setting],[UPnP support not compiled if undefined, otherwise value (0 or 1) determines default state]) + AC_DEFINE([USE_UPNP], [1], [Define to 1 if UPnP support should be compiled in.]) if test "$TARGET_OS" = "windows"; then - MINIUPNPC_CPPFLAGS="$MINIUPNPC_CPPFLAGS -DSTATICLIB -DMINIUPNP_STATICLIB" + MINIUPNPC_CPPFLAGS="$MINIUPNPC_CPPFLAGS -DMINIUPNP_STATICLIB" fi else AC_MSG_RESULT([no]) @@ -1793,15 +1785,8 @@ if test "$have_natpmp" = "no"; then else if test "$use_natpmp" != "no"; then AC_MSG_RESULT([yes]) - AC_MSG_CHECKING([whether to build with NAT-PMP enabled by default]) use_natpmp=yes - natpmp_setting=0 - if test "$use_natpmp_default" != "no"; then - use_natpmp_default=yes - natpmp_setting=1 - fi - AC_MSG_RESULT($use_natpmp_default) - AC_DEFINE_UNQUOTED([USE_NATPMP], [$natpmp_setting], [NAT-PMP support not compiled if undefined, otherwise value (0 or 1) determines default state]) + AC_DEFINE([USE_NATPMP], [1], [Define to 1 if UPnP support should be compiled in.]) if test "$TARGET_OS" = "windows"; then NATPMP_CPPFLAGS="$NATPMP_CPPFLAGS -DSTATICLIB -DNATPMP_STATICLIB" fi @@ -1870,7 +1855,7 @@ else AC_MSG_RESULT([no]) fi -if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$enable_fuzz_binary$use_bench$use_tests" = "nonononononononono"; then +if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$enable_fuzz_binary$use_bench$use_tests" = "nononononononononono"; then AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-gui --enable-fuzz(-binary) --enable-bench or --enable-tests]) fi @@ -2004,15 +1989,7 @@ CPPFLAGS_TEMP="$CPPFLAGS" unset CPPFLAGS CPPFLAGS="$CPPFLAGS_TEMP" -LDFLAGS_TEMP="$LDFLAGS" -unset LDFLAGS -LDFLAGS="$LDFLAGS_TEMP" - -LIBS_TEMP="$LIBS" -unset LIBS -LIBS="$LIBS_TEMP" - -ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --enable-module-recovery --enable-module-schnorrsig" +ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --enable-module-recovery --disable-module-ecdh" AC_CONFIG_SUBDIRS([src/secp256k1]) AC_OUTPUT diff --git a/contrib/README.md b/contrib/README.md index 0252d668d9af..3c6e97806116 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -26,9 +26,6 @@ The [Debian](/contrib/debian) subfolder contains the copyright file. All other packaging related files can be found in the [bitcoin-core/packaging](https://github.com/bitcoin-core/packaging) repository. -### [Builder keys](/contrib/builder-keys) -PGP keys used for signing Bitcoin Core [release](/doc/release-process.md) results. - ### [MacDeploy](/contrib/macdeploy) ### Scripts and notes for Mac builds. diff --git a/contrib/builder-keys/README.md b/contrib/builder-keys/README.md deleted file mode 100644 index a6179d6012cd..000000000000 --- a/contrib/builder-keys/README.md +++ /dev/null @@ -1,33 +0,0 @@ -## PGP keys of builders and Developers - -The file `keys.txt` contains fingerprints of the public keys of builders and -active developers. - -The associated keys are mainly used to sign git commits or the build results -of Guix builds. - -The most recent version of each pgp key can be found on most pgp key servers. - -Fetch the latest version from the key server to see if any key was revoked in -the meantime. -To fetch the latest version of all pgp keys in your gpg homedir, - -```sh -gpg --refresh-keys -``` - -To fetch keys of builders and active developers, feed the list of fingerprints -of the primary keys into gpg: - -On \*NIX: -```sh -while read fingerprint keyholder_name; do gpg --keyserver hkps://keys.openpgp.org --recv-keys ${fingerprint}; done < ./keys.txt -``` - -On Windows (requires Gpg4win >= 4.0.0): -``` -FOR /F "tokens=1" %i IN (keys.txt) DO gpg --keyserver hkps://keys.openpgp.org --recv-keys %i -``` - -Add your key to the list if you provided Guix attestations for two major or -minor releases of Bitcoin Core. diff --git a/contrib/builder-keys/keys.txt b/contrib/builder-keys/keys.txt deleted file mode 100644 index b2b5b031f8a6..000000000000 --- a/contrib/builder-keys/keys.txt +++ /dev/null @@ -1,57 +0,0 @@ -982A193E3CE0EED535E09023188CBB2648416AD5 0xB10C (0xb10c) -9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C Aaron Clauson (sipsorcery) -617C90010B3BD370B0AC7D424BB42E31C79111B8 Akira Takizawa (akx20000) -E944AE667CF960B1004BC32FCA662BE18B877A60 Andreas Schildbach (aschildbach) -152812300785C96444D3334D17565732E08E5E41 Andrew Chow (achow101) -590B7292695AFFA5B672CBB2E13FC145CD3F4304 Antoine Poinsot (darosior) -0AD83877C1F0CD1EE9BD660AD7CC770B81FD22A8 Ben Carman (benthecarman) -912FD3228387123DC97E0E57D5566241A0295FA9 BtcDrak (btcdrak) -04017A2A6D9A0CCDC81D8EC296AB007F1A7ED999 Carl Dong (dongcarl) -C519EBCF3B926298946783EFF6430754120EC2F4 Christian Decker (cdecker) -18AE2F798E0D239755DA4FD24B79F986CBDF8736 Chun Kuan Le (ken2812221) -101598DC823C1B5F9A6624ABA5E0907A0380E6C3 CoinForensics (CoinForensics) -F20F56EF6A067F70E8A5C99FFF95FAA971697405 centaur (centaur) -C060A6635913D98A3587D7DB1C2491FFEB0EF770 Cory Fields (cfields) -BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random (devrandom) -6D3170C1DC2C6FD0AEEBCA6743811D1A26623924 Douglas Roark (droark) -948444FCE03B05BA5AB0591EC37B1C1D44C786EE Duncan Dean (dunxen) -1C6621605EC50319C463D56C7F81D87985D61612 Emanuele Cisbani (cisba) -9A1689B60D1B3CCE9262307A2F40A9BF167FBA47 Erik Mossberg (erkmos) -D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece (fivepiece) -6F993B250557E7B016ADE5713BDCDA2D87A881D9 Fuzzbawls (Fuzzbawls) -01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen (gavinandresen) -6B002C6EA3F91B1B0DF0C9BC8F617F1200A6D25C Gloria Zhao (glozow) -D1DBF2C4B96F2DEBF4C16654410108112E7EA81F Hennadii Stepanov (hebasto) -2688F5A9A4BE0F295E921E8A25F27A38A47AD566 James O'Beirne (jamesob) -D3F22A3A4C366C2DCB66D3722DA9C5A7FA81EA35 Jarol Rodriguez (jarolrod) -7480909378D544EA6B6DCEB7535B12980BB8A4D3 Jeffri H Frontz (jhfrontz) -D3CC177286005BB8FF673294C5242A1AB3936517 jl2012 (jl2012) -82921A4B88FD454B7EB8CE3C796C4109063D4EAF Jon Atack (jonatack) -32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC Jonas Schnelli (jonasschnelli) -4B4E840451149DD7FB0D633477DFAB5C3108B9A8 Jorge Timon (jtimon) -C42AFF7C61B3E44A1454CD3557AF762DB3353322 Karl-Johan Alm (kallewoof) -70A1D47DD44F59DF8B22244333E472FE870C7E5D Kristaps Kaupe (kristapsk) -30DE693AE0DE9E37B3E7EB6BBFF0F67810C1EED1 Lisa Neigut (niftynei) -E463A93F5F3117EEDE6C7316BD02942421F4889F Luke Dashjr (luke-jr) -B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B Marco Falke (marco) -07DF3E57A548CCFB7530709189BBB8663E2E65CE Matt Corallo (BlueMatt) -CA03882CB1FC067B5D3ACFE4D300116E1C875A3D MeshCollider (meshcollider) -E777299FC265DD04793070EB944D35F9AC3DB76A Michael Ford (fanquake) -AD5764F4ADCE1B99BDFD179E12335A271D4D62EC Michael Tidwell (miketwenty1) -9692B91BBF0E8D34DFD33B1882C5C009628ECF0C Michagogo (michagogo) -C57E4B42223FDE851D4F69DD28DF2724F241D8EE midnightmagic (midnightmagic) -F4FC70F07310028424EFC20A8E4256593F177720 Oliver Gugger (guggero, Oliver Gugger) -D62A803E27E7F43486035ADBBCD04D8E9CCCAC2A Paul Rabahy (prab) -37EC7D7B0A217CDB4B4E007E7FAB114267E4FA04 Peter Todd (petertodd) -D762373D24904A3E42F33B08B9A408E71DAAC974 Pieter Wuille [Location: Leuven, Belgium] (sipa) -133EAC179436F14A5CF1B794860FEB804E669320 Pieter Wuille (sipa) -6A8F9C266528E25AEB1D7731C2371D91CB716EA7 Sebastian Falbesoner (theStack) -A8FC55F3B04BA3146F3492E79303B33A305224CB Sebastian Kung (TheCharlatan) -ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost (sjors) -867345026B6763E8B07EE73AB6737117397F5C4F Stephan Oeste (Emzy) -9EDAFF80E080659604F4A76B2EBB056FD847F8A7 Stephan Oeste (Emzy) -6DEEF79B050C4072509B743F8C275BC595448867 Tomas Kanocz (KanoczTomas) -AEC1884398647C47413C1C3FB1179EB7347DC10D Warren Togami (wtogami) -74E2DEF5D77260B98BC19438099BAD163C70FBFA Will Clark (will8clark) -79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF Willy Ko (willyko) -71A3B16735405025D447E8F274810B012346C9A6 Wladimir J. van der Laan (laanwj) diff --git a/contrib/completions/bash/bitcoin-cli.bash-completion b/contrib/completions/bash/bitcoin-cli.bash-completion index ddea58a05cc1..89e01bc09ae6 100644 --- a/contrib/completions/bash/bitcoin-cli.bash-completion +++ b/contrib/completions/bash/bitcoin-cli.bash-completion @@ -1,5 +1,5 @@ # bash programmable completion for bitcoin-cli(1) -# Copyright (c) 2012-2019 The Bitcoin Core developers +# Copyright (c) 2012-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/completions/bash/bitcoin-tx.bash-completion b/contrib/completions/bash/bitcoin-tx.bash-completion index a83d2979ed3a..51a9fe31cb71 100644 --- a/contrib/completions/bash/bitcoin-tx.bash-completion +++ b/contrib/completions/bash/bitcoin-tx.bash-completion @@ -1,5 +1,5 @@ # bash programmable completion for bitcoin-tx(1) -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/completions/bash/bitcoind.bash-completion b/contrib/completions/bash/bitcoind.bash-completion index ec1d9512d475..c11d99ef3169 100644 --- a/contrib/completions/bash/bitcoind.bash-completion +++ b/contrib/completions/bash/bitcoind.bash-completion @@ -1,5 +1,5 @@ # bash programmable completion for bitcoind(1) and bitcoin-qt(1) -# Copyright (c) 2012-2019 The Bitcoin Core developers +# Copyright (c) 2012-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/debian/copyright b/contrib/debian/copyright index 95a281ce054e..ca430170a1d6 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -5,24 +5,16 @@ Upstream-Contact: Satoshi Nakamoto Source: https://github.com/bitcoin/bitcoin Files: * -Copyright: 2009-2022, Bitcoin Core Developers +Copyright: 2009-2023, Bitcoin Core Developers License: Expat -Comment: The Bitcoin Core Developers encompasses the current developers listed on bitcoin.org, - as well as the numerous contributors to the project. +Comment: The Bitcoin Core Developers encompasses all contributors to the + project, listed in the release notes or the git log. Files: debian/* Copyright: 2010-2011, Jonas Smedegaard 2011, Matt Corallo License: GPL-2+ -Files: src/secp256k1/build-aux/m4/ax_jni_include_dir.m4 -Copyright: 2008 Don Anderson -License: GNU-All-permissive-License - -Files: src/secp256k1/build-aux/m4/ax_prog_cc_for_build.m4 -Copyright: 2008 Paolo Bonzini -License: GNU-All-permissive-License - Files: src/qt/res/icons/add.png src/qt/res/icons/address-book.png src/qt/res/icons/chevron.png @@ -112,12 +104,6 @@ License: Expat TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -License: GNU-All-permissive-License - Copying and distribution of this file, with or without modification, are - permitted in any medium without royalty provided the copyright notice - and this notice are preserved. This file is offered as-is, without any - warranty. - License: GPL-2+ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py index 98eee67f4300..420bf7ff3300 100755 --- a/contrib/devtools/clang-format-diff.py +++ b/contrib/devtools/clang-format-diff.py @@ -146,7 +146,7 @@ def main(): stdout=subprocess.PIPE, stderr=None, stdin=subprocess.PIPE, - universal_newlines=True) + text=True) stdout, stderr = p.communicate() if p.returncode != 0: sys.exit(p.returncode) diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 680de1f1b3e4..3dddffe32427 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2021 The Bitcoin Core developers +# Copyright (c) 2016-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/devtools/gen-manpages.py b/contrib/devtools/gen-manpages.py index 38cdb3acb8d1..2860e7db9944 100755 --- a/contrib/devtools/gen-manpages.py +++ b/contrib/devtools/gen-manpages.py @@ -23,7 +23,7 @@ # If not otherwise specified, get top directory from git. topdir = os.getenv('TOPDIR') if not topdir: - r = subprocess.run([git, 'rev-parse', '--show-toplevel'], stdout=subprocess.PIPE, check=True, universal_newlines=True) + r = subprocess.run([git, 'rev-parse', '--show-toplevel'], stdout=subprocess.PIPE, check=True, text=True) topdir = r.stdout.rstrip() # Get input and output directories. @@ -36,7 +36,7 @@ for relpath in BINARIES: abspath = os.path.join(builddir, relpath) try: - r = subprocess.run([abspath, "--version"], stdout=subprocess.PIPE, check=True, universal_newlines=True) + r = subprocess.run([abspath, "--version"], stdout=subprocess.PIPE, check=True, text=True) except IOError: print(f'{abspath} not found or not an executable', file=sys.stderr) sys.exit(1) diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 05c0af029ec8..6cd022ef1710 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2021 The Bitcoin Core developers +# Copyright (c) 2015-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' @@ -34,7 +34,7 @@ def check_ELF_RELRO(binary) -> bool: flags = binary.get(lief.ELF.DYNAMIC_TAGS.FLAGS) if flags.value & lief.ELF.DYNAMIC_FLAGS.BIND_NOW: have_bindnow = True - except: + except Exception: have_bindnow = False return have_gnu_relro and have_bindnow @@ -146,6 +146,12 @@ def check_PE_control_flow(binary) -> bool: return True return False +def check_PE_Canary(binary) -> bool: + ''' + Check for use of stack canary + ''' + return binary.has_symbol('__stack_chk_fail') + def check_MACHO_NOUNDEFS(binary) -> bool: ''' Check for no undefined references. @@ -203,6 +209,7 @@ def check_MACHO_control_flow(binary) -> bool: ('NX', check_NX), ('RELOC_SECTION', check_PE_RELOC_SECTION), ('CONTROL_FLOW', check_PE_control_flow), + ('Canary', check_PE_Canary), ] BASE_MACHO = [ diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 4b1cceb57ceb..f26236dd59ed 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -15,19 +15,19 @@ import lief #type:ignore -# Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases +# Debian 10 (Buster) EOL: 2024. https://wiki.debian.org/LTS # -# - g++ version 6.3.0 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=g%2B%2B) -# - libc version 2.24 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=libc6) +# - libgcc version 8.3.0 (https://packages.debian.org/search?suite=buster&arch=any&searchon=names&keywords=libgcc1) +# - libc version 2.28 (https://packages.debian.org/search?suite=buster&arch=any&searchon=names&keywords=libc6) # -# Ubuntu 16.04 (Xenial) EOL: 2026. https://wiki.ubuntu.com/Releases +# Ubuntu 18.04 (Bionic) EOL: 2028. https://wiki.ubuntu.com/ReleaseTeam # -# - g++ version 5.3.1 -# - libc version 2.23 +# - libgcc version 8.4.0 (https://packages.ubuntu.com/bionic/libgcc1) +# - libc version 2.27 (https://packages.ubuntu.com/bionic/libc6) # # CentOS Stream 8 EOL: 2024. https://wiki.centos.org/About/Product # -# - g++ version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) +# - libgcc version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) # - libc version 2.28 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) # # See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. @@ -35,10 +35,10 @@ MAX_VERSIONS = { 'GCC': (4,8,0), 'GLIBC': { - lief.ELF.ARCH.x86_64: (2,18), - lief.ELF.ARCH.ARM: (2,18), - lief.ELF.ARCH.AARCH64:(2,18), - lief.ELF.ARCH.PPC64: (2,18), + lief.ELF.ARCH.x86_64: (2,27), + lief.ELF.ARCH.ARM: (2,27), + lief.ELF.ARCH.AARCH64:(2,27), + lief.ELF.ARCH.PPC64: (2,27), lief.ELF.ARCH.RISCV: (2,27), }, 'LIBATOMIC': (1,0), diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 84283e3522b8..54718fd7a1f8 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2021 The Bitcoin Core developers +# Copyright (c) 2015-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' @@ -39,7 +39,7 @@ def call_security_check(cc, source, executable, options): env_flags += filter(None, os.environ.get(var, '').split(' ')) subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) - p = subprocess.run([os.path.join(os.path.dirname(__file__), 'security-check.py'), executable], stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.run([os.path.join(os.path.dirname(__file__), 'security-check.py'), executable], stdout=subprocess.PIPE, text=True) return (p.returncode, p.stdout.rstrip()) def get_arch(cc, source, executable): @@ -94,19 +94,19 @@ def test_PE(self): cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--disable-nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--disable-nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE','-fno-stack-protector']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE','-fstack-protector-all', '-lssp']), (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE','-fstack-protector-all', '-lssp']), (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE','-fstack-protector-all', '-lssp']), (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) # -pie -fPIE does nothing unless --dynamicbase is also supplied - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE','-fstack-protector-all', '-lssp']), (1, executable+': failed HIGH_ENTROPY_VA CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE','-fstack-protector-all', '-lssp']), (1, executable+': failed CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full','-fstack-protector-all', '-lssp']), (0, '')) clean_files(source, executable) diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index 85d3a6d07bab..e304880140e6 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' @@ -23,13 +23,13 @@ def call_symbol_check(cc: List[str], source, executable, options): env_flags += filter(None, os.environ.get(var, '').split(' ')) subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) - p = subprocess.run([os.path.join(os.path.dirname(__file__), 'symbol-check.py'), executable], stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.run([os.path.join(os.path.dirname(__file__), 'symbol-check.py'), executable], stdout=subprocess.PIPE, text=True) os.remove(source) os.remove(executable) return (p.returncode, p.stdout.rstrip()) def get_machine(cc: List[str]): - p = subprocess.run([*cc,'-dumpmachine'], stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.run([*cc,'-dumpmachine'], stdout=subprocess.PIPE, text=True) return p.stdout.rstrip() class TestSymbolChecks(unittest.TestCase): @@ -38,31 +38,6 @@ def test_ELF(self): executable = 'test1' cc = determine_wellknown_cmd('CC', 'gcc') - # there's no way to do this test for RISC-V at the moment; we build for - # RISC-V in a glibc 2.27 environment and we allow all symbols from 2.27. - if 'riscv' in get_machine(cc): - self.skipTest("test not available for RISC-V") - - # nextup was introduced in GLIBC 2.24, so is newer than our supported - # glibc (2.18), and available in our release build environment (2.24). - with open(source, 'w', encoding="utf8") as f: - f.write(''' - #define _GNU_SOURCE - #include - - double nextup(double x); - - int main() - { - nextup(3.14); - return 0; - } - ''') - - self.assertEqual(call_symbol_check(cc, source, executable, ['-lm']), - (1, executable + ': symbol nextup from unsupported version GLIBC_2.24(3)\n' + - executable + ': failed IMPORTED_SYMBOLS')) - # -lutil is part of the libc6 package so a safe bet that it's installed # it's also out of context enough that it's unlikely to ever become a real dependency source = 'test2.c' diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index f26b6795d16e..e0bd15493f88 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C @@ -188,9 +188,9 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ ${SDK_PATH+SDK_PATH="$SDK_PATH"} \ x86_64_linux_CC=x86_64-linux-gnu-gcc \ x86_64_linux_CXX=x86_64-linux-gnu-g++ \ - x86_64_linux_AR=x86_64-linux-gnu-ar \ - x86_64_linux_RANLIB=x86_64-linux-gnu-ranlib \ - x86_64_linux_NM=x86_64-linux-gnu-nm \ + x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \ + x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \ + x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \ x86_64_linux_STRIP=x86_64-linux-gnu-strip \ FORCE_USE_SYSTEM_CLANG=1 @@ -238,13 +238,6 @@ case "$HOST" in *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac -# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.18, by -# avoiding a PowerPC64 optimisation available in glibc 2.22 and later. -# https://sourceware.org/binutils/docs-2.35/ld/PowerPC64-ELF64.html -case "$HOST" in - *powerpc64*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,--no-tls-get-addr-optimize" ;; -esac - # Make $HOST-specific native binaries from depends available in $PATH export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" mkdir -p "$DISTSRC" diff --git a/contrib/guix/libexec/codesign.sh b/contrib/guix/libexec/codesign.sh index 9a5d3a1ce542..6ffa0f07b2e9 100755 --- a/contrib/guix/libexec/codesign.sh +++ b/contrib/guix/libexec/codesign.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2021 The Bitcoin Core developers +# Copyright (c) 2021-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C @@ -77,6 +77,7 @@ mkdir -p "$DISTSRC" osslsigncode attach-signature \ -in "$infile" \ -out "${OUTDIR}/${infile_base/-unsigned}" \ + -CAfile "$GUIX_ENVIRONMENT/etc/ssl/certs/ca-certificates.crt" \ -sigin codesignatures/win/"$infile_base".pem done ;; diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 8e5c89cc5e0a..d83ff08713d5 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -28,6 +28,7 @@ (gnu packages shells) (gnu packages tls) (gnu packages version-control) + (guix build-system cmake) (guix build-system gnu) (guix build-system python) (guix build-system trivial) @@ -139,15 +140,17 @@ chain for " target " development.")) ;; https://gcc.gnu.org/install/configure.html (define (hardened-gcc gcc) (package-with-extra-configure-variable ( - package-with-extra-configure-variable gcc - "--enable-default-ssp" "yes") - "--enable-default-pie" "yes")) + package-with-extra-configure-variable ( + package-with-extra-configure-variable gcc + "--enable-initfini-array" "yes") + "--enable-default-ssp" "yes") + "--enable-default-pie" "yes")) (define* (make-bitcoin-cross-toolchain target #:key (base-gcc-for-libc base-gcc) (base-kernel-headers base-linux-kernel-headers) - (base-libc (make-glibc-with-bind-now (make-glibc-without-werror glibc-2.24))) + (base-libc (hardened-glibc glibc-2.27)) (base-gcc (make-gcc-rpath-link (hardened-gcc base-gcc)))) "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values desirable for building Bitcoin Core release binaries." @@ -205,60 +208,58 @@ chain for " target " development.")) (package-with-extra-patches lief (search-our-patches "lief-fix-ppc64-nx-default.patch"))) -(define-public lief +;; Our python-lief package can be removed once we are using +;; guix 83bfdb409787cb2737e68b093a319b247b7858e6 or later. +;; Note we currently use cmake-minimal. +(define-public python-lief (package - (name "python-lief") - (version "0.12.1") - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/lief-project/LIEF.git") - (commit version))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "1xzbh3bxy4rw1yamnx68da1v5s56ay4g081cyamv67256g0qy2i1")))) - (build-system python-build-system) - (arguments - `(#:phases - (modify-phases %standard-phases - (add-after 'unpack 'parallel-jobs - ;; build with multiple cores - (lambda _ - (substitute* "setup.py" (("self.parallel if self.parallel else 1") (number->string (parallel-job-count))))))))) - (native-inputs - `(("cmake" ,cmake))) - (home-page "https://github.com/lief-project/LIEF") - (synopsis "Library to Instrument Executable Formats") - (description "Python library to to provide a cross platform library which can -parse, modify and abstract ELF, PE and MachO formats.") - (license license:asl2.0))) + (name "python-lief") + (version "0.12.3") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/lief-project/LIEF") + (commit version))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "11i6hqmcjh56y554kqhl61698n9v66j2qk1c1g63mv2w07h2z661")))) + (build-system python-build-system) + (native-inputs (list cmake-minimal)) + (arguments + (list + #:tests? #f ;needs network + #:phases #~(modify-phases %standard-phases + (replace 'build + (lambda _ + (invoke + "python" "setup.py" "--sdk" "build" + (string-append + "-j" (number->string (parallel-job-count))))))))) + (home-page "https://github.com/lief-project/LIEF") + (synopsis "Library to instrument executable formats") + (description + "@code{python-lief} is a cross platform library which can parse, modify +and abstract ELF, PE and MachO formats.") + (license license:asl2.0))) (define osslsigncode (package (name "osslsigncode") - (version "2.0") + (version "2.5") (source (origin (method url-fetch) (uri (string-append "https://github.com/mtrojnar/" name "/archive/" version ".tar.gz")) (sha256 (base32 - "0byri6xny770wwb2nciq44j5071122l14bvv65axdd70nfjf0q2s")))) - (build-system gnu-build-system) - (native-inputs - `(("pkg-config" ,pkg-config) - ("autoconf" ,autoconf) - ("automake" ,automake) - ("libtool" ,libtool))) + "03by9706gg0an6dn48pljx38vcb76ziv11bgm8ilwsf293x2k4hv")))) + (build-system cmake-build-system) (inputs - `(("openssl" ,openssl))) + `(("openssl", openssl))) (arguments - `(#:configure-flags - `("--without-gsf" - "--without-curl" - "--disable-dependency-tracking"))) + '(#:configure-flags + (list "-DCMAKE_DISABLE_FIND_PACKAGE_CURL=TRUE"))) (home-page "https://github.com/mtrojnar/osslsigncode") (synopsis "Authenticode signing and timestamping tool") (description "osslsigncode is a small tool that implements part of the @@ -534,36 +535,18 @@ and endian independent.") inspecting signatures in Mach-O binaries.") (license license:expat)))) -(define (make-glibc-without-werror glibc) - (package-with-extra-configure-variable glibc "enable_werror" "no")) - -(define (make-glibc-with-stack-protector glibc) - (package-with-extra-configure-variable glibc "--enable-stack-protector" "all")) - -(define (make-glibc-with-bind-now glibc) - (package-with-extra-configure-variable glibc "--enable-bind-now" "yes")) - -(define-public glibc-2.24 - (package - (inherit glibc-2.31) - (version "2.24") - (source (origin - (method git-fetch) - (uri (git-reference - (url "https://sourceware.org/git/glibc.git") - (commit "0d7f1ed30969886c8dde62fbf7d2c79967d4bace"))) - (file-name (git-file-name "glibc" "0d7f1ed30969886c8dde62fbf7d2c79967d4bace")) - (sha256 - (base32 - "0g5hryia5v1k0qx97qffgwzrz4lr4jw3s5kj04yllhswsxyjbic3")) - (patches (search-our-patches "glibc-ldd-x86_64.patch" - "glibc-versioned-locpath.patch" - "glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch" - "glibc-2.24-no-build-time-cxx-header-run.patch" - "glibc-2.24-fcommon.patch" - "glibc-2.24-guix-prefix.patch")))))) +;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html +;; We don't use --disable-werror directly, as that would be passed through to bash, +;; and cause it's build to fail. +(define (hardened-glibc glibc) + (package-with-extra-configure-variable ( + package-with-extra-configure-variable ( + package-with-extra-configure-variable glibc + "enable_werror" "no") + "--enable-stack-protector" "all") + "--enable-bind-now" "yes")) -(define-public glibc-2.27/bitcoin-patched +(define-public glibc-2.27 (package (inherit glibc-2.31) (version "2.27") @@ -571,22 +554,23 @@ inspecting signatures in Mach-O binaries.") (method git-fetch) (uri (git-reference (url "https://sourceware.org/git/glibc.git") - (commit "23158b08a0908f381459f273a984c6fd328363cb"))) - (file-name (git-file-name "glibc" "23158b08a0908f381459f273a984c6fd328363cb")) + (commit "73886db6218e613bd6d4edf529f11e008a6c2fa6"))) + (file-name (git-file-name "glibc" "73886db6218e613bd6d4edf529f11e008a6c2fa6")) (sha256 (base32 - "1b2n1gxv9f4fd5yy68qjbnarhf8mf4vmlxk10i3328c1w5pmp0ca")) + "0azpb9cvnbv25zg8019rqz48h8i2257ngyjg566dlnp74ivrs9vq")) (patches (search-our-patches "glibc-ldd-x86_64.patch" + "glibc-versioned-locpath.patch" "glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch" - "glibc-2.27-dont-redefine-nss-database.patch" + "glibc-2.27-fcommon.patch" "glibc-2.27-guix-prefix.patch")))))) (packages->manifest (append (list ;; The Basics - bash + bash-minimal which - coreutils + coreutils-minimal util-linux ;; File(system) inspection file @@ -614,25 +598,21 @@ inspecting signatures in Mach-O binaries.") gcc-toolchain-10 (list gcc-toolchain-10 "static") ;; Scripting - python-3 + python-minimal ;; (3.9) ;; Git git-minimal ;; Tests - (fix-ppc64-nx-default lief)) + (fix-ppc64-nx-default python-lief)) (let ((target (getenv "HOST"))) (cond ((string-suffix? "-mingw32" target) ;; Windows (list zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") (make-nsis-for-gcc-10 nsis-x86_64) + nss-certs osslsigncode)) ((string-contains target "-linux-") - (list (cond ((string-contains target "riscv64-") - (make-bitcoin-cross-toolchain target - #:base-libc (make-glibc-with-stack-protector - (make-glibc-with-bind-now (make-glibc-without-werror glibc-2.27/bitcoin-patched))))) - (else - (make-bitcoin-cross-toolchain target))))) + (list (make-bitcoin-cross-toolchain target))) ((string-contains target "darwin") - (list clang-toolchain-10 binutils cmake xorriso python-signapple)) + (list clang-toolchain-10 binutils cmake-minimal xorriso python-signapple)) (else '()))))) diff --git a/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch b/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch deleted file mode 100644 index 5c4d0c6ebe19..000000000000 --- a/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch +++ /dev/null @@ -1,62 +0,0 @@ -https://sourceware.org/git/?p=glibc.git;a=commit;h=a68ba2f3cd3cbe32c1f31e13c20ed13487727b32 - -commit 6b02af31e9a721bb15a11380cd22d53b621711f8 -Author: Szabolcs Nagy -Date: Wed Oct 18 17:26:23 2017 +0100 - - [AARCH64] Rewrite elf_machine_load_address using _DYNAMIC symbol - - This patch rewrites aarch64 elf_machine_load_address to use special _DYNAMIC - symbol instead of _dl_start. - - The static address of _DYNAMIC symbol is stored in the first GOT entry. - Here is the change which makes this solution work (part of binutils 2.24): - https://sourceware.org/ml/binutils/2013-06/msg00248.html - - i386, x86_64 targets use the same method to do this as well. - - The original implementation relies on a trick that R_AARCH64_ABS32 relocation - being resolved at link time and the static address fits in the 32bits. - However, in LP64, normally, the address is defined to be 64 bit. - - Here is the C version one which should be portable in all cases. - - * sysdeps/aarch64/dl-machine.h (elf_machine_load_address): Use - _DYNAMIC symbol to calculate load address. - -diff --git a/sysdeps/aarch64/dl-machine.h b/sysdeps/aarch64/dl-machine.h -index e86d8b5b63..5a5b8a5de5 100644 ---- a/sysdeps/aarch64/dl-machine.h -+++ b/sysdeps/aarch64/dl-machine.h -@@ -49,26 +49,11 @@ elf_machine_load_address (void) - /* To figure out the load address we use the definition that for any symbol: - dynamic_addr(symbol) = static_addr(symbol) + load_addr - -- The choice of symbol is arbitrary. The static address we obtain -- by constructing a non GOT reference to the symbol, the dynamic -- address of the symbol we compute using adrp/add to compute the -- symbol's address relative to the PC. -- This depends on 32bit relocations being resolved at link time -- and that the static address fits in the 32bits. */ -- -- ElfW(Addr) static_addr; -- ElfW(Addr) dynamic_addr; -- -- asm (" \n" --" adrp %1, _dl_start; \n" --" add %1, %1, #:lo12:_dl_start \n" --" ldr %w0, 1f \n" --" b 2f \n" --"1: \n" --" .word _dl_start \n" --"2: \n" -- : "=r" (static_addr), "=r" (dynamic_addr)); -- return dynamic_addr - static_addr; -+ _DYNAMIC sysmbol is used here as its link-time address stored in -+ the special unrelocated first GOT entry. */ -+ -+ extern ElfW(Dyn) _DYNAMIC[] attribute_hidden; -+ return (ElfW(Addr)) &_DYNAMIC - elf_machine_dynamic (); - } - - /* Set up the loaded object described by L so its unrelocated PLT diff --git a/contrib/guix/patches/glibc-2.24-guix-prefix.patch b/contrib/guix/patches/glibc-2.24-guix-prefix.patch deleted file mode 100644 index 875e8cd61191..000000000000 --- a/contrib/guix/patches/glibc-2.24-guix-prefix.patch +++ /dev/null @@ -1,25 +0,0 @@ -Without ffile-prefix-map, the debug symbols will contain paths for the -guix store which will include the hashes of each package. However, the -hash for the same package will differ when on different architectures. -In order to be reproducible regardless of the architecture used to build -the package, map all guix store prefixes to something fixed, e.g. /usr. - -We might be able to drop this in favour of using --with-nonshared-cflags -when we being using newer versions of glibc. - ---- a/Makeconfig -+++ b/Makeconfig -@@ -950,6 +950,10 @@ object-suffixes-for-libc += .oS - # shared objects. We don't want to use CFLAGS-os because users may, for - # example, make that processor-specific. - CFLAGS-.oS = $(CFLAGS-.o) $(PIC-ccflag) -+ -+# Map Guix store paths to /usr -+CFLAGS-.oS += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` -+ - CPPFLAGS-.oS = $(CPPFLAGS-.o) -DPIC -DLIBC_NONSHARED=1 - libtype.oS = lib%_nonshared.a - endif --- -2.35.1 - diff --git a/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch b/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch deleted file mode 100644 index 11fe7fdc99c0..000000000000 --- a/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch +++ /dev/null @@ -1,100 +0,0 @@ -https://sourceware.org/git/?p=glibc.git;a=commit;h=fc3e1337be1c6935ab58bd13520f97a535cf70cc - -commit dc23a45db566095e83ff0b7a57afc87fb5ca89a1 -Author: Florian Weimer -Date: Wed Sep 21 10:45:32 2016 +0200 - - Avoid running $(CXX) during build to obtain header file paths - - This reduces the build time somewhat and is particularly noticeable - during rebuilds with few code changes. - -diff --git a/Makerules b/Makerules -index 7e4077ee50..c338850de5 100644 ---- a/Makerules -+++ b/Makerules -@@ -121,14 +121,10 @@ ifneq (,$(CXX)) - # will be used instead of /usr/include/stdlib.h and /usr/include/math.h. - before-compile := $(common-objpfx)cstdlib $(common-objpfx)cmath \ - $(before-compile) --cstdlib=$(shell echo "\#include " | $(CXX) -M -MP -x c++ - \ -- | sed -n "/cstdlib:/{s/:$$//;p}") --$(common-objpfx)cstdlib: $(cstdlib) -+$(common-objpfx)cstdlib: $(c++-cstdlib-header) - $(INSTALL_DATA) $< $@T - $(move-if-change) $@T $@ --cmath=$(shell echo "\#include " | $(CXX) -M -MP -x c++ - \ -- | sed -n "/cmath:/{s/:$$//;p}") --$(common-objpfx)cmath: $(cmath) -+$(common-objpfx)cmath: $(c++-cmath-header) - $(INSTALL_DATA) $< $@T - $(move-if-change) $@T $@ - endif -diff --git a/config.make.in b/config.make.in -index 95c6f36876..04a8b3ed7f 100644 ---- a/config.make.in -+++ b/config.make.in -@@ -45,6 +45,8 @@ defines = @DEFINES@ - sysheaders = @sysheaders@ - sysincludes = @SYSINCLUDES@ - c++-sysincludes = @CXX_SYSINCLUDES@ -+c++-cstdlib-header = @CXX_CSTDLIB_HEADER@ -+c++-cmath-header = @CXX_CMATH_HEADER@ - all-warnings = @all_warnings@ - enable-werror = @enable_werror@ - -diff --git a/configure b/configure -index 17625e1041..6ff252744b 100755 ---- a/configure -+++ b/configure -@@ -635,6 +635,8 @@ BISON - INSTALL_INFO - PERL - BASH_SHELL -+CXX_CMATH_HEADER -+CXX_CSTDLIB_HEADER - CXX_SYSINCLUDES - SYSINCLUDES - AUTOCONF -@@ -5054,6 +5056,18 @@ fi - - - -+# Obtain some C++ header file paths. This is used to make a local -+# copy of those headers in Makerules. -+if test -n "$CXX"; then -+ find_cxx_header () { -+ echo "#include <$1>" | $CXX -M -MP -x c++ - | sed -n "/$1:/{s/:\$//;p}" -+ } -+ CXX_CSTDLIB_HEADER="$(find_cxx_header cstdlib)" -+ CXX_CMATH_HEADER="$(find_cxx_header cmath)" -+fi -+ -+ -+ - # Test if LD_LIBRARY_PATH contains the notation for the current directory - # since this would lead to problems installing/building glibc. - # LD_LIBRARY_PATH contains the current directory if one of the following -diff --git a/configure.ac b/configure.ac -index 33bcd62180..9938ab0dc2 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -1039,6 +1039,18 @@ fi - AC_SUBST(SYSINCLUDES) - AC_SUBST(CXX_SYSINCLUDES) - -+# Obtain some C++ header file paths. This is used to make a local -+# copy of those headers in Makerules. -+if test -n "$CXX"; then -+ find_cxx_header () { -+ echo "#include <$1>" | $CXX -M -MP -x c++ - | sed -n "/$1:/{s/:\$//;p}" -+ } -+ CXX_CSTDLIB_HEADER="$(find_cxx_header cstdlib)" -+ CXX_CMATH_HEADER="$(find_cxx_header cmath)" -+fi -+AC_SUBST(CXX_CSTDLIB_HEADER) -+AC_SUBST(CXX_CMATH_HEADER) -+ - # Test if LD_LIBRARY_PATH contains the notation for the current directory - # since this would lead to problems installing/building glibc. - # LD_LIBRARY_PATH contains the current directory if one of the following diff --git a/contrib/guix/patches/glibc-2.27-dont-redefine-nss-database.patch b/contrib/guix/patches/glibc-2.27-dont-redefine-nss-database.patch deleted file mode 100644 index 16a595d613c1..000000000000 --- a/contrib/guix/patches/glibc-2.27-dont-redefine-nss-database.patch +++ /dev/null @@ -1,87 +0,0 @@ -commit 78a90c2f74a2012dd3eff302189e47ff6779a757 -Author: Andreas Schwab -Date: Fri Mar 2 23:07:14 2018 +0100 - - Fix multiple definitions of __nss_*_database (bug 22918) - - (cherry picked from commit eaf6753f8aac33a36deb98c1031d1bad7b593d2d) - -diff --git a/nscd/gai.c b/nscd/gai.c -index d081747797..576fd0045b 100644 ---- a/nscd/gai.c -+++ b/nscd/gai.c -@@ -45,3 +45,6 @@ - #ifdef HAVE_LIBIDN - # include - #endif -+ -+/* Some variables normally defined in libc. */ -+service_user *__nss_hosts_database attribute_hidden; -diff --git a/nss/nsswitch.c b/nss/nsswitch.c -index d5e655974f..b0f0c11a3e 100644 ---- a/nss/nsswitch.c -+++ b/nss/nsswitch.c -@@ -62,7 +62,7 @@ static service_library *nss_new_service (name_database *database, - - /* Declare external database variables. */ - #define DEFINE_DATABASE(name) \ -- extern service_user *__nss_##name##_database attribute_hidden; \ -+ service_user *__nss_##name##_database attribute_hidden; \ - weak_extern (__nss_##name##_database) - #include "databases.def" - #undef DEFINE_DATABASE -diff --git a/nss/nsswitch.h b/nss/nsswitch.h -index eccb535ef5..63573b9ebc 100644 ---- a/nss/nsswitch.h -+++ b/nss/nsswitch.h -@@ -226,10 +226,10 @@ libc_hidden_proto (__nss_hostname_digits_dots) - #define MAX_NR_ADDRS 48 - - /* Prototypes for __nss_*_lookup2 functions. */ --#define DEFINE_DATABASE(arg) \ -- service_user *__nss_##arg##_database attribute_hidden; \ -- int __nss_##arg##_lookup2 (service_user **, const char *, \ -- const char *, void **); \ -+#define DEFINE_DATABASE(arg) \ -+ extern service_user *__nss_##arg##_database attribute_hidden; \ -+ int __nss_##arg##_lookup2 (service_user **, const char *, \ -+ const char *, void **); \ - libc_hidden_proto (__nss_##arg##_lookup2) - #include "databases.def" - #undef DEFINE_DATABASE -diff --git a/posix/tst-rfc3484-2.c b/posix/tst-rfc3484-2.c -index f509534ca9..8c64ac59ff 100644 ---- a/posix/tst-rfc3484-2.c -+++ b/posix/tst-rfc3484-2.c -@@ -58,6 +58,7 @@ _res_hconf_init (void) - #undef USE_NSCD - #include "../sysdeps/posix/getaddrinfo.c" - -+service_user *__nss_hosts_database attribute_hidden; - - /* This is the beginning of the real test code. The above defines - (among other things) the function rfc3484_sort. */ -diff --git a/posix/tst-rfc3484-3.c b/posix/tst-rfc3484-3.c -index ae44087a10..1c61aaf844 100644 ---- a/posix/tst-rfc3484-3.c -+++ b/posix/tst-rfc3484-3.c -@@ -58,6 +58,7 @@ _res_hconf_init (void) - #undef USE_NSCD - #include "../sysdeps/posix/getaddrinfo.c" - -+service_user *__nss_hosts_database attribute_hidden; - - /* This is the beginning of the real test code. The above defines - (among other things) the function rfc3484_sort. */ -diff --git a/posix/tst-rfc3484.c b/posix/tst-rfc3484.c -index 7f191abbbc..8f45848e44 100644 ---- a/posix/tst-rfc3484.c -+++ b/posix/tst-rfc3484.c -@@ -58,6 +58,7 @@ _res_hconf_init (void) - #undef USE_NSCD - #include "../sysdeps/posix/getaddrinfo.c" - -+service_user *__nss_hosts_database attribute_hidden; - - /* This is the beginning of the real test code. The above defines - (among other things) the function rfc3484_sort. */ diff --git a/contrib/guix/patches/glibc-2.24-fcommon.patch b/contrib/guix/patches/glibc-2.27-fcommon.patch similarity index 80% rename from contrib/guix/patches/glibc-2.24-fcommon.patch rename to contrib/guix/patches/glibc-2.27-fcommon.patch index 2bc32ede9056..f3baacab98ee 100644 --- a/contrib/guix/patches/glibc-2.24-fcommon.patch +++ b/contrib/guix/patches/glibc-2.27-fcommon.patch @@ -18,15 +18,15 @@ Date: Fri May 6 11:03:04 2022 +0100 https://sourceware.org/git/?p=glibc.git;a=commit;h=7650321ce037302bfc2f026aa19e0213b8d02fe6 diff --git a/Makeconfig b/Makeconfig -index ee379f5852..63c4a2f234 100644 +index 86a71e5802..aa2166be60 100644 --- a/Makeconfig +++ b/Makeconfig -@@ -824,7 +824,7 @@ ifeq "$(strip $(+cflags))" "" - +cflags := $(default_cflags) +@@ -896,7 +896,7 @@ ifeq "$(strip $(+cflags))" "" endif # $(+cflags) == "" --+cflags += $(cflags-cpu) $(+gccwarn) $(+merge-constants) $(+math-flags) -++cflags += $(cflags-cpu) $(+gccwarn) $(+merge-constants) $(+math-flags) -fcommon + +cflags += $(cflags-cpu) $(+gccwarn) $(+merge-constants) $(+math-flags) \ +- $(+stack-protector) ++ $(+stack-protector) -fcommon +gcc-nowarn := -w # Don't duplicate options if we inherited variables from the parent. diff --git a/contrib/guix/patches/glibc-2.27-guix-prefix.patch b/contrib/guix/patches/glibc-2.27-guix-prefix.patch index d777af74f018..6648bc6c053d 100644 --- a/contrib/guix/patches/glibc-2.27-guix-prefix.patch +++ b/contrib/guix/patches/glibc-2.27-guix-prefix.patch @@ -20,6 +20,3 @@ when we being using newer versions of glibc. libtype.o := lib%.a object-suffixes += .o ifeq (yes,$(build-shared)) --- -2.35.1 - diff --git a/contrib/guix/patches/glibc-ldd-x86_64.patch b/contrib/guix/patches/glibc-ldd-x86_64.patch index b1b6d5a54863..a23b095caa75 100644 --- a/contrib/guix/patches/glibc-ldd-x86_64.patch +++ b/contrib/guix/patches/glibc-ldd-x86_64.patch @@ -1,8 +1,8 @@ By default, 'RTDLLIST' in 'ldd' refers to 'lib64/ld-linux-x86-64.so', whereas it's in 'lib/' for us. This patch fixes that. ---- glibc-2.17/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed 2012-12-25 04:02:13.000000000 +0100 -+++ glibc-2.17/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed 2013-09-15 23:08:03.000000000 +0200 +--- a/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed ++++ b/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed @@ -1,3 +1,3 @@ /LD_TRACE_LOADED_OBJECTS=1/a\ add_env="$add_env LD_LIBRARY_VERSION=\\$verify_out" diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh deleted file mode 100755 index 2850c4b993ef..000000000000 --- a/contrib/install_db4.sh +++ /dev/null @@ -1,260 +0,0 @@ -#!/bin/sh -# Copyright (c) 2017-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -# Install libdb4.8 (Berkeley DB). - -export LC_ALL=C -set -e - -if [ -z "${1}" ]; then - echo "Usage: $0 [ ...]" - echo - echo "Must specify a single argument: the directory in which db4 will be built." - echo "This is probably \`pwd\` if you're at the root of the bitcoin repository." - exit 1 -fi - -expand_path() { - cd "${1}" && pwd -P -} - -BDB_PREFIX="$(expand_path "${1}")/db4"; shift; -BDB_VERSION='db-4.8.30.NC' -BDB_HASH='12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef' -BDB_URL="https://download.oracle.com/berkeley-db/${BDB_VERSION}.tar.gz" - -check_exists() { - command -v "$1" >/dev/null -} - -sha256_check() { - # Args: - # - if check_exists sha256sum; then - echo "${1} ${2}" | sha256sum -c - elif check_exists sha256; then - if [ "$(uname)" = "FreeBSD" ]; then - sha256 -c "${1}" "${2}" - else - echo "${1} ${2}" | sha256 -c - fi - else - echo "${1} ${2}" | shasum -a 256 -c - fi -} - -http_get() { - # Args: - # - # It's acceptable that we don't require SSL here because we manually verify - # content hashes below. - # - if [ -f "${2}" ]; then - echo "File ${2} already exists; not downloading again" - elif check_exists curl; then - curl --insecure --retry 5 "${1}" -o "${2}" - elif check_exists wget; then - wget --no-check-certificate "${1}" -O "${2}" - else - echo "Simple transfer utilities 'curl' and 'wget' not found. Please install one of them and try again." - exit 1 - fi - - sha256_check "${3}" "${2}" -} - -# Ensure the commands we use exist on the system -if ! check_exists patch; then - echo "Command-line tool 'patch' not found. Install patch and try again." - exit 1 -fi - -mkdir -p "${BDB_PREFIX}" -http_get "${BDB_URL}" "${BDB_VERSION}.tar.gz" "${BDB_HASH}" -tar -xzvf ${BDB_VERSION}.tar.gz -C "$BDB_PREFIX" -cd "${BDB_PREFIX}/${BDB_VERSION}/" - -# Apply a patch necessary when building with clang and c++11 (see https://community.oracle.com/thread/3952592) -patch --ignore-whitespace -p1 << 'EOF' -commit 3311d68f11d1697565401eee6efc85c34f022ea7 -Author: fanquake -Date: Mon Aug 17 20:03:56 2020 +0800 - - Fix C++11 compatibility - -diff --git a/dbinc/atomic.h b/dbinc/atomic.h -index 0034dcc..7c11d4a 100644 ---- a/dbinc/atomic.h -+++ b/dbinc/atomic.h -@@ -70,7 +70,7 @@ typedef struct { - * These have no memory barriers; the caller must include them when necessary. - */ - #define atomic_read(p) ((p)->value) --#define atomic_init(p, val) ((p)->value = (val)) -+#define atomic_init_db(p, val) ((p)->value = (val)) - - #ifdef HAVE_ATOMIC_SUPPORT - -@@ -144,7 +144,7 @@ typedef LONG volatile *interlocked_val; - #define atomic_inc(env, p) __atomic_inc(p) - #define atomic_dec(env, p) __atomic_dec(p) - #define atomic_compare_exchange(env, p, o, n) \ -- __atomic_compare_exchange((p), (o), (n)) -+ __atomic_compare_exchange_db((p), (o), (n)) - static inline int __atomic_inc(db_atomic_t *p) - { - int temp; -@@ -176,7 +176,7 @@ static inline int __atomic_dec(db_atomic_t *p) - * http://gcc.gnu.org/onlinedocs/gcc-4.1.0/gcc/Atomic-Builtins.html - * which configure could be changed to use. - */ --static inline int __atomic_compare_exchange( -+static inline int __atomic_compare_exchange_db( - db_atomic_t *p, atomic_value_t oldval, atomic_value_t newval) - { - atomic_value_t was; -@@ -206,7 +206,7 @@ static inline int __atomic_compare_exchange( - #define atomic_dec(env, p) (--(p)->value) - #define atomic_compare_exchange(env, p, oldval, newval) \ - (DB_ASSERT(env, atomic_read(p) == (oldval)), \ -- atomic_init(p, (newval)), 1) -+ atomic_init_db(p, (newval)), 1) - #else - #define atomic_inc(env, p) __atomic_inc(env, p) - #define atomic_dec(env, p) __atomic_dec(env, p) -diff --git a/mp/mp_fget.c b/mp/mp_fget.c -index 5fdee5a..0b75f57 100644 ---- a/mp/mp_fget.c -+++ b/mp/mp_fget.c -@@ -617,7 +617,7 @@ alloc: /* Allocate a new buffer header and data space. */ - - /* Initialize enough so we can call __memp_bhfree. */ - alloc_bhp->flags = 0; -- atomic_init(&alloc_bhp->ref, 1); -+ atomic_init_db(&alloc_bhp->ref, 1); - #ifdef DIAGNOSTIC - if ((uintptr_t)alloc_bhp->buf & (sizeof(size_t) - 1)) { - __db_errx(env, -@@ -911,7 +911,7 @@ alloc: /* Allocate a new buffer header and data space. */ - MVCC_MPROTECT(bhp->buf, mfp->stat.st_pagesize, - PROT_READ); - -- atomic_init(&alloc_bhp->ref, 1); -+ atomic_init_db(&alloc_bhp->ref, 1); - MUTEX_LOCK(env, alloc_bhp->mtx_buf); - alloc_bhp->priority = bhp->priority; - alloc_bhp->pgno = bhp->pgno; -diff --git a/mp/mp_mvcc.c b/mp/mp_mvcc.c -index 34467d2..f05aa0c 100644 ---- a/mp/mp_mvcc.c -+++ b/mp/mp_mvcc.c -@@ -276,7 +276,7 @@ __memp_bh_freeze(dbmp, infop, hp, bhp, need_frozenp) - #else - memcpy(frozen_bhp, bhp, SSZA(BH, buf)); - #endif -- atomic_init(&frozen_bhp->ref, 0); -+ atomic_init_db(&frozen_bhp->ref, 0); - if (mutex != MUTEX_INVALID) - frozen_bhp->mtx_buf = mutex; - else if ((ret = __mutex_alloc(env, MTX_MPOOL_BH, -@@ -428,7 +428,7 @@ __memp_bh_thaw(dbmp, infop, hp, frozen_bhp, alloc_bhp) - #endif - alloc_bhp->mtx_buf = mutex; - MUTEX_LOCK(env, alloc_bhp->mtx_buf); -- atomic_init(&alloc_bhp->ref, 1); -+ atomic_init_db(&alloc_bhp->ref, 1); - F_CLR(alloc_bhp, BH_FROZEN); - } - -diff --git a/mp/mp_region.c b/mp/mp_region.c -index e6cece9..ddbe906 100644 ---- a/mp/mp_region.c -+++ b/mp/mp_region.c -@@ -224,7 +224,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg) - MTX_MPOOL_FILE_BUCKET, 0, &htab[i].mtx_hash)) != 0) - return (ret); - SH_TAILQ_INIT(&htab[i].hash_bucket); -- atomic_init(&htab[i].hash_page_dirty, 0); -+ atomic_init_db(&htab[i].hash_page_dirty, 0); - } - - /* -@@ -269,7 +269,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg) - hp->mtx_hash = (mtx_base == MUTEX_INVALID) ? MUTEX_INVALID : - mtx_base + i; - SH_TAILQ_INIT(&hp->hash_bucket); -- atomic_init(&hp->hash_page_dirty, 0); -+ atomic_init_db(&hp->hash_page_dirty, 0); - #ifdef HAVE_STATISTICS - hp->hash_io_wait = 0; - hp->hash_frozen = hp->hash_thawed = hp->hash_frozen_freed = 0; -diff --git a/mutex/mut_method.c b/mutex/mut_method.c -index 2588763..5c6d516 100644 ---- a/mutex/mut_method.c -+++ b/mutex/mut_method.c -@@ -426,7 +426,7 @@ atomic_compare_exchange(env, v, oldval, newval) - MUTEX_LOCK(env, mtx); - ret = atomic_read(v) == oldval; - if (ret) -- atomic_init(v, newval); -+ atomic_init_db(v, newval); - MUTEX_UNLOCK(env, mtx); - - return (ret); -diff --git a/mutex/mut_tas.c b/mutex/mut_tas.c -index f3922e0..e40fcdf 100644 ---- a/mutex/mut_tas.c -+++ b/mutex/mut_tas.c -@@ -46,7 +46,7 @@ __db_tas_mutex_init(env, mutex, flags) - - #ifdef HAVE_SHARED_LATCHES - if (F_ISSET(mutexp, DB_MUTEX_SHARED)) -- atomic_init(&mutexp->sharecount, 0); -+ atomic_init_db(&mutexp->sharecount, 0); - else - #endif - if (MUTEX_INIT(&mutexp->tas)) { -@@ -486,7 +486,7 @@ __db_tas_mutex_unlock(env, mutex) - F_CLR(mutexp, DB_MUTEX_LOCKED); - /* Flush flag update before zeroing count */ - MEMBAR_EXIT(); -- atomic_init(&mutexp->sharecount, 0); -+ atomic_init_db(&mutexp->sharecount, 0); - } else { - DB_ASSERT(env, sharecount > 0); - MEMBAR_EXIT(); -EOF - -# The packaged config.guess and config.sub are ancient (2009) and can cause build issues. -# Replace them with modern versions. -# See https://github.com/bitcoin/bitcoin/issues/16064 -CONFIG_GUESS_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=4550d2f15b3a7ce2451c1f29500b9339430c877f' -CONFIG_GUESS_HASH='c8f530e01840719871748a8071113435bdfdf75b74c57e78e47898edea8754ae' -CONFIG_SUB_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=4550d2f15b3a7ce2451c1f29500b9339430c877f' -CONFIG_SUB_HASH='3969f7d5f6967ccc6f792401b8ef3916a1d1b1d0f0de5a4e354c95addb8b800e' - -rm -f "dist/config.guess" -rm -f "dist/config.sub" - -http_get "${CONFIG_GUESS_URL}" dist/config.guess "${CONFIG_GUESS_HASH}" -http_get "${CONFIG_SUB_URL}" dist/config.sub "${CONFIG_SUB_HASH}" - -cd build_unix/ - -"${BDB_PREFIX}/${BDB_VERSION}/dist/configure" \ - --enable-cxx --disable-shared --disable-replication --with-pic --prefix="${BDB_PREFIX}" \ - "${@}" - -make install - -echo -echo "db4 build complete." -echo -# shellcheck disable=SC2016 -echo 'When compiling bitcoind, run `./configure` in the following way:' -echo -echo " export BDB_PREFIX='${BDB_PREFIX}'" -# shellcheck disable=SC2016 -echo ' ./configure BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" BDB_CFLAGS="-I${BDB_PREFIX}/include" ...' diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index b72c7b0d0885..24f6b29a26f8 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -2,7 +2,7 @@ # # linearize-data.py: Construct a linear, no-fork version of the chain. # -# Copyright (c) 2013-2021 The Bitcoin Core developers +# Copyright (c) 2013-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index 5959300e7474..695bafad3428 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -2,7 +2,7 @@ # # linearize-hashes.py: List blocks in a linear, no-fork version of the chain. # -# Copyright (c) 2013-2019 The Bitcoin Core developers +# Copyright (c) 2013-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -78,7 +78,7 @@ def get_block_hashes(settings, max_blocks_per_call=10000): if rpc.response_is_error(resp_obj): print('JSON-RPC: error at height', height+x, ': ', resp_obj['error'], file=sys.stderr) sys.exit(1) - assert(resp_obj['id'] == x) # assume replies are in-sequence + assert resp_obj['id'] == x # assume replies are in-sequence if settings['rev_hash_bytes'] == 'true': resp_obj['result'] = bytes.fromhex(resp_obj['result'])[::-1].hex() print(resp_obj['result']) diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index f393331084e3..626381cf4302 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2014-2021 The Bitcoin Core developers +# Copyright (c) 2014-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 2420539b7cad..f8677ba7b880 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -187,7 +187,7 @@ def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]: if verbose: print(f"Inspecting with otool: {binaryPath}") otoolbin=os.getenv("OTOOL", "otool") - otool = run([otoolbin, "-L", binaryPath], stdout=PIPE, stderr=PIPE, universal_newlines=True) + otool = run([otoolbin, "-L", binaryPath], stdout=PIPE, stderr=PIPE, text=True) if otool.returncode != 0: sys.stderr.write(otool.stderr) sys.stderr.flush() @@ -577,17 +577,17 @@ if config.dmg is not None: tempname: str = appname + ".temp.dmg" - run(["hdiutil", "create", tempname, "-srcfolder", "dist", "-format", "UDRW", "-size", str(size), "-volname", appname], check=True, universal_newlines=True) + run(["hdiutil", "create", tempname, "-srcfolder", "dist", "-format", "UDRW", "-size", str(size), "-volname", appname], check=True, text=True) if verbose: print("Attaching temp image...") - output = run(["hdiutil", "attach", tempname, "-readwrite"], check=True, universal_newlines=True, stdout=PIPE).stdout + output = run(["hdiutil", "attach", tempname, "-readwrite"], check=True, text=True, stdout=PIPE).stdout print("+ Finalizing .dmg disk image +") - run(["hdiutil", "detach", f"/Volumes/{appname}"], universal_newlines=True) + run(["hdiutil", "detach", f"/Volumes/{appname}"], text=True) - run(["hdiutil", "convert", tempname, "-format", "UDZO", "-o", appname, "-imagekey", "zlib-level=9"], check=True, universal_newlines=True) + run(["hdiutil", "convert", tempname, "-format", "UDZO", "-o", appname, "-imagekey", "zlib-level=9"], check=True, text=True) os.unlink(tempname) diff --git a/contrib/message-capture/message-capture-parser.py b/contrib/message-capture/message-capture-parser.py index 33759ee71377..d6ddc1c149a2 100755 --- a/contrib/message-capture/message-capture-parser.py +++ b/contrib/message-capture/message-capture-parser.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Parse message capture binary files. To be used in conjunction with -capturemessages.""" diff --git a/contrib/seeds/README.md b/contrib/seeds/README.md index b2ea7522ac9f..b0bbe964934e 100644 --- a/contrib/seeds/README.md +++ b/contrib/seeds/README.md @@ -13,6 +13,6 @@ data. Run the following commands from the `/contrib/seeds` directory: curl https://bitcoin.sipa.be/seeds.txt.gz | gzip -dc > seeds_main.txt curl https://bitcoin.sipa.be/asmap-filled.dat > asmap-filled.dat - python3 makeseeds.py -a asmap-filled.dat < seeds_main.txt > nodes_main.txt + python3 makeseeds.py -a asmap-filled.dat -s seeds_main.txt > nodes_main.txt cat nodes_main_manual.txt >> nodes_main.txt python3 generate-seeds.py . > ../../src/chainparamsseeds.h diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py index 44345e39874c..a6f435af0d3a 100755 --- a/contrib/seeds/generate-seeds.py +++ b/contrib/seeds/generate-seeds.py @@ -70,13 +70,13 @@ def name_to_bip155(addr): if i == 0 or i == (len(addr)-1): # skip empty component at beginning or end continue x += 1 # :: skips to suffix - assert(x < 2) + assert x < 2 else: # two bytes per component val = int(comp, 16) sub[x].append(val >> 8) sub[x].append(val & 0xff) nullbytes = 16 - len(sub[0]) - len(sub[1]) - assert((x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0)) + assert (x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0) addr_bytes = bytes(sub[0] + ([0] * nullbytes) + sub[1]) if addr_bytes[0] == 0xfc: # Assume that seeds with fc00::/8 addresses belong to CJDNS, diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py index eda58c370f29..23d38ee48dbc 100755 --- a/contrib/seeds/makeseeds.py +++ b/contrib/seeds/makeseeds.py @@ -173,6 +173,7 @@ def ip_stats(ips: List[Dict]) -> str: def parse_args(): argparser = argparse.ArgumentParser(description='Generate a list of bitcoin node seed ip addresses.') argparser.add_argument("-a","--asmap", help='the location of the asmap asn database file (required)', required=True) + argparser.add_argument("-s","--seeds", help='the location of the DNS seeds file (required)', required=True) return argparser.parse_args() def main(): @@ -184,7 +185,8 @@ def main(): print('Done.', file=sys.stderr) print('Loading and parsing DNS seeds…', end='', file=sys.stderr, flush=True) - lines = sys.stdin.readlines() + with open(args.seeds, 'r', encoding='utf8') as f: + lines = f.readlines() ips = [parseline(line) for line in lines] print('Done.', file=sys.stderr) diff --git a/contrib/signet/getcoins.py b/contrib/signet/getcoins.py index a069f5fad3de..19751ae2695f 100755 --- a/contrib/signet/getcoins.py +++ b/contrib/signet/getcoins.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -129,7 +129,7 @@ def bitcoin_cli(rpc_command_and_params): # Convert SVG image to PPM, and load it try: - rv = subprocess.run([args.imagemagick, 'svg:-', '-depth', '8', 'ppm:-'], input=res.content, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + rv = subprocess.run([args.imagemagick, 'svg:-', '-depth', '8', 'ppm:-'], input=res.content, check=True, capture_output=True) except FileNotFoundError: raise SystemExit(f"The binary {args.imagemagick} could not be found. Please make sure ImageMagick (or a compatible fork) is installed and that the correct path is specified.") @@ -142,7 +142,7 @@ def bitcoin_cli(rpc_command_and_params): try: res = session.post(args.faucet, data=data) -except: +except Exception: raise SystemExit(f"Unexpected error when contacting faucet: {sys.exc_info()[0]}") # Display the output as per the returned status code diff --git a/contrib/tracing/README.md b/contrib/tracing/README.md index a409a23ef8db..206bec164788 100644 --- a/contrib/tracing/README.md +++ b/contrib/tracing/README.md @@ -286,3 +286,53 @@ Spent a05880b8c77971ed0b9f73062c7c4cdb0ff3856ab14cbf8bc481ed571cd34b83:1 Added eb689865f7d957938978d6207918748f74e6aa074f47874724327089445b0960:0 5589696005 2094513 No Added eb689865f7d957938978d6207918748f74e6aa074f47874724327089445b0960:1 1565556 2094513 No ``` + +### mempool_monitor.py + +A BCC Python script producing mempool statistics and an event log. Based on the +`mempool:added`, `mempool:removed`, `mempool:replaced`, and `mempool:rejected` +tracepoints. + +Statistics include incidence and rate for each event type since the script was +started (`total`) as well as during the last minute (`1 min`) and ten minutes +(`10 min`). The event log shows mempool events in real time, each entry +comprising a timestamp along with all event data available via the event's +tracepoint. + +```console +$ python3 contrib/tracing/mempool_monitor.py ./src/bitcoind +``` + +``` + Mempool Monitor + Press CTRL-C to stop. + + ┌─Event count───────────────────────┐ ┌─Event rate──────────────────────────┐ + │ Event total 1 min 10 min │ │ Event total 1 min 10 min │ + │ added 1425tx 201tx 1425tx │ │ added 4.7tx/s 3.4tx/s 4.7tx/s │ + │ removed 35tx 4tx 35tx │ │ removed 0.1tx/s 0.1tx/s 0.1tx/s │ + │ replaced 35tx 4tx 35tx │ │ replaced 0.1tx/s 0.1tx/s 0.1tx/s │ + │ rejected 0tx 0tx 0tx │ │ rejected 0.0tx/s 0.0tx/s 0.0tx/s │ + └───────────────────────────────────┘ └─────────────────────────────────────┘ + + ┌─Event log────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ 13:10:30Z added f9064ca5bfc87cdd191faa42bf697217cd920b2b94838c1f1192e4f06c4fd217 with feerate 8.92 sat/vB (981 sat, 110 vbytes) │ + │ 13:10:30Z added 53ffa3afbe57b1bfe423e1755ca2b52c5b6cb4aa91b8b7ee9cb694953f47f234 with feerate 5.00 sat/vB (550 sat, 110 vbytes) │ + │ 13:10:30Z added 4177df5e19465eb5e53c3f8b6830a293f57474921bc6c2ae89375e0986e1f0f9 with feerate 2.98 sat/vB (429 sat, 144 vbytes) │ + │ 13:10:30Z added 931a10d83f0a268768da75dc4b9e199f2f055f12979ae5491cc304ee10f890ea with feerate 3.55 sat/vB (500 sat, 141 vbytes) │ + │ 13:10:30Z added 4cf32b295723cc4ab73f2a2e51d4bb276c0042760a4c00a3eb9595b8ebb24721 with feerate 89.21 sat/vB (12668 sat, 142 vbytes) │ + │ 13:10:31Z replaced d1eecf9d662121322f4f31f0c2267a752d14bb3956e6016ba96e87f47890e1db with feerate 27.12 sat/vB received 23.3 seconds ago (7213 sat, 266 vbytes) with c412db908│ + │ 9b7ed53f3e5e36d2819dd291278b59ccaabaeb17fd37c3d87fdcd57 with feerate 28.12 sat/vB (8351 sat, 297 vbytes) │ + │ 13:10:31Z added c412db9089b7ed53f3e5e36d2819dd291278b59ccaabaeb17fd37c3d87fdcd57 with feerate 28.12 sat/vB (8351 sat, 297 vbytes) │ + │ 13:10:31Z added b8388a5bdc421b11460bdf477d5a85a1a39c2784e7dd7bffabe688740424ea57 with feerate 25.21 sat/vB (3554 sat, 141 vbytes) │ + │ 13:10:31Z added 4ddb88bc90a122cd9eae8a664e73bdf5bebe75f3ef901241b4a251245854a98e with feerate 24.15 sat/vB (5072 sat, 210 vbytes) │ + │ 13:10:31Z added 19101e4161bca5271ad5d03e7747f2faec7793b274dc2f3c4cf516b7cef1aac3 with feerate 7.06 sat/vB (1080 sat, 153 vbytes) │ + │ 13:10:31Z removed d1eecf9d662121322f4f31f0c2267a752d14bb3956e6016ba96e87f47890e1db with feerate 27.12 sat/vB (7213 sat, 266 vbytes): replaced │ + │ 13:10:31Z added 6c511c60d9b95b9eff81df6ecba5c86780f513fe62ce3ad6be2c5340d957025a with feerate 4.00 sat/vB (440 sat, 110 vbytes) │ + │ 13:10:31Z added 44d66f7f004bd52c46be4dff3067cab700e51c7866a84282bd8aab560a5bfb79 with feerate 3.15 sat/vB (448 sat, 142 vbytes) │ + │ 13:10:31Z added b17b7c9ec5acfbbf12f0eeef8e29826fad3105bb95eef7a47d2f1f22b4784643 with feerate 4.10 sat/vB (1348 sat, 329 vbytes) │ + │ 13:10:31Z added b7a4ad93554e57454e8a8049bfc0bd803fa962bd3f0a08926aa72e7cb23e2276 with feerate 1.01 sat/vB (205 sat, 202 vbytes) │ + │ 13:10:32Z added c78e87be86c828137a6e7e00a177c03b52202ce4c39029b99904c2a094b9da87 with feerate 11.00 sat/vB (1562 sat, 142 vbytes) │ + │ │ + └──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` diff --git a/contrib/tracing/log_utxocache_flush.py b/contrib/tracing/log_utxocache_flush.py index 8c073bea0d18..6c568998e992 100755 --- a/contrib/tracing/log_utxocache_flush.py +++ b/contrib/tracing/log_utxocache_flush.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2021 The Bitcoin Core developers +# Copyright (c) 2021-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/tracing/mempool_monitor.py b/contrib/tracing/mempool_monitor.py new file mode 100755 index 000000000000..9d427d4632a4 --- /dev/null +++ b/contrib/tracing/mempool_monitor.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Example logging Bitcoin Core mempool events using the mempool:added, + mempool:removed, mempool:replaced, and mempool:rejected tracepoints. """ + +import curses +import sys +from datetime import datetime, timezone + +from bcc import BPF, USDT + +# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into +# a sandboxed Linux kernel VM. +PROGRAM = """ +# include + +// The longest rejection reason is 118 chars and is generated in case of SCRIPT_ERR_EVAL_FALSE by +// strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError())) +#define MAX_REJECT_REASON_LENGTH 118 +// The longest string returned by RemovalReasonToString() is 'sizelimit' +#define MAX_REMOVAL_REASON_LENGTH 9 +#define HASH_LENGTH 32 + +struct added_event +{ + u8 hash[HASH_LENGTH]; + u64 vsize; + s64 fee; +}; + +struct removed_event +{ + u8 hash[HASH_LENGTH]; + char reason[MAX_REMOVAL_REASON_LENGTH]; + u64 vsize; + s64 fee; + u64 entry_time; +}; + +struct rejected_event +{ + u8 hash[HASH_LENGTH]; + char reason[MAX_REJECT_REASON_LENGTH]; +}; + +struct replaced_event +{ + u8 replaced_hash[HASH_LENGTH]; + u64 replaced_vsize; + s64 replaced_fee; + u64 replaced_entry_time; + u8 replacement_hash[HASH_LENGTH]; + u64 replacement_vsize; + s64 replacement_fee; +}; + +// BPF perf buffer to push the data to user space. +BPF_PERF_OUTPUT(added_events); +BPF_PERF_OUTPUT(removed_events); +BPF_PERF_OUTPUT(rejected_events); +BPF_PERF_OUTPUT(replaced_events); + +int trace_added(struct pt_regs *ctx) { + struct added_event added = {}; + + bpf_usdt_readarg_p(1, ctx, &added.hash, HASH_LENGTH); + bpf_usdt_readarg(2, ctx, &added.vsize); + bpf_usdt_readarg(3, ctx, &added.fee); + + added_events.perf_submit(ctx, &added, sizeof(added)); + return 0; +} + +int trace_removed(struct pt_regs *ctx) { + struct removed_event removed = {}; + + bpf_usdt_readarg_p(1, ctx, &removed.hash, HASH_LENGTH); + bpf_usdt_readarg_p(2, ctx, &removed.reason, MAX_REMOVAL_REASON_LENGTH); + bpf_usdt_readarg(3, ctx, &removed.vsize); + bpf_usdt_readarg(4, ctx, &removed.fee); + bpf_usdt_readarg(5, ctx, &removed.entry_time); + + removed_events.perf_submit(ctx, &removed, sizeof(removed)); + return 0; +} + +int trace_rejected(struct pt_regs *ctx) { + struct rejected_event rejected = {}; + + bpf_usdt_readarg_p(1, ctx, &rejected.hash, HASH_LENGTH); + bpf_usdt_readarg_p(2, ctx, &rejected.reason, MAX_REJECT_REASON_LENGTH); + + rejected_events.perf_submit(ctx, &rejected, sizeof(rejected)); + return 0; +} + +int trace_replaced(struct pt_regs *ctx) { + struct replaced_event replaced = {}; + + bpf_usdt_readarg_p(1, ctx, &replaced.replaced_hash, HASH_LENGTH); + bpf_usdt_readarg(2, ctx, &replaced.replaced_vsize); + bpf_usdt_readarg(3, ctx, &replaced.replaced_fee); + bpf_usdt_readarg(4, ctx, &replaced.replaced_entry_time); + bpf_usdt_readarg_p(5, ctx, &replaced.replacement_hash, HASH_LENGTH); + bpf_usdt_readarg(6, ctx, &replaced.replacement_vsize); + bpf_usdt_readarg(7, ctx, &replaced.replacement_fee); + + replaced_events.perf_submit(ctx, &replaced, sizeof(replaced)); + return 0; +} +""" + + +def main(bitcoind_path): + bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + + # attaching the trace functions defined in the BPF program + # to the tracepoints + bitcoind_with_usdts.enable_probe(probe="mempool:added", fn_name="trace_added") + bitcoind_with_usdts.enable_probe(probe="mempool:removed", fn_name="trace_removed") + bitcoind_with_usdts.enable_probe(probe="mempool:replaced", fn_name="trace_replaced") + bitcoind_with_usdts.enable_probe(probe="mempool:rejected", fn_name="trace_rejected") + bpf = BPF(text=PROGRAM, usdt_contexts=[bitcoind_with_usdts]) + + events = [] + + def get_timestamp(): + return datetime.now(timezone.utc) + + def handle_added(_, data, size): + event = bpf["added_events"].event(data) + events.append((get_timestamp(), "added", event)) + + def handle_removed(_, data, size): + event = bpf["removed_events"].event(data) + events.append((get_timestamp(), "removed", event)) + + def handle_rejected(_, data, size): + event = bpf["rejected_events"].event(data) + events.append((get_timestamp(), "rejected", event)) + + def handle_replaced(_, data, size): + event = bpf["replaced_events"].event(data) + events.append((get_timestamp(), "replaced", event)) + + bpf["added_events"].open_perf_buffer(handle_added) + # By default, open_perf_buffer uses eight pages for a buffer, making for a total + # buffer size of 32k on most machines. In practice, this size is insufficient: + # Each `mempool:removed` event takes up 57 bytes in the buffer (32 bytes for txid, + # 9 bytes for removal reason, and 8 bytes each for vsize and fee). Full blocks + # contain around 2k transactions, requiring a buffer size of around 114kB. To cover + # this amount, 32 4k pages are required. + bpf["removed_events"].open_perf_buffer(handle_removed, page_cnt=32) + bpf["rejected_events"].open_perf_buffer(handle_rejected) + bpf["replaced_events"].open_perf_buffer(handle_replaced) + + curses.wrapper(loop, bpf, events) + + +def loop(screen, bpf, events): + dashboard = Dashboard(screen) + while True: + try: + bpf.perf_buffer_poll(timeout=50) + dashboard.render(events) + except KeyboardInterrupt: + exit() + + +class Dashboard: + """Visualization of mempool state using ncurses.""" + + INFO_WIN_HEIGHT = 2 + EVENT_WIN_HEIGHT = 7 + + def __init__(self, screen): + screen.nodelay(True) + curses.curs_set(False) + self._screen = screen + self._time_started = datetime.now(timezone.utc) + self._timestamps = {"added": [], "removed": [], "rejected": [], "replaced": []} + self._event_history = {"added": 0, "removed": 0, "rejected": 0, "replaced": 0} + self._init_windows() + + def _init_windows(self): + """Initialize all windows.""" + self._init_info_win() + self._init_event_count_win() + self._init_event_rate_win() + self._init_event_log_win() + + @staticmethod + def create_win(x, y, height, width, title=None): + """Helper function to create generic windows and decorate them with box and title if requested.""" + win = curses.newwin(height, width, x, y) + if title: + win.box() + win.addstr(0, 2, title, curses.A_BOLD) + return win + + def _init_info_win(self): + """Create and populate the info window.""" + self._info_win = Dashboard.create_win( + x=0, y=1, height=Dashboard.INFO_WIN_HEIGHT, width=22 + ) + self._info_win.addstr(0, 0, "Mempool Monitor", curses.A_REVERSE) + self._info_win.addstr(1, 0, "Press CTRL-C to stop.", curses.A_NORMAL) + self._info_win.refresh() + + def _init_event_count_win(self): + """Create and populate the event count window.""" + self._event_count_win = Dashboard.create_win( + x=3, y=1, height=Dashboard.EVENT_WIN_HEIGHT, width=37, title="Event count" + ) + header = " {:<8} {:>8} {:>7} {:>7} " + self._event_count_win.addstr( + 1, 1, header.format("Event", "total", "1 min", "10 min"), curses.A_UNDERLINE + ) + self._event_count_win.refresh() + + def _init_event_rate_win(self): + """Create and populate the event rate window.""" + self._event_rate_win = Dashboard.create_win( + x=3, y=40, height=Dashboard.EVENT_WIN_HEIGHT, width=42, title="Event rate" + ) + header = " {:<8} {:>9} {:>9} {:>9} " + self._event_rate_win.addstr( + 1, 1, header.format("Event", "total", "1 min", "10 min"), curses.A_UNDERLINE + ) + self._event_rate_win.refresh() + + def _init_event_log_win(self): + """Create windows showing event log. This comprises a dummy boxed window and an + inset window so line breaks don't overwrite box.""" + # dummy boxed window + num_rows, num_cols = self._screen.getmaxyx() + space_above = Dashboard.INFO_WIN_HEIGHT + 1 + Dashboard.EVENT_WIN_HEIGHT + 1 + box_win_height = num_rows - space_above + box_win_width = num_cols - 2 + win_box = Dashboard.create_win( + x=space_above, + y=1, + height=box_win_height, + width=box_win_width, + title="Event log", + ) + # actual logging window + log_lines = box_win_height - 2 # top and bottom box lines + log_line_len = box_win_width - 2 - 1 # box lines and left padding + win = win_box.derwin(log_lines, log_line_len, 1, 2) + win.idlok(True) + win.scrollok(True) + win_box.refresh() + win.refresh() + self._event_log_win_box = win_box + self._event_log_win = win + + def calculate_metrics(self, events): + """Calculate count and rate metrics.""" + count, rate = {}, {} + for event_ts, event_type, event_data in events: + self._timestamps[event_type].append(event_ts) + for event_type, ts in self._timestamps.items(): + # remove timestamps older than ten minutes but keep track of their + # count for the 'total' metric + # + self._event_history[event_type] += len( + [t for t in ts if Dashboard.timestamp_age(t) >= 600] + ) + ts = [t for t in ts if Dashboard.timestamp_age(t) < 600] + self._timestamps[event_type] = ts + # count metric + count_1m = len([t for t in ts if Dashboard.timestamp_age(t) < 60]) + count_10m = len(ts) + count_total = self._event_history[event_type] + len(ts) + count[event_type] = (count_total, count_1m, count_10m) + # rate metric + runtime = Dashboard.timestamp_age(self._time_started) + rate_1m = count_1m / min(60, runtime) + rate_10m = count_10m / min(600, runtime) + rate_total = count_total / runtime + rate[event_type] = (rate_total, rate_1m, rate_10m) + return count, rate + + def _update_event_count(self, count): + """Update the event count window.""" + w = self._event_count_win + row_format = " {:<8} {:>6}tx {:>5}tx {:>5}tx " + for line, metric in enumerate(["added", "removed", "replaced", "rejected"]): + w.addstr(2 + line, 1, row_format.format(metric, *count[metric])) + w.refresh() + + def _update_event_rate(self, rate): + """Update the event rate window.""" + w = self._event_rate_win + row_format = " {:<8} {:>5.1f}tx/s {:>5.1f}tx/s {:>5.1f}tx/s " + for line, metric in enumerate(["added", "removed", "replaced", "rejected"]): + w.addstr(2 + line, 1, row_format.format(metric, *rate[metric])) + w.refresh() + + def _update_event_log(self, events): + """Update the event log window.""" + w = self._event_log_win + for event in events: + w.addstr(Dashboard.parse_event(event) + "\n") + w.refresh() + + def render(self, events): + """Render the dashboard.""" + count, rate = self.calculate_metrics(events) + self._update_event_count(count) + self._update_event_rate(rate) + self._update_event_log(events) + events.clear() + + @staticmethod + def parse_event(event): + """Converts events into human-readable messages""" + + ts_dt, type_, data = event + ts = ts_dt.strftime("%H:%M:%SZ") + if type_ == "added": + return ( + f"{ts} added {bytes(data.hash)[::-1].hex()}" + f" with feerate {data.fee/data.vsize:.2f} sat/vB" + f" ({data.fee} sat, {data.vsize} vbytes)" + ) + + if type_ == "removed": + return ( + f"{ts} removed {bytes(data.hash)[::-1].hex()}" + f" with feerate {data.fee/data.vsize:.2f} sat/vB" + f" ({data.fee} sat, {data.vsize} vbytes)" + f" received {ts_dt.timestamp()-data.entry_time:.1f} seconds ago" + f": {data.reason.decode('UTF-8')}" + ) + + if type_ == "rejected": + return ( + f"{ts} rejected {bytes(data.hash)[::-1].hex()}" + f": {data.reason.decode('UTF-8')}" + ) + + if type_ == "replaced": + return ( + f"{ts} replaced {bytes(data.replaced_hash)[::-1].hex()}" + f" with feerate {data.replaced_fee/data.replaced_vsize:.2f} sat/vB" + f" received {ts_dt.timestamp()-data.replaced_entry_time:.1f} seconds ago" + f" ({data.replaced_fee} sat, {data.replaced_vsize} vbytes)" + f" with {bytes(data.replacement_hash)[::-1].hex()}" + f" with feerate {data.replacement_fee/data.replacement_vsize:.2f} sat/vB" + f" ({data.replacement_fee} sat, {data.replacement_vsize} vbytes)" + ) + + raise NotImplementedError("Unsupported event type: {type_}") + + @staticmethod + def timestamp_age(timestamp): + """Return age of timestamp in seconds.""" + return (datetime.now(timezone.utc) - timestamp).total_seconds() + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("USAGE: ", sys.argv[0], "path/to/bitcoind") + exit(1) + + path = sys.argv[1] + main(path) diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index d6856b42749e..faba1f6ae6cb 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -13,19 +13,8 @@ # # Note that suppressions may depend on OS and/or library versions. # Tested on: -# * aarch64 (Ubuntu 22.04 system libs, clang, without gui) -# * x86_64 (Ubuntu 22.04 system libs, clang, without gui) -{ - Suppress libstdc++ warning - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:*/libstdc++.* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - obj:*/ld-*.so -} +# * aarch64 (Debian Bookworm system libs, clang, without gui) +# * x86_64 (Debian Bookworm system libs, clang, without gui) { Suppress libdb warning - https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=662917 Memcheck:Cond @@ -70,12 +59,6 @@ ... fun:_Z8ShutdownR11NodeContext } -{ - Ignore GUI warning - Memcheck:Leak - ... - obj:/usr/lib64/libgdk-3.so.0.2404.7 -} { Suppress leveldb leak Memcheck:Leak @@ -98,14 +81,6 @@ fun:_Znwm fun:_Z11LogInstancev } -{ - Suppress secp256k1_context_create still reachable memory warning - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - ... - fun:secp256k1_context_create -} { Suppress BCLog::Logger::StartLogging() still reachable memory warning Memcheck:Leak diff --git a/contrib/verify-binaries/README.md b/contrib/verify-binaries/README.md new file mode 100644 index 000000000000..c62d760e1a7e --- /dev/null +++ b/contrib/verify-binaries/README.md @@ -0,0 +1,88 @@ +### Verify Binaries + +#### Preparation + +As of Bitcoin Core v22.0, releases are signed by a number of public keys on the basis +of the [guix.sigs repository](https://github.com/bitcoin-core/guix.sigs/). When +verifying binary downloads, you (the end user) decide which of these public keys you +trust and then use that trust model to evaluate the signature on a file that contains +hashes of the release binaries. The downloaded binaries are then hashed and compared to +the signed checksum file. + +First, you have to figure out which public keys to recognize. Browse the [list of frequent +builder-keys](https://github.com/bitcoin-core/guix.sigs/tree/main/builder-keys) and +decide which of these keys you would like to trust. For each key you want to trust, you +must obtain that key for your local GPG installation. + +You can obtain these keys by + - through a browser using a key server (e.g. keyserver.ubuntu.com), + - manually using the `gpg --keyserver --recv-keys ` command, or + - you can run the packaged `verify.py ... --import-keys` script to + have it automatically retrieve unrecognized keys. + +#### Usage + +This script attempts to download the checksum file (`SHA256SUMS`) and corresponding +signature file `SHA256SUMS.asc` from https://bitcoincore.org and https://bitcoin.org. + +It first checks if the checksum file is valid based upon a plurality of signatures, and +then downloads the release files specified in the checksum file, and checks if the +hashes of the release files are as expected. + +If we encounter pubkeys in the signature file that we do not recognize, the script +can prompt the user as to whether they'd like to download the pubkeys. To enable +this behavior, use the `--import-keys` flag. + +The script returns 0 if everything passes the checks. It returns 1 if either the +signature check or the hash check doesn't pass. An exit code of >2 indicates an error. + +See the `Config` object for various options. + +#### Examples + +Validate releases with default settings: +```sh +./contrib/verify-binaries/verify.py pub 22.0 +./contrib/verify-binaries/verify.py pub 22.0-rc3 +``` + +Get JSON output and don't prompt for user input (no auto key import): + +```sh +./contrib/verify-binaries/verify.py --json pub 22.0-x86 +``` + +Rely only on local GPG state and manually specified keys, while requiring a +threshold of at least 10 trusted signatures: +```sh +./contrib/verify-binaries/verify.py \ + --trusted-keys 74E2DEF5D77260B98BC19438099BAD163C70FBFA,9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C \ + --min-good-sigs 10 pub 22.0-x86 +``` + +If you only want to download the binaries for a certain platform, add the corresponding suffix, e.g.: + +```sh +./contrib/verify-binaries/verify.py pub 24.0.1-darwin +./contrib/verify-binaries/verify.py pub 23.1-rc1-win64 +``` + +If you do not want to keep the downloaded binaries, specify the cleanup option. + +```sh +./contrib/verify-binaries/verify.py pub --cleanup 22.0 +``` + +Use the bin subcommand to verify all files listed in a local checksum file + +```sh +./contrib/verify-binaries/verify.py bin SHA256SUMS +``` + +Verify only a subset of the files listed in a local checksum file + +```sh +./contrib/verify-binaries/verify.py bin ~/Downloads/SHA256SUMS \ + ~/Downloads/bitcoin-24.0.1-x86_64-linux-gnu.tar.gz \ + ~/Downloads/bitcoin-24.0.1-arm-linux-gnueabihf.tar.gz +``` diff --git a/contrib/verify-binaries/test.py b/contrib/verify-binaries/test.py new file mode 100755 index 000000000000..22d718ece334 --- /dev/null +++ b/contrib/verify-binaries/test.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +import json +import sys +import subprocess +from pathlib import Path + + +def main(): + """Tests ordered roughly from faster to slower.""" + expect_code(run_verify("", "pub", '0.32'), 4, "Nonexistent version should fail") + expect_code(run_verify("", "pub", '0.32.awefa.12f9h'), 11, "Malformed version should fail") + expect_code(run_verify('--min-good-sigs 20', "pub", "22.0"), 9, "--min-good-sigs 20 should fail") + + print("- testing verification (22.0)", flush=True) + _220 = run_verify("--json", "pub", "22.0") + try: + result = json.loads(_220.stdout.decode()) + except Exception: + print("failed on 22.0 --json:") + print_process_failure(_220) + raise + + expect_code(_220, 0, "22.0 should succeed") + v = result['verified_binaries'] + assert result['good_trusted_sigs'] + assert v['bitcoin-22.0-aarch64-linux-gnu.tar.gz'] == 'ac718fed08570a81b3587587872ad85a25173afa5f9fbbd0c03ba4d1714cfa3e' + assert v['bitcoin-22.0-osx64.tar.gz'] == '2744d199c3343b2d94faffdfb2c94d75a630ba27301a70e47b0ad30a7e0155e9' + assert v['bitcoin-22.0-x86_64-linux-gnu.tar.gz'] == '59ebd25dd82a51638b7a6bb914586201e67db67b919b2a1ff08925a7936d1b16' + + +def run_verify(global_args: str, command: str, command_args: str) -> subprocess.CompletedProcess: + maybe_here = Path.cwd() / 'verify.py' + path = maybe_here if maybe_here.exists() else Path.cwd() / 'contrib' / 'verify-binaries' / 'verify.py' + + if command == "pub": + command += " --cleanup" + + return subprocess.run( + f"{path} {global_args} {command} {command_args}", + stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + + +def expect_code(completed: subprocess.CompletedProcess, expected_code: int, msg: str): + if completed.returncode != expected_code: + print(f"{msg!r} failed: got code {completed.returncode}, expected {expected_code}") + print_process_failure(completed) + sys.exit(1) + else: + print(f"✓ {msg!r} passed") + + +def print_process_failure(completed: subprocess.CompletedProcess): + print(f"stdout:\n{completed.stdout.decode()}") + print(f"stderr:\n{completed.stderr.decode()}") + + +if __name__ == '__main__': + main() diff --git a/contrib/verify-binaries/verify.py b/contrib/verify-binaries/verify.py new file mode 100755 index 000000000000..d0749f503f86 --- /dev/null +++ b/contrib/verify-binaries/verify.py @@ -0,0 +1,713 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Script for verifying Bitcoin Core release binaries. + +This script attempts to download the sum file SHA256SUMS and corresponding +signature file SHA256SUMS.asc from bitcoincore.org and bitcoin.org and +compares them. + +The sum-signature file is signed by a number of builder keys. This script +ensures that there is a minimum threshold of signatures from pubkeys that +we trust. This trust is articulated on the basis of configuration options +here, but by default is based upon local GPG trust settings. + +The builder keys are available in the guix.sigs repo: + + https://github.com/bitcoin-core/guix.sigs/tree/main/builder-keys + +If a minimum good, trusted signature threshold is met on the sum file, we then +download the files specified in SHA256SUMS, and check if the hashes of these +files match those that are specified. The script returns 0 if everything passes +the checks. It returns 1 if either the signature check or the hash check +doesn't pass. If an error occurs the return value is >= 2. + +Logging output goes to stderr and final binary verification data goes to stdout. + +JSON output can by obtained by setting env BINVERIFY_JSON=1. +""" +import argparse +import difflib +import json +import logging +import os +import subprocess +import typing as t +import re +import sys +import shutil +import tempfile +import textwrap +import urllib.request +import urllib.error +import enum +from hashlib import sha256 +from pathlib import PurePath, Path + +# The primary host; this will fail if we can't retrieve files from here. +HOST1 = "https://bitcoincore.org" +HOST2 = "https://bitcoin.org" +VERSIONPREFIX = "bitcoin-core-" +SUMS_FILENAME = 'SHA256SUMS' +SIGNATUREFILENAME = f"{SUMS_FILENAME}.asc" + + +class ReturnCode(enum.IntEnum): + SUCCESS = 0 + INTEGRITY_FAILURE = 1 + FILE_GET_FAILED = 4 + FILE_MISSING_FROM_ONE_HOST = 5 + FILES_NOT_EQUAL = 6 + NO_BINARIES_MATCH = 7 + NOT_ENOUGH_GOOD_SIGS = 9 + BINARY_DOWNLOAD_FAILED = 10 + BAD_VERSION = 11 + + +def set_up_logger(is_verbose: bool = True) -> logging.Logger: + """Set up a logger that writes to stderr.""" + log = logging.getLogger(__name__) + log.setLevel(logging.INFO if is_verbose else logging.WARNING) + console = logging.StreamHandler(sys.stderr) # log to stderr + console.setLevel(logging.DEBUG) + formatter = logging.Formatter('[%(levelname)s] %(message)s') + console.setFormatter(formatter) + log.addHandler(console) + return log + + +log = set_up_logger() + + +def indent(output: str) -> str: + return textwrap.indent(output, ' ') + + +def bool_from_env(key, default=False) -> bool: + if key not in os.environ: + return default + raw = os.environ[key] + + if raw.lower() in ('1', 'true'): + return True + elif raw.lower() in ('0', 'false'): + return False + raise ValueError(f"Unrecognized environment value {key}={raw!r}") + + +VERSION_FORMAT = ".[.][-rc[0-9]][-platform]" +VERSION_EXAMPLE = "22.0-x86_64 or 23.1-rc1-darwin" + +def parse_version_string(version_str): + parts = version_str.split('-') + version_base = parts[0] + version_rc = "" + version_os = "" + if len(parts) == 2: # "-rcN" or "version-platform" + if "rc" in parts[1]: + version_rc = parts[1] + else: + version_os = parts[1] + elif len(parts) == 3: # "-rcN-platform" + version_rc = parts[1] + version_os = parts[2] + + return version_base, version_rc, version_os + + +def download_with_wget(remote_file, local_file): + result = subprocess.run(['wget', '-O', local_file, remote_file], + stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + return result.returncode == 0, result.stdout.decode().rstrip() + + +def download_lines_with_urllib(url) -> t.Tuple[bool, t.List[str]]: + """Get (success, text lines of a file) over HTTP.""" + try: + return (True, [ + line.strip().decode() for line in urllib.request.urlopen(url).readlines()]) + except urllib.error.HTTPError as e: + log.warning(f"HTTP request to {url} failed (HTTPError): {e}") + except Exception as e: + log.warning(f"HTTP request to {url} failed ({e})") + return (False, []) + + +def verify_with_gpg( + filename, + signature_filename, + output_filename: t.Optional[str] = None +) -> t.Tuple[int, str]: + with tempfile.NamedTemporaryFile() as status_file: + args = [ + 'gpg', '--yes', '--verify', '--verify-options', 'show-primary-uid-only', "--status-file", status_file.name, + '--output', output_filename if output_filename else '', signature_filename, filename] + + env = dict(os.environ, LANGUAGE='en') + result = subprocess.run(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, env=env) + + gpg_data = status_file.read().decode().rstrip() + + log.debug(f'Result from GPG ({result.returncode}): {result.stdout.decode()}') + log.debug(f"{gpg_data}") + return result.returncode, gpg_data + + +def remove_files(filenames): + for filename in filenames: + os.remove(filename) + + +class SigData: + """GPG signature data as parsed from GPG stdout.""" + def __init__(self): + self.key = None + self.name = "" + self.trusted = False + self.status = "" + + def __bool__(self): + return self.key is not None + + def __repr__(self): + return ( + "SigData(%r, %r, trusted=%s, status=%r)" % + (self.key, self.name, self.trusted, self.status)) + + +def parse_gpg_result( + output: t.List[str] +) -> t.Tuple[t.List[SigData], t.List[SigData], t.List[SigData]]: + """Returns good, unknown, and bad signatures from GPG stdout.""" + good_sigs: t.List[SigData] = [] + unknown_sigs: t.List[SigData] = [] + bad_sigs: t.List[SigData] = [] + total_resolved_sigs = 0 + + # Ensure that all lines we match on include a prefix that prevents malicious input + # from fooling the parser. + def line_begins_with(patt: str, line: str) -> t.Optional[re.Match]: + return re.match(r'^(\[GNUPG:\])\s+' + patt, line) + + curr_sigs = unknown_sigs + curr_sigdata = SigData() + + for line in output: + if line_begins_with(r"NEWSIG(?:\s|$)", line): + total_resolved_sigs += 1 + if curr_sigdata: + curr_sigs.append(curr_sigdata) + curr_sigdata = SigData() + newsig_split = line.split() + if len(newsig_split) == 3: + curr_sigdata.name = newsig_split[2] + + elif line_begins_with(r"GOODSIG(?:\s|$)", line): + curr_sigdata.key, curr_sigdata.name = line.split(maxsplit=3)[2:4] + curr_sigs = good_sigs + + elif line_begins_with(r"EXPKEYSIG(?:\s|$)", line): + curr_sigdata.key, curr_sigdata.name = line.split(maxsplit=3)[2:4] + curr_sigs = good_sigs + curr_sigdata.status = "expired" + + elif line_begins_with(r"REVKEYSIG(?:\s|$)", line): + curr_sigdata.key, curr_sigdata.name = line.split(maxsplit=3)[2:4] + curr_sigs = good_sigs + curr_sigdata.status = "revoked" + + elif line_begins_with(r"BADSIG(?:\s|$)", line): + curr_sigdata.key, curr_sigdata.name = line.split(maxsplit=3)[2:4] + curr_sigs = bad_sigs + + elif line_begins_with(r"ERRSIG(?:\s|$)", line): + curr_sigdata.key, _, _, _, _, _ = line.split()[2:8] + curr_sigs = unknown_sigs + + elif line_begins_with(r"TRUST_(UNDEFINED|NEVER)(?:\s|$)", line): + curr_sigdata.trusted = False + + elif line_begins_with(r"TRUST_(MARGINAL|FULLY|ULTIMATE)(?:\s|$)", line): + curr_sigdata.trusted = True + + # The last one won't have been added, so add it now + assert curr_sigdata + curr_sigs.append(curr_sigdata) + + all_found = len(good_sigs + bad_sigs + unknown_sigs) + if all_found != total_resolved_sigs: + raise RuntimeError( + f"failed to evaluate all signatures: found {all_found} " + f"but expected {total_resolved_sigs}") + + return (good_sigs, unknown_sigs, bad_sigs) + + +def files_are_equal(filename1, filename2): + with open(filename1, 'rb') as file1: + contents1 = file1.read() + with open(filename2, 'rb') as file2: + contents2 = file2.read() + eq = contents1 == contents2 + + if not eq: + with open(filename1, 'r', encoding='utf-8') as f1, \ + open(filename2, 'r', encoding='utf-8') as f2: + f1lines = f1.readlines() + f2lines = f2.readlines() + + diff = indent( + ''.join(difflib.unified_diff(f1lines, f2lines))) + log.warning(f"found diff in files ({filename1}, {filename2}):\n{diff}\n") + + return eq + + +def get_files_from_hosts_and_compare( + hosts: t.List[str], path: str, filename: str, require_all: bool = False +) -> ReturnCode: + """ + Retrieve the same file from a number of hosts and ensure they have the same contents. + The first host given will be treated as the "primary" host, and is required to succeed. + + Args: + filename: for writing the file locally. + """ + assert len(hosts) > 1 + primary_host = hosts[0] + other_hosts = hosts[1:] + got_files = [] + + def join_url(host: str) -> str: + return host.rstrip('/') + '/' + path.lstrip('/') + + url = join_url(primary_host) + success, output = download_with_wget(url, filename) + if not success: + log.error( + f"couldn't fetch file ({url}). " + "Have you specified the version number in the following format?\n" + f"{VERSION_FORMAT} " + f"(example: {VERSION_EXAMPLE})\n" + f"wget output:\n{indent(output)}") + return ReturnCode.FILE_GET_FAILED + else: + log.info(f"got file {url} as {filename}") + got_files.append(filename) + + for i, host in enumerate(other_hosts): + url = join_url(host) + fname = filename + f'.{i + 2}' + success, output = download_with_wget(url, fname) + + if require_all and not success: + log.error( + f"{host} failed to provide file ({url}), but {primary_host} did?\n" + f"wget output:\n{indent(output)}") + return ReturnCode.FILE_MISSING_FROM_ONE_HOST + elif not success: + log.warning( + f"{host} failed to provide file ({url}). " + f"Continuing based solely upon {primary_host}.") + else: + log.info(f"got file {url} as {fname}") + got_files.append(fname) + + for i, got_file in enumerate(got_files): + if got_file == got_files[-1]: + break # break on last file, nothing after it to compare to + + compare_to = got_files[i + 1] + if not files_are_equal(got_file, compare_to): + log.error(f"files not equal: {got_file} and {compare_to}") + return ReturnCode.FILES_NOT_EQUAL + + return ReturnCode.SUCCESS + + +def check_multisig(sums_file: str, sigfilename: str, args: argparse.Namespace) -> t.Tuple[int, str, t.List[SigData], t.List[SigData], t.List[SigData]]: + # check signature + # + # We don't write output to a file because this command will almost certainly + # fail with GPG exit code '2' (and so not writing to --output) because of the + # likely presence of multiple untrusted signatures. + retval, output = verify_with_gpg(sums_file, sigfilename) + + if args.verbose: + log.info(f"gpg output:\n{indent(output)}") + + good, unknown, bad = parse_gpg_result(output.splitlines()) + + if unknown and args.import_keys: + # Retrieve unknown keys and then try GPG again. + for unsig in unknown: + if prompt_yn(f" ? Retrieve key {unsig.key} ({unsig.name})? (y/N) "): + ran = subprocess.run( + ["gpg", "--keyserver", args.keyserver, "--recv-keys", unsig.key]) + + if ran.returncode != 0: + log.warning(f"failed to retrieve key {unsig.key}") + + # Reparse the GPG output now that we have more keys + retval, output = verify_with_gpg(sums_file, sigfilename) + good, unknown, bad = parse_gpg_result(output.splitlines()) + + return retval, output, good, unknown, bad + + +def prompt_yn(prompt) -> bool: + """Return true if the user inputs 'y'.""" + got = '' + while got not in ['y', 'n']: + got = input(prompt).lower() + return got == 'y' + +def verify_shasums_signature( + signature_file_path: str, sums_file_path: str, args: argparse.Namespace +) -> t.Tuple[ + ReturnCode, t.List[SigData], t.List[SigData], t.List[SigData], t.List[SigData] +]: + min_good_sigs = args.min_good_sigs + gpg_allowed_codes = [0, 2] # 2 is returned when untrusted signatures are present. + + gpg_retval, gpg_output, good, unknown, bad = check_multisig(sums_file_path, signature_file_path, args) + + if gpg_retval not in gpg_allowed_codes: + if gpg_retval == 1: + log.critical(f"Bad signature (code: {gpg_retval}).") + else: + log.critical(f"unexpected GPG exit code ({gpg_retval})") + + log.error(f"gpg output:\n{indent(gpg_output)}") + return (ReturnCode.INTEGRITY_FAILURE, [], [], [], []) + + # Decide which keys we trust, though not "trust" in the GPG sense, but rather + # which pubkeys convince us that this sums file is legitimate. In other words, + # which pubkeys within the Bitcoin community do we trust for the purposes of + # binary verification? + trusted_keys = set() + if args.trusted_keys: + trusted_keys |= set(args.trusted_keys.split(',')) + + # Tally signatures and make sure we have enough goods to fulfill + # our threshold. + good_trusted = [sig for sig in good if sig.trusted or sig.key in trusted_keys] + good_untrusted = [sig for sig in good if sig not in good_trusted] + num_trusted = len(good_trusted) + len(good_untrusted) + log.info(f"got {num_trusted} good signatures") + + if num_trusted < min_good_sigs: + log.info("Maybe you need to import " + f"(`gpg --keyserver {args.keyserver} --recv-keys `) " + "some of the following keys: ") + log.info('') + for sig in unknown: + log.info(f" {sig.key} ({sig.name})") + log.info('') + log.error( + "not enough trusted sigs to meet threshold " + f"({num_trusted} vs. {min_good_sigs})") + + return (ReturnCode.NOT_ENOUGH_GOOD_SIGS, [], [], [], []) + + for sig in good_trusted: + log.info(f"GOOD SIGNATURE: {sig}") + + for sig in good_untrusted: + log.info(f"GOOD SIGNATURE (untrusted): {sig}") + + for sig in [sig for sig in good if sig.status == 'expired']: + log.warning(f"key {sig.key} for {sig.name} is expired") + + for sig in bad: + log.warning(f"BAD SIGNATURE: {sig}") + + for sig in unknown: + log.warning(f"UNKNOWN SIGNATURE: {sig}") + + return (ReturnCode.SUCCESS, good_trusted, good_untrusted, unknown, bad) + + +def parse_sums_file(sums_file_path: str, filename_filter: t.List[str]) -> t.List[t.List[str]]: + # extract hashes/filenames of binaries to verify from hash file; + # each line has the following format: " " + with open(sums_file_path, 'r', encoding='utf8') as hash_file: + return [line.split()[:2] for line in hash_file if len(filename_filter) == 0 or any(f in line for f in filename_filter)] + + +def verify_binary_hashes(hashes_to_verify: t.List[t.List[str]]) -> t.Tuple[ReturnCode, t.Dict[str, str]]: + offending_files = [] + files_to_hashes = {} + + for hash_expected, binary_filename in hashes_to_verify: + with open(binary_filename, 'rb') as binary_file: + hash_calculated = sha256(binary_file.read()).hexdigest() + if hash_calculated != hash_expected: + offending_files.append(binary_filename) + else: + files_to_hashes[binary_filename] = hash_calculated + + if offending_files: + joined_files = '\n'.join(offending_files) + log.critical( + "Hashes don't match.\n" + f"Offending files:\n{joined_files}") + return (ReturnCode.INTEGRITY_FAILURE, files_to_hashes) + + return (ReturnCode.SUCCESS, files_to_hashes) + + +def verify_published_handler(args: argparse.Namespace) -> ReturnCode: + WORKINGDIR = Path(tempfile.gettempdir()) / f"bitcoin_verify_binaries.{args.version}" + + def cleanup(): + log.info("cleaning up files") + os.chdir(Path.home()) + shutil.rmtree(WORKINGDIR) + + # determine remote dir dependent on provided version string + try: + version_base, version_rc, os_filter = parse_version_string(args.version) + version_tuple = [int(i) for i in version_base.split('.')] + except Exception as e: + log.debug(e) + log.error(f"unable to parse version; expected format is {VERSION_FORMAT}") + log.error(f" e.g. {VERSION_EXAMPLE}") + return ReturnCode.BAD_VERSION + + remote_dir = f"/bin/{VERSIONPREFIX}{version_base}/" + if version_rc: + remote_dir += f"test.{version_rc}/" + remote_sigs_path = remote_dir + SIGNATUREFILENAME + remote_sums_path = remote_dir + SUMS_FILENAME + + # create working directory + os.makedirs(WORKINGDIR, exist_ok=True) + os.chdir(WORKINGDIR) + + hosts = [HOST1, HOST2] + + got_sig_status = get_files_from_hosts_and_compare( + hosts, remote_sigs_path, SIGNATUREFILENAME, args.require_all_hosts) + if got_sig_status != ReturnCode.SUCCESS: + return got_sig_status + + # Multi-sig verification is available after 22.0. + if version_tuple[0] < 22: + log.error("Version too old - single sig not supported. Use a previous " + "version of this script from the repo.") + return ReturnCode.BAD_VERSION + + got_sums_status = get_files_from_hosts_and_compare( + hosts, remote_sums_path, SUMS_FILENAME, args.require_all_hosts) + if got_sums_status != ReturnCode.SUCCESS: + return got_sums_status + + # Verify the signature on the SHA256SUMS file + sigs_status, good_trusted, good_untrusted, unknown, bad = verify_shasums_signature(SIGNATUREFILENAME, SUMS_FILENAME, args) + if sigs_status != ReturnCode.SUCCESS: + if sigs_status == ReturnCode.INTEGRITY_FAILURE: + cleanup() + return sigs_status + + # Extract hashes and filenames + hashes_to_verify = parse_sums_file(SUMS_FILENAME, [os_filter]) + if not hashes_to_verify: + log.error("no files matched the platform specified") + return ReturnCode.NO_BINARIES_MATCH + + # remove binaries that are known not to be hosted by bitcoincore.org + fragments_to_remove = ['-unsigned', '-debug', '-codesignatures'] + for fragment in fragments_to_remove: + nobinaries = [i for i in hashes_to_verify if fragment in i[1]] + if nobinaries: + remove_str = ', '.join(i[1] for i in nobinaries) + log.info( + f"removing *{fragment} binaries ({remove_str}) from verification " + f"since {HOST1} does not host *{fragment} binaries") + hashes_to_verify = [i for i in hashes_to_verify if fragment not in i[1]] + + # download binaries + for _, binary_filename in hashes_to_verify: + log.info(f"downloading {binary_filename} to {WORKINGDIR}") + success, output = download_with_wget( + HOST1 + remote_dir + binary_filename, binary_filename) + + if not success: + log.error( + f"failed to download {binary_filename}\n" + f"wget output:\n{indent(output)}") + return ReturnCode.BINARY_DOWNLOAD_FAILED + + # verify hashes + hashes_status, files_to_hashes = verify_binary_hashes(hashes_to_verify) + if hashes_status != ReturnCode.SUCCESS: + return hashes_status + + + if args.cleanup: + cleanup() + else: + log.info(f"did not clean up {WORKINGDIR}") + + if args.json: + output = { + 'good_trusted_sigs': [str(s) for s in good_trusted], + 'good_untrusted_sigs': [str(s) for s in good_untrusted], + 'unknown_sigs': [str(s) for s in unknown], + 'bad_sigs': [str(s) for s in bad], + 'verified_binaries': files_to_hashes, + } + print(json.dumps(output, indent=2)) + else: + for filename in files_to_hashes: + print(f"VERIFIED: {filename}") + + return ReturnCode.SUCCESS + + +def verify_binaries_handler(args: argparse.Namespace) -> ReturnCode: + binary_to_basename = {} + for file in args.binary: + binary_to_basename[PurePath(file).name] = file + + sums_sig_path = None + if args.sums_sig_file: + sums_sig_path = Path(args.sums_sig_file) + else: + log.info(f"No signature file specified, assuming it is {args.sums_file}.asc") + sums_sig_path = Path(args.sums_file).with_suffix(".asc") + + # Verify the signature on the SHA256SUMS file + sigs_status, good_trusted, good_untrusted, unknown, bad = verify_shasums_signature(str(sums_sig_path), args.sums_file, args) + if sigs_status != ReturnCode.SUCCESS: + return sigs_status + + # Extract hashes and filenames + hashes_to_verify = parse_sums_file(args.sums_file, [k for k, n in binary_to_basename.items()]) + if not hashes_to_verify: + log.error(f"No files in {args.sums_file} match the specified binaries") + return ReturnCode.NO_BINARIES_MATCH + + # Make sure all files are accounted for + sums_file_path = Path(args.sums_file) + missing_files = [] + files_to_hash = [] + if len(binary_to_basename) > 0: + for file_hash, file in hashes_to_verify: + files_to_hash.append([file_hash, binary_to_basename[file]]) + del binary_to_basename[file] + if len(binary_to_basename) > 0: + log.error(f"Not all specified binaries are in {args.sums_file}") + return ReturnCode.NO_BINARIES_MATCH + else: + log.info(f"No binaries specified, assuming all files specified in {args.sums_file} are located relatively") + for file_hash, file in hashes_to_verify: + file_path = Path(sums_file_path.parent.joinpath(file)) + if file_path.exists(): + files_to_hash.append([file_hash, str(file_path)]) + else: + missing_files.append(file) + + # verify hashes + hashes_status, files_to_hashes = verify_binary_hashes(files_to_hash) + if hashes_status != ReturnCode.SUCCESS: + return hashes_status + + if args.json: + output = { + 'good_trusted_sigs': [str(s) for s in good_trusted], + 'good_untrusted_sigs': [str(s) for s in good_untrusted], + 'unknown_sigs': [str(s) for s in unknown], + 'bad_sigs': [str(s) for s in bad], + 'verified_binaries': files_to_hashes, + "missing_binaries": missing_files, + } + print(json.dumps(output, indent=2)) + else: + for filename in files_to_hashes: + print(f"VERIFIED: {filename}") + for filename in missing_files: + print(f"MISSING: {filename}") + + return ReturnCode.SUCCESS + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '-v', '--verbose', action='store_true', + default=bool_from_env('BINVERIFY_VERBOSE'), + ) + parser.add_argument( + '-q', '--quiet', action='store_true', + default=bool_from_env('BINVERIFY_QUIET'), + ) + parser.add_argument( + '--import-keys', action='store_true', + default=bool_from_env('BINVERIFY_IMPORTKEYS'), + help='if specified, ask to import each unknown builder key' + ) + parser.add_argument( + '--min-good-sigs', type=int, action='store', nargs='?', + default=int(os.environ.get('BINVERIFY_MIN_GOOD_SIGS', 3)), + help=( + 'The minimum number of good signatures to require successful termination.'), + ) + parser.add_argument( + '--keyserver', action='store', nargs='?', + default=os.environ.get('BINVERIFY_KEYSERVER', 'hkps://keys.openpgp.org'), + help='which keyserver to use', + ) + parser.add_argument( + '--trusted-keys', action='store', nargs='?', + default=os.environ.get('BINVERIFY_TRUSTED_KEYS', ''), + help='A list of trusted signer GPG keys, separated by commas. Not "trusted keys" in the GPG sense.', + ) + parser.add_argument( + '--json', action='store_true', + default=bool_from_env('BINVERIFY_JSON'), + help='If set, output the result as JSON', + ) + + subparsers = parser.add_subparsers(title="Commands", required=True, dest="command") + + pub_parser = subparsers.add_parser("pub", help="Verify a published release.") + pub_parser.set_defaults(func=verify_published_handler) + pub_parser.add_argument( + 'version', type=str, help=( + f'version of the bitcoin release to download; of the format ' + f'{VERSION_FORMAT}. Example: {VERSION_EXAMPLE}') + ) + pub_parser.add_argument( + '--cleanup', action='store_true', + default=bool_from_env('BINVERIFY_CLEANUP'), + help='if specified, clean up files afterwards' + ) + pub_parser.add_argument( + '--require-all-hosts', action='store_true', + default=bool_from_env('BINVERIFY_REQUIRE_ALL_HOSTS'), + help=( + f'If set, require all hosts ({HOST1}, {HOST2}) to provide signatures. ' + '(Sometimes bitcoin.org lags behind bitcoincore.org.)') + ) + + bin_parser = subparsers.add_parser("bin", help="Verify local binaries.") + bin_parser.set_defaults(func=verify_binaries_handler) + bin_parser.add_argument("--sums-sig-file", "-s", help="Path to the SHA256SUMS.asc file to verify") + bin_parser.add_argument("sums_file", help="Path to the SHA256SUMS file to verify") + bin_parser.add_argument( + "binary", nargs="*", + help="Path to a binary distribution file to verify. Can be specified multiple times for multiple files to verify." + ) + + args = parser.parse_args() + if args.quiet: + log.setLevel(logging.WARNING) + + return args.func(args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/contrib/verify-commits/README.md b/contrib/verify-commits/README.md index b8b15280bae2..020890c3660e 100644 --- a/contrib/verify-commits/README.md +++ b/contrib/verify-commits/README.md @@ -27,6 +27,10 @@ Note that the above isn't a good UI/UX yet, and needs significant improvements to make it more convenient and reduce the chance of errors; pull-reqs improving this process would be much appreciated. +Unless `--clean-merge 0` is specified, `verify-commits.py` will attempt to verify that +each merge commit applies cleanly (with some exceptions). This requires using at least +git v2.38.0. + Configuration files ------------------- diff --git a/contrib/verify-commits/allow-incorrect-sha512-commits b/contrib/verify-commits/allow-incorrect-sha512-commits index c572806f2612..e69de29bb2d1 100644 --- a/contrib/verify-commits/allow-incorrect-sha512-commits +++ b/contrib/verify-commits/allow-incorrect-sha512-commits @@ -1,2 +0,0 @@ -f8feaa4636260b599294c7285bcf1c8b7737f74e -8040ae6fc576e9504186f2ae3ff2c8125de1095c diff --git a/contrib/verify-commits/allow-revsig-commits b/contrib/verify-commits/allow-revsig-commits index 0bb299b8faef..e69de29bb2d1 100644 --- a/contrib/verify-commits/allow-revsig-commits +++ b/contrib/verify-commits/allow-revsig-commits @@ -1,645 +0,0 @@ -a06ede9a138d0fb86b0de17c42b936d9fe6e2158 -923dc447eaa8e017985b2afbbb12dd1283fbea0e -71148b8947fe8b4d756822420a7f31c380159425 -6696b4635ceb9b47aaa63244bff9032fa7b08354 -812714fd80e96e28cd288c553c83838cecbfc2d9 -8a445c5651edb9a1f51497055b7ddf4402be9188 -e126d0c12ca66278d9e7b12187c5ff4fc02a7e6c -3908fc4728059719bed0e1c7b1c8b388c2d4a8da -8b66bf74e2a349e71eaa183af81fa63eaee76ad2 -05950427d310654774031764a7141a1a4fd9c6e4 -07fd147b9f12e9205afd66a624edce357977d615 -12e31127948fa4bb01c3bddc1b8c85b432f7465b -8c87f175d335e9d9e93f987d871ae9f05f6a10a7 -46b249e578e8a3dfbe85bc7253a12e82ef4b658b -a55716abe5662ec74c2f8af93023f1e7cca901fc -f646275b90b1de93bc62b4c4d045d75ac0b96eee -c252685aa5867631e9a5ef07ccae7c7c25cae8ff -a7d55c93385359952d85decd5037843ac70ba3d4 -7dac1e5e9e887f5f6ff146e812a05bd3bf281eae -2a524b8e8fe69ef487fd8ea1b4f7a03f473ed201 -ce5c1f4acae43477989cdf9a82ed33703919cda2 -2db4cbcc437f51f5dac82cc4de46f383b92e6f11 -7aa700424cbda387536373d8dfec88aee43f950e -b99a093afed880f23fb279c443cc6ae5e379cc43 -b83264d9c7a8ddb79f64bd9540caddc8632ef31f -57e337d40e94ba33d8cd265c134d6ef857b32b59 -a1dcf2e1087beaf3981739fd2bb74f35ecad630a -d38b0d7a6b6056cba26999b702815775e2437d87 -815640ec6af9a38d6a2da4a4400056e2f4105080 -09c4fd157c5b88df2d97fad4826c79b094db90c9 -2efcfa5acfacb958973d9e8125e1d81f102e2dfd -dc6dee41f7cf2ba93fcd0fea7c157e4b2775d439 -ad826b3df9f763b49f1e3e3d50c4efdd438c7547 -c1a52276848d8caa9a9789dff176408c1aa6b1ed -3bf06e9bac57b5b5a746677b75e297a7b154bdbd -72ae6f8cf0224370e8121d6769b21e612ca15d6f -a143b88dbd4971ecfdd1d39a494489c8f2db0344 -76fec09d878d6dbf214bdb6228d480bd9195db4c -93566e0c37c5ae104095474fea89f00dcb40f551 -407d9232ef5cb1ebf6cff21f3d13e07ea4158eeb -9346f8429957e356d21c665bab59fe45bcf1f74e -6eeac6e30d65f9a972067c1ea8c49978c8e631ac -dc6b9406bdfab2af8c86cb080cb3e6cf8f2385d8 -9f554e03ebe5701c1b75ff03b3d6152095c0cad3 -05009935f9ac070197113954d680bc2c9150b9b3 -508404de98a8a5435f52916cef8f328e82651961 -ed0cc50afed146c27f6d8129c683c225fb940093 -6429cfa8a70308241c576aeb92ffe3db5203b2ef -6898213409811b140843c3d89af43328c3b22fad -5b2ea29cf4fd298346437bb16a54407f8c1f9dca -e2a1a1ee895149c544d4ae295466611f0cec3094 -e82fb872ff5cc8fd22d43327c1ee3e755f61c562 -19b0f33de0efd9da788e8e4f3fdc2a9e159abdb1 -89de1538ce1f8c00f80e8d11f43e1b77e24d7dea -de07fdcf77e97b8613091285e4d0a734f5de7492 -01680195f8aa586c55c44767397380def3a23b54 -05e1c85fb687c82ae477c72d4a7e2d6b0c692167 -c072b8fd95cd4fa84f08189a0cd8b173ea2dbb8e -9a0ed08b40b15ae2b791aa8549b53e69934b4ea7 -53f8f226bd1d627c4a6dec5862a1d4ea5a933e45 -9d0f43b7ca7241d8a018fd35dd3bc01555235ec6 -f12d2b5a8ac397e4bcaefcc19898f8ff5705dea5 -8250de13587ed05ca45df3e12c5dc9bcb1500e2c -d727f77e390426e9e463336bda08d50c451c7086 -484312bda2d43e3ea60047be076332299463adf8 -c7e05b35ab0a791c7a8e2d863e716fdec6f3f671 -b9c1cd81848da9de1baf9c2f29c19c50e549de13 -8ea7d31e384975019733b5778feabbd9955c79d8 -f798b891bcecea9548eedacae70eeb9906c1ddbf -ebefe7a00b46579cdd1e033a8c7fd8ce9aa578e4 -ad087638ee4864d6244ec9381ff764bfa6ee5086 -66db2d62d59817320c9182fc18e75a93b76828ea -7ce9ac5c83b1844a518ef2e12e87aae3cacdfe58 -4286f43025149cf44207c3ad98e4a1f068520ada -cd0c5135ab2291aaa5410ac919bad3fc87249a4a -66ed450d771a8fc01c159a8402648ebd1c35eb4c -a82f03393a32842d49236e8666ee57805ca701f8 -f972b04d63eb8af79ff3cec1dc561ed13dfa6053 -ec45cc5e27668171b55271b0c735194c70e7da41 -715e9fd7454f7a48d7adba7d42f662c20a3e3367 -2e0a99037dcc35bc63ba0d54371bc678af737c8e -7fa8d758598407f3bf0beb0118dc122ea5340736 -6a22373771edbc3c7513cacb9355f880c73c2cbf -b89ef131147f71a96152a7b5c4374266cdf539b2 -01d8359983e2f77b5118fede3ffa947072c666c8 -58f0c929a3d70a4bff79cc200f1c186f71ef1675 -950be19727a581970591d8f8138dfe4725750382 -425278d17bd0edf8a3a7cc81e55016f7fd8e7726 -c028c7b7557da2baff7af8840108e8be4db8e0c6 -47a7cfb0aa2498f6801026d258a59f9de48f60b0 -f6b7df3155ddb4cedfbcf5d3eb3383d4614b3a85 -d72098038f3b55a714ed8adb34fab547b15eb0d5 -c49c825bd9f4764536b45df5a684d97173673fc7 -33799afe83eec4200ff140e9bf5eae83701a4d7f -5c3f8ddcaa1164079105c452429fccf8127b01b6 -1f01443567b03ac75a91c810f1733f5c21b5699d -b3e42b6d02e8d19658a9135e427ebceab5367779 -69b3a6dd9d9a0adf5506c8b9fde42187356bd4a8 -bafd075c5e6a1088ef0f1aa0b0b224e026a3d3e0 -7daa3adb242d9c8728fdb15c6af6596aaad5502f -514993554c370f4cf30a109ac28d5d64893dbf0a -c8d2473e6cb042e7275a10c49d3f6a4a91bf0166 -386f4385ab04b0b2c3d47bddc0dc0f2de7354964 -9f33dba05c01ecc5c56eb1284ab7d64d42f55171 -7466a26cab5d66665991433947964a638f5b957e -b43aba89e356ff95b706e80d4802f60fc46a569a -02b7e8319aef2a870264ad4fa2e3bb18664dcc36 -f686002a8eba820a40ac2f34a6e8f57b2b5cc54c -2b1c50b9352ab1dc40b0f877db23c1fa4048fae3 -2405ce1df043f778b8efb9205009500cbc17313a -4ad3b3c72c73d61e0a0cab541dca20acf651320d -4ba3d4f4393d81148422d24d222fe7ed00130194 -8ee5c7b747171e335793c74cd9d2f7491da58164 -872c921c0a208b04bd0713758e52fcab5b7c1684 -00d1680498c5550e7db1f359202d3433a092fafd -585db41e9ab7a6fb262c8bad7f427cdbdc497188 -18462960c0f13bd07d8f52b61e7d7bc17e991eea -0630974647dacaf25e7fcb7f9cbb785bb078ede6 -0f58d7f3d62f012f2584f5e781fc73de4763dd9e -3d16f581538b0974853e820508e8b3093269d2fd -66e91420ab233cf1dac64504e0dc129019bf8c0d -d8d9162f5bad39b2720dd2b2da237c6159e4755f -29fad97c320c892ab6a480c81e2078ec22ab354b -791c3ea61b4e49fd46a1a71b84ca99ddf69d2ff7 -a312e201ba56742499a5480b5f2115f01505c217 -ce56fdd2e8cdf94fd0ab76d71adbfa755e23ce7d -480f42630cbd598c04fa59ee0e406f56904ecffb -6012f1caf744ac9b53383d7d10a8f1b70ca2c0e1 -ded6a2afa549f693dcabb430ce0862f8631360c8 -07090c5339436f856e79a8036d1c85deeb453803 -0e265916d1c6a63e4a3821dab9db597b5ec64b46 -e4ffcacc2187d3419c8ea12b82fb06d82d8751d2 -e117cfe45eee9169409e74a44ef4a866be25bc35 -dcfe218626b05204e9fbc95ba5d95ca0eb72ec9b -23481fa50301201ef5a60675ef899aa6ce94ca03 -27c59dc502f29cf1d76290556c21e366145e3b2e -4a62ddd01873d18dbca96c81d756be1020249b45 -a233fb4f1d037e68ff70eef3a9f5b7bf1d631918 -b2089c51cc4af2f7e1c0ec75be9449ee222b1d69 -c997f8808256521397f1c003bb1e9896fee6eaa0 -5dc00f68c49c46a380a98d06233f90528b8e2557 -fe53d5f3636aed064823bc220d828c7ff08d1d52 -935eb8de039dec65669a96a1c3b86f4b03a1b86c -0277173b1defb63216d40a8d8805ae6d5d563c26 -2a30e67d20f76bbcd9a7d445f616f005316e0a1a -d32528e733f2711b34dbc41fbb2bb0f153bf7e9a -4cad91663df381d0dff8526f3b4aa74569dfb626 -1b06ed136f17b526360617a70026aed5ded5746c -895fbd768f0c89cea3f78acac58b233d4e3a145e -f0295becbf3ef1fb78095306408789253fe0c114 -8d573198638e52e2dbd9abc609861430f9d2bcc3 -9d9c4185fadaf243bb97c226e2fef16b65299699 -eebe4580bc8d6484d79ecb24dd87412221cf2ea7 -9cf6393a4f82b9c81d3b4b468a17a89db10531a2 -598a9c4e4dcd03c6d80fba005de729a6a3aeba7e -6970b30c6f1d2be7947295fe18f2390649b17a4b -f359afcc410432ed5d30001acda0c66741ee8935 -126000ba9e7ff16271be2f4eef3df99ade8d624f -b5e4b9b5100ec15217d43edb5f4149439f4b20a5 -b987ca4ee495a7fff82f0ac14ef0753bfb7586e2 -b03013396cb2f4bf25746388b3982a2c3616e16b -9a97f39afaa890caa7987c6bc001b9a66e3e74e8 -cad504bf4c302f7a72e0a0e191f3fdbafda7340f -45cf8a03cb57b8639a8d47323bde46ba22d9eeaf -b7450cdbd89a1c862f4d4d8bf093f8a0b5448f9c -0910cbe4ef31eb95fd76c7c2f820419fe64a3150 -92a810d04b906722c9efe60e3997243c71ff3d4c -45173fa6fca9537abb0a0554f731d14b9f89c456 -fd4ca17360e6fc0c9bb76bf6b5b07c9102c12728 -ddff3447f29b62d79a33f728791f42fa9436216e -36a5a4404836da323c755523fbd27563a8e84f94 -c991b304dee368f506cfee27ddaa333f1f82c518 -d38d1a3e75aa97ffa8755ddd431754a6d0942964 -a332a7d5a15214015f9553fdb2bcf80a1a4b8dc0 -604e08c83cf58ca7e7cda2ab284c1ace7bb12977 -18a1bbad98bd4321f15e7921d9aec91661499d90 -8049241e226c16bd07b029c0cb4b62ac40f0c923 -797441ee995aac59f55d59a93ecb55e8ecbe7dbc -62fdf9b07087b80d2142799bdd2324f61483359d -f60b4ad57912b78a96af08046a503f7905610a8c -13e31dd6548d64a5992f439e74bb424bf88aca04 -fbce66a982679b5409a295be5c99a2eef429cabf -9f2c2dba21855b8cb9b193b1819be73fa4a23a99 -a89221873a3ee2451c73b41bbe2d99d36f439d31 -3d6ad407770e13958e157bf026cae0bfb9254899 -901ba3e3819405306414628306746552b0aa1d28 -7a43fbb959c38e025e558e472ad57de357539894 -0d89fa0877930c6c8a539a656c1009ad8ab6755b -54aedc013744c86b11157423fa3cffc9a51eef02 -f0c1f8abb0182da557d07372b938f3a0a4bb906f -4ed818060ecf4a38a02c8cb48f6cbc78d2ee7708 -3bdf242fc68a8d767932c6214455d4d413effbc9 -5e468994fbb349e8eefc996954a31a67a34aaa15 -41aa9c4a801a01eca1fad22a7095372d23dace60 -2adbddb03840ad71e843c6c4a207a13e871cd1d4 -13e352dc53dec0127c5f94a60055d0ca829420dc -95e14dc81dd30ee0d396ad08dca9a6980d16eee1 -61fb80660f73e5aa5b69302ecc7ac33da206ba5a -05a761932edd05cf94ffe938908baf058f38632a -ee92243e66f2df03b3a759a8ffb75dc06f0cea0d -22cdf93c062eeaa0f8f9d6220f01b67240073dfb -76b33491596736ca804e3a29bd8398d7a1516ab7 -6e4e98ee8ce2da3cca2e2fd210e9e8dbc9b1c936 -c838283ecdfb9490425bb071b7c22e542de46c7c -5e3f5e4f25b65b583d3bfefac9e1148035781089 -f7388e93d3dd91a90239aedac4ec58404f103a2e -0a2f46b0158b6fc7244a585913b0925c0acf707f -dd561667cb7ccbbfed3134b05a565971ef6f5873 -6f01dcf63873a5e42798635ab4026c9a5f9fa213 -70fec9e36bcd1a3d93df019be084aaf89cecd7d7 -f9b74ef3fc74fd7d2aa94560820341f03cda8e12 -998c3046fab2b52bc9f141cfb588a18c05506a86 -89cc4f905e30b913ca20e4192d538cc5cbe2c38d -87d90efd69b64f769116956a5db89e536e9e3714 -5aeaa9ccd1568a77e075dbe2bd2435bd60c87c91 -bfb270acfa30713dc8c968bb9ee40cf5a2360359 -1b8c88451b0554502435d3883c528ad0aad1b09b -57ee73990f1ce29916adfd99f93eae1ccea1a43b -808c84f89d0edcef9ddaab0b849a382719f6ec9e -14b860bf64020451ced823b859da8cb912278ab9 -c63364610f4a041df1c1bd81d01b1f6856160749 -92eadc395071876d77f3babddc056b4325bdbabc -e93fff1463ae906fc986bf98c3b118c82f171546 -9ccafb1d7bdd172a9b963444072a844da379c4f7 -b4a509a3f817121c3df98ddfd96b2769e18a3e5a -dbc4ae03963014ab4b7957d62ba59dbd8f938c33 -8ddf60db7ad636b6a31b590251c671ded635fa1d -f199b8a33d9443a258a1f49a1a29674cd9ee9a20 -e542728cde676f218c552d841d0af29b92f9800b -763231051596b8e3455b839911ad6a3a1f1c3c74 -ff4cd6075b12fb32b9a906deea3ed033e3f9560a -9c3c9cdae3e20b5bdea91a0631edac5116bbc89f -93d20a734d2ee873832bed8ca5c05cf8e539c53c -ef8340d25f7c5dd5682bdecea97ce84cfce1493c -69c7ecef405d168f658a9cc7996da84c17f61e66 -4ce2f3d0d33346e9f0e96851689ee6550b2a72e3 -44e1fd926cfb0df0fbd8c41de8cd65ed8d5d6e18 -d6d2c8503c4039b682196d83a67dc28359c10c5c -ae233c4ec3d14a97c6195059f52873cdba2b4755 -0f399a9ff227896265cafab9b2e9fab6cdb9b5b9 -f4ed44ab4a8f9a87ba678d5fd1449fbf636103dc -7fcd61b2613c211bb042a82a889655178be6a212 -42973f834445d7735738bdba8847812ba3c34d95 -8df48b36ed3201d938b9974ecbee455d7dc2fb84 -96ac26e56627f0c24213fcd3a1cce9fc95f1f661 -cce94c518a46b7b0006f984bbe4d69e8749182d2 -801dd40666d1e6009920ad3ff755c7bb993b2a62 -ce829855cfca103dde55661fa1524e66b139d063 -b148803b181e30213e8a7f3bd89c8239e9dcb866 -c377feaad87f8109f85da6caf62602b30c20effc -b37cab65c63e051ebc5b491da9bd687581df94df -16e41844e7d6c5876d2caaeef6010656950c6ec5 -ee50c9e48786dea0d9df2e45805c25565c100fe3 -11dacc6154c42bc6fe3ba94c1823f8a46e4fe81a -791a0e6ddade27d1b69f4861a6640de60b9553cf -638e6c59da4fad987c437592174b188510193b2e -52f8877525d5238f3440e73710507be889d14127 -2a56baf395bf11835d784c4f8634f4525deed6a1 -bc561b4b7d6a3f71649d37d5eb9047c29efa2b13 -31809d6f8514c4a8d5677e947e3f1ebb0db210b9 -a31e9ad4f027955d43c04a05517244647e250161 -777519bd96f68c18150a0f5942f8f97a91937f5e -4eb1f39d421024d9666cec61deaf96715ffae4c6 -50fae68d416b4b8ec4ca192923dfd5ae9ea42773 -ce665863b137ac4a7470cf006a92aa7694faca71 -81f8c0378b2ab5ea0d7b65635cb529bd3c69127c -108222b9c323a05cc9339368f10ddd0859f62b43 -28f788e47e58f2b462351d6989348a4e1a241b2b -d81dccf191a48a6b59c3747d7b4ccbe3535dde40 -a90e6d2bffc422ddcdb771c53aac0bceb970a2c4 -91e49c51f1aecc9e1d75457f4920d52a4b0a133c -60dd9cc470584960431de425e2a9ffbed0e8034a -ede386c2193fc31351e193b3a8cf30030d6be62c -a084767b40c0d3ba8fa8f8d60f1e8d99a9dc3457 -3f726c99f819f97f2ab21b94d34c6b3129cd883a -77fc469fc78cdd87c29f398d46ac58dbb9ef62c0 -4ae6d0fbef60ccbecf8f23bb482e201b3678f7a3 -8858b6ddd3bce9daa08da6e05de3ca863a399c15 -22e301a3d56dc9e6878380ee92c7d19ca43119d2 -c484ec6c9b85ca4e331e395c564ae232fd0681dd -a46a671e253528e450bd57645c400bf761da07ab -655970d9c60ae6850daf452457e14e21047c0e1b -b6a48914c50631914192aa11b19205436a9c664d -7db65c363a0cc6ca7cdb04de9a973ab70013baad -6366941275344dac7e2130b0c972e90117d37ed0 -4fb2586661471a1572c2df2a5a091011d45eb7c4 -d7be7b39fa1021ec4518186afe145ee948e12a94 -85aec87b11ec41295558175c63f1f5a849460fdf -aeb31756276034dd506fdf97c8aaade0e7e584f5 -ac016e17d20253129a0287cee7e1d06b7ef15966 -bf74d377fb8e20140da6eac1407414928384bcea -2c811e08db651a4aed6ea0f7c1972d60de6de8ab -e5d26e47c7a482c072a7fe47bb84c56854734184 -96a63a3e0cefe920819bd42add0041837b1214a1 -e526ca6284b9e13be1b912b80dd73a34e739b539 -ecd21357f16106e541e9c2854ead2a906659b938 -4b5a7ce0c301ad971f383eb60f61bf9b4026efda -929fd7276c0f0c30b9416f61a6f5f35d763d81e4 -fa8a0639f7b0ce04030b72b4d5be4f0aa36fc5cb -f1f1605c22a6283bbfd757055fcf2b584a857709 -0c173a15ca1bf20999f74987988985508c9de463 -df0793f324e33066cc746c0cb1d053d35733d626 -2b0179d8a9b75397937126b36114df0dddeab40c -bf0a08be281dc42241e7f264c2a20515eb4781bb -3895e25a77363ae8b49358fb793f50fa8b271e2d -1fc783fc08bc078239537535f174ab8a489772c0 -1d4805ce04645f3203b0cfd3d66ea710e7433eb4 -d3b58704d1d325875fc605580c1c02b825c1bbcc -ed88e3194c4bc43aeafef929da7b419d03dea1ad -dd07f47b79628668e29cc0143b21e790100ee445 -65cc7aacfbfc7b747926375280a1d839e88d576b -080ec5209172ac9605f1434559dbb3c1e012b10a -416af3edf5b5ab265acf95568f2bc9eabd3d96de -e0a7801223fd573863939e76cb633f1dcc2d22c4 -4bc853b50fd9127687eb9e4f3b679dd261a4fa96 -c68a9a69278aa194fed96bd9733d32af3690a11e -c38f540298f0e188df5ed68fd56c623b9ac8331b -643fa0b22d70e459d7f7ec3d728ae4811dc5158f -e053e05c130549f43953f1d70e724dc9ce3e1b85 -75e898c094eea533d1dfaf141c6afccc3072c49f -2805d606bc46bf5589093a1b92d3542c13ce50c2 -32751807c9c06011eb689cba56b401a6302699c0 -30853e16d332816752dafcfca92147c7ffef5b54 -bea5b00cfe95cd37832305c0f93c339a22a7d79d -c871f323b418fac27bf834843ca26985010df53f -329fc1dce7a1c372c8b10c2f2f8732b2c60daff0 -1aefc94dd78d6e0c9209cb09fc16f53dedf42108 -8e5725666b519b61fcdc3141da5c6a57c1959909 -a4ca0b042365061020627a8c045cddacea3312ec -8bd16ee12fc8ef6723e0572c29b979c15b92b4f4 -87abe20fc118721cc5efdbd94a8462468cd1da2b -4b766fcdd4ca16399075d1e081a321b3b05ce516 -f6241b3e420e19f3f0507cbbc872fe9218916a02 -7ee523604851af62c0a47c07ee023a8710ef32f7 -776ba233e939fe41a74c6b2632b93a0679a32c71 -6a796b2b53fe542e0f340f250f4f20d69efed8d0 -23d78c4dd01bc74ba35db3e3df95280f6f1b2e22 -f4b15e2de97c4f8cdbb40bef4c9d0ab2807974d9 -fff72de5bf8ac7b70208e655f237b80e70e18851 -170bc2c381f86a523de2fc8b71d62ade66303c0d -314ebdfcb38d4b4c977579f787d5e1a20d068c94 -e9274839bf316b1972d80d28e45759f898edbf86 -75171f099e82e3527d7c3469b15891bd92227ec2 -3c5e6c94caf40395e031fbde44a0cca46fdd76ec -dc8fc0c73bebbc1c48ac5540026030c9cc00ec23 -492d22f92919d8d9d59568318c26c1e2ac4890cc -80c3a734298e824f9321c4efdd446086a3baad89 -47535d7c3ec79c5978cdcc03a5351ddbbb22538d -1b25b6df0f08f7474228c5b6ed13b58682e1e440 -c530c15180631cea95e9c292cf7fabde9dca9db3 -2723bcdce3248417e98e6c43207bef74d34076c1 -ed22eb4a62bd8d5369aaec87d4cbdc03c9f16368 -9111df9673beb6d6616d491a5478f09b5f14d040 -d86bb075bf6d1e78c1e4f3dd38b0ea828ef5ecfe -50a1cc0f0aef1514b917a5a3f4476967170b429d -6ce733747e160ca699711f2c47e686284ca9aa07 -b44adf92342ad4f9c343ba29c081a91687932936 -88799ea1b1c08f4bc1a487c9e3c2effd5e1650ae -080d7c700fc3291560d79fc590e05b8e2bad984f -12af74b289f8cdc6caf850dc6c802f9936b1e8b3 -8e4f7e72410df3ba430082c7cf385f26fd75b033 -8ac80412867118172dc4172494304e19969e9489 -f2734c2828f69d9cfd535e5eab0592a7674b2b61 -0b9fb682890b8fe10cec54072b809a5efe57d33d -5b029aaedb5fcf7cadd249607dd28eb3f233ab8c -79af9fbd8c3c0e54702a9c92b171f134bd4466c8 -c412fd805ddf3282dc2e1f28e30f51ffcb1f1da2 -111849345bb5140f86b48e730ceab4bff45fa2e9 -a0b1e57b20a17177ed5a9a54e4a8aab597a546b4 -ca209230c8e73745cf8cfc79f500c9c46e103306 -a230b0588788dbe1ac84622aea169c577b381241 -dfef6b6af08097f0676a2323085558fbbd3c48c6 -3192e5278abca7c1f3b4a2a7f77a0ce941c73985 -7c7ddd9ead99a8b5033a1a5d4698032c9e2b3a92 -10b930dde8f14e9cb661810e97a33bbf144fc55c -9225de2cf652fe2bf6e50636824cdb641546f57d -598ef9c44b3ea2cc142c175f077b493f39f5ba22 -c49355c7170a64bdd7864cc3ba9a64916b67fe7c -857d1e171e051b254a617f27b39f6a551054cee2 -21833f9456f6ad5bc06321ad6d9590f42ce0195c -8910b4717e5bb946ee6988f7fe9fd461f53a5935 -5703dff0939f05c7457cebd6fc61d88ab13afe41 -8bfa13b15b84cb372950fb7b25a1080173060b6a -ac23a7c1f19b3d8c326ffe75c8e13edf285f90fe -19be26afe3d04783a92d032b55bf3fb1e2ae63cc -f7ec7cfd38b543ba81ac7bed5b77f9a19739460b -36afd4db4442c45d4078b1a7ad16a1872b5bee0d -88c2ae3ed2bb5d367dd408c9255cd8f1e7a36c7d -a13a417cdcfdfd1f1b3bf997bb6ffe6e69b096b9 -d6064a89ac97dc0d2ce9da3982e1a4e25afaeda8 -7146d96de3e15a80cafbab2af48ff6f65d8e41bb -5628c70f2a44567695e5331fe2293c5b7f35b629 -7ff4a538a8682cdf02a4bcd6f15499c841001b73 -aa5fa642b0e7ce2ea55e2298886f212f11a8894e -8efd1c820b9a782d8608d54d924658536178295c -50a226563cd8d7c0a5e8448e87fede0eb72a8354 -b860915f8b0dae98e57a254d11575ea41f5c5a79 -d304fef3746039183f51b3ac8f4774dcf3a64f59 -53ab12d9318d5d195ccc77028b0e3ae66dc6e1fd -668de70be039a4f1ffcf20aeae2a22ee71fc55a8 -0fea960ca917b73aff853fe88476174c8a313863 -f89502306dcf6393a2c7b0efbb0fa728fc582137 -ff58b1c3bdff5e5f687f10f9e40ce495ca49674e -0b96abc35f1a9d46a27eeddd7df418d107c29c57 -b0b57a17306a7e963a4fe463f84e2b150a00a859 -4105cb6fd964ad13099ca83b1fdf3d35f3961f74 -23281a4dc3afc42a001346caec4dbb8193f0bb53 -8daf103fa138f9a184448ebf1c2e03b9dbd96f21 -02e5308c1b9f3771bbe49bc5036215fa2bd66aa9 -a65ced1a66575c652baf5084644b8647f531be8c -2456a835f0bc7796d9ff71f64837fa6790e2b7cc -9ec1330b455c1ab2eb6b89f8a2ab885677d4ae8a -0b738075bd43fbd4410e30a51e0498cbfd2b7513 -98c80e374b84e5a9c2d5c36889a0b1ebed5b814b -25720fc394e27a951bcad26095fb5a711bfacb8f -4cfd57d2e38207d78722ce8c9274ba8dd700d1cc -0fc1c31a878e93d938c67db3f958e82e3c39659f -df1ab5b4d67b46b5e9e840b1fbe0ff02520831f9 -5bc3b6cede8dabdf3f4f27ddb03723cbb7cde51a -c2ea1e6561caba3abffce361abc800822b9e0efe -caa2f106d704ec3ade63498031dd58d34510bc76 -dce853ef76ef90c46d84294225088d595467d08c -dbc8a8c86ae50059fddb2d6834fa5f0c9bbf9b71 -0f921e6a0492c4e9f037a9ed91f474885032d68c -041331e1da23e4136fd046ed870cdcc177464176 -e6ba5068f107ac234576e77cedbd748b665369c2 -76fcd9d5034143a5b041766552670d19f926097d -72bf1b3d0962304850a3ef5fe375db4bff1d0a39 -919db037f1f5cc73cdcaef92dd9cb0e7f5c8dec3 -c36229b0b2e9d4554053f5c9fc451ac29a493b1f -9e4bb312e6958d2baa309ba670e5eed1523c6f47 -d7ba4a233bd5a6f8fadee681c68a995e23fe36d7 -98514988a3d3e8b7dbf0463884a5c38f5ed5562d -5412c08c3cf13577566064edd04da021c37b7cbe -31bcc667863f368157efa1143a78623a5db8f0d1 -7bd1aa566fb4a4fe194f209085649f2c722b0cff -c4522e71c7e1d8ecfd70112e9375b9d00d6733a8 -e22f409f18881b63a8e747036584a71217f40e6e -97ec6e5c9098a1240655cfcab05b6cd5eedb6cd1 -bc121b0eb19713ec72002b5be03ba5ac35903a17 -c98f6b3d93a2cc1b49a6db425ea2b661089d0f9e -0de7fd36de57a68e543b4c1f184fba192c398c73 -e662d281b837c25b2b70525aa8fe8af894339823 -44adf683ad232db8ce0cb89b3e236a1f5944cfb0 -cb2ed300a89ebf9f0654da869ced665ed8b2abe7 -0a6d48d9ed60b0b02177059ab116f8f46d2cbed3 -b42291334651fff46dbfe5947a726f65cb9d7dfe -e5364991daecb73aca3bb5ac37f2619d7a89211b -4a2b170c075ce703cbdc82519a48016a9ee3f99c -924de0bd75a7f75df65d7d15f9d1587a2e794abf -1253f8692fc3a11be9430685cd405236a68df6c3 -2b799ae9e1e0a540f9a5971ddf27d83254668279 -c9bdf9a75f9fde8cd011e4aa94be4ed4347078a3 -3d69ecb4edeb80003a1a41442e320898a30dbd9c -f08222e882b18c1f279308636e03beceece2dbf1 -23e03f8d26d7bd03273a5dcbdcfe3905dfb49ffb -03dd707dc027fbf6f24120213f8eb66571600374 -d0754799698de2c032abcb8198ee5d5401063213 -072116fceb2294b97d1c40f79305f2e3ff71812b -e66cc1d58e16bf1650dd6479fed64ecaca8c6098 -f137753a2dcd8229f89d1d1ac28039364e5850b4 -61d191fbf953700ba8aeadc9c8cf4c195efbd10c -76f3c02fb01a6df98fbd8c16ac21d159d4649d37 -6013c73b3312e11b447ed387426749014716f820 -6faffb8a83db3f209a303a4464dbdd597faad5a4 -cc9e8aca5f950c78dcfeff63c441ba993c1fe12f -8ca69a2a88a77eb06149fa049ab1a7e6de38b321 -2f71490d21796594ca6f55e375558944de9db5a0 -08cc5fd666456cb476467473ed1880c90c92dedb -e31a43c725ebe641d7c219c3886eee18eebf0bb8 -52b5a8785de760a204b2b0aab19dfaf79c2c3ff0 -483e8e4f4875a1a621ec9e9df2880d3037d95ed7 -1e5799c52535a3fc20e885916f1e7ed33ecc7f46 -a82e5d8220bbc8b5d786bed99b0876f530b9b7cc -7fe6c5c993706e8395cdaf7977bee793c06f48f3 -2a0836f6d5e7c1d7e97bedb0e0ea33dcaf981f77 -ddc308068d69c6c9aa629ee3c4ce75e1d1cf08b5 -ec139a5621a9c9f03e1988391a3c7c6c5d849776 -c01a6c48b982d625fd9f4f69005878781d3d56fa -95a983d56dbda457e3bf8766d59bac74c7aa5699 -760741a00833876976389ed7a6b73f36ee5b4c13 -6e5e5abba6f8bbbe61c22795df440dfafcfdc378 -cf2cecb18779ce83de9adebf382dff1c19b12840 -af9b7a9f2f73b1a2f9728106774dd13e8d1cdd8d -115735d547fdeade822f547eb3e8c8f9961a9b07 -c2c69edf37b5c02aafa01d0407dadbf5ef8751b5 -a072d1a83787e786d074a4b5871b0b961781f7c6 -ed2cd59e258f756b2eaed7909a60956ade6ef7ee -ae5575ba41c8a782805afb1c08730343cfc22397 -6ff2c8d29f6b5a5c2ce63f0a16f3bb0dbd049451 -a80de15113166354cdf208e3d8b6e25f4511a591 -06bd4f637f15e769f088d9051a5af94bbb0217a3 -6700cc993cc07fb0f5b8b577ff8c4afcf0b18274 -37f9a1f627c0995d89b62923e75cd092600894f9 -8844ef15ded02d5ed86fb95aaf251235fcef2396 -1b87e5b5b184a0a6c683eda23b36393822b57f03 -e2bf830bb6c1bfa038c943dd6f5d92a406bd723f -423ca302a3ee87000530da3c105f269b8fabece7 -4e14afe42fdd468d5de11df8cc13defdcb8e83f8 -3e90fe6534206412ea22beaa445cf20d28fbe718 -88b77c7da0a672c89e24df37ea6e9085b4e2a05c -0ad104190465d8d65c2344bbe10dcf3df025d86c -5c7df7022bcd360e6af00b9458b1a3fd54e1cc9a -59ad56851a342d2c62f6b38bf15002b23ab439e1 -d8cd7b137fb075616f31d2b43b85fa2e27ea7477 -655937ebcbf681ededf86b1f0f60aac45c73393d -abdfd2d0e3ebec7dbead89317ee9192189a35809 -e439aeb30c0439001a781c5979aec41e1fc2aa50 -b9b26d9c3615d15669ae0a049c1dede39a9e59a9 -fdf146f3293c487afdc4d6d9f6b64099aa8bd28a -16e3b175781caacee403a2dc40cd6c70448e12ef -b30c62d4b954df05bf404cfbeb5b728282201496 -b3f377daaa86cd7755a552fa3adfeb195835f58e -0a8f519a0626d7cd385114ce610d23215c051b3f -544f3234384b2f6c290e987ad18576e1b50d7db9 -91482e5bf22d283d32c9f83c8057f10971848107 -e754c6e33194e9ed69ba5350c5139b0423b645fe -dc1e54206d76e5fa378d28a18ae1fb2bcf714485 -b2863c0685a5c12f829095cbabaf26ccc49e46ec -b14db5abab405a708f0166293f1ea12222a6bf03 -8010ded6da56842c09b14665343cd189d7e08401 -d387507aeca652a5569825af65243536f2ce26ea -27bf14f6f3e0fb1f348f13c1b54fc6b67b3bef6e -f8d470e24606297dab95e30b1d39ff664fbda31d -b25a4c2284babdf1e8cf0ec3b1402200dd25f33f -1329ef1f00e4fad83937ddd8721d0292ccfe7808 -9a1ad2c5cbdfa3114d05df57103c34f72e087f26 -1e90862f5d0b5f7dcc18fb018b2bbaa323dcca1d -ad552a54c56a420be84b47154882c3e4c76f9bdd -90b1c7e5c50551f39d4983008d1d5ab3b085803e -d6b2235ca45e072961e25a35e6a159e97c9e556b -2643fa50869f22672cbc72ac497d9c30234075b8 -01f909828d126d5443bc28758f51781eccdf5848 -f54f3738c8ce839c413d7b6b719be2ff341536ca -418ae49ee1eac2c9d6cd4ba83c036a41f1afe922 -5a666428b0f11d62af2002bd54a45ff2f79f30cd -a07e8caa5d5000286604458e6887f57fec7fdcbb -8b262eb2d80bfa27ae8501078ce47bc1407e9c55 -5df84de583c900e00fef63bedaef32786f205a33 -4ba6da55743a55189164e29e45ac9e73a074d808 -88430cbab4dca36b6a867364cab319cde6a9ebca -e0f7515f5500968c86e5a9f4912d83d4abc5b2b9 -9b8b1079ddab64ac955766536c38d23dc57bc499 -af20f9b1d485582b8c8aa8294bac4f2c540246d2 -7be9a9a570c1140048f8781ced1111e1d930e517 -2bac3e484114c30548e286972525dd799dbd0a5b -df529dcc65e8037c5a3a3ad74545be294a770f07 -6acd8700bc0ee1d10207a362c1e07372ba274041 -ffc6e48b2983189dc0ce7de0a038e5329bc07b1b -252ae7111cbff09a4cbc5caee9e02b6ed3580476 -b3ecb7bab6074377d87c700bf0c5d351e5d3174f -d9fdac130a5ed1d96fcac6bb87c10bec9d596b17 -a07e8caa5d5000286604458e6887f57fec7fdcbb -8b262eb2d80bfa27ae8501078ce47bc1407e9c55 -5df84de583c900e00fef63bedaef32786f205a33 -4ba6da55743a55189164e29e45ac9e73a074d808 -5bea05bc1d17aa43cbdf3a3413241f8132790d93 -c17f11f7b43ad3bd9e242c67db1f3679558a0581 -5ea932a51083837cdd27715e10a3a0d5d553af24 -033c78671b91b12d589ebff6c5ede8d94d7500f8 -ef8a634358848847e006c43ce621bc17a612fd1f -ba216b5fa63e7e6cae847d1e3621f5c54840f898 -26fee4f6bd9aec62c6caa60683ad66574cf16aa6 -6ab0e4cf49549640b903bf5fce0e6035b8116397 -326a5652e0d25fdb60c337ef4f1c98a63e0748f0 -424be03305143cbe5da5d5adb54d73d3dc3747b6 -38c201f47c0bc388a05cdb35d6137150fa90193e -12ed800ab870e0fc527a84d6e4584b10c8d239f5 -aeed345c9bade5d52a3fbf0a943203f6c82e6344 -c6223b3daab0328ca742b1cc3c15e89e698630bb -877678710800a4d78afc12519424f232f1a583d3 -6c4fecfaf7beefad0d1c3f8520bf50bb515a0716 -98212745c8acb5cc4e688bbb3979bfd46b25f98a -b9bceaf1c081a84d9fcc680372614e797b168a9e -1afc22a7667a7a5c66b4b5d7f50832356dd5ec12 -3255d6347b1f9eccbec3d6d93d4a424087a3b35b -ec20f01ba0945a3113797ac98a6b3500e24603d4 -75b5643c47c3b382ed97a9f5e2bdc883a0f98709 -fee0d803fb55c8d85b5cd1ff69d799c5ad522e18 -565494619d809655fa94d274bb2202d25553e485 -ad6fce67b9bb6eee864c8431ad3291aebaa2e5d2 -99c7db8731cc77f143b52f544b3fdd93033ed20d -b4d03be3cac04da8b5d5fa17e29c5220b75d970b -ef37f2033c4ae104585cd980141262f95d33166e -5cfdda2503c995cdd563b1a2a29162ac298d173d -c5904e871479514b2e2e18b4fdbbe468c4e5ec8e -10b22e3141a603ec891d2cfc7100c29c7409aabe -afd2fca911c4a5e3a4d1f0993a226d40f250aff4 -505955052e60e0681865f3064e005ca0d3aa90bf -8fdd23a224ba236874ef662c4ca311b002dbcab3 -1c011ff430106b5f727f2eaa0f7f4883cd2122a3 -ec8a50b8d786a8cd1192e692ab19b46979add582 -f90603ac6d24f5263649675d51233f1fce8b2ecd -b7d6623c76e1468f2a93db5a3120580e2784d74a -66270a416edb1610f276124483feceef9cba93ff -e4fcbf797ed3b472d352ac3794ec82f581209c50 -479afa0f8486146a35f1fb96be1826061ecbcf23 -2a09a3891fde052a585dc019eea9fba26d42445d -90a002ea647dcea57a2ed4294eab77897168ba1d -30c21306c17165c3925fea4ac9d1a4763c6d2a99 -b3eb0d6485510f2bdf36d256ab60ce29b8213744 -efbcf2b1d5ff4ee7132eae9c9e203d2b875c545b -b33ca14f594e2cf2a16ef27778169deb7cc9f4dc -d636f3943d39ec893dab2d2546f77f3f2607769d -cafe24f039e117d53288387c2720f44f27deecd0 -de8db47b7ff351f3287c5efb85102ba8836058d6 -d76e84a21416ef77e78138e326d4d249454e79dc -7a74f88a26cf251ba36b26f604f1ac9940fd9c92 -1ad3d4e1261f4a444d982a1470c257c78233bda3 -8d9f45ea6a5e4220e44d34139438eea75a07530b -c98ebf1bfb29a8203b5090412afeb333384213cd -f18bb49547095020a30e81b648075bc7e707515c -76f268b9bd1b69eb7784c5324abbb67f3e395b97 -e801084decf4542d57cf5ddb95820643766a172a -be3e042c20e2f3449b7b55d1cab0a80b0c6f00af -400fdd08cc95f1e85afafd07ddd9c0bed11483ea -098b01dc58ff555c473ae58c92c34b03a77eda5f -7cc2c670e3d7cf26454ac8547a94ec2c8ca90b34 -1088b02f0ccd7358d2b7076bb9e122d59d502d02 -f94b7d5bfa911ea7125920589723ee63a3eec9f0 -b4b057a3e0712dd16b50cbcfe7d613e4413ffa1c -b40ceed98a112f4f0d07351ce07270d9ff2bf796 -4cb8757aae1ae31e5519d81e854f44ed062d9836 -f2f7e97e8cc24cf7a2b7954cb74ecfd0f91a95ad -ae786098bc58b1ca92f596a698b23aeade9cd2cd -c33652576ce21694b33a94832378f737dd6959fb -e317c0d19201ff75fa7afedf93a9d1cd2c560af2 -bee35299716cc72cb7d5bd4daa9fddca05c58378 -318ea50a1c2f612e750a93e36620dd0c4531e9cf -b6ee855b411ee9bc39f935d0da3298a773a2ed37 -daf3e7def7b9e5db7a32f5a20b5c4e09e3f0dd18 -bc64b5aa0fc543fe8fd3dbaec275f89df44dc409 -3f57c55dba6ef1fda2bdf6fd9abd8ca7eb6828e4 -431a548faaf51c7a5fc89b6e479187a1c0e29805 -e4bbd3d230f22401ba0a0a72c8ed41ee1bd098a0 -c45da32047cac54afc99cf9b8a539389c577ded1 -ab1f1d32469180b3d011e9625d67c86a22b55903 -a550f6e415fd8aec8c45d4704712a408c37ecd18 -c73af5416b66f09cec0eb106f5a10f9bb6ef9cb1 -a077a90da88f12d9f10c8b85840bdb847a98b0a0 -c5e9e428a9198c8c4076f239b5eaa8dc95e7985b -b7365f0545b1a6862e3277b2b2139ee0d5aee1cf -4bd0e9b90a39c5c6a016b83882ae44cb4d28f1f8 -7438ceac716fdfe6621728c05e718eaa89dd89aa -4e3efd47e0d50c6cd1dc81ccc9669a5b2658f495 diff --git a/contrib/verify-commits/allow-unclean-merge-commits b/contrib/verify-commits/allow-unclean-merge-commits index 7aab274b9a84..e69de29bb2d1 100644 --- a/contrib/verify-commits/allow-unclean-merge-commits +++ b/contrib/verify-commits/allow-unclean-merge-commits @@ -1,4 +0,0 @@ -6052d509105790a26b3ad5df43dd61e7f1b24a12 -3798e5de334c3deb5f71302b782f6b8fbd5087f1 -326ffed09bfcc209a2efd6a2ebc69edf6bd200b5 -97d83739db0631be5d4ba86af3616014652c00ec diff --git a/contrib/verify-commits/gpg.sh b/contrib/verify-commits/gpg.sh index db5bfce208ef..cfd68e45b8da 100755 --- a/contrib/verify-commits/gpg.sh +++ b/contrib/verify-commits/gpg.sh @@ -5,12 +5,9 @@ export LC_ALL=C INPUT=$(cat /dev/stdin) -VALID=false -REVSIG=false -IFS=' -' if [ "$BITCOIN_VERIFY_COMMITS_ALLOW_SHA1" = 1 ]; then - GPG_RES="$(printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" + printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null + exit $? else # Note how we've disabled SHA1 with the --weak-digest option, disabling # signatures - including selfsigs - that use SHA1. While you might think that @@ -20,12 +17,12 @@ else # an attacker could construct a pull-req that results in a commit object that # they've created a collision for. Not the most likely attack, but preventing # it is pretty easy so we do so as a "belt-and-suspenders" measure. - GPG_RES="" for LINE in $(gpg --version); do case "$LINE" in "gpg (GnuPG) 1.4.1"*|"gpg (GnuPG) 2.0."*) echo "Please upgrade to at least gpg 2.1.10 to check for weak signatures" > /dev/stderr - GPG_RES="$(printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" + printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null + exit $? ;; # We assume if you're running 2.1+, you're probably running 2.1.10+ # gpg will fail otherwise @@ -33,33 +30,6 @@ else # gpg will fail otherwise esac done - [ "$GPG_RES" = "" ] && GPG_RES="$(printf '%s\n' "$INPUT" | gpg --trust-model always --weak-digest sha1 "$@" 2>/dev/null)" -fi -for LINE in $GPG_RES; do - case "$LINE" in - "[GNUPG:] VALIDSIG "*) - while read KEY; do - [ "${LINE#?GNUPG:? VALIDSIG * * * * * * * * * }" = "$KEY" ] && VALID=true - done < ./contrib/verify-commits/trusted-keys - ;; - "[GNUPG:] REVKEYSIG "*) - [ "$BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG" != 1 ] && exit 1 - REVSIG=true - GOODREVSIG="[GNUPG:] GOODSIG ${LINE#* * *}" - ;; - "[GNUPG:] EXPKEYSIG "*) - [ "$BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG" != 1 ] && exit 1 - REVSIG=true - GOODREVSIG="[GNUPG:] GOODSIG ${LINE#* * *}" - ;; - esac -done -if ! $VALID; then - exit 1 -fi -if $VALID && $REVSIG; then - printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null | grep "^\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" - echo "$GOODREVSIG" -else - printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null + printf '%s\n' "$INPUT" | gpg --trust-model always --weak-digest sha1 "$@" 2>/dev/null + exit $? fi diff --git a/contrib/verify-commits/trusted-git-root b/contrib/verify-commits/trusted-git-root index 1c421959618e..7ec318e1ea79 100644 --- a/contrib/verify-commits/trusted-git-root +++ b/contrib/verify-commits/trusted-git-root @@ -1 +1 @@ -577bd51a4b8de066466a445192c1c653872657e2 +437dfe1c26e752c280014a30f809e62c684ad99e diff --git a/contrib/verify-commits/trusted-keys b/contrib/verify-commits/trusted-keys index 5ca65e7b0d18..94daf28b156e 100644 --- a/contrib/verify-commits/trusted-keys +++ b/contrib/verify-commits/trusted-keys @@ -1,5 +1,3 @@ -71A3B16735405025D447E8F274810B012346C9A6 -B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B E777299FC265DD04793070EB944D35F9AC3DB76A D1DBF2C4B96F2DEBF4C16654410108112E7EA81F 152812300785C96444D3334D17565732E08E5E41 diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py index 2ff14c1f86d6..a1fe78a6436c 100755 --- a/contrib/verify-commits/verify-commits.py +++ b/contrib/verify-commits/verify-commits.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Verify commits against a trusted keys list.""" @@ -92,8 +92,10 @@ def main(): unclean_merge_allowed = f.read().splitlines() with open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf8") as f: incorrect_sha512_allowed = f.read().splitlines() + with open(dirname + "/trusted-keys", "r", encoding="utf8") as f: + trusted_keys = f.read().splitlines() - # Set commit and branch and set variables + # Set commit and variables current_commit = args.commit if ' ' in current_commit: print("Commit must not contain spaces", file=sys.stderr) @@ -102,7 +104,6 @@ def main(): no_sha1 = True prev_commit = "" initial_commit = current_commit - branch = subprocess.check_output([GIT, 'show', '-s', '--format=%H', initial_commit]).decode('utf8').splitlines()[0] # Iterate through commits while True: @@ -113,17 +114,41 @@ def main(): if current_commit == verified_root: print('There is a valid path from "{}" to {} where all commits are signed!'.format(initial_commit, verified_root)) sys.exit(0) - if current_commit == verified_sha512_root: - if verify_tree: + else: + # Make sure this commit isn't older than trusted roots + check_root_older_res = subprocess.run([GIT, "merge-base", "--is-ancestor", verified_root, current_commit]) + if check_root_older_res.returncode != 0: + print(f"\"{current_commit}\" predates the trusted root, stopping!") + sys.exit(0) + + if verify_tree: + if current_commit == verified_sha512_root: print("All Tree-SHA512s matched up to {}".format(verified_sha512_root), file=sys.stderr) - verify_tree = False - no_sha1 = False + verify_tree = False + no_sha1 = False + else: + # Skip the tree check if we are older than the trusted root + check_root_older_res = subprocess.run([GIT, "merge-base", "--is-ancestor", verified_sha512_root, current_commit]) + if check_root_older_res.returncode != 0: + print(f"\"{current_commit}\" predates the trusted SHA512 root, disabling tree verification.") + verify_tree = False + no_sha1 = False + os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_SHA1'] = "0" if no_sha1 else "1" - os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG'] = "1" if current_commit in revsig_allowed else "0" + allow_revsig = current_commit in revsig_allowed # Check that the commit (and parents) was signed with a trusted key - if subprocess.call([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', current_commit], stdout=subprocess.DEVNULL): + valid_sig = False + verify_res = subprocess.run([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', "--raw", current_commit], capture_output=True) + for line in verify_res.stderr.decode().splitlines(): + if line.startswith("[GNUPG:] VALIDSIG "): + key = line.split(" ")[-1] + valid_sig = key in trusted_keys + elif (line.startswith("[GNUPG:] REVKEYSIG ") or line.startswith("[GNUPG:] EXPKEYSIG ")) and not allow_revsig: + valid_sig = False + break + if not valid_sig: if prev_commit != "": print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr) print("Parents are:", file=sys.stderr) @@ -153,15 +178,24 @@ def main(): allow_unclean = current_commit in unclean_merge_allowed if len(parents) == 2 and check_merge and not allow_unclean: current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit]).decode('utf8').splitlines()[0] - subprocess.call([GIT, 'checkout', '--force', '--quiet', parents[0]]) - subprocess.call([GIT, 'merge', '--no-ff', '--quiet', '--no-gpg-sign', parents[1]], stdout=subprocess.DEVNULL) - recreated_tree = subprocess.check_output([GIT, 'show', '--format=format:%T', 'HEAD']).decode('utf8').splitlines()[0] + + # This merge-tree functionality requires git >= 2.38. The + # --write-tree option was added in order to opt-in to the new + # behavior. Older versions of git will not recognize the option and + # will instead exit with code 128. + try: + recreated_tree = subprocess.check_output([GIT, "merge-tree", "--write-tree", parents[0], parents[1]]).decode('utf8').splitlines()[0] + except subprocess.CalledProcessError as e: + if e.returncode == 128: + print("git v2.38+ is required for this functionality.", file=sys.stderr) + sys.exit(1) + else: + raise e + if current_tree != recreated_tree: print("Merge commit {} is not clean".format(current_commit), file=sys.stderr) - subprocess.call([GIT, 'diff', current_commit]) - subprocess.call([GIT, 'checkout', '--force', '--quiet', branch]) + subprocess.call([GIT, 'diff', recreated_tree, current_tree]) sys.exit(1) - subprocess.call([GIT, 'checkout', '--force', '--quiet', branch]) prev_commit = current_commit current_commit = parents[0] diff --git a/contrib/verifybinaries/README.md b/contrib/verifybinaries/README.md deleted file mode 100644 index c50d4bef7151..000000000000 --- a/contrib/verifybinaries/README.md +++ /dev/null @@ -1,41 +0,0 @@ -### Verify Binaries - -#### Preparation: - -Make sure you obtain the proper release signing key and verify the fingerprint with several independent sources. - -```sh -$ gpg --fingerprint "Bitcoin Core binary release signing key" -pub 4096R/36C2E964 2015-06-24 [expires: YYYY-MM-DD] - Key fingerprint = 01EA 5486 DE18 A882 D4C2 6845 90C8 019E 36C2 E964 -uid Wladimir J. van der Laan (Bitcoin Core binary release signing key) -``` - -#### Usage: - -This script attempts to download the signature file `SHA256SUMS.asc` from https://bitcoin.org. - -It first checks if the signature passes, and then downloads the files specified in the file, and checks if the hashes of these files match those that are specified in the signature file. - -The script returns 0 if everything passes the checks. It returns 1 if either the signature check or the hash check doesn't pass. If an error occurs the return value is 2. - - -```sh -./verify.py bitcoin-core-0.11.2 -./verify.py bitcoin-core-0.12.0 -./verify.py bitcoin-core-0.13.0-rc3 -``` - -If you only want to download the binaries of certain platform, add the corresponding suffix, e.g.: - -```sh -./verify.py bitcoin-core-0.11.2-osx -./verify.py 0.12.0-linux -./verify.py bitcoin-core-0.13.0-rc3-win64 -``` - -If you do not want to keep the downloaded binaries, specify anything as the second parameter. - -```sh -./verify.py bitcoin-core-0.13.0 delete -``` diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py deleted file mode 100755 index b5e4f1318b95..000000000000 --- a/contrib/verifybinaries/verify.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Script for verifying Bitcoin Core release binaries - -This script attempts to download the signature file SHA256SUMS.asc from -bitcoincore.org and bitcoin.org and compares them. -It first checks if the signature passes, and then downloads the files -specified in the file, and checks if the hashes of these files match those -that are specified in the signature file. -The script returns 0 if everything passes the checks. It returns 1 if either -the signature check or the hash check doesn't pass. If an error occurs the -return value is >= 2. -""" -from hashlib import sha256 -import os -import subprocess -import sys -from textwrap import indent - -WORKINGDIR = "/tmp/bitcoin_verify_binaries" -HASHFILE = "hashes.tmp" -HOST1 = "https://bitcoincore.org" -HOST2 = "https://bitcoin.org" -VERSIONPREFIX = "bitcoin-core-" -SIGNATUREFILENAME = "SHA256SUMS.asc" - - -def parse_version_string(version_str): - if version_str.startswith(VERSIONPREFIX): # remove version prefix - version_str = version_str[len(VERSIONPREFIX):] - - parts = version_str.split('-') - version_base = parts[0] - version_rc = "" - version_os = "" - if len(parts) == 2: # "-rcN" or "version-platform" - if "rc" in parts[1]: - version_rc = parts[1] - else: - version_os = parts[1] - elif len(parts) == 3: # "-rcN-platform" - version_rc = parts[1] - version_os = parts[2] - - return version_base, version_rc, version_os - - -def download_with_wget(remote_file, local_file=None): - if local_file: - wget_args = ['wget', '-O', local_file, remote_file] - else: - # use timestamping mechanism if local filename is not explicitly set - wget_args = ['wget', '-N', remote_file] - - result = subprocess.run(wget_args, - stderr=subprocess.STDOUT, stdout=subprocess.PIPE) - return result.returncode == 0, result.stdout.decode().rstrip() - - -def files_are_equal(filename1, filename2): - with open(filename1, 'rb') as file1: - contents1 = file1.read() - with open(filename2, 'rb') as file2: - contents2 = file2.read() - return contents1 == contents2 - - -def verify_with_gpg(signature_filename, output_filename): - result = subprocess.run(['gpg', '--yes', '--decrypt', '--output', - output_filename, signature_filename], - stderr=subprocess.STDOUT, stdout=subprocess.PIPE) - return result.returncode, result.stdout.decode().rstrip() - - -def remove_files(filenames): - for filename in filenames: - os.remove(filename) - - -def main(args): - # sanity check - if len(args) < 1: - print("Error: need to specify a version on the command line") - return 3 - - # determine remote dir dependent on provided version string - version_base, version_rc, os_filter = parse_version_string(args[0]) - remote_dir = f"/bin/{VERSIONPREFIX}{version_base}/" - if version_rc: - remote_dir += f"test.{version_rc}/" - remote_sigfile = remote_dir + SIGNATUREFILENAME - - # create working directory - os.makedirs(WORKINGDIR, exist_ok=True) - os.chdir(WORKINGDIR) - - # fetch first signature file - sigfile1 = SIGNATUREFILENAME - success, output = download_with_wget(HOST1 + remote_sigfile, sigfile1) - if not success: - print("Error: couldn't fetch signature file. " - "Have you specified the version number in the following format?") - print(f"[{VERSIONPREFIX}][-rc[0-9]][-platform] " - f"(example: {VERSIONPREFIX}0.21.0-rc3-osx)") - print("wget output:") - print(indent(output, '\t')) - return 4 - - # fetch second signature file - sigfile2 = SIGNATUREFILENAME + ".2" - success, output = download_with_wget(HOST2 + remote_sigfile, sigfile2) - if not success: - print("bitcoin.org failed to provide signature file, " - "but bitcoincore.org did?") - print("wget output:") - print(indent(output, '\t')) - remove_files([sigfile1]) - return 5 - - # ensure that both signature files are equal - if not files_are_equal(sigfile1, sigfile2): - print("bitcoin.org and bitcoincore.org signature files were not equal?") - print(f"See files {WORKINGDIR}/{sigfile1} and {WORKINGDIR}/{sigfile2}") - return 6 - - # check signature and extract data into file - retval, output = verify_with_gpg(sigfile1, HASHFILE) - if retval != 0: - if retval == 1: - print("Bad signature.") - elif retval == 2: - print("gpg error. Do you have the Bitcoin Core binary release " - "signing key installed?") - print("gpg output:") - print(indent(output, '\t')) - remove_files([sigfile1, sigfile2, HASHFILE]) - return 1 - - # extract hashes/filenames of binaries to verify from hash file; - # each line has the following format: " " - with open(HASHFILE, 'r', encoding='utf8') as hash_file: - hashes_to_verify = [ - line.split()[:2] for line in hash_file if os_filter in line] - remove_files([HASHFILE]) - if not hashes_to_verify: - print("error: no files matched the platform specified") - return 7 - - # download binaries - for _, binary_filename in hashes_to_verify: - print(f"Downloading {binary_filename}") - download_with_wget(HOST1 + remote_dir + binary_filename) - - # verify hashes - offending_files = [] - for hash_expected, binary_filename in hashes_to_verify: - with open(binary_filename, 'rb') as binary_file: - hash_calculated = sha256(binary_file.read()).hexdigest() - if hash_calculated != hash_expected: - offending_files.append(binary_filename) - if offending_files: - print("Hashes don't match.") - print("Offending files:") - print('\n'.join(offending_files)) - return 1 - verified_binaries = [entry[1] for entry in hashes_to_verify] - - # clean up files if desired - if len(args) >= 2: - print("Clean up the binaries") - remove_files([sigfile1, sigfile2] + verified_binaries) - else: - print(f"Keep the binaries in {WORKINGDIR}") - - print("Verified hashes of") - print('\n'.join(verified_binaries)) - return 0 - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/depends/Makefile b/depends/Makefile index 11fdd6dd53e1..316911763350 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -32,6 +32,8 @@ SOURCES_PATH ?= $(BASEDIR)/sources WORK_PATH = $(BASEDIR)/work BASE_CACHE ?= $(BASEDIR)/built SDK_PATH ?= $(BASEDIR)/SDKs +NO_BOOST ?= +NO_LIBEVENT ?= NO_QT ?= NO_QR ?= NO_BDB ?= @@ -43,6 +45,7 @@ NO_USDT ?= NO_NATPMP ?= MULTIPROCESS ?= LTO ?= +NO_HARDEN ?= FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources C_STANDARD ?= c11 @@ -144,8 +147,12 @@ include packages/packages.mk # 2. Before including packages/*.mk (excluding packages/packages.mk), since # they rely on the build_id variables # -build_id:=$(shell env CC='$(build_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(build_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') -$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(host_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') +build_id:=$(shell env CC='$(build_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(build_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' NO_HARDEN='$(NO_HARDEN)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') +$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(host_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' NO_HARDEN='$(NO_HARDEN)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') + +boost_packages_$(NO_BOOST) = $(boost_packages) + +libevent_packages_$(NO_LIBEVENT) = $(libevent_packages) qrencode_packages_$(NO_QR) = $(qrencode_$(host_os)_packages) @@ -162,7 +169,7 @@ zmq_packages_$(NO_ZMQ) = $(zmq_packages) multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages) usdt_packages_$(NO_USDT) = $(usdt_$(host_os)_packages) -packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) $(usdt_packages_) +packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(boost_packages_) $(libevent_packages_) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) $(usdt_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) ifneq ($(zmq_packages_),) @@ -247,6 +254,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@no_natpmp@|$(NO_NATPMP)|' \ -e 's|@multiprocess@|$(MULTIPROCESS)|' \ -e 's|@lto@|$(LTO)|' \ + -e 's|@no_harden@|$(NO_HARDEN)|' \ -e 's|@debug@|$(DEBUG)|' \ $< > $@ touch $@ diff --git a/depends/README.md b/depends/README.md index a8831eb0fce5..1064b7d18a66 100644 --- a/depends/README.md +++ b/depends/README.md @@ -98,6 +98,8 @@ The following can be set when running make: `make FOO=bar` - `FALLBACK_DOWNLOAD_PATH`: If a source file can't be fetched, try here before giving up - `C_STANDARD`: Set the C standard version used. Defaults to `c11`. - `CXX_STANDARD`: Set the C++ standard version used. Defaults to `c++17`. +- `NO_BOOST`: Don't download/build/cache Boost +- `NO_LIBEVENT`: Don't download/build/cache Libevent - `NO_QT`: Don't download/build/cache Qt and its dependencies - `NO_QR`: Don't download/build/cache packages needed for enabling qrencode - `NO_ZMQ`: Don't download/build/cache packages needed for enabling ZeroMQ @@ -105,7 +107,8 @@ The following can be set when running make: `make FOO=bar` - `NO_BDB`: Don't download/build/cache BerkeleyDB - `NO_SQLITE`: Don't download/build/cache SQLite - `NO_UPNP`: Don't download/build/cache packages needed for enabling UPnP -- `NO_NATPMP`: Don't download/build/cache packages needed for enabling NAT-PMP +- `NO_NATPMP`: Don't download/build/cache packages needed for enabling NAT-PMP +- `NO_USDT`: Don't download/build/cache packages needed for enabling USDT tracepoints - `ALLOW_HOST_PACKAGES`: Packages that are missed in dependencies (due to `NO_*` option or build script logic) are searched for among the host system packages using `pkg-config`. It allows building with packages of other (newer) versions @@ -120,6 +123,7 @@ The following can be set when running make: `make FOO=bar` resides in the `depends` directory, and the log file is printed out automatically in case of build error. After successful build log files are moved along with package archives - `LTO`: Use LTO when building packages. +- `NO_HARDEN=1`: Don't use hardening options when building packages If some packages are not built, for example `make NO_WALLET=1`, the appropriate options will be passed to bitcoin's configure. In this case, `--disable-wallet`. diff --git a/depends/config.site.in b/depends/config.site.in index 8f6849214d16..05c2ccbac104 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -82,6 +82,10 @@ if test -z "$enable_lto" && test -n "@lto@"; then enable_lto=yes fi +if test -z "$enable_hardening" && test -n "@no_harden@"; then + enable_hardening=no +fi + PKG_CONFIG="$(which pkg-config) --static" PKG_CONFIG_PATH="${depends_prefix}/share/pkgconfig:${depends_prefix}/lib/pkgconfig" diff --git a/depends/funcs.mk b/depends/funcs.mk index a00f3802363a..f0bbf4a168a6 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -76,7 +76,7 @@ $(1)_extracted=$$($(1)_extract_dir)/.stamp_extracted $(1)_preprocessed=$$($(1)_extract_dir)/.stamp_preprocessed $(1)_cleaned=$$($(1)_extract_dir)/.stamp_cleaned $(1)_built=$$($(1)_build_dir)/.stamp_built -$(1)_configured=$$($(1)_build_dir)/.stamp_configured +$(1)_configured=$(host_prefix)/.$(1)_stamp_configured $(1)_staged=$$($(1)_staging_dir)/.stamp_staged $(1)_postprocessed=$$($(1)_staging_prefix_dir)/.stamp_postprocessed $(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path)) @@ -87,9 +87,9 @@ $(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path)) $(1)_fetch_cmds ?= $(call fetch_file,$(1),$(subst \:,:,$$($(1)_download_path_fixed)),$$($(1)_download_file),$($(1)_file_name),$($(1)_sha256_hash)) $(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash) $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_TAR) --no-same-owner --strip-components=1 -xf $$($(1)_source) $(1)_preprocess_cmds ?= true -$(1)_build_cmds ?= -$(1)_config_cmds ?= -$(1)_stage_cmds ?= +$(1)_build_cmds ?= true +$(1)_config_cmds ?= true +$(1)_stage_cmds ?= true $(1)_set_vars ?= @@ -137,6 +137,7 @@ $(1)_config_env+=$($(1)_config_env_$(host_arch)_$(host_os)) $($(1)_config_env_$( $(1)_config_env+=PKG_CONFIG_LIBDIR=$($($(1)_type)_prefix)/lib/pkgconfig $(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig +$(1)_config_env+=PKG_CONFIG_SYSROOT_DIR=/ $(1)_config_env+=CMAKE_MODULE_PATH=$($($(1)_type)_prefix)/lib/cmake $(1)_config_env+=PATH=$(build_prefix)/bin:$(PATH) $(1)_build_env+=PATH=$(build_prefix)/bin:$(PATH) @@ -213,18 +214,18 @@ $($(1)_preprocessed): | $($(1)_extracted) $($(1)_configured): | $($(1)_dependencies) $($(1)_preprocessed) echo Configuring $(1)... rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), $(build_TAR) --no-same-owner -xf $($(package)_cached); ) - mkdir -p $$(@D) - +{ cd $$(@D); $($(1)_config_env) $($(1)_config_cmds); } $$($(1)_logging) + mkdir -p $$($(1)_build_dir) + +{ cd $$($(1)_build_dir); export $($(1)_config_env); $($(1)_config_cmds); } $$($(1)_logging) touch $$@ $($(1)_built): | $($(1)_configured) echo Building $(1)... mkdir -p $$(@D) - +{ cd $$(@D); $($(1)_build_env) $($(1)_build_cmds); } $$($(1)_logging) + +{ cd $$(@D); export $($(1)_build_env); $($(1)_build_cmds); } $$($(1)_logging) touch $$@ $($(1)_staged): | $($(1)_built) echo Staging $(1)... mkdir -p $($(1)_staging_dir)/$(host_prefix) - +{ cd $($(1)_build_dir); $($(1)_stage_env) $($(1)_stage_cmds); } $$($(1)_logging) + +{ cd $($(1)_build_dir); export $($(1)_stage_env); $($(1)_stage_cmds); } $$($(1)_logging) rm -rf $($(1)_extract_dir) touch $$@ $($(1)_postprocessed): | $($(1)_staged) @@ -233,7 +234,9 @@ $($(1)_postprocessed): | $($(1)_staged) touch $$@ $($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed) echo Caching $(1)... - cd $$($(1)_staging_dir)/$(host_prefix); find . | sort | $(build_TAR) --no-recursion -czf $$($(1)_staging_dir)/$$(@F) -T - + cd $$($(1)_staging_dir)/$(host_prefix); \ + find . ! -name '.stamp_postprocessed' -print0 | TZ=UTC xargs -0r touch -h -m -t 200001011200; \ + find . ! -name '.stamp_postprocessed' | LC_ALL=C sort | $(build_TAR) --numeric-owner --no-recursion -czf $$($(1)_staging_dir)/$$(@F) -T - mkdir -p $$(@D) rm -rf $$(@D) && mkdir -p $$(@D) mv $$($(1)_staging_dir)/$$(@F) $$(@) diff --git a/depends/gen_id b/depends/gen_id index 7caf8d764d13..3341310e460a 100755 --- a/depends/gen_id +++ b/depends/gen_id @@ -2,7 +2,7 @@ # Usage: env [ CC=... ] [ C_STANDARD=...] [ CXX=... ] [CXX_STANDARD=...] \ # [ AR=... ] [ RANLIB=... ] [ STRIP=... ] [ DEBUG=... ] \ -# [ LTO=... ] ./build-id [ID_SALT]... +# [ LTO=... ] [ NO_HARDEN=... ] ./build-id [ID_SALT]... # # Prints to stdout a SHA256 hash representing the current toolset, used by # depends/Makefile as a build id for caching purposes (detecting when the @@ -70,6 +70,10 @@ echo "LTO=${LTO}" echo "END LTO" + echo "BEGIN NO_HARDEN" + echo "NO_HARDEN=${NO_HARDEN}" + echo "END NO_HARDEN" + echo "END ALL" ) | if [ -n "$DEBUG" ] && command -v tee > /dev/null 2>&1; then # When debugging and `tee` is available, output the preimage to stderr diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 8fcea35d9875..522a6b17efc1 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -99,8 +99,8 @@ darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ -isysroot$(OSX_SDK) \ - -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ - -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include + -Xclang -internal-externc-isystem -Xclang $(clang_resource_dir)/include \ + -Xclang -internal-externc-isystem -Xclang $(OSX_SDK)/usr/include darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ -u LIBRARY_PATH \ @@ -109,8 +109,8 @@ darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -isysroot$(OSX_SDK) \ -stdlib=libc++ \ -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 \ - -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ - -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include + -Xclang -internal-externc-isystem -Xclang $(clang_resource_dir)/include \ + -Xclang -internal-externc-isystem -Xclang $(OSX_SDK)/usr/include darwin_CFLAGS=-pipe -std=$(C_STANDARD) darwin_CXXFLAGS=-pipe -std=$(CXX_STANDARD) diff --git a/depends/hosts/default.mk b/depends/hosts/default.mk index 7c76331ab4a4..bad4568bcb1d 100644 --- a/depends/hosts/default.mk +++ b/depends/hosts/default.mk @@ -28,8 +28,13 @@ host_$1=$$($(host_arch)_$(host_os)_$1) endef define add_host_flags_func +ifeq ($(filter $(origin $1),undefined default),) +$(host_arch)_$(host_os)_$1 = +$(host_arch)_$(host_os)_$(release_type)_$1 = $($1) +else $(host_arch)_$(host_os)_$1 += $($(host_os)_$1) $(host_arch)_$(host_os)_$(release_type)_$1 += $($(host_os)_$(release_type)_$1) +endif host_$1 = $$($(host_arch)_$(host_os)_$1) host_$(release_type)_$1 = $$($(host_arch)_$(host_os)_$(release_type)_$1) endef diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk index 262587690c03..d60733605976 100644 --- a/depends/packages/bdb.mk +++ b/depends/packages/bdb.mk @@ -15,7 +15,7 @@ $(package)_config_opts_netbsd=--with-pic $(package)_config_opts_openbsd=--with-pic $(package)_config_opts_android=--with-pic $(package)_cflags+=-Wno-error=implicit-function-declaration -Wno-error=format-security -$(package)_cppflags_freebsd=-D_XOPEN_SOURCE=600 +$(package)_cppflags_freebsd=-D_XOPEN_SOURCE=600 -D__BSD_VISIBLE=1 $(package)_cppflags_netbsd=-D_XOPEN_SOURCE=600 $(package)_cppflags_openbsd=-D_XOPEN_SOURCE=600 $(package)_cppflags_mingw32=-DUNICODE -D_UNICODE diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index e3625d97f566..ebc097d686f5 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -1,8 +1,8 @@ package=boost -$(package)_version=1.80.0 +$(package)_version=1.81.0 $(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/$($(package)_version)/source/ $(package)_file_name=boost_$(subst .,_,$($(package)_version)).tar.bz2 -$(package)_sha256_hash=1e19565d82e43bc59209a168f5ac899d3ba471d55c7610c677d4ccf2c9c500c0 +$(package)_sha256_hash=71feeed900fbccca04a3b4f2f84a7c217186f28a940ed8b7ed4725986baf99fa define $(package)_stage_cmds mkdir -p $($(package)_staging_prefix_dir)/include && \ diff --git a/depends/packages/fontconfig.mk b/depends/packages/fontconfig.mk index c8b2fc33d573..444acfe36d74 100644 --- a/depends/packages/fontconfig.mk +++ b/depends/packages/fontconfig.mk @@ -9,6 +9,7 @@ $(package)_patches=gperf_header_regen.patch define $(package)_set_vars $(package)_config_opts=--disable-docs --disable-static --disable-libxml2 --disable-iconv $(package)_config_opts += --disable-dependency-tracking --enable-option-checking + $(package)_cflags += -Wno-implicit-function-declaration endef define $(package)_preprocess_cmds diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 5bd12522a7d5..9650f77db920 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -17,6 +17,10 @@ define $(package)_set_vars $(package)_config_opts_openbsd=--with-pic $(package)_config_opts_android=--with-pic $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0601 + + ifeq ($(NO_HARDEN),) + $(package)_cppflags+=-D_FORTIFY_SOURCE=3 + endif endef define $(package)_preprocess_cmds diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk index 9b66207fc506..6da5693b3fca 100644 --- a/depends/packages/libmultiprocess.mk +++ b/depends/packages/libmultiprocess.mk @@ -24,5 +24,5 @@ define $(package)_build_cmds endef define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install + $(MAKE) DESTDIR=$($(package)_staging_dir) install-lib endef diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk index 6e600c5720f5..e647afba5f35 100644 --- a/depends/packages/native_libmultiprocess.mk +++ b/depends/packages/native_libmultiprocess.mk @@ -1,8 +1,8 @@ package=native_libmultiprocess -$(package)_version=d576d975debdc9090bd2582f83f49c76c0061698 +$(package)_version=1af83d15239ccfa7e47b8764029320953dd7fdf1 $(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=9f8b055c8bba755dc32fe799b67c20b91e7b13e67cadafbc54c0f1def057a370 +$(package)_sha256_hash=e5587d3feedc7f8473f178a89b94163a11076629825d664964799bbbd5844da5 $(package)_dependencies=native_capnp define $(package)_config_cmds @@ -14,5 +14,5 @@ define $(package)_build_cmds endef define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install + $(MAKE) DESTDIR=$($(package)_staging_dir) install-bin endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 998cc0221c64..b3600b72d0b5 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -1,4 +1,8 @@ -packages:=boost libevent +packages:= + +boost_packages = boost + +libevent_packages = libevent qrencode_linux_packages = qrencode qrencode_android_packages = qrencode diff --git a/depends/packages/qrencode.mk b/depends/packages/qrencode.mk index d1687883bcd5..2afd95d7c4fa 100644 --- a/depends/packages/qrencode.mk +++ b/depends/packages/qrencode.mk @@ -1,15 +1,16 @@ package=qrencode -$(package)_version=3.4.4 +$(package)_version=4.1.1 $(package)_download_path=https://fukuchi.org/works/qrencode/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=efe5188b1ddbcbf98763b819b146be6a90481aac30cfc8d858ab78a19cde1fa5 +$(package)_sha256_hash=e455d9732f8041cf5b9c388e345a641fd15707860f928e94507b1961256a6923 define $(package)_set_vars -$(package)_config_opts=--disable-shared --without-tools --without-tests --disable-sdltest +$(package)_config_opts=--disable-shared --without-tools --without-tests --without-png $(package)_config_opts += --disable-gprof --disable-gcov --disable-mudflap $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts_linux=--with-pic $(package)_config_opts_android=--with-pic +$(package)_cflags += -Wno-int-conversion -Wno-implicit-function-declaration endef define $(package)_preprocess_cmds diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index d9ae918d7119..136ce325798a 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -33,6 +33,7 @@ $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) define $(package)_set_vars +$(package)_config_env = QT_MAC_SDK_NO_VERSION_CHECK=1 $(package)_config_opts_release = -release $(package)_config_opts_release += -silent $(package)_config_opts_debug = -debug @@ -160,9 +161,15 @@ $(package)_config_opts_linux += -dbus-runtime ifneq ($(LTO),) $(package)_config_opts_linux += -ltcg endif -$(package)_config_opts_linux += -platform linux-g++ -xplatform bitcoin-linux-g++ -ifneq (,$(findstring -stdlib=libc++,$($(1)_cxx))) -$(package)_config_opts_x86_64_linux = -xplatform linux-clang-libc++ + +ifneq (,$(findstring clang,$($(package)_cxx))) + ifneq (,$(findstring -stdlib=libc++,$($(package)_cxx))) + $(package)_config_opts_linux += -platform linux-clang-libc++ -xplatform linux-clang-libc++ + else + $(package)_config_opts_linux += -platform linux-clang -xplatform linux-clang + endif +else + $(package)_config_opts_linux += -platform linux-g++ -xplatform bitcoin-linux-g++ endif $(package)_config_opts_mingw32 = -no-opengl @@ -261,9 +268,6 @@ define $(package)_preprocess_cmds endef define $(package)_config_cmds - export PKG_CONFIG_SYSROOT_DIR=/ && \ - export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \ - export QT_MAC_SDK_NO_VERSION_CHECK=1 && \ cd qtbase && \ ./configure -top-level $($(package)_config_opts) endef diff --git a/depends/packages/sqlite.mk b/depends/packages/sqlite.mk index 820d724214a2..a8ec89c6c647 100644 --- a/depends/packages/sqlite.mk +++ b/depends/packages/sqlite.mk @@ -6,10 +6,15 @@ $(package)_sha256_hash=5af07de982ba658fd91a03170c945f99c971f6955bc79df3266544373 define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-readline --disable-dynamic-extensions --enable-option-checking +$(package)_config_opts+= --disable-rtree --disable-fts4 --disable-fts5 $(package)_config_opts_linux=--with-pic $(package)_config_opts_freebsd=--with-pic $(package)_config_opts_netbsd=--with-pic $(package)_config_opts_openbsd=--with-pic +$(package)_config_opts_debug=--enable-debug +$(package)_cflags+=-DSQLITE_DQS=0 -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED +$(package)_cflags+=-DSQLITE_OMIT_SHARED_CACHE -DSQLITE_OMIT_JSON -DSQLITE_LIKE_DOESNT_MATCH_BLOBS +$(package)_cflags+=-DSQLITE_OMIT_DECLTYPE -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_AUTOINIT endef define $(package)_preprocess_cmds diff --git a/depends/packages/systemtap.mk b/depends/packages/systemtap.mk index a57f1b6d36a5..541ebeee0140 100644 --- a/depends/packages/systemtap.mk +++ b/depends/packages/systemtap.mk @@ -1,12 +1,13 @@ package=systemtap -$(package)_version=4.7 -$(package)_download_path=https://sourceware.org/systemtap/ftp/releases/ +$(package)_version=4.8 +$(package)_download_path=https://sourceware.org/ftp/systemtap/releases/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=43a0a3db91aa4d41e28015b39a65e62059551f3cc7377ebf3a3a5ca7339e7b1f -$(package)_patches=remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch +$(package)_sha256_hash=cbd50a4eba5b261394dc454c12448ddec73e55e6742fda7f508f9fbc1331c223 +$(package)_patches=remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch fix_variadic_warning.patch define $(package)_preprocess_cmds patch -p1 < $($(package)_patch_dir)/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch && \ + patch -p1 < $($(package)_patch_dir)/fix_variadic_warning.patch && \ mkdir -p $($(package)_staging_prefix_dir)/include/sys && \ cp includes/sys/sdt.h $($(package)_staging_prefix_dir)/include/sys/sdt.h endef diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index 267ed1125318..d7152327934f 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -20,12 +20,12 @@ endef define $(package)_preprocess_cmds patch -p1 < $($(package)_patch_dir)/remove_libstd_link.patch && \ - patch -p1 < $($(package)_patch_dir)/netbsd_kevent_void.patch && \ - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config + patch -p1 < $($(package)_patch_dir)/netbsd_kevent_void.patch endef define $(package)_config_cmds ./autogen.sh && \ + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config && \ $($(package)_autoconf) endef diff --git a/depends/patches/systemtap/fix_variadic_warning.patch b/depends/patches/systemtap/fix_variadic_warning.patch new file mode 100644 index 000000000000..93cc2d6081d7 --- /dev/null +++ b/depends/patches/systemtap/fix_variadic_warning.patch @@ -0,0 +1,16 @@ +Could be dropped after a migration to C++20. +See: https://github.com/bitcoin/bitcoin/issues/26916. + +diff --git a/includes/sys/sdt.h b/includes/sys/sdt.h +index 4075a5f..7c6138c 100644 +--- a/includes/sys/sdt.h ++++ b/includes/sys/sdt.h +@@ -276,7 +276,7 @@ __extension__ extern unsigned long long __sdt_unsp; + _SDT_ASM_1(.purgem _SDT_TYPE_) \ + _SDT_ASM_1(.purgem _SDT_TYPE) + +-#define _SDT_ASM_BODY(provider, name, pack_args, args, ...) \ ++#define _SDT_ASM_BODY(provider, name, pack_args, args) \ + _SDT_DEF_MACROS \ + _SDT_ASM_1(990: _SDT_NOP) \ + _SDT_ASM_3( .pushsection .note.stapsdt,_SDT_ASM_AUTOGROUP,"note") \ diff --git a/doc/REST-interface.md b/doc/REST-interface.md index a704b969df8a..2d7d0e376936 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -134,11 +134,15 @@ Returns various information about the transaction mempool. Only supports JSON as output format. Refer to the `getmempoolinfo` RPC help for details. -`GET /rest/mempool/contents.json` +`GET /rest/mempool/contents.json?verbose=&mempool_sequence=` Returns the transactions in the mempool. Only supports JSON as output format. -Refer to the `getrawmempool` RPC help for details. +Refer to the `getrawmempool` RPC help for details. Defaults to setting +`verbose=true` and `mempool_sequence=false`. + +*Query parameters for `verbose` and `mempool_sequence` available in 25.0 and up.* + Risks ------------- diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index a8e643a2ab0e..aa10e4a8919b 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -36,13 +36,30 @@ pkg install sqlite3 ``` ###### Legacy Wallet Support -`db5` is only required to support legacy wallets. -Skip if you don't intend to use legacy wallets. +BerkeleyDB is only required if legacy wallet support is required. + +It is required to use Berkeley DB 4.8. You **cannot** use the BerkeleyDB library +from ports. However, you can build DB 4.8 yourself [using depends](/depends). -```bash -pkg install db5 ``` ---- +gmake -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_NATPMP=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 +``` + +When the build is complete, the Berkeley DB installation location will be displayed: + +``` +to: /path/to/bitcoin/depends/x86_64-unknown-freebsd[release-number] +``` + +Finally, set `BDB_PREFIX` to this path according to your shell: + +``` +csh: setenv BDB_PREFIX [path displayed above] +``` + +``` +sh/bash: export BDB_PREFIX=[path displayed above] +``` #### GUI Dependencies ###### Qt5 @@ -72,7 +89,7 @@ There is an included test suite that is useful for testing code changes when dev To run the test suite (recommended), you will need to have Python 3 installed: ```bash -pkg install python3 +pkg install python3 databases/py-sqlite3 ``` --- @@ -91,12 +108,12 @@ This explicitly enables the GUI and disables legacy wallet support, assuming `sq ##### Descriptor & Legacy Wallet. No GUI: This enables support for both wallet types and disables the GUI, assuming -`sqlite3` and `db5` are both installed. +`sqlite3` and `db4` are both installed. ```bash ./autogen.sh -./configure --with-gui=no --with-incompatible-bdb \ - BDB_LIBS="-ldb_cxx-5" \ - BDB_CFLAGS="-I/usr/local/include/db5" \ +./configure --with-gui=no \ + BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ + BDB_CFLAGS="-I${BDB_PREFIX}/include" \ MAKE=gmake ``` diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index afbb5c8e7584..96ee714341ff 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -1,6 +1,6 @@ # OpenBSD Build Guide -**Updated for OpenBSD [7.1](https://www.openbsd.org/71.html)** +**Updated for OpenBSD [7.3](https://www.openbsd.org/73.html)** This guide describes how to build bitcoind, command-line utilities, and GUI on OpenBSD. @@ -41,16 +41,18 @@ pkg_add sqlite3 BerkeleyDB is only required to support legacy wallets. It is recommended to use Berkeley DB 4.8. You cannot use the BerkeleyDB library -from ports. However you can build it yourself, [using the installation script included in contrib/](/contrib/install_db4.sh), like so, from the root of the repository. +from ports. However you can build it yourself, [using depends](/depends). ```bash -./contrib/install_db4.sh `pwd` +gmake -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_NATPMP=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 +... +to: /path/to/bitcoin/depends/x86_64-unknown-openbsd ``` Then set `BDB_PREFIX`: ```bash -export BDB_PREFIX="$PWD/db4" +export BDB_PREFIX="/path/to/bitcoin/depends/x86_64-unknown-openbsd" ``` #### GUI Dependencies @@ -78,10 +80,6 @@ export AUTOMAKE_VERSION=1.16 ### 1. Configuration -Note that external signer support is currently not available on OpenBSD, since -the used header-only library Boost.Process fails to compile (certain system -calls and preprocessor defines like `waitid()` and `WEXITED` are missing). - There are many ways to configure Bitcoin Core, here are a few common examples: ##### Descriptor Wallet and GUI: diff --git a/doc/build-unix.md b/doc/build-unix.md index 874015707a25..0960ae157773 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -72,7 +72,7 @@ executables, which are based on BerkeleyDB 4.8. If you do not care about wallet To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) -Optional port mapping libraries (see: `--with-miniupnpc`, `--enable-upnp-default`, and `--with-natpmp`, `--enable-natpmp-default`): +Optional port mapping libraries (see: `--with-miniupnpc` and `--with-natpmp`): sudo apt install libminiupnpc-dev libnatpmp-dev @@ -133,7 +133,7 @@ pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley D To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) -Optional port mapping libraries (see: `--with-miniupnpc`, `--enable-upnp-default`, and `--with-natpmp`, `--enable-natpmp-default`): +Optional port mapping libraries (see: `--with-miniupnpc` and `--with-natpmp`): sudo dnf install miniupnpc-devel libnatpmp-devel @@ -176,38 +176,34 @@ miniupnpc [miniupnpc](https://miniupnp.tuxfamily.org) may be used for UPnP port mapping. It can be downloaded from [here]( https://miniupnp.tuxfamily.org/files/). UPnP support is compiled in and -turned off by default. See the configure options for UPnP behavior desired: - - --without-miniupnpc No UPnP support, miniupnp not required - --disable-upnp-default (the default) UPnP support turned off by default at runtime - --enable-upnp-default UPnP support turned on by default at runtime +turned off by default. libnatpmp --------- [libnatpmp](https://miniupnp.tuxfamily.org/libnatpmp.html) may be used for NAT-PMP port mapping. It can be downloaded from [here](https://miniupnp.tuxfamily.org/files/). NAT-PMP support is compiled in and -turned off by default. See the configure options for NAT-PMP behavior desired: - - --without-natpmp No NAT-PMP support, libnatpmp not required - --disable-natpmp-default (the default) NAT-PMP support turned off by default at runtime - --enable-natpmp-default NAT-PMP support turned on by default at runtime +turned off by default. Berkeley DB ----------- The legacy wallet uses Berkeley DB. To ensure backwards compatibility it is -recommended to use Berkeley DB 4.8. If you have to build it yourself, you can -use [the installation script included in contrib/](/contrib/install_db4.sh) -like so: - -```shell -./contrib/install_db4.sh `pwd` +recommended to use Berkeley DB 4.8. If you have to build it yourself, and don't +want to use any other libraries built in depends, you can do: +```bash +make -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_NATPMP=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 +... +to: /path/to/bitcoin/depends/x86_64-pc-linux-gnu ``` +and configure using the following: +```bash +export BDB_PREFIX="/path/to/bitcoin/depends/x86_64-pc-linux-gnu" -from the root of the repository. - -Otherwise, you can build Bitcoin Core from self-compiled [depends](/depends/README.md). +./configure \ + BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ + BDB_CFLAGS="-I${BDB_PREFIX}/include" +``` **Note**: You only need Berkeley DB if the legacy wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). diff --git a/doc/build-windows.md b/doc/build-windows.md index e35d3bcbd09a..027e8f80f50f 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -6,8 +6,8 @@ Below are some notes on how to build Bitcoin Core for Windows. The options known to work for building Bitcoin Core on Windows are: * On Linux, using the [Mingw-w64](https://www.mingw-w64.org/) cross compiler tool chain. -* On Windows, using [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about) and Mingw-w64. -* On Windows, using [Microsoft Visual Studio](https://www.visualstudio.com). See [README.md](/build_msvc/README.md). +* On Windows, using [Windows Subsystem for Linux (WSL)](https://learn.microsoft.com/en-us/windows/wsl/about) and Mingw-w64. +* On Windows, using [Microsoft Visual Studio](https://visualstudio.microsoft.com). See [README.md](/build_msvc/README.md). Other options which may work, but which have not been extensively tested are (please contribute instructions): @@ -16,7 +16,7 @@ Other options which may work, but which have not been extensively tested are (pl Installing Windows Subsystem for Linux --------------------------------------- -Follow the upstream installation instructions, available [here](https://docs.microsoft.com/windows/wsl/install-win10). +Follow the upstream installation instructions, available [here](https://learn.microsoft.com/en-us/windows/wsl/install). Cross-compilation for Ubuntu and Windows Subsystem for Linux ------------------------------------------------------------ @@ -66,7 +66,7 @@ Note that for WSL the Bitcoin Core source path MUST be somewhere in the default example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. This means you cannot use a directory that is located directly on the host Windows file system to perform the build. -Additional WSL Note: WSL support for [launching Win32 applications](https://docs.microsoft.com/en-us/archive/blogs/wsl/windows-and-ubuntu-interoperability#launching-win32-applications-from-within-wsl) +Additional WSL Note: WSL support for [launching Win32 applications](https://learn.microsoft.com/en-us/archive/blogs/wsl/windows-and-ubuntu-interoperability#launching-win32-applications-from-within-wsl) results in `Autoconf` configure scripts being able to execute Windows Portable Executable files. This can cause unexpected behaviour during the build, such as Win32 error dialogs for missing libraries. The recommended approach is to temporarily disable WSL support for Win32 applications. diff --git a/doc/dependencies.md b/doc/dependencies.md index ef8faff06c41..182d026c581e 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -10,16 +10,16 @@ You can find installation instructions in the `build-*.md` file for your platfor | [Automake](https://www.gnu.org/software/automake/) | [1.13](https://github.com/bitcoin/bitcoin/pull/18290) | | [Clang](https://clang.llvm.org) | [8.0](https://github.com/bitcoin/bitcoin/pull/24164) | | [GCC](https://gcc.gnu.org) | [8.1](https://github.com/bitcoin/bitcoin/pull/23060) | -| [Python](https://www.python.org) (tests) | [3.6](https://github.com/bitcoin/bitcoin/pull/19504) | +| [Python](https://www.python.org) (tests) | [3.7](https://github.com/bitcoin/bitcoin/pull/26226) | | [systemtap](https://sourceware.org/systemtap/) ([tracing](tracing.md))| N/A | ## Required | Dependency | Releases | Version used | Minimum required | Runtime | | --- | --- | --- | --- | --- | -| [Boost](../depends/packages/boost.mk) | [link](https://www.boost.org/users/download/) | [1.80.0](https://github.com/bitcoin/bitcoin/pull/25873) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | +| [Boost](../depends/packages/boost.mk) | [link](https://www.boost.org/users/download/) | [1.81.0](https://github.com/bitcoin/bitcoin/pull/26557) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | | [libevent](../depends/packages/libevent.mk) | [link](https://github.com/libevent/libevent/releases) | [2.1.12-stable](https://github.com/bitcoin/bitcoin/pull/21991) | [2.1.8](https://github.com/bitcoin/bitcoin/pull/24681) | No | -| glibc | [link](https://www.gnu.org/software/libc/) | N/A | [2.18](https://github.com/bitcoin/bitcoin/pull/23511) | Yes | +| glibc | [link](https://www.gnu.org/software/libc/) | N/A | [2.27](https://github.com/bitcoin/bitcoin/pull/27029) | Yes | | Linux Kernel | [link](https://www.kernel.org/) | N/A | 3.2.0 | Yes | ## Optional @@ -29,14 +29,14 @@ You can find installation instructions in the `build-*.md` file for your platfor | --- | --- | --- | --- | --- | | [Fontconfig](../depends/packages/fontconfig.mk) | [link](https://www.freedesktop.org/wiki/Software/fontconfig/) | [2.12.6](https://github.com/bitcoin/bitcoin/pull/23495) | 2.6 | Yes | | [FreeType](../depends/packages/freetype.mk) | [link](https://freetype.org) | [2.11.0](https://github.com/bitcoin/bitcoin/commit/01544dd78ccc0b0474571da854e27adef97137fb) | 2.3.0 | Yes | -| [qrencode](../depends/packages/qrencode.mk) | [link](https://fukuchi.org/works/qrencode/) | [3.4.4](https://github.com/bitcoin/bitcoin/pull/6373) | | No | +| [qrencode](../depends/packages/qrencode.mk) | [link](https://fukuchi.org/works/qrencode/) | [4.1.1](https://github.com/bitcoin/bitcoin/pull/27312) | | No | | [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.5](https://github.com/bitcoin/bitcoin/pull/25719) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | ### Networking | Dependency | Releases | Version used | Minimum required | Runtime | | --- | --- | --- | --- | --- | | [libnatpmp](../depends/packages/libnatpmp.mk) | [link](https://github.com/miniupnp/libnatpmp/) | commit [07004b9...](https://github.com/bitcoin/bitcoin/pull/25917) | | No | -| [MiniUPnPc](../depends/packages/miniupnpc.mk) | [link](https://miniupnp.tuxfamily.org/) | [2.2.2](https://github.com/bitcoin/bitcoin/pull/20421) | 1.9 | No | +| [MiniUPnPc](../depends/packages/miniupnpc.mk) | [link](https://miniupnp.tuxfamily.org/) | [2.2.2](https://github.com/bitcoin/bitcoin/pull/20421) | 2.1 | No | ### Notifications | Dependency | Releases | Version used | Minimum required | Runtime | @@ -47,4 +47,4 @@ You can find installation instructions in the `build-*.md` file for your platfor | Dependency | Releases | Version used | Minimum required | Runtime | | --- | --- | --- | --- | --- | | [Berkeley DB](../depends/packages/bdb.mk) (legacy wallet) | [link](https://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) | 4.8.30 | 4.8.x | No | -| [SQLite](../depends/packages/sqlite.mk) | [link](https://sqlite.org) | [3.32.1](https://github.com/bitcoin/bitcoin/pull/19077) | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | No | +| [SQLite](../depends/packages/sqlite.mk) | [link](https://sqlite.org) | [3.38.5](https://github.com/bitcoin/bitcoin/pull/25378) | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | No | diff --git a/doc/design/assumeutxo.md b/doc/design/assumeutxo.md index ea51b1b87f6b..469c5515360e 100644 --- a/doc/design/assumeutxo.md +++ b/doc/design/assumeutxo.md @@ -3,9 +3,9 @@ Assumeutxo is a feature that allows fast bootstrapping of a validating bitcoind instance with a very similar security model to assumevalid. -The RPC commands `dumptxoutset` and `loadtxoutset` are used to respectively generate -and load UTXO snapshots. The utility script `./contrib/devtools/utxo_snapshot.sh` may -be of use. +The RPC commands `dumptxoutset` and `loadtxoutset` (yet to be merged) are used to +respectively generate and load UTXO snapshots. The utility script +`./contrib/devtools/utxo_snapshot.sh` may be of use. ## General background @@ -22,10 +22,6 @@ be of use. chainstate running asynchronously in the background. We also use this flag to control which index entries are added to setBlockIndexCandidates during LoadBlockIndex(). -- Indexing implementations via BaseIndex can no longer assume that indexation happens - sequentially, since background validation chainstates can submit BlockConnected - events out of order with the active chain. - - The concept of UTXO snapshots is treated as an implementation detail that lives behind the ChainstateManager interface. The external presentation of the changes required to facilitate the use of UTXO snapshots is the understanding that there are @@ -76,9 +72,15 @@ original chainstate remains in use as active. Once the snapshot chainstate is loaded and validated, it is promoted to active chainstate and a sync to tip begins. A new chainstate directory is created in the -datadir for the snapshot chainstate called `chainstate_snapshot`. When this directory -is present in the datadir, the snapshot chainstate will be detected and loaded as -active on node startup (via `DetectSnapshotChainstate()`). +datadir for the snapshot chainstate called `chainstate_snapshot`. + +When this directory is present in the datadir, the snapshot chainstate will be detected +and loaded as active on node startup (via `DetectSnapshotChainstate()`). + +A special file is created within that directory, `base_blockhash`, which contains the +serialized `uint256` of the base block of the snapshot. This is used to reinitialize +the snapshot chainstate on subsequent inits. Otherwise, the directory is a normal +leveldb database. | | | | ---------- | ----------- | @@ -88,7 +90,7 @@ active on node startup (via `DetectSnapshotChainstate()`). The snapshot begins to sync to tip from its base block, technically in parallel with the original chainstate, but it is given priority during block download and is allocated most of the cache (see `MaybeRebalanceCaches()` and usages) as our chief -consideration is getting to network tip. +goal is getting to network tip. **Failure consideration:** if shutdown happens at any point during this phase, both chainstates will be detected during the next init and the process will resume. @@ -107,33 +109,32 @@ sequentially. ### Background chainstate hits snapshot base block Once the tip of the background chainstate hits the base block of the snapshot -chainstate, we stop use of the background chainstate by setting `m_stop_use` (not yet -committed - see #15606), in `CompleteSnapshotValidation()`, which is checked in -`ActivateBestChain()`). We hash the background chainstate's UTXO set contents and -ensure it matches the compiled value in `CMainParams::m_assumeutxo_data`. - -The background chainstate data lingers on disk until shutdown, when in -`ChainstateManager::Reset()`, the background chainstate is cleaned up with -`ValidatedSnapshotShutdownCleanup()`, which renames the `chainstate_[hash]` datadir as -`chainstate`. +chainstate, we stop use of the background chainstate by setting `m_disabled`, in +`CompleteSnapshotValidation()`, which is checked in `ActivateBestChain()`). We hash the +background chainstate's UTXO set contents and ensure it matches the compiled value in +`CMainParams::m_assumeutxo_data`. | | | | ---------- | ----------- | -| number of chainstates | 2 (ibd has `m_stop_use=true`) | +| number of chainstates | 2 (ibd has `m_disabled=true`) | | active chainstate | snapshot | -**Failure consideration:** if bitcoind unexpectedly halts after `m_stop_use` is set on -the background chainstate but before `CompleteSnapshotValidation()` can finish, the -need to complete snapshot validation will be detected on subsequent init by -`ChainstateManager::CheckForUncleanShutdown()`. +The background chainstate data lingers on disk until the program is restarted. ### Bitcoind restarts sometime after snapshot validation has completed -When bitcoind initializes again, what began as the snapshot chainstate is now -indistinguishable from a chainstate that has been built from the traditional IBD -process, and will be initialized as such. +After a shutdown and subsequent restart, `LoadChainstate()` cleans up the background +chainstate with `ValidatedSnapshotCleanup()`, which renames the `chainstate_snapshot` +datadir as `chainstate` and removes the now unnecessary background chainstate data. | | | | ---------- | ----------- | | number of chainstates | 1 | -| active chainstate | ibd | +| active chainstate | ibd (was snapshot, but is now fully validated) | + +What began as the snapshot chainstate is now indistinguishable from a chainstate that +has been built from the traditional IBD process, and will be initialized as such. + +A file will be left in `chainstate/base_blockhash`, which indicates that the +chainstate, even though now fully validated, was originally started from a snapshot +with the corresponding base blockhash. diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 00c68911efac..ceaba8cb99b1 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -109,6 +109,10 @@ code. - `++i` is preferred over `i++`. - `nullptr` is preferred over `NULL` or `(void*)0`. - `static_assert` is preferred over `assert` where possible. Generally; compile-time checking is preferred over run-time checking. + - Use a named cast or functional cast, not a C-Style cast. When casting + between integer types, use functional casts such as `int(x)` or `int{x}` + instead of `(int) x`. When casting between more complex types, use static_cast. + Use reinterpret_cast and const_cast as appropriate. For function calls a namespace should be specified explicitly, unless such functions have been declared within it. Otherwise, [argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl), also known as ADL, could be @@ -213,11 +217,14 @@ apt install clang-tidy bear clang Then, pass clang as compiler to configure, and use bear to produce the `compile_commands.json`: ```sh -./autogen.sh && ./configure CC=clang CXX=clang++ -make clean && bear make -j $(nproc) # For bear 2.x -make clean && bear -- make -j $(nproc) # For bear 3.x +./autogen.sh && ./configure CC=clang CXX=clang++ --enable-suppress-external-warnings +make clean && bear --config src/.bear-tidy-config -- make -j $(nproc) ``` +The output is denoised of errors from external dependencies and includes with +`--enable-suppress-external-warnings` and `--config src/.bear-tidy-config`. Both +options may be omitted to view the full list of errors. + To run clang-tidy on all source files: ```sh @@ -556,8 +563,19 @@ address sanitizer, libtsan for the thread sanitizer, and libubsan for the undefined sanitizer. If you are missing required libraries, the configure script will fail with a linker error when testing the sanitizer flags. -The test suite should pass cleanly with the `thread` and `undefined` sanitizers, -but there are a number of known problems when using the `address` sanitizer. The +The test suite should pass cleanly with the `thread` and `undefined` sanitizers. You +may need to use a suppressions file, see `test/sanitizer_suppressions`. They may be +used as follows: +```bash +export LSAN_OPTIONS="suppressions=$(pwd)/test/sanitizer_suppressions/lsan" +export TSAN_OPTIONS="suppressions=$(pwd)/test/sanitizer_suppressions/tsan:halt_on_error=1:second_deadlock_stack=1" +export UBSAN_OPTIONS="suppressions=$(pwd)/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" +``` + +See the CI config for more examples, and upstream documentation for more information +about any additional options. + +There are a number of known problems when using the `address` sanitizer. The address sanitizer is known to fail in [sha256_sse4::Transform](/src/crypto/sha256_sse4.cpp) which makes it unusable unless you also use `--disable-asm` when running configure. We would like to fix @@ -842,12 +860,12 @@ Strings and formatting buffer overflows, and surprises with `\0` characters. Also, some C string manipulations tend to act differently depending on platform, or even the user locale. -- Use `ParseInt32`, `ParseInt64`, `ParseUInt32`, `ParseUInt64`, `ParseDouble` from `utilstrencodings.h` for number parsing. +- Use `ToIntegral` from [`strencodings.h`](/src/util/strencodings.h) for number parsing. In legacy code you might also find `ParseInt*` family of functions, `ParseDouble` or `LocaleIndependentAtoi`. - *Rationale*: These functions do overflow checking and avoid pesky locale issues. - Avoid using locale dependent functions if possible. You can use the provided - [`lint-locale-dependence.sh`](/test/lint/lint-locale-dependence.sh) + [`lint-locale-dependence.py`](/test/lint/lint-locale-dependence.py) to check for accidental use of locale dependent functions. - *Rationale*: Unnecessary locale dependence can cause bugs that are very tricky to isolate and fix. diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 9abfbc9213ec..84ebb0986d26 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -136,10 +136,10 @@ You may also need to take care of giving the correct path for `clang` and `clang++`, like `CC=/path/to/clang CXX=/path/to/clang++` if the non-systems `clang` does not come first in your path. -Full configure that was tested on macOS Catalina with `brew` installed `llvm`: +Full configure that was tested on macOS with `brew` installed `llvm`: ```sh -./configure --enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=/usr/local/opt/llvm/bin/clang CXX=/usr/local/opt/llvm/bin/clang++ --disable-asm +./configure --enable-fuzz --with-sanitizers=fuzzer,address,undefined --disable-asm CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ ``` Read the [libFuzzer documentation](https://llvm.org/docs/LibFuzzer.html) for more information. This [libFuzzer tutorial](https://github.com/google/fuzzing/blob/master/tutorial/libFuzzerTutorial.md) might also be of interest. @@ -283,8 +283,8 @@ $ sudo apt-get install libtool libtool-bin wget automake autoconf bison gdb ``` At this point, you must install the .NET core. The process differs, depending on your Linux distribution. -See [this link](https://docs.microsoft.com/en-us/dotnet/core/install/linux) for details. -On ubuntu 20.04, the following should work: +See [this link](https://learn.microsoft.com/en-us/dotnet/core/install/linux) for details. +On Ubuntu 20.04, the following should work: ```sh $ wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb diff --git a/doc/i2p.md b/doc/i2p.md index 8433bbeaa292..0432136554d9 100644 --- a/doc/i2p.md +++ b/doc/i2p.md @@ -16,8 +16,7 @@ enabled is required. Options include: Java - [i2pd (I2P Daemon)](https://github.com/PurpleI2P/i2pd) ([documentation](https://i2pd.readthedocs.io/en/latest)), a lighter - alternative in C++ (successfully tested with version 2.23 and up; version 2.36 - or later recommended) + alternative in C++ - [i2p-zero](https://github.com/i2p-zero/i2p-zero) - [other alternatives](https://en.wikipedia.org/wiki/I2P#Routers) @@ -33,12 +32,10 @@ Core configuration options: none) -i2pacceptincoming - If set and -i2psam is also set then incoming I2P connections are - accepted via the SAM proxy. If this is not set but -i2psam is set - then only outgoing connections will be made to the I2P network. - Ignored if -i2psam is not set. Listening for incoming I2P - connections is done through the SAM proxy, not by binding to a - local address and port (default: 1) + Whether to accept inbound I2P connections (default: 1). Ignored if + -i2psam is not set. Listening for inbound I2P connections is + done through the SAM proxy, not by binding to a local address and + port. ``` In a typical situation, this suffices: @@ -47,27 +44,6 @@ In a typical situation, this suffices: bitcoind -i2psam=127.0.0.1:7656 ``` -The first time Bitcoin Core connects to the I2P router, if -`-i2pacceptincoming=1`, then it will automatically generate a persistent I2P -address and its corresponding private key. The private key will be saved in a -file named `i2p_private_key` in the Bitcoin Core data directory. The persistent -I2P address is used for accepting incoming connections and for making outgoing -connections if `-i2pacceptincoming=1`. If `-i2pacceptincoming=0` then only -outbound I2P connections are made and a different transient I2P address is used -for each connection to improve privacy. - -## Persistent vs transient I2P addresses - -In I2P connections, the connection receiver sees the I2P address of the -connection initiator. This is unlike the Tor network where the recipient does -not know who is connecting to them and can't tell if two connections are from -the same peer or not. - -If an I2P node is not accepting incoming connections, then Bitcoin Core uses -random, one-time, transient I2P addresses for itself for outbound connections -to make it harder to discriminate, fingerprint or analyze it based on its I2P -address. - ## Additional configuration options related to I2P ``` @@ -100,7 +76,29 @@ In general, a node can be run with both onion and I2P hidden services (or any/all of IPv4/IPv6/onion/I2P/CJDNS), which can provide a potential fallback if one of the networks has issues. -## I2P-related information in Bitcoin Core +## Persistent vs transient I2P addresses + +The first time Bitcoin Core connects to the I2P router, it automatically +generates a persistent I2P address and its corresponding private key by default +or if `-i2pacceptincoming=1` is set. The private key is saved in a file named +`i2p_private_key` in the Bitcoin Core data directory. The persistent I2P +address is used for making outbound connections and accepting inbound +connections. + +In the I2P network, the receiver of an inbound connection sees the address of +the initiator. This is unlike the Tor network, where the recipient does not +know who is connecting to it. + +If your node is configured by setting `-i2pacceptincoming=0` to not accept +inbound I2P connections, then it will use a random transient I2P address for +itself on each outbound connection to make it harder to discriminate, +fingerprint or analyze it based on its I2P address. + +I2P addresses are designed to be long-lived. Waiting for tunnels to be built +for every peer connection adds delay to connection setup time. Therefore, I2P +listening should only be turned off if really needed. + +## Fetching I2P-related information from Bitcoin Core There are several ways to see your I2P address in Bitcoin Core if accepting incoming I2P connections (`-i2pacceptincoming`): @@ -136,14 +134,19 @@ port (`TO_PORT`) is always set to 0 and is not in the control of Bitcoin Core. ## Bandwidth -I2P routers may route a large amount of general network traffic with their -default settings. Check your router's configuration to limit the amount of this -traffic relayed, if desired. +By default, your node shares bandwidth and transit tunnels with the I2P network +in order to increase your anonymity with cover traffic, help the I2P router used +by your node integrate optimally with the network, and give back to the network. +It's important that the nodes of a popular application like Bitcoin contribute +as much to the I2P network as they consume. -With `i2pd`, the amount of bandwidth being shared with the wider network can be -adjusted with the `bandwidth`, `share` and `transittunnels` options in your -`i2pd.conf` file. For example, to limit total I2P traffic to 256KB/s and share -50% of this limit for a maximum of 20 transit tunnels: +It is possible, though strongly discouraged, to change your I2P router +configuration to limit the amount of I2P traffic relayed by your node. + +With `i2pd`, this can be done by adjusting the `bandwidth`, `share` and +`transittunnels` options in your `i2pd.conf` file. For example, to limit total +I2P traffic to 256KB/s and share 50% of this limit for a maximum of 20 transit +tunnels: ``` bandwidth = 256 @@ -153,9 +156,15 @@ share = 50 transittunnels = 20 ``` -If you prefer not to relay any public I2P traffic and only permit I2P traffic -from programs which are connecting via the SAM proxy, e.g. Bitcoin Core, you -can set the `notransit` option to `true`. - Similar bandwidth configuration options for the Java I2P router can be found in `http://127.0.0.1:7657/config` under the "Bandwidth" tab. + +Before doing this, please see the "Participating Traffic Considerations" section +in [Embedding I2P in your Application](https://geti2p.net/en/docs/applications/embedding). + +In most cases, the default router settings should work fine. + +## Bundling I2P in a Bitcoin application + +Please see the "General Guidance for Developers" section in https://geti2p.net/en/docs/api/samv3 +if you are developing a downstream application that may be bundling I2P with Bitcoin. diff --git a/doc/init.md b/doc/init.md index 399b819bf414..7f7902771802 100644 --- a/doc/init.md +++ b/doc/init.md @@ -70,7 +70,7 @@ NOTE: When using the systemd .service file, the creation of the aforementioned directories and the setting of their permissions is automatically handled by systemd. Directories are given a permission of 710, giving the bitcoin group access to files under it _if_ the files themselves give permission to the -bitcoin group to do so (e.g. when `-sysperms` is specified). This does not allow +bitcoin group to do so. This does not allow for the listing of files under the directory. NOTE: It is not currently possible to override `datadir` in diff --git a/doc/managing-wallets.md b/doc/managing-wallets.md index 366d7ec54bee..22e006c9639e 100644 --- a/doc/managing-wallets.md +++ b/doc/managing-wallets.md @@ -88,7 +88,7 @@ In the RPC, the destination parameter must include the name of the file. Otherwi $ bitcoin-cli -rpcwallet="wallet-01" backupwallet /home/node01/Backups/backup-01.dat ``` -In the GUI, the wallet is selected in the `Wallet` drop-down list in the upper right corner. If this list is not present, the wallet can be loaded in `File` ->`Open wallet` if necessary. Then, the backup can be done in `File` -> `Backup Wallet...`. +In the GUI, the wallet is selected in the `Wallet` drop-down list in the upper right corner. If this list is not present, the wallet can be loaded in `File` ->`Open Wallet` if necessary. Then, the backup can be done in `File` -> `Backup Wallet…`. This backup file can be stored on one or multiple offline devices, which must be reliable enough to work in an emergency and be malware free. Backup files can be regularly tested to avoid problems in the future. @@ -108,7 +108,7 @@ Wallets created before version 0.13 are not HD and must be backed up every 100 k ### 1.6 Restoring the Wallet From a Backup -To restore a wallet, the `restorewallet` RPC must be used. +To restore a wallet, the `restorewallet` RPC or the `Restore Wallet` GUI menu item (`File` -> `Restore Wallet…`) must be used. ``` $ bitcoin-cli restorewallet "restored-wallet" /home/node01/Backups/backup-01.dat @@ -144,5 +144,5 @@ unforeseen configurations which result in some scripts being excluded. If a migr unexpectedly or otherwise misses any scripts, please create an issue on GitHub. A backup of the original wallet can be found in the wallet directory with the name `-.legacy.bak`. -The backup can be restored using the `restorewallet` command as discussed in the -[Restoring the Wallet From a Backup](#16-restoring-the-wallet-from-a-backup) section +The backup can be restored using the methods discussed in the +[Restoring the Wallet From a Backup](#16-restoring-the-wallet-from-a-backup) section. diff --git a/doc/reduce-memory.md b/doc/reduce-memory.md index 296b172bdeaf..25205258b842 100644 --- a/doc/reduce-memory.md +++ b/doc/reduce-memory.md @@ -16,11 +16,11 @@ The size of some in-memory caches can be reduced. As caches trade off memory usa - The minimum value for `-maxmempool` is 5. - A lower maximum mempool size means that transactions will be evicted sooner. This will affect any uses of `bitcoind` that process unconfirmed transactions. -- To completely disable mempool functionality there is the option `-blocksonly`. This will make the client opt out of receiving (and thus relaying) transactions completely, except as part of blocks. +- Since `0.14.0`, unused memory allocated to the mempool (default: 300MB) is shared with the UTXO cache, so when trying to reduce memory usage you should limit the mempool, with the `-maxmempool` command line argument. - - Do not use this when using the client to broadcast transactions as any transaction sent will stick out like a sore thumb, affecting privacy. When used with the wallet it should be combined with `-walletbroadcast=0` and `-spendzeroconfchange=0`. Another mechanism for broadcasting outgoing transactions (if any) should be used. +- To disable most of the mempool functionality there is the `-blocksonly` option. This will reduce the default memory usage to 5MB and make the client opt out of receiving (and thus relaying) transactions, except from peers who have the `relay` permission set (e.g. whitelisted peers), and as part of blocks. -- Since `0.14.0`, unused memory allocated to the mempool (default: 300MB) is shared with the UTXO cache, so when trying to reduce memory usage you should limit the mempool, with the `-maxmempool` command line argument. + - Do not use this when using the client to broadcast transactions as any transaction sent will stick out like a sore thumb, affecting privacy. When used with the wallet it should be combined with `-walletbroadcast=0` and `-spendzeroconfchange=0`. Another mechanism for broadcasting outgoing transactions (if any) should be used. ## Number of peers diff --git a/doc/release-note-26194.md b/doc/release-note-26194.md new file mode 100644 index 000000000000..b72dbf9a238b --- /dev/null +++ b/doc/release-note-26194.md @@ -0,0 +1,4 @@ +Add `next_index` in `listdescriptors` RPC +----------------- + +- Added a new `next_index` field in the response in `listdescriptors` to have the same format as `importdescriptors` diff --git a/doc/release-notes-23395.md b/doc/release-notes-23395.md new file mode 100644 index 000000000000..b9d7d9409c26 --- /dev/null +++ b/doc/release-notes-23395.md @@ -0,0 +1,8 @@ +Notable changes +=============== + +New settings +------------ + +- The `shutdownnotify` option is used to specify a command to execute synchronously +before Bitcoin Core has begun its shutdown sequence. (#23395) diff --git a/doc/release-notes-25375.md b/doc/release-notes-25375.md new file mode 100644 index 000000000000..504a2644f461 --- /dev/null +++ b/doc/release-notes-25375.md @@ -0,0 +1,11 @@ +Updated RPCs +-------- + +The `minconf` option, which allows a user to specify the minimum number +of confirmations a UTXO being spent has, and the `maxconf` option, +which allows specifying the maximum number of confirmations, have been +added to the following RPCs: +- `fundrawtransaction` +- `send` +- `walletcreatefundedpsbt` +- `sendall` diff --git a/doc/release-notes-25574.md b/doc/release-notes-25574.md new file mode 100644 index 000000000000..312a99d95baa --- /dev/null +++ b/doc/release-notes-25574.md @@ -0,0 +1,13 @@ +Updated settings +---------------- + +If the `-checkblocks` or `-checklevel` options are explicitly provided by the +user, but the verification checks cannot be completed due to an insufficient +dbcache, Bitcoin Core will now return an error at startup. (#25574) + +RPC +--- +The `-verifychain` RPC will now return `false` if the checks didn't fail, +but couldn't be completed at the desired depth and level. This could be due +to missing data while pruning, due to an insufficient dbcache or due to +the node being shutdown before the call could finish. (#25574) diff --git a/doc/release-notes-25943.md b/doc/release-notes-25943.md new file mode 100644 index 000000000000..81b0a48b5d3b --- /dev/null +++ b/doc/release-notes-25943.md @@ -0,0 +1,4 @@ +New RPC Argument +-------- +- `sendrawtransaction` has a new, optional argument, `maxburnamount` with a default value of `0`. Any transaction containing an unspendable output with a value greater than `maxburnamount` will not be submitted. At present, the outputs deemed unspendable are those with scripts that begin with an `OP_RETURN` code (known as 'datacarriers'), scripts that exceed the maximum script size, and scripts that contain invalid opcodes. + diff --git a/doc/release-notes-25957.md b/doc/release-notes-25957.md new file mode 100644 index 000000000000..c71afa2c2e17 --- /dev/null +++ b/doc/release-notes-25957.md @@ -0,0 +1,9 @@ +Wallet +------ + +- Rescans for descriptor wallets are now significantly faster if compact + block filters (BIP158) are available. Since those are not constructed + by default, the configuration option "-blockfilterindex=1" has to be + provided to take advantage of the optimization. This improves the + performance of the RPC calls `rescanblockchain`, `importdescriptors` + and `restorewallet`. (#25957) diff --git a/doc/release-notes-26213.md b/doc/release-notes-26213.md new file mode 100644 index 000000000000..e78d718ca998 --- /dev/null +++ b/doc/release-notes-26213.md @@ -0,0 +1,8 @@ +Low-level changes +================= + +- Previously `setban`, `addpeeraddress`, `walletcreatefundedpsbt`, methods + allowed non-boolean and non-null values to be passed as boolean parameters. + Any string, number, array, or object value that was passed would be treated + as false. After this change, passing any value except `true`, `false`, or + `null` now triggers a JSON value is not of expected type error. (#26213) diff --git a/doc/release-notes-26265.md b/doc/release-notes-26265.md new file mode 100644 index 000000000000..ca2313d95694 --- /dev/null +++ b/doc/release-notes-26265.md @@ -0,0 +1,6 @@ +P2P and network changes +--------- + +- Transactions of non-witness size 65 and above are now allowed by mempool + and relay policy. This is to better reflect the actual afforded protections + against CVE-2017-12842 and open up additional use-cases of smaller transaction sizes. (#26265) diff --git a/doc/release-notes-26471.md b/doc/release-notes-26471.md new file mode 100644 index 000000000000..2cb74804cade --- /dev/null +++ b/doc/release-notes-26471.md @@ -0,0 +1,13 @@ +Updated settings +---------------- + +- Setting `-blocksonly` will now reduce the maximum mempool memory + to 5MB (users may still use `-maxmempool` to override). Previously, + the default 300MB would be used, leading to unexpected memory usage + for users running with `-blocksonly` expecting it to eliminate + mempool memory usage. + + As unused mempool memory is shared with dbcache, this also reduces + the dbcache size for users running with `-blocksonly`, potentially + impacting performance. + diff --git a/doc/release-notes-26618.md b/doc/release-notes-26618.md new file mode 100644 index 000000000000..9d1ef3bd2ee0 --- /dev/null +++ b/doc/release-notes-26618.md @@ -0,0 +1,4 @@ +RPC Wallet +---------- + +- RPC `unloadwallet` now fails if a rescan is in progress. (#26618) diff --git a/doc/release-notes-26628.md b/doc/release-notes-26628.md new file mode 100644 index 000000000000..48a07c1e8187 --- /dev/null +++ b/doc/release-notes-26628.md @@ -0,0 +1,4 @@ +JSON-RPC +--- + +The JSON-RPC server now rejects requests where a parameter is specified multiple times with the same name, instead of silently overwriting earlier parameter values with later ones. (#26628) diff --git a/doc/release-notes-26646.md b/doc/release-notes-26646.md new file mode 100644 index 000000000000..7f94505a01ab --- /dev/null +++ b/doc/release-notes-26646.md @@ -0,0 +1,8 @@ +JSON-RPC +-------- + +The `testmempoolaccept` RPC now returns 2 additional results within the "fees" result: +"effective-feerate" is the feerate including fees and sizes of transactions validated together if +package validation was used, and also includes any modified fees from prioritisetransaction. The +"effective-includes" result lists the wtxids of transactions whose modified fees and sizes were used +in the effective-feerate (#26646). diff --git a/doc/release-notes-26896.md b/doc/release-notes-26896.md new file mode 100644 index 000000000000..ff4ab44e2705 --- /dev/null +++ b/doc/release-notes-26896.md @@ -0,0 +1,7 @@ +Build System +------------ + +The --enable-upnp-default and --enable-natpmp-default options +have been removed. If you want to use port mapping, you can +configure it using a .conf file, or by passing the relevant +options at runtime. \ No newline at end of file diff --git a/doc/release-notes-26899.md b/doc/release-notes-26899.md new file mode 100644 index 000000000000..ceb9ec2f7a11 --- /dev/null +++ b/doc/release-notes-26899.md @@ -0,0 +1,5 @@ +Updated settings +---------------- + +- Setting `-maxconnections=0` will now disable `-dnsseed` + and `-listen` (users may still set them to override). \ No newline at end of file diff --git a/doc/release-notes-27037.md b/doc/release-notes-27037.md new file mode 100644 index 000000000000..ee30e6401021 --- /dev/null +++ b/doc/release-notes-27037.md @@ -0,0 +1,5 @@ +RPC +--- + +- `decodescript` may now infer a Miniscript descriptor under P2WSH context if it is not lacking + information. diff --git a/doc/release-notes-27068.md b/doc/release-notes-27068.md new file mode 100644 index 000000000000..3f5c5dba37bd --- /dev/null +++ b/doc/release-notes-27068.md @@ -0,0 +1,6 @@ +Wallet +------ + +- Wallet passphrases may now contain null characters. + Prior to this change, only characters up to the first + null character were recognized and accepted. (#27068) \ No newline at end of file diff --git a/doc/release-notes-27217.md b/doc/release-notes-27217.md new file mode 100644 index 000000000000..ab1452fc9420 --- /dev/null +++ b/doc/release-notes-27217.md @@ -0,0 +1,6 @@ +Wallet +------ + +* Address Purposes strings are now restricted to the currently known values of "send", "receive", and "refund". + Wallets that have unrecognized purpose strings will have loading warnings, and the `listlabels` + RPC will raise an error if an unrecognized purpose is requested. diff --git a/doc/release-notes-27279.md b/doc/release-notes-27279.md new file mode 100644 index 000000000000..b664aee755d3 --- /dev/null +++ b/doc/release-notes-27279.md @@ -0,0 +1,10 @@ +Wallet +------ + +- In the createwallet, loadwallet, unloadwallet, and restorewallet RPCs, the + "warning" string field is deprecated in favor of a "warnings" field that + returns a JSON array of strings to better handle multiple warning messages and + for consistency with other wallet RPCs. The "warning" field will be fully + removed from these RPCs in v26. It can be temporarily re-enabled during the + deprecation period by launching bitcoind with the configuration option + `-deprecatedrpc=walletwarningfield`. (#27279) diff --git a/doc/release-notes/release-notes-22.1.md b/doc/release-notes/release-notes-22.1.md new file mode 100644 index 000000000000..d304b7e57aef --- /dev/null +++ b/doc/release-notes/release-notes-22.1.md @@ -0,0 +1,128 @@ +22.1 Release Notes +================== + +Bitcoin Core version 22.1 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.14+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 22.0 onwards, macOS versions earlier than 10.14 are no longer supported. + +Notable changes +=============== + +Updated settings +---------------- + +- In previous releases, the meaning of the command line option + `-persistmempool` (without a value provided) incorrectly disabled mempool + persistence. `-persistmempool` is now treated like other boolean options to + mean `-persistmempool=1`. Passing `-persistmempool=0`, `-persistmempool=1` + and `-nopersistmempool` is unaffected. (#23061) + +### P2P + +### RPC and other APIs + +- #25237 rpc: Capture UniValue by ref for rpcdoccheck +- #25983 Prevent data race for pathHandlers +- #26275 Fix crash on deriveaddresses when index is 2147483647 (2^31-1) + +### Wallet + +- #22781 wallet: fix the behavior of IsHDEnabled +- #22949 fee: Round up fee calculation to avoid a lower than expected feerate +- #23333 wallet: fix segfault by avoiding invalid default-ctored external_spk_managers entry + +### Build system + +- #22820 build, qt: Fix typo in QtInputSupport check +- #23045 build: Restrict check for CRC32C intrinsic to aarch64 +- #23148 build: Fix guix linker-loader path and add check_ELF_interpreter +- #23314 build: explicitly disable libsecp256k1 openssl based tests +- #23580 build: patch qt to explicitly define previously implicit header include +- #24215 guix: ignore additional failing certvalidator test +- #24256 build: Bump depends packages (zmq, libXau) +- #25201 windeploy: Renewed windows code signing certificate +- #25985 Revert "build: Use Homebrew's sqlite package if it is available" +- #26633 depends: update qt 5.12 url to archive location + +### GUI + +- #gui631 Disallow encryption of watchonly wallets +- #gui680 Fixes MacOS 13 segfault by preventing certain notifications +- #24498 qt: Avoid crash on startup if int specified in settings.json + +### Tests + +- #23716 test: replace hashlib.ripemd160 with an own implementation +- #24239 test: fix ceildiv division by using integers + +### Utilities + +- #22390 system: skip trying to set the locale on NetBSD +- #22895 don't call GetBlockPos in ReadBlockFromDisk without cs_main lock +- #24104 fs: Make compatible with boost 1.78 + +### Miscellaneous + +- #23335 refactor: include a missing header in fs.cpp +- #23504 ci: Replace soon EOL hirsute with jammy +- #26321 Adjust .tx/config for new Transifex CLI + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Andrew Chow +- BlackcoinDev +- Carl Dong +- Hennadii Stepanov +- Joan Karadimov +- John Moffett +- Jon Atack +- Kittywhiskers Van Gogh +- Marco Falke +- Martin Zumsande +- Michael Ford +- muxator +- Pieter Wuille +- Ryan Ofsky +- Saibato +- Sebastian Falbesoner +- W. J. van der Laan + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-23.1.md b/doc/release-notes/release-notes-23.1.md new file mode 100644 index 000000000000..31d9b7f068d8 --- /dev/null +++ b/doc/release-notes/release-notes-23.1.md @@ -0,0 +1,90 @@ +23.1 Release Notes +================== + +Bitcoin Core version 23.1 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +### P2P + +- #25314 p2p: always set nTime for self-advertisements + +### RPC and other APIs + +- #25220 rpc: fix incorrect warning for address type p2sh-segwit in createmultisig +- #25237 rpc: Capture UniValue by ref for rpcdoccheck +- #25983 Prevent data race for pathHandlers +- #26275 Fix crash on deriveaddresses when index is 2147483647 (2^31-1) + +### Build system + +- #25201 windeploy: Renewed windows code signing certificate +- #25788 guix: patch NSIS to remove .reloc sections from installer stubs +- #25861 guix: use --build={arch}-guix-linux-gnu in cross toolchain +- #25985 Revert "build: Use Homebrew's sqlite package if it is available" + +### GUI + +- #24668 build, qt: bump Qt5 version to 5.15.3 +- gui#631 Disallow encryption of watchonly wallets +- gui#680 Fixes MacOS 13 segfault by preventing certain notifications + +### Tests + +- #24454 tests: Fix calculation of external input weights + +### Miscellaneous + +- #26321 Adjust .tx/config for new Transifex CLI + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Andrew Chow +- brunoerg +- Hennadii Stepanov +- John Moffett +- MacroFake +- Martin Zumsande +- Michael Ford +- muxator +- Pavol Rusnak +- Sebastian Falbesoner +- W. J. van der Laan + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-process.md b/doc/release-process.md index 17a03f7dcdb0..f1fd72f8acfe 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -46,10 +46,10 @@ Release Process - Clear the release notes and move them to the wiki (see "Write the release notes" below). - Translations on Transifex: - Pull translations from Transifex into the master branch. - - Create [a new resource](https://www.transifex.com/bitcoin/bitcoin/content/) named after the major version with the slug `[bitcoin.qt-translation-x]`, where `RRR` is the major branch number padded with zeros. Use `src/qt/locale/bitcoin_en.xlf` to create it. - - In the project workflow settings, ensure that [Translation Memory Fill-up](https://docs.transifex.com/translation-memory/enabling-autofill) is enabled and that [Translation Memory Context Matching](https://docs.transifex.com/translation-memory/translation-memory-with-context) is disabled. + - Create [a new resource](https://www.transifex.com/bitcoin/bitcoin/content/) named after the major version with the slug `qt-translation-x`, where `RRR` is the major branch number padded with zeros. Use `src/qt/locale/bitcoin_en.xlf` to create it. + - In the project workflow settings, ensure that [Translation Memory Fill-up](https://help.transifex.com/en/articles/6224817-setting-up-translation-memory-fill-up) is enabled and that [Translation Memory Context Matching](https://help.transifex.com/en/articles/6224753-translation-memory-with-context) is disabled. - Update the Transifex slug in [`.tx/config`](/.tx/config) to the slug of the resource created in the first step. This identifies which resource the translations will be synchronized from. - - Make an announcement that translators can start translating for the new version. You can use one of the [previous announcements](https://www.transifex.com/bitcoin/bitcoin/announcements/) as a template. + - Make an announcement that translators can start translating for the new version. You can use one of the [previous announcements](https://www.transifex.com/bitcoin/communication/) as a template. - Change the auto-update URL for the resource to `master`, e.g. `https://raw.githubusercontent.com/bitcoin/bitcoin/master/src/qt/locale/bitcoin_en.xlf`. (Do this only after the previous steps, to prevent an auto-update from interfering.) #### After branch-off (on the major release branch) @@ -134,7 +134,6 @@ Follow the relevant Guix README.md sections: ### Verify other builders' signatures to your own (optional) -- [Add other builders keys to your gpg keyring, and/or refresh keys](/contrib/builder-keys/README.md) - [Verifying build output attestations](/contrib/guix/README.md#verifying-build-output-attestations) ### Commit your non codesigned signature to guix.sigs @@ -193,7 +192,6 @@ popd ### Verify other builders' signatures to your own (optional) -- [Add other builders keys to your gpg keyring, and/or refresh keys](/contrib/builder-keys/README.md) - [Verifying build output attestations](/contrib/guix/README.md#verifying-build-output-attestations) ### Commit your codesigned signature to guix.sigs (for the signed macOS/Windows binaries) diff --git a/doc/tor.md b/doc/tor.md index 08d031d08443..581d124f7aa7 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -92,19 +92,13 @@ out by default (if not, add them): ControlPort 9051 CookieAuthentication 1 CookieAuthFileGroupReadable 1 +DataDirectoryGroupReadable 1 ``` Add or uncomment those, save, and restart Tor (usually `systemctl restart tor` or `sudo systemctl restart tor` on most systemd-based systems, including recent Debian and Ubuntu, or just restart the computer). -On some systems (such as Arch Linux), you may also need to add the following -line: - -``` -DataDirectoryGroupReadable 1 -``` - ### Authentication Connecting to Tor's control socket API requires one of two authentication diff --git a/doc/tracing.md b/doc/tracing.md index 6e60901496b2..d26cf52fc3bb 100644 --- a/doc/tracing.md +++ b/doc/tracing.md @@ -211,6 +211,58 @@ Arguments passed: 4. The expected transaction fee as an `int64` 5. The position of the change output as an `int32` +### Context `mempool` + +#### Tracepoint `mempool:added` + +Is called when a transaction is added to the node's mempool. Passes information +about the transaction. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Transaction virtual size as `uint64` +3. Transaction fee as `int64` + +#### Tracepoint `mempool:removed` + +Is called when a transaction is removed from the node's mempool. Passes information +about the transaction. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Removal reason as `pointer to C-style String` (max. length 9 characters) +3. Transaction virtual size as `uint64` +4. Transaction fee as `int64` +5. Transaction mempool entry time (epoch) as `uint64` + +#### Tracepoint `mempool:replaced` + +Is called when a transaction in the node's mempool is getting replaced by another. +Passes information about the replaced and replacement transactions. + +Arguments passed: +1. Replaced transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Replaced transaction virtual size as `uint64` +3. Replaced transaction fee as `int64` +4. Replaced transaction mempool entry time (epoch) as `uint64` +5. Replacement transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +6. Replacement transaction virtual size as `uint64` +7. Replacement transaction fee as `int64` + +Note: In cases where a single replacement transaction replaces multiple +existing transactions in the mempool, the tracepoint is called once for each +replaced transaction, with data of the replacement transaction being the same +in each call. + +#### Tracepoint `mempool:rejected` + +Is called when a transaction is not permitted to enter the mempool. Passes +information about the rejected transaction. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Reject reason as `pointer to C-style String` (max. length 118 characters) + ## Adding tracepoints to Bitcoin Core To add a new tracepoint, `#include ` in the compilation unit where diff --git a/share/rpcauth/rpcauth.py b/share/rpcauth/rpcauth.py index d441d5f21d4c..cc7bba1f8b19 100755 --- a/share/rpcauth/rpcauth.py +++ b/share/rpcauth/rpcauth.py @@ -4,22 +4,20 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. from argparse import ArgumentParser -from base64 import urlsafe_b64encode from getpass import getpass -from os import urandom - +from secrets import token_hex, token_urlsafe import hmac def generate_salt(size): """Create size byte hex salt""" - return urandom(size).hex() + return token_hex(size) def generate_password(): """Create 32 byte b64 password""" - return urlsafe_b64encode(urandom(32)).decode('utf-8') + return token_urlsafe(32) def password_to_hmac(salt, password): - m = hmac.new(bytearray(salt, 'utf-8'), bytearray(password, 'utf-8'), 'SHA256') + m = hmac.new(salt.encode('utf-8'), password.encode('utf-8'), 'SHA256') return m.hexdigest() def main(): @@ -38,8 +36,8 @@ def main(): password_hmac = password_to_hmac(salt, args.password) print('String to be appended to bitcoin.conf:') - print('rpcauth={0}:{1}${2}'.format(args.username, salt, password_hmac)) - print('Your password:\n{0}'.format(args.password)) + print(f'rpcauth={args.username}:{salt}${password_hmac}') + print(f'Your password:\n{args.password}') if __name__ == '__main__': main() diff --git a/src/.clang-tidy b/src/.clang-tidy index 9d78ccc95965..39566c0cb7fc 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -5,24 +5,17 @@ bugprone-use-after-move, misc-unused-using-decls, modernize-use-default-member-init, modernize-use-nullptr, -performance-for-range-copy, -performance-move-const-arg, -performance-unnecessary-copy-initialization, -readability-redundant-declaration, -readability-redundant-string-init, -' -WarningsAsErrors: ' -bugprone-argument-comment, -bugprone-use-after-move, -misc-unused-using-decls, -modernize-use-default-member-init, -modernize-use-nullptr, -performance-for-range-copy, -performance-move-const-arg, -performance-unnecessary-copy-initialization, +performance-*, +-performance-inefficient-string-concatenation, +-performance-no-int-to-ptr, +-performance-noexcept-move-constructor, +-performance-unnecessary-value-param, +readability-const-return-type, readability-redundant-declaration, readability-redundant-string-init, ' +WarningsAsErrors: '*' CheckOptions: - key: performance-move-const-arg.CheckTriviallyCopyableMove value: false +HeaderFilterRegex: '.' diff --git a/src/Makefile.am b/src/Makefile.am index e73245daeb38..1d7004ac863b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -20,6 +20,7 @@ noinst_LTLIBRARIES = bin_PROGRAMS = noinst_PROGRAMS = +check_PROGRAMS = TESTS = BENCHMARKS = @@ -133,6 +134,7 @@ BITCOIN_CORE_H = \ clientversion.h \ coins.h \ common/bloom.h \ + common/init.h \ common/run_command.h \ common/url.h \ compat/assumptions.h \ @@ -152,7 +154,6 @@ BITCOIN_CORE_H = \ deploymentstatus.h \ external_signer.h \ flatfile.h \ - fs.h \ headerssync.h \ httprpc.h \ httpserver.h \ @@ -172,11 +173,14 @@ BITCOIN_CORE_H = \ interfaces/ipc.h \ interfaces/node.h \ interfaces/wallet.h \ + kernel/blockmanager_opts.h \ kernel/chain.h \ + kernel/chainparams.h \ kernel/chainstatemanager_opts.h \ kernel/checks.h \ kernel/coinstats.h \ kernel/context.h \ + kernel/cs_main.h \ kernel/mempool_entry.h \ kernel/mempool_limits.h \ kernel/mempool_options.h \ @@ -197,13 +201,16 @@ BITCOIN_CORE_H = \ netbase.h \ netgroup.h \ netmessagemaker.h \ + node/blockmanager_args.h \ node/blockstorage.h \ node/caches.h \ node/chainstate.h \ node/chainstatemanager_args.h \ node/coin.h \ + node/coins_view_args.h \ node/connection_types.h \ node/context.h \ + node/database_args.h \ node/eviction.h \ node/interface_ui.h \ node/mempool_args.h \ @@ -274,8 +281,11 @@ BITCOIN_CORE_H = \ util/check.h \ util/epochguard.h \ util/error.h \ + util/exception.h \ util/fastrange.h \ util/fees.h \ + util/fs.h \ + util/fs_helpers.h \ util/getuniquepath.h \ util/golombrice.h \ util/hash_type.h \ @@ -319,7 +329,6 @@ BITCOIN_CORE_H = \ wallet/external_signer_scriptpubkeyman.h \ wallet/feebumper.h \ wallet/fees.h \ - wallet/ismine.h \ wallet/load.h \ wallet/receive.h \ wallet/rpc/util.h \ @@ -329,6 +338,7 @@ BITCOIN_CORE_H = \ wallet/spend.h \ wallet/sqlite.h \ wallet/transaction.h \ + wallet/types.h \ wallet/wallet.h \ wallet/walletdb.h \ wallet/wallettool.h \ @@ -375,18 +385,22 @@ libbitcoin_node_a_SOURCES = \ kernel/checks.cpp \ kernel/coinstats.cpp \ kernel/context.cpp \ + kernel/cs_main.cpp \ kernel/mempool_persist.cpp \ mapport.cpp \ net.cpp \ net_processing.cpp \ netgroup.cpp \ + node/blockmanager_args.cpp \ node/blockstorage.cpp \ node/caches.cpp \ node/chainstate.cpp \ node/chainstatemanager_args.cpp \ node/coin.cpp \ + node/coins_view_args.cpp \ node/connection_types.cpp \ node/context.cpp \ + node/database_args.cpp \ node/eviction.cpp \ node/interface_ui.cpp \ node/interfaces.cpp \ @@ -633,6 +647,7 @@ libbitcoin_common_a_SOURCES = \ chainparams.cpp \ coins.cpp \ common/bloom.cpp \ + common/init.cpp \ common/interfaces.cpp \ common/run_command.cpp \ compressor.cpp \ @@ -641,6 +656,7 @@ libbitcoin_common_a_SOURCES = \ deploymentinfo.cpp \ external_signer.cpp \ init/common.cpp \ + kernel/chainparams.cpp \ key.cpp \ key_io.cpp \ merkleblock.cpp \ @@ -653,8 +669,9 @@ libbitcoin_common_a_SOURCES = \ policy/policy.cpp \ protocol.cpp \ psbt.cpp \ - rpc/rawtransaction_util.cpp \ rpc/external_signer.cpp \ + rpc/rawtransaction_util.cpp \ + rpc/request.cpp \ rpc/util.cpp \ scheduler.cpp \ script/descriptor.cpp \ @@ -678,11 +695,9 @@ libbitcoin_util_a_SOURCES = \ support/lockedpool.cpp \ chainparamsbase.cpp \ clientversion.cpp \ - fs.cpp \ logging.cpp \ random.cpp \ randomenv.cpp \ - rpc/request.cpp \ support/cleanse.cpp \ sync.cpp \ util/asmap.cpp \ @@ -690,7 +705,10 @@ libbitcoin_util_a_SOURCES = \ util/bytevectorhash.cpp \ util/check.cpp \ util/error.cpp \ + util/exception.cpp \ util/fees.cpp \ + util/fs.cpp \ + util/fs_helpers.cpp \ util/getuniquepath.cpp \ util/hasher.cpp \ util/sock.cpp \ @@ -848,11 +866,12 @@ bitcoin_chainstate_SOURCES = bitcoin-chainstate.cpp bitcoin_chainstate_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) bitcoin_chainstate_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -# $(LIBTOOL_APP_LDFLAGS) deliberately omitted here so that we can test linking -# bitcoin-chainstate against libbitcoinkernel as a shared or static library by -# setting --{en,dis}able-shared. -bitcoin_chainstate_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(PTHREAD_FLAGS) +bitcoin_chainstate_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(PTHREAD_FLAGS) $(LIBTOOL_APP_LDFLAGS) -static bitcoin_chainstate_LDADD = $(LIBBITCOINKERNEL) + +# libtool is unable to calculate this indirect dependency, presumably because it's a subproject. +# libsecp256k1 only needs to be linked in when libbitcoinkernel is static. +bitcoin_chainstate_LDADD += $(LIBSECP256K1) # # bitcoinkernel library # @@ -871,15 +890,6 @@ libbitcoinkernel_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp # to export from the library. libbitcoinkernel_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -fvisibility=default -# TODO: For now, Specify -static in both CXXFLAGS and LDFLAGS when building for -# windows targets so libtool will only build a static version of this -# library. There are unresolved problems when building dll's for mingw-w64 -# and attempting to statically embed libstdc++, libpthread, etc. -if TARGET_WINDOWS -libbitcoinkernel_la_LDFLAGS += -static -libbitcoinkernel_la_CXXFLAGS += -static -endif - # TODO: libbitcoinkernel is a work in progress consensus engine library, as more # and more modules are decoupled from the consensus engine, this list will # shrink to only those which are absolutely necessary. @@ -900,12 +910,13 @@ libbitcoinkernel_la_SOURCES = \ deploymentinfo.cpp \ deploymentstatus.cpp \ flatfile.cpp \ - fs.cpp \ hash.cpp \ kernel/chain.cpp \ kernel/checks.cpp \ + kernel/chainparams.cpp \ kernel/coinstats.cpp \ kernel/context.cpp \ + kernel/cs_main.cpp \ kernel/mempool_persist.cpp \ key.cpp \ logging.cpp \ @@ -940,6 +951,9 @@ libbitcoinkernel_la_SOURCES = \ txmempool.cpp \ uint256.cpp \ util/check.cpp \ + util/exception.cpp \ + util/fs.cpp \ + util/fs_helpers.cpp \ util/getuniquepath.cpp \ util/hasher.cpp \ util/moneystr.cpp \ diff --git a/src/Makefile.minisketch.include b/src/Makefile.minisketch.include index b337f483498e..1363bec34eac 100644 --- a/src/Makefile.minisketch.include +++ b/src/Makefile.minisketch.include @@ -31,7 +31,7 @@ if ENABLE_TESTS if !ENABLE_FUZZ MINISKETCH_TEST = minisketch/test TESTS += $(MINISKETCH_TEST) -noinst_PROGRAMS += $(MINISKETCH_TEST) +check_PROGRAMS += $(MINISKETCH_TEST) minisketch_test_SOURCES = $(MINISKETCH_TEST_SOURCES_INT) minisketch_test_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS) diff --git a/src/Makefile.qt_locale.include b/src/Makefile.qt_locale.include index 8638bd024cd4..c44cf61a1a82 100644 --- a/src/Makefile.qt_locale.include +++ b/src/Makefile.qt_locale.include @@ -31,6 +31,7 @@ QT_TS = \ qt/locale/bitcoin_gl.ts \ qt/locale/bitcoin_gl_ES.ts \ qt/locale/bitcoin_gu.ts \ + qt/locale/bitcoin_ha.ts \ qt/locale/bitcoin_he.ts \ qt/locale/bitcoin_hr.ts \ qt/locale/bitcoin_hu.ts \ @@ -43,6 +44,7 @@ QT_TS = \ qt/locale/bitcoin_kl.ts \ qt/locale/bitcoin_km.ts \ qt/locale/bitcoin_ko.ts \ + qt/locale/bitcoin_ku.ts \ qt/locale/bitcoin_ku_IQ.ts \ qt/locale/bitcoin_ky.ts \ qt/locale/bitcoin_la.ts \ @@ -58,6 +60,7 @@ QT_TS = \ qt/locale/bitcoin_ne.ts \ qt/locale/bitcoin_nl.ts \ qt/locale/bitcoin_no.ts \ + qt/locale/bitcoin_pa.ts \ qt/locale/bitcoin_pam.ts \ qt/locale/bitcoin_pl.ts \ qt/locale/bitcoin_pt.ts \ @@ -78,10 +81,13 @@ QT_TS = \ qt/locale/bitcoin_ta.ts \ qt/locale/bitcoin_te.ts \ qt/locale/bitcoin_th.ts \ + qt/locale/bitcoin_tk.ts \ + qt/locale/bitcoin_tl.ts \ qt/locale/bitcoin_tr.ts \ qt/locale/bitcoin_ug.ts \ qt/locale/bitcoin_uk.ts \ qt/locale/bitcoin_ur.ts \ + qt/locale/bitcoin_uz.ts \ qt/locale/bitcoin_uz@Cyrl.ts \ qt/locale/bitcoin_uz@Latn.ts \ qt/locale/bitcoin_vi.ts \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 74c30f1caf9c..15d5a17cec83 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -147,6 +147,7 @@ BITCOIN_TESTS =\ test/timedata_tests.cpp \ test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ + test/translation_tests.cpp \ test/txindex_tests.cpp \ test/txpackage_tests.cpp \ test/txreconciliation_tests.cpp \ @@ -162,7 +163,8 @@ BITCOIN_TESTS =\ test/validation_flush_tests.cpp \ test/validation_tests.cpp \ test/validationinterface_tests.cpp \ - test/versionbits_tests.cpp + test/versionbits_tests.cpp \ + test/xoroshiro128plusplus_tests.cpp if ENABLE_WALLET BITCOIN_TESTS += \ @@ -174,12 +176,12 @@ BITCOIN_TESTS += \ wallet/test/wallet_crypto_tests.cpp \ wallet/test/wallet_transaction_tests.cpp \ wallet/test/coinselector_tests.cpp \ - wallet/test/availablecoins_tests.cpp \ wallet/test/init_tests.cpp \ wallet/test/ismine_tests.cpp \ wallet/test/rpc_util_tests.cpp \ wallet/test/scriptpubkeyman_tests.cpp \ - wallet/test/walletload_tests.cpp + wallet/test/walletload_tests.cpp \ + wallet/test/group_outputs_tests.cpp FUZZ_SUITE_LD_COMMON +=\ $(SQLITE_LIBS) \ @@ -249,6 +251,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/chain.cpp \ test/fuzz/checkqueue.cpp \ test/fuzz/coins_view.cpp \ + test/fuzz/coinscache_sim.cpp \ test/fuzz/connman.cpp \ test/fuzz/crypto.cpp \ test/fuzz/crypto_aes256.cpp \ @@ -269,6 +272,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/flatfile.cpp \ test/fuzz/float.cpp \ test/fuzz/golomb_rice.cpp \ + test/fuzz/headerssync.cpp \ test/fuzz/hex.cpp \ test/fuzz/http_request.cpp \ test/fuzz/i2p.cpp \ @@ -294,6 +298,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/parse_numbers.cpp \ test/fuzz/parse_script.cpp \ test/fuzz/parse_univalue.cpp \ + test/fuzz/partially_downloaded_block.cpp \ test/fuzz/policy_estimator.cpp \ test/fuzz/policy_estimator_io.cpp \ test/fuzz/pow.cpp \ @@ -376,7 +381,7 @@ if TARGET_WINDOWS else if ENABLE_BENCH @echo "Running bench/bench_bitcoin (one iteration sanity check, only high priority)..." - $(BENCH_BINARY) -sanity-check -priority-level=high > /dev/null + $(BENCH_BINARY) -sanity-check -priority-level=high endif endif $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index a4e8b3f842f8..aefefe789a78 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -10,15 +10,19 @@ EXTRA_LIBRARIES += \ TEST_UTIL_H = \ test/util/blockfilter.h \ test/util/chainstate.h \ + test/util/coins.h \ + test/util/json.h \ test/util/logging.h \ test/util/mining.h \ test/util/net.h \ + test/util/random.h \ test/util/script.h \ test/util/setup_common.h \ test/util/str.h \ test/util/transaction_utils.h \ test/util/txmempool.h \ - test/util/validation.h + test/util/validation.h \ + test/util/xoroshiro128plusplus.h if ENABLE_WALLET TEST_UTIL_H += wallet/test/util.h @@ -28,6 +32,8 @@ libtest_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libtest_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libtest_util_a_SOURCES = \ test/util/blockfilter.cpp \ + test/util/coins.cpp \ + test/util/json.cpp \ test/util/logging.cpp \ test/util/mining.cpp \ test/util/net.cpp \ diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 7106d819b0f3..8ecccd4d22e6 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2021 The Bitcoin Core developers +// Copyright (c) 2009-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,8 +9,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -18,6 +18,8 @@ #include #include #include +#include +#include #include #include #include @@ -34,10 +36,9 @@ bool SerializeDB(Stream& stream, const Data& data) { // Write and commit header, data try { - CHashWriter hasher(stream.GetType(), stream.GetVersion()); - stream << Params().MessageStart() << data; - hasher << Params().MessageStart() << data; - stream << hasher.GetHash(); + HashedSourceWriter hashwriter{stream}; + hashwriter << Params().MessageStart() << data; + stream << hashwriter.GetHash(); } catch (const std::exception& e) { return error("%s: Serialize or I/O error - %s", __func__, e.what()); } @@ -191,7 +192,7 @@ std::optional LoadAddrman(const NetGroupManager& netgroupman, con const auto path_addr{args.GetDataDirNet() / "peers.dat"}; try { DeserializeFileDB(path_addr, *addrman, CLIENT_VERSION); - LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->size(), Ticks(SteadyClock::now() - start)); + LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->Size(), Ticks(SteadyClock::now() - start)); } catch (const DbNotFoundError&) { // Addrman can be in an inconsistent state after failure, reset it addrman = std::make_unique(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); diff --git a/src/addrdb.h b/src/addrdb.h index 627ef3ac3ce2..08d86d0f01be 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -6,9 +6,9 @@ #ifndef BITCOIN_ADDRDB_H #define BITCOIN_ADDRDB_H -#include #include // For banmap_t #include +#include #include #include diff --git a/src/addrman.cpp b/src/addrman.cpp index f16ff2230b93..f5ca9a5c3431 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2021 The Bitcoin Core developers +// Copyright (c) 2012-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -291,6 +291,7 @@ void AddrManImpl::Unserialize(Stream& s_) mapAddr[info] = n; info.nRandomPos = vRandom.size(); vRandom.push_back(n); + m_network_counts[info.GetNetwork()].n_new++; } nIdCount = nNew; @@ -310,6 +311,7 @@ void AddrManImpl::Unserialize(Stream& s_) mapAddr[info] = nIdCount; vvTried[nKBucket][nKBucketPos] = nIdCount; nIdCount++; + m_network_counts[info.GetNetwork()].n_tried++; } else { nLost++; } @@ -425,6 +427,8 @@ AddrInfo* AddrManImpl::Create(const CAddress& addr, const CNetAddr& addrSource, mapAddr[addr] = nId; mapInfo[nId].nRandomPos = vRandom.size(); vRandom.push_back(nId); + nNew++; + m_network_counts[addr.GetNetwork()].n_new++; if (pnId) *pnId = nId; return &mapInfo[nId]; @@ -464,6 +468,7 @@ void AddrManImpl::Delete(int nId) assert(info.nRefCount == 0); SwapRandom(info.nRandomPos, vRandom.size() - 1); + m_network_counts[info.GetNetwork()].n_new--; vRandom.pop_back(); mapAddr.erase(info); mapInfo.erase(nId); @@ -481,7 +486,7 @@ void AddrManImpl::ClearNew(int nUBucket, int nUBucketPos) assert(infoDelete.nRefCount > 0); infoDelete.nRefCount--; vvNew[nUBucket][nUBucketPos] = -1; - LogPrint(BCLog::ADDRMAN, "Removed %s from new[%i][%i]\n", infoDelete.ToString(), nUBucket, nUBucketPos); + LogPrint(BCLog::ADDRMAN, "Removed %s from new[%i][%i]\n", infoDelete.ToStringAddrPort(), nUBucket, nUBucketPos); if (infoDelete.nRefCount == 0) { Delete(nIdDelete); } @@ -504,6 +509,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) } } nNew--; + m_network_counts[info.GetNetwork()].n_new--; assert(info.nRefCount == 0); @@ -522,6 +528,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) infoOld.fInTried = false; vvTried[nKBucket][nKBucketPos] = -1; nTried--; + m_network_counts[infoOld.GetNetwork()].n_tried--; // find which new bucket it belongs to int nUBucket = infoOld.GetNewBucket(nKey, m_netgroupman); @@ -533,14 +540,16 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) infoOld.nRefCount = 1; vvNew[nUBucket][nUBucketPos] = nIdEvict; nNew++; + m_network_counts[infoOld.GetNetwork()].n_new++; LogPrint(BCLog::ADDRMAN, "Moved %s from tried[%i][%i] to new[%i][%i] to make space\n", - infoOld.ToString(), nKBucket, nKBucketPos, nUBucket, nUBucketPos); + infoOld.ToStringAddrPort(), nKBucket, nKBucketPos, nUBucket, nUBucketPos); } assert(vvTried[nKBucket][nKBucketPos] == -1); vvTried[nKBucket][nKBucketPos] = nId; nTried++; info.fInTried = true; + m_network_counts[info.GetNetwork()].n_tried++; } bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, std::chrono::seconds time_penalty) @@ -591,7 +600,6 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, std::c } else { pinfo = Create(addr, source, &nId); pinfo->nTime = std::max(NodeSeconds{0s}, pinfo->nTime - time_penalty); - nNew++; } int nUBucket = pinfo->GetNewBucket(nKey, source, m_netgroupman); @@ -610,7 +618,7 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, std::c pinfo->nRefCount++; vvNew[nUBucket][nUBucketPos] = nId; LogPrint(BCLog::ADDRMAN, "Added %s mapped to AS%i to new[%i][%i]\n", - addr.ToString(), m_netgroupman.GetMappedAS(addr), nUBucket, nUBucketPos); + addr.ToStringAddrPort(), m_netgroupman.GetMappedAS(addr), nUBucket, nUBucketPos); } else { if (pinfo->nRefCount == 0) { Delete(nId); @@ -661,15 +669,15 @@ bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, NodeSecond // Output the entry we'd be colliding with, for debugging purposes auto colliding_entry = mapInfo.find(vvTried[tried_bucket][tried_bucket_pos]); LogPrint(BCLog::ADDRMAN, "Collision with %s while attempting to move %s to tried table. Collisions=%d\n", - colliding_entry != mapInfo.end() ? colliding_entry->second.ToString() : "", - addr.ToString(), + colliding_entry != mapInfo.end() ? colliding_entry->second.ToStringAddrPort() : "", + addr.ToStringAddrPort(), m_tried_collisions.size()); return false; } else { // move nId to the tried tables MakeTried(info, nId); LogPrint(BCLog::ADDRMAN, "Moved %s mapped to AS%i to tried[%i][%i]\n", - addr.ToString(), m_netgroupman.GetMappedAS(addr), tried_bucket, tried_bucket_pos); + addr.ToStringAddrPort(), m_netgroupman.GetMappedAS(addr), tried_bucket, tried_bucket_pos); return true; } } @@ -681,7 +689,7 @@ bool AddrManImpl::Add_(const std::vector& vAddr, const CNetAddr& sourc added += AddSingle(*it, source, time_penalty) ? 1 : 0; } if (added > 0) { - LogPrint(BCLog::ADDRMAN, "Added %i addresses (of %i) from %s: %i tried, %i new\n", added, vAddr.size(), source.ToString(), nTried, nNew); + LogPrint(BCLog::ADDRMAN, "Added %i addresses (of %i) from %s: %i tried, %i new\n", added, vAddr.size(), source.ToStringAddr(), nTried, nNew); } return added > 0; } @@ -738,7 +746,7 @@ std::pair AddrManImpl::Select_(bool newOnly) const const AddrInfo& info{it_found->second}; // With probability GetChance() * fChanceFactor, return the entry. if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { - LogPrint(BCLog::ADDRMAN, "Selected %s from tried\n", info.ToString()); + LogPrint(BCLog::ADDRMAN, "Selected %s from tried\n", info.ToStringAddrPort()); return {info, info.m_last_try}; } // Otherwise start over with a (likely) different bucket, and increased chance factor. @@ -766,7 +774,7 @@ std::pair AddrManImpl::Select_(bool newOnly) const const AddrInfo& info{it_found->second}; // With probability GetChance() * fChanceFactor, return the entry. if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { - LogPrint(BCLog::ADDRMAN, "Selected %s from new\n", info.ToString()); + LogPrint(BCLog::ADDRMAN, "Selected %s from new\n", info.ToStringAddrPort()); return {info, info.m_last_try}; } // Otherwise start over with a (likely) different bucket, and increased chance factor. @@ -883,7 +891,7 @@ void AddrManImpl::ResolveCollisions_() // Give address at least 60 seconds to successfully connect if (current_time - info_old.m_last_try > 60s) { - LogPrint(BCLog::ADDRMAN, "Replacing %s with %s in tried table\n", info_old.ToString(), info_new.ToString()); + LogPrint(BCLog::ADDRMAN, "Replacing %s with %s in tried table\n", info_old.ToStringAddrPort(), info_new.ToStringAddrPort()); // Replaces an existing address already in the tried table with the new address Good_(info_new, false, current_time); @@ -893,7 +901,7 @@ void AddrManImpl::ResolveCollisions_() // If the collision hasn't resolved in some reasonable amount of time, // just evict the old entry -- we must not be able to // connect to it for some reason. - LogPrint(BCLog::ADDRMAN, "Unable to test; replacing %s with %s in tried table anyway\n", info_old.ToString(), info_new.ToString()); + LogPrint(BCLog::ADDRMAN, "Unable to test; replacing %s with %s in tried table anyway\n", info_old.ToStringAddrPort(), info_new.ToStringAddrPort()); Good_(info_new, false, current_time); erase_collision = true; } @@ -962,6 +970,28 @@ std::optional AddrManImpl::FindAddressEntry_(const CAddress& ad } } +size_t AddrManImpl::Size_(std::optional net, std::optional in_new) const +{ + AssertLockHeld(cs); + + if (!net.has_value()) { + if (in_new.has_value()) { + return *in_new ? nNew : nTried; + } else { + return vRandom.size(); + } + } + if (auto it = m_network_counts.find(*net); it != m_network_counts.end()) { + auto net_count = it->second; + if (in_new.has_value()) { + return *in_new ? net_count.n_new : net_count.n_tried; + } else { + return net_count.n_new + net_count.n_tried; + } + } + return 0; +} + void AddrManImpl::Check() const { AssertLockHeld(cs); @@ -986,6 +1016,7 @@ int AddrManImpl::CheckAddrman() const std::unordered_set setTried; std::unordered_map mapNew; + std::unordered_map local_counts; if (vRandom.size() != (size_t)(nTried + nNew)) return -7; @@ -1000,12 +1031,14 @@ int AddrManImpl::CheckAddrman() const if (info.nRefCount) return -2; setTried.insert(n); + local_counts[info.GetNetwork()].n_tried++; } else { if (info.nRefCount < 0 || info.nRefCount > ADDRMAN_NEW_BUCKETS_PER_ADDRESS) return -3; if (!info.nRefCount) return -4; mapNew[n] = info.nRefCount; + local_counts[info.GetNetwork()].n_new++; } const auto it{mapAddr.find(info)}; if (it == mapAddr.end() || it->second != n) { @@ -1065,13 +1098,27 @@ int AddrManImpl::CheckAddrman() const if (nKey.IsNull()) return -16; + // It's possible that m_network_counts may have all-zero entries that local_counts + // doesn't have if addrs from a network were being added and then removed again in the past. + if (m_network_counts.size() < local_counts.size()) { + return -20; + } + for (const auto& [net, count] : m_network_counts) { + if (local_counts[net].n_new != count.n_new || local_counts[net].n_tried != count.n_tried) { + return -21; + } + } + return 0; } -size_t AddrManImpl::size() const +size_t AddrManImpl::Size(std::optional net, std::optional in_new) const { - LOCK(cs); // TODO: Cache this in an atomic to avoid this overhead - return vRandom.size(); + LOCK(cs); + Check(); + auto ret = Size_(net, in_new); + Check(); + return ret; } bool AddrManImpl::Add(const std::vector& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) @@ -1112,7 +1159,7 @@ std::pair AddrManImpl::SelectTriedCollision() { LOCK(cs); Check(); - const auto ret = SelectTriedCollision_(); + auto ret = SelectTriedCollision_(); Check(); return ret; } @@ -1121,7 +1168,7 @@ std::pair AddrManImpl::Select(bool newOnly) const { LOCK(cs); Check(); - const auto addrRet = Select_(newOnly); + auto addrRet = Select_(newOnly); Check(); return addrRet; } @@ -1130,7 +1177,7 @@ std::vector AddrManImpl::GetAddr(size_t max_addresses, size_t max_pct, { LOCK(cs); Check(); - const auto addresses = GetAddr_(max_addresses, max_pct, network); + auto addresses = GetAddr_(max_addresses, max_pct, network); Check(); return addresses; } @@ -1178,17 +1225,16 @@ void AddrMan::Unserialize(Stream& s_) } // explicit instantiation -template void AddrMan::Serialize(CHashWriter& s) const; -template void AddrMan::Serialize(CAutoFile& s) const; +template void AddrMan::Serialize(HashedSourceWriter& s) const; template void AddrMan::Serialize(CDataStream& s) const; template void AddrMan::Unserialize(CAutoFile& s); template void AddrMan::Unserialize(CHashVerifier& s); template void AddrMan::Unserialize(CDataStream& s); template void AddrMan::Unserialize(CHashVerifier& s); -size_t AddrMan::size() const +size_t AddrMan::Size(std::optional net, std::optional in_new) const { - return m_impl->size(); + return m_impl->Size(net, in_new); } bool AddrMan::Add(const std::vector& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) diff --git a/src/addrman.h b/src/addrman.h index 5099c8c7a37e..4985fc764cf4 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -1,5 +1,5 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2021 The Bitcoin Core developers +// Copyright (c) 2012-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -99,8 +99,14 @@ class AddrMan template void Unserialize(Stream& s_); - //! Return the number of (unique) addresses in all tables. - size_t size() const; + /** + * Return size information about addrman. + * + * @param[in] net Select addresses only from specified network (nullopt = all) + * @param[in] in_new Select addresses only from one table (true = new, false = tried, nullopt = both) + * @return Number of unique addresses that match specified options. + */ + size_t Size(std::optional net = std::nullopt, std::optional in_new = std::nullopt) const; /** * Attempt to add one or more addresses to addrman's new table. diff --git a/src/addrman_impl.h b/src/addrman_impl.h index 376e79f49f5e..94fe81aca9bc 100644 --- a/src/addrman_impl.h +++ b/src/addrman_impl.h @@ -1,4 +1,4 @@ -// Copyright (c) 2021 The Bitcoin Core developers +// Copyright (c) 2021-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -112,7 +112,7 @@ class AddrManImpl template void Unserialize(Stream& s_) EXCLUSIVE_LOCKS_REQUIRED(!cs); - size_t size() const EXCLUSIVE_LOCKS_REQUIRED(!cs); + size_t Size(std::optional net, std::optional in_new) const EXCLUSIVE_LOCKS_REQUIRED(!cs); bool Add(const std::vector& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) EXCLUSIVE_LOCKS_REQUIRED(!cs); @@ -215,6 +215,14 @@ class AddrManImpl /** Reference to the netgroup manager. netgroupman must be constructed before addrman and destructed after. */ const NetGroupManager& m_netgroupman; + struct NewTriedCount { + size_t n_new; + size_t n_tried; + }; + + /** Number of entries in addrman per network and new/tried table. */ + std::unordered_map m_network_counts GUARDED_BY(cs); + //! Find an entry. AddrInfo* Find(const CService& addr, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); @@ -257,6 +265,8 @@ class AddrManImpl std::optional FindAddressEntry_(const CAddress& addr) EXCLUSIVE_LOCKS_REQUIRED(cs); + size_t Size_(std::optional net, std::optional in_new) const EXCLUSIVE_LOCKS_REQUIRED(cs); + //! Consistency check, taking into account m_consistency_check_ratio. //! Will std::abort if an inconsistency is detected. void Check() const EXCLUSIVE_LOCKS_REQUIRED(cs); diff --git a/src/arith_uint256.cpp b/src/arith_uint256.cpp index e614102de3f9..3776cfb6defc 100644 --- a/src/arith_uint256.cpp +++ b/src/arith_uint256.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/arith_uint256.h b/src/arith_uint256.h index b7b3b3a2852a..c710fe947124 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -58,7 +58,7 @@ class base_uint explicit base_uint(const std::string& str); - const base_uint operator~() const + base_uint operator~() const { base_uint ret; for (int i = 0; i < WIDTH; i++) @@ -66,7 +66,7 @@ class base_uint return ret; } - const base_uint operator-() const + base_uint operator-() const { base_uint ret; for (int i = 0; i < WIDTH; i++) @@ -171,7 +171,7 @@ class base_uint return *this; } - const base_uint operator++(int) + base_uint operator++(int) { // postfix operator const base_uint ret = *this; @@ -188,7 +188,7 @@ class base_uint return *this; } - const base_uint operator--(int) + base_uint operator--(int) { // postfix operator const base_uint ret = *this; @@ -199,16 +199,16 @@ class base_uint int CompareTo(const base_uint& b) const; bool EqualTo(uint64_t b) const; - friend inline const base_uint operator+(const base_uint& a, const base_uint& b) { return base_uint(a) += b; } - friend inline const base_uint operator-(const base_uint& a, const base_uint& b) { return base_uint(a) -= b; } - friend inline const base_uint operator*(const base_uint& a, const base_uint& b) { return base_uint(a) *= b; } - friend inline const base_uint operator/(const base_uint& a, const base_uint& b) { return base_uint(a) /= b; } - friend inline const base_uint operator|(const base_uint& a, const base_uint& b) { return base_uint(a) |= b; } - friend inline const base_uint operator&(const base_uint& a, const base_uint& b) { return base_uint(a) &= b; } - friend inline const base_uint operator^(const base_uint& a, const base_uint& b) { return base_uint(a) ^= b; } - friend inline const base_uint operator>>(const base_uint& a, int shift) { return base_uint(a) >>= shift; } - friend inline const base_uint operator<<(const base_uint& a, int shift) { return base_uint(a) <<= shift; } - friend inline const base_uint operator*(const base_uint& a, uint32_t b) { return base_uint(a) *= b; } + friend inline base_uint operator+(const base_uint& a, const base_uint& b) { return base_uint(a) += b; } + friend inline base_uint operator-(const base_uint& a, const base_uint& b) { return base_uint(a) -= b; } + friend inline base_uint operator*(const base_uint& a, const base_uint& b) { return base_uint(a) *= b; } + friend inline base_uint operator/(const base_uint& a, const base_uint& b) { return base_uint(a) /= b; } + friend inline base_uint operator|(const base_uint& a, const base_uint& b) { return base_uint(a) |= b; } + friend inline base_uint operator&(const base_uint& a, const base_uint& b) { return base_uint(a) &= b; } + friend inline base_uint operator^(const base_uint& a, const base_uint& b) { return base_uint(a) ^= b; } + friend inline base_uint operator>>(const base_uint& a, int shift) { return base_uint(a) >>= shift; } + friend inline base_uint operator<<(const base_uint& a, int shift) { return base_uint(a) <<= shift; } + friend inline base_uint operator*(const base_uint& a, uint32_t b) { return base_uint(a) *= b; } friend inline bool operator==(const base_uint& a, const base_uint& b) { return memcmp(a.pn, b.pn, sizeof(a.pn)) == 0; } friend inline bool operator!=(const base_uint& a, const base_uint& b) { return memcmp(a.pn, b.pn, sizeof(a.pn)) != 0; } friend inline bool operator>(const base_uint& a, const base_uint& b) { return a.CompareTo(b) > 0; } diff --git a/src/banman.cpp b/src/banman.cpp index 3cd646c148da..ece949d99761 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2021 The Bitcoin Core developers +// Copyright (c) 2009-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/banman.h b/src/banman.h index 77b043f081b5..5a5f5677b008 100644 --- a/src/banman.h +++ b/src/banman.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2021 The Bitcoin Core developers +// Copyright (c) 2009-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_BANMAN_H @@ -7,9 +7,9 @@ #include #include -#include #include // For banmap_t #include +#include #include #include diff --git a/src/base58.cpp b/src/base58.cpp index 11c1ce7397f4..cf5d62f164f3 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2021 The Bitcoin Core developers +// Copyright (c) 2014-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/base58.h b/src/base58.h index d2a8d5e3bc47..2f4d0b74b1b3 100644 --- a/src/base58.h +++ b/src/base58.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2020 The Bitcoin Core developers +// Copyright (c) 2009-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bech32.cpp b/src/bech32.cpp index 8e0025b8f42b..ba3c419d8b68 100644 --- a/src/bech32.cpp +++ b/src/bech32.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2017, 2021 Pieter Wuille -// Copyright (c) 2021 The Bitcoin Core developers +// Copyright (c) 2021-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index 019b1333458f..d6b52eb587ad 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2021 The Bitcoin Core developers +// Copyright (c) 2020-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index 3d08b7201b71..78748bc5bde5 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2020 The Bitcoin Core developers +// Copyright (c) 2016-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/bech32.cpp b/src/bench/bech32.cpp index 1a166e70812b..9922653766fd 100644 --- a/src/bench/bech32.cpp +++ b/src/bench/bech32.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2021 The Bitcoin Core developers +// Copyright (c) 2018-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 1a3a0062865c..84b66bc4b222 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2015-2021 The Bitcoin Core developers +// Copyright (c) 2015-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include -#include #include +#include #include #include @@ -83,7 +83,7 @@ void BenchRunner::RunAll(const Args& args) std::smatch baseMatch; if (args.sanity_check) { - std::cout << "Running with --sanity-check option, benchmark results will be useless." << std::endl; + std::cout << "Running with -sanity-check option, output is being suppressed as benchmark results will be useless." << std::endl; } std::vector benchmarkResults; @@ -106,6 +106,7 @@ void BenchRunner::RunAll(const Args& args) Bench bench; if (args.sanity_check) { bench.epochs(1).epochIterations(1); + bench.output(nullptr); } bench.name(name); if (args.min_time > 0ms) { diff --git a/src/bench/bench.h b/src/bench/bench.h index 63e1bf67e21e..78196134e781 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -1,11 +1,11 @@ -// Copyright (c) 2015-2021 The Bitcoin Core developers +// Copyright (c) 2015-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_BENCH_BENCH_H #define BITCOIN_BENCH_BENCH_H -#include +#include #include #include diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 1ac8db19fd88..7faaa1fb14e6 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2021 The Bitcoin Core developers +// Copyright (c) 2015-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include @@ -31,7 +31,7 @@ static void SetupBenchArgs(ArgsManager& argsman) argsman.AddArg("-min-time=", strprintf("Minimum runtime per benchmark, in milliseconds (default: %d)", DEFAULT_MIN_TIME_MS), ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS); argsman.AddArg("-output-csv=", "Generate CSV file with the most important benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-output-json=", "Generate JSON file with all benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-sanity-check", "Run benchmarks for only one iteration", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-sanity-check", "Run benchmarks for only one iteration with no output", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-priority-level=", strprintf("Run benchmarks of one or multiple priority level(s) (%s), default: '%s'", benchmark::ListPriorities(), DEFAULT_PRIORITY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); } diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 69258377d5be..8dd4117a3ec5 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -1,10 +1,11 @@ -// Copyright (c) 2011-2021 The Bitcoin Core developers +// Copyright (c) 2011-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include +#include #include #include #include @@ -45,5 +46,18 @@ static void AssembleBlock(benchmark::Bench& bench) PrepareBlock(test_setup->m_node, P2WSH_OP_TRUE); }); } +static void BlockAssemblerAddPackageTxns(benchmark::Bench& bench) +{ + FastRandomContext det_rand{true}; + auto testing_setup{MakeNoLogFileContext()}; + testing_setup->PopulateMempool(det_rand, /*num_transactions=*/1000, /*submit=*/true); + node::BlockAssembler::Options assembler_options; + assembler_options.test_block_validity = false; + + bench.run([&] { + PrepareBlock(testing_setup->m_node, P2WSH_OP_TRUE, assembler_options); + }); +} BENCHMARK(AssembleBlock, benchmark::PriorityLevel::HIGH); +BENCHMARK(BlockAssemblerAddPackageTxns, benchmark::PriorityLevel::LOW); diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index 5d55ed933209..4a3ec67c2b81 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2021 The Bitcoin Core developers +// Copyright (c) 2016-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,7 +18,6 @@ // (https://github.com/bitcoin/bitcoin/issues/7883#issuecomment-224807484) static void CCoinsCaching(benchmark::Bench& bench) { - const ECCVerifyHandle verify_handle; ECC_Start(); FillableSigningProvider keystore; diff --git a/src/bench/chacha20.cpp b/src/bench/chacha20.cpp index 9584dd58bb40..115cd064bd9d 100644 --- a/src/bench/chacha20.cpp +++ b/src/bench/chacha20.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2020 The Bitcoin Core developers +// Copyright (c) 2019-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,9 +14,9 @@ static const uint64_t BUFFER_SIZE_LARGE = 1024*1024; static void CHACHA20(benchmark::Bench& bench, size_t buffersize) { std::vector key(32,0); - ChaCha20 ctx(key.data(), key.size()); + ChaCha20 ctx(key.data()); ctx.SetIV(0); - ctx.Seek(0); + ctx.Seek64(0); std::vector in(buffersize,0); std::vector out(buffersize,0); bench.batch(in.size()).unit("byte").run([&] { diff --git a/src/bench/chacha_poly_aead.cpp b/src/bench/chacha_poly_aead.cpp index 15b3a4f31061..db88841c3275 100644 --- a/src/bench/chacha_poly_aead.cpp +++ b/src/bench/chacha_poly_aead.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2020 The Bitcoin Core developers +// Copyright (c) 2019-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 747279c1614b..ee76f7b76743 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2020 The Bitcoin Core developers +// Copyright (c) 2016-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 8517c9fee29f..8ad6fde6bffe 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2020 The Bitcoin Core developers +// Copyright (c) 2015-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -25,12 +25,10 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) // We shouldn't ever be running with the checkqueue on a single core machine. if (GetNumCores() <= 1) return; - const ECCVerifyHandle verify_handle; ECC_Start(); struct PrevectorJob { prevector p; - PrevectorJob() = default; explicit PrevectorJob(FastRandomContext& insecure_rand){ p.resize(insecure_rand.randrange(PREVECTOR_SIZE*2)); } @@ -38,10 +36,6 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) { return true; } - void swap(PrevectorJob& x) noexcept - { - p.swap(x.p); - }; }; CCheckQueue queue {QUEUE_BATCH_SIZE}; // The main thread should be counted to prevent thread oversubscription, and @@ -61,7 +55,7 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) // Make insecure_rand here so that each iteration is identical. CCheckQueueControl control(&queue); for (auto vChecks : vBatches) { - control.Add(vChecks); + control.Add(std::move(vChecks)); } // control waits for completion by RAII, but // it is done explicitly here for clarity diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 53d89039a75b..265d4bf6559a 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2021 The Bitcoin Core developers +// Copyright (c) 2012-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -45,7 +45,7 @@ static void CoinSelection(benchmark::Bench& bench) { NodeContext node; auto chain = interfaces::MakeChain(node); - CWallet wallet(chain.get(), "", gArgs, CreateDummyWalletDatabase()); + CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); std::vector> wtxs; LOCK(wallet.cs_wallet); @@ -75,8 +75,9 @@ static void CoinSelection(benchmark::Bench& bench) /*tx_noinputs_size=*/ 0, /*avoid_partial=*/ false, }; + auto group = wallet::GroupOutputs(wallet, available_coins, coin_selection_params, {{filter_standard}})[filter_standard]; bench.run([&] { - auto result = AttemptSelection(wallet, 1003 * COIN, filter_standard, available_coins, coin_selection_params, /*allow_mixed_output_types=*/true); + auto result = AttemptSelection(1003 * COIN, group, coin_selection_params, /*allow_mixed_output_types=*/true); assert(result); assert(result->GetSelectedValue() == 1003 * COIN); assert(result->GetInputSet().size() == 2); @@ -91,7 +92,7 @@ static void add_coin(const CAmount& nValue, int nInput, std::vector tx.vout[nInput].nValue = nValue; COutput output(COutPoint(tx.GetHash(), nInput), tx.vout.at(nInput), /*depth=*/ 0, /*input_bytes=*/ -1, /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, /*time=*/ 0, /*from_me=*/ true, /*fees=*/ 0); set.emplace_back(); - set.back().Insert(output, /*ancestors=*/ 0, /*descendants=*/ 0, /*positive_only=*/ false); + set.back().Insert(std::make_shared(output), /*ancestors=*/ 0, /*descendants=*/ 0); } // Copied from src/wallet/test/coinselector_tests.cpp static CAmount make_hard_case(int utxos, std::vector& utxo_pool) @@ -99,9 +100,9 @@ static CAmount make_hard_case(int utxos, std::vector& utxo_pool) utxo_pool.clear(); CAmount target = 0; for (int i = 0; i < utxos; ++i) { - target += (CAmount)1 << (utxos+i); - add_coin((CAmount)1 << (utxos+i), 2*i, utxo_pool); - add_coin(((CAmount)1 << (utxos+i)) + ((CAmount)1 << (utxos-1-i)), 2*i + 1, utxo_pool); + target += CAmount{1} << (utxos+i); + add_coin(CAmount{1} << (utxos+i), 2*i, utxo_pool); + add_coin((CAmount{1} << (utxos+i)) + (CAmount{1} << (utxos-1-i)), 2*i + 1, utxo_pool); } return target; } diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 162b02f344da..cf8d807d7b13 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2021 The Bitcoin Core developers +// Copyright (c) 2016-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,7 +18,7 @@ /* Number of bytes to hash per iteration */ static const uint64_t BUFFER_SIZE = 1000*1000; -static void RIPEMD160(benchmark::Bench& bench) +static void BenchRIPEMD160(benchmark::Bench& bench) { uint8_t hash[CRIPEMD160::OUTPUT_SIZE]; std::vector in(BUFFER_SIZE,0); @@ -150,7 +150,7 @@ static void MuHashPrecompute(benchmark::Bench& bench) }); } -BENCHMARK(RIPEMD160, benchmark::PriorityLevel::HIGH); +BENCHMARK(BenchRIPEMD160, benchmark::PriorityLevel::HIGH); BENCHMARK(SHA1, benchmark::PriorityLevel::HIGH); BENCHMARK(SHA256, benchmark::PriorityLevel::HIGH); BENCHMARK(SHA512, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/descriptors.cpp b/src/bench/descriptors.cpp index 972a6ff9531a..5d28d2690925 100644 --- a/src/bench/descriptors.cpp +++ b/src/bench/descriptors.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,7 +13,6 @@ static void ExpandDescriptor(benchmark::Bench& bench) { - const ECCVerifyHandle verify_handle; ECC_Start(); const auto desc_str = "sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))"; diff --git a/src/bench/duplicate_inputs.cpp b/src/bench/duplicate_inputs.cpp index 559854ff4838..b3799ad1b72c 100644 --- a/src/bench/duplicate_inputs.cpp +++ b/src/bench/duplicate_inputs.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2021 The Bitcoin Core developers +// Copyright (c) 2011-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/examples.cpp b/src/bench/examples.cpp index abef69cc42df..671902ef2fb6 100644 --- a/src/bench/examples.cpp +++ b/src/bench/examples.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2020 The Bitcoin Core developers +// Copyright (c) 2015-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/gcs_filter.cpp b/src/bench/gcs_filter.cpp index b795ebff3913..0af4ee98fe9d 100644 --- a/src/bench/gcs_filter.cpp +++ b/src/bench/gcs_filter.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2018-2020 The Bitcoin Core developers +// Copyright (c) 2018-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include -static const GCSFilter::ElementSet GenerateGCSTestElements() +static GCSFilter::ElementSet GenerateGCSTestElements() { GCSFilter::ElementSet elements; diff --git a/src/bench/hashpadding.cpp b/src/bench/hashpadding.cpp index ac5aeebe51d3..e9d2c25fe358 100644 --- a/src/bench/hashpadding.cpp +++ b/src/bench/hashpadding.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2020 The Bitcoin Core developers +// Copyright (c) 2015-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/load_external.cpp b/src/bench/load_external.cpp index be01b2a4838c..0fd842c7c367 100644 --- a/src/bench/load_external.cpp +++ b/src/bench/load_external.cpp @@ -27,7 +27,7 @@ static void LoadExternalBlockFile(benchmark::Bench& bench) // Create a single block as in the blocks files (magic bytes, block size, // block data) as a stream object. const fs::path blkfile{testing_setup.get()->m_path_root / "blk.dat"}; - CDataStream ss(SER_DISK, 0); + DataStream ss{}; auto params{testing_setup->m_node.chainman->GetParams()}; ss << params.MessageStart(); ss << static_cast(benchmark::data::block413567.size()); diff --git a/src/bench/lockedpool.cpp b/src/bench/lockedpool.cpp index ac8262654c1f..6851ed0bd88a 100644 --- a/src/bench/lockedpool.cpp +++ b/src/bench/lockedpool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2020 The Bitcoin Core developers +// Copyright (c) 2016-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,9 +17,7 @@ static void BenchLockedPool(benchmark::Bench& bench) const size_t synth_size = 1024*1024; Arena b(synth_base, synth_size, 16); - std::vector addr; - for (int x=0; x addr{ASIZE, nullptr}; uint32_t s = 0x12345678; bench.run([&] { int idx = s & (addr.size() - 1); diff --git a/src/bench/logging.cpp b/src/bench/logging.cpp index 49a9e59893ab..9aedb26236df 100644 --- a/src/bench/logging.cpp +++ b/src/bench/logging.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,9 +6,17 @@ #include #include +// All but 2 of the benchmarks should have roughly similar performance: +// +// LogPrintWithoutCategory should be ~3 orders of magnitude faster, as nothing is logged. +// +// LogWithoutWriteToFile should be ~2 orders of magnitude faster, as it avoids disk writes. static void Logging(benchmark::Bench& bench, const std::vector& extra_args, const std::function& log) { + // Reset any enabled logging categories from a previous benchmark run. + LogInstance().DisableCategory(BCLog::LogFlags::ALL); + TestingSetup test_setup{ CBaseChainParams::REGTEST, extra_args, @@ -17,32 +25,67 @@ static void Logging(benchmark::Bench& bench, const std::vector& ext bench.run([&] { log(); }); } -static void LoggingYoThreadNames(benchmark::Bench& bench) +static void LogPrintLevelWithThreadNames(benchmark::Bench& bench) { - Logging(bench, {"-logthreadnames=1"}, [] { LogPrintf("%s\n", "test"); }); + Logging(bench, {"-logthreadnames=1", "-debug=net"}, [] { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "%s\n", "test"); }); } -static void LoggingNoThreadNames(benchmark::Bench& bench) + +static void LogPrintLevelWithoutThreadNames(benchmark::Bench& bench) { - Logging(bench, {"-logthreadnames=0"}, [] { LogPrintf("%s\n", "test"); }); + Logging(bench, {"-logthreadnames=0", "-debug=net"}, [] { + LogPrintLevel(BCLog::NET, BCLog::Level::Error, "%s\n", "test"); }); } -static void LoggingYoCategory(benchmark::Bench& bench) + +static void LogPrintWithCategory(benchmark::Bench& bench) { Logging(bench, {"-logthreadnames=0", "-debug=net"}, [] { LogPrint(BCLog::NET, "%s\n", "test"); }); } -static void LoggingNoCategory(benchmark::Bench& bench) + +static void LogPrintWithoutCategory(benchmark::Bench& bench) { Logging(bench, {"-logthreadnames=0", "-debug=0"}, [] { LogPrint(BCLog::NET, "%s\n", "test"); }); } -static void LoggingNoFile(benchmark::Bench& bench) + +static void LogPrintfCategoryWithThreadNames(benchmark::Bench& bench) +{ + Logging(bench, {"-logthreadnames=1", "-debug=net"}, [] { + LogPrintfCategory(BCLog::NET, "%s\n", "test"); + }); +} + +static void LogPrintfCategoryWithoutThreadNames(benchmark::Bench& bench) +{ + Logging(bench, {"-logthreadnames=0", "-debug=net"}, [] { + LogPrintfCategory(BCLog::NET, "%s\n", "test"); + }); +} + +static void LogPrintfWithThreadNames(benchmark::Bench& bench) +{ + Logging(bench, {"-logthreadnames=1"}, [] { LogPrintf("%s\n", "test"); }); +} + +static void LogPrintfWithoutThreadNames(benchmark::Bench& bench) +{ + Logging(bench, {"-logthreadnames=0"}, [] { LogPrintf("%s\n", "test"); }); +} + +static void LogWithoutWriteToFile(benchmark::Bench& bench) { + // Disable writing the log to a file, as used for unit tests and fuzzing in `MakeNoLogFileContext`. Logging(bench, {"-nodebuglogfile", "-debug=1"}, [] { LogPrintf("%s\n", "test"); LogPrint(BCLog::NET, "%s\n", "test"); }); } -BENCHMARK(LoggingYoThreadNames, benchmark::PriorityLevel::HIGH); -BENCHMARK(LoggingNoThreadNames, benchmark::PriorityLevel::HIGH); -BENCHMARK(LoggingYoCategory, benchmark::PriorityLevel::HIGH); -BENCHMARK(LoggingNoCategory, benchmark::PriorityLevel::HIGH); -BENCHMARK(LoggingNoFile, benchmark::PriorityLevel::HIGH); +BENCHMARK(LogPrintLevelWithThreadNames, benchmark::PriorityLevel::HIGH); +BENCHMARK(LogPrintLevelWithoutThreadNames, benchmark::PriorityLevel::HIGH); +BENCHMARK(LogPrintWithCategory, benchmark::PriorityLevel::HIGH); +BENCHMARK(LogPrintWithoutCategory, benchmark::PriorityLevel::HIGH); +BENCHMARK(LogPrintfCategoryWithThreadNames, benchmark::PriorityLevel::HIGH); +BENCHMARK(LogPrintfCategoryWithoutThreadNames, benchmark::PriorityLevel::HIGH); +BENCHMARK(LogPrintfWithThreadNames, benchmark::PriorityLevel::HIGH); +BENCHMARK(LogPrintfWithoutThreadNames, benchmark::PriorityLevel::HIGH); +BENCHMARK(LogWithoutWriteToFile, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 2b133f58f40d..735dc92dfbe1 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2021 The Bitcoin Core developers +// Copyright (c) 2011-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index 67564508d975..80c959cdfbc1 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2021 The Bitcoin Core developers +// Copyright (c) 2011-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/merkle_root.cpp b/src/bench/merkle_root.cpp index 4140d67bc7ab..55409335bdd9 100644 --- a/src/bench/merkle_root.cpp +++ b/src/bench/merkle_root.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2020 The Bitcoin Core developers +// Copyright (c) 2016-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/nanobench.h b/src/bench/nanobench.h index 916a7d61effd..8b3dc6c71c01 100644 --- a/src/bench/nanobench.h +++ b/src/bench/nanobench.h @@ -7,7 +7,7 @@ // // Licensed under the MIT License . // SPDX-License-Identifier: MIT -// Copyright (c) 2019-2021 Martin Ankerl +// Copyright (c) 2019-2023 Martin Leitner-Ankerl // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -31,19 +31,20 @@ #define ANKERL_NANOBENCH_H_INCLUDED // see https://semver.org/ -#define ANKERL_NANOBENCH_VERSION_MAJOR 4 // incompatible API changes -#define ANKERL_NANOBENCH_VERSION_MINOR 3 // backwards-compatible changes -#define ANKERL_NANOBENCH_VERSION_PATCH 6 // backwards-compatible bug fixes +#define ANKERL_NANOBENCH_VERSION_MAJOR 4 // incompatible API changes +#define ANKERL_NANOBENCH_VERSION_MINOR 3 // backwards-compatible changes +#define ANKERL_NANOBENCH_VERSION_PATCH 10 // backwards-compatible bug fixes /////////////////////////////////////////////////////////////////////////////////////////////////// // public facing api - as minimal as possible /////////////////////////////////////////////////////////////////////////////////////////////////// -#include // high_resolution_clock -#include // memcpy -#include // for std::ostream* custom output target in Config -#include // all names -#include // holds all results +#include // high_resolution_clock +#include // memcpy +#include // for std::ostream* custom output target in Config +#include // all names +#include // holds context information of results +#include // holds all results #define ANKERL_NANOBENCH(x) ANKERL_NANOBENCH_PRIVATE_##x() @@ -91,7 +92,7 @@ #define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 0 #if defined(__linux__) && !defined(ANKERL_NANOBENCH_DISABLE_PERF_COUNTERS) # include -# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0) +# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0) // PERF_COUNT_HW_REF_CPU_CYCLES only available since kernel 3.3 // PERF_FLAG_FD_CLOEXEC since kernel 3.14 # undef ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS @@ -144,43 +145,45 @@ class BigO; * * `{{#result}}` Marks the begin of the result layer. Whatever comes after this will be instantiated as often as * a benchmark result is available. Within it, you can use these tags: * - * * `{{title}}` See Bench::title(). + * * `{{title}}` See Bench::title. * - * * `{{name}}` Benchmark name, usually directly provided with Bench::run(), but can also be set with Bench::name(). + * * `{{name}}` Benchmark name, usually directly provided with Bench::run, but can also be set with Bench::name. * - * * `{{unit}}` Unit, e.g. `byte`. Defaults to `op`, see Bench::title(). + * * `{{unit}}` Unit, e.g. `byte`. Defaults to `op`, see Bench::unit. * - * * `{{batch}}` Batch size, see Bench::batch(). + * * `{{batch}}` Batch size, see Bench::batch. * - * * `{{complexityN}}` Value used for asymptotic complexity calculation. See Bench::complexityN(). + * * `{{complexityN}}` Value used for asymptotic complexity calculation. See Bench::complexityN. * - * * `{{epochs}}` Number of epochs, see Bench::epochs(). + * * `{{epochs}}` Number of epochs, see Bench::epochs. * * * `{{clockResolution}}` Accuracy of the clock, i.e. what's the smallest time possible to measure with the clock. * For modern systems, this can be around 20 ns. This value is automatically determined by nanobench at the first * benchmark that is run, and used as a static variable throughout the application's runtime. * - * * `{{clockResolutionMultiple}}` Configuration multiplier for `clockResolution`. See Bench::clockResolutionMultiple(). + * * `{{clockResolutionMultiple}}` Configuration multiplier for `clockResolution`. See Bench::clockResolutionMultiple. * This is the target runtime for each measurement (epoch). That means the more accurate your clock is, the faster * will be the benchmark. Basing the measurement's runtime on the clock resolution is the main reason why nanobench is so fast. * * * `{{maxEpochTime}}` Configuration for a maximum time each measurement (epoch) is allowed to take. Note that at least - * a single iteration will be performed, even when that takes longer than maxEpochTime. See Bench::maxEpochTime(). + * a single iteration will be performed, even when that takes longer than maxEpochTime. See Bench::maxEpochTime. * - * * `{{minEpochTime}}` Minimum epoch time, usually not set. See Bench::minEpochTime(). + * * `{{minEpochTime}}` Minimum epoch time, defaults to 1ms. See Bench::minEpochTime. * - * * `{{minEpochIterations}}` See Bench::minEpochIterations(). + * * `{{minEpochIterations}}` See Bench::minEpochIterations. * - * * `{{epochIterations}}` See Bench::epochIterations(). + * * `{{epochIterations}}` See Bench::epochIterations. * - * * `{{warmup}}` Number of iterations used before measuring starts. See Bench::warmup(). + * * `{{warmup}}` Number of iterations used before measuring starts. See Bench::warmup. * - * * `{{relative}}` True or false, depending on the setting you have used. See Bench::relative(). + * * `{{relative}}` True or false, depending on the setting you have used. See Bench::relative. + * + * * `{{context(variableName)}}` See Bench::context. * * Apart from these tags, it is also possible to use some mathematical operations on the measurement data. The operations * are of the form `{{command(name)}}`. Currently `name` can be one of `elapsed`, `iterations`. If performance counters * are available (currently only on current Linux systems), you also have `pagefaults`, `cpucycles`, - * `contextswitches`, `instructions`, `branchinstructions`, and `branchmisses`. All the measuers (except `iterations`) are + * `contextswitches`, `instructions`, `branchinstructions`, and `branchmisses`. All the measures (except `iterations`) are * provided for a single iteration (so `elapsed` is the time a single iteration took). The following tags are available: * * * `{{median()}}` Calculate median of a measurement data set, e.g. `{{median(elapsed)}}`. @@ -201,7 +204,7 @@ class BigO; * This measurement is a bit hard to interpret, but it is very robust against outliers. E.g. a value of 5% means that half of the * measurements deviate less than 5% from the median, and the other deviate more than 5% from the median. * - * * `{{sum()}}` Sums of all the measurements. E.g. `{{sum(iterations)}}` will give you the total number of iterations + * * `{{sum()}}` Sum of all the measurements. E.g. `{{sum(iterations)}}` will give you the total number of iterations * measured in this benchmark. * * * `{{minimum()}}` Minimum of all measurements. @@ -244,21 +247,21 @@ class BigO; * For the layer tags *result* and *measurement* you additionally can use these special markers: * * * ``{{#-first}}`` - Begin marker of a template that will be instantiated *only for the first* entry in the layer. Use is only - * allowed between the begin and end marker of the layer allowed. So between ``{{#result}}`` and ``{{/result}}``, or between + * allowed between the begin and end marker of the layer. So between ``{{#result}}`` and ``{{/result}}``, or between * ``{{#measurement}}`` and ``{{/measurement}}``. Finish the template with ``{{/-first}}``. * * * ``{{^-first}}`` - Begin marker of a template that will be instantiated *for each except the first* entry in the layer. This, - * this is basically the inversion of ``{{#-first}}``. Use is only allowed between the begin and end marker of the layer allowed. + * this is basically the inversion of ``{{#-first}}``. Use is only allowed between the begin and end marker of the layer. * So between ``{{#result}}`` and ``{{/result}}``, or between ``{{#measurement}}`` and ``{{/measurement}}``. * * * ``{{/-first}}`` - End marker for either ``{{#-first}}`` or ``{{^-first}}``. * * * ``{{#-last}}`` - Begin marker of a template that will be instantiated *only for the last* entry in the layer. Use is only - * allowed between the begin and end marker of the layer allowed. So between ``{{#result}}`` and ``{{/result}}``, or between + * allowed between the begin and end marker of the layer. So between ``{{#result}}`` and ``{{/result}}``, or between * ``{{#measurement}}`` and ``{{/measurement}}``. Finish the template with ``{{/-last}}``. * * * ``{{^-last}}`` - Begin marker of a template that will be instantiated *for each except the last* entry in the layer. This, - * this is basically the inversion of ``{{#-last}}``. Use is only allowed between the begin and end marker of the layer allowed. + * this is basically the inversion of ``{{#-last}}``. Use is only allowed between the begin and end marker of the layer. * So between ``{{#result}}`` and ``{{/result}}``, or between ``{{#measurement}}`` and ``{{/measurement}}``. * * * ``{{/-last}}`` - End marker for either ``{{#-last}}`` or ``{{^-last}}``. @@ -316,12 +319,12 @@ char const* csv() noexcept; See the tutorial at :ref:`tutorial-template-html` for an example. @endverbatim - @see ankerl::nanobench::render() + @see also ankerl::nanobench::render() */ char const* htmlBoxplot() noexcept; /*! - @brief Output in pyperf compatible JSON format, which can be used for more analyzations. + @brief Output in pyperf compatible JSON format, which can be used for more analyzation. @verbatim embed:rst See the tutorial at :ref:`tutorial-template-pyperf` for an example how to further analyze the output. @endverbatim @@ -378,30 +381,32 @@ struct PerfCountSet { ANKERL_NANOBENCH(IGNORE_PADDED_PUSH) struct Config { // actual benchmark config - std::string mBenchmarkTitle = "benchmark"; - std::string mBenchmarkName = "noname"; - std::string mUnit = "op"; - double mBatch = 1.0; - double mComplexityN = -1.0; - size_t mNumEpochs = 11; - size_t mClockResolutionMultiple = static_cast(1000); - std::chrono::nanoseconds mMaxEpochTime = std::chrono::milliseconds(100); - std::chrono::nanoseconds mMinEpochTime{}; - uint64_t mMinEpochIterations{1}; - uint64_t mEpochIterations{0}; // If not 0, run *exactly* these number of iterations per epoch. - uint64_t mWarmup = 0; - std::ostream* mOut = nullptr; - std::chrono::duration mTimeUnit = std::chrono::nanoseconds{1}; - std::string mTimeUnitName = "ns"; - bool mShowPerformanceCounters = true; - bool mIsRelative = false; + std::string mBenchmarkTitle = "benchmark"; // NOLINT(misc-non-private-member-variables-in-classes) + std::string mBenchmarkName = "noname"; // NOLINT(misc-non-private-member-variables-in-classes) + std::string mUnit = "op"; // NOLINT(misc-non-private-member-variables-in-classes) + double mBatch = 1.0; // NOLINT(misc-non-private-member-variables-in-classes) + double mComplexityN = -1.0; // NOLINT(misc-non-private-member-variables-in-classes) + size_t mNumEpochs = 11; // NOLINT(misc-non-private-member-variables-in-classes) + size_t mClockResolutionMultiple = static_cast(1000); // NOLINT(misc-non-private-member-variables-in-classes) + std::chrono::nanoseconds mMaxEpochTime = std::chrono::milliseconds(100); // NOLINT(misc-non-private-member-variables-in-classes) + std::chrono::nanoseconds mMinEpochTime = std::chrono::milliseconds(1); // NOLINT(misc-non-private-member-variables-in-classes) + uint64_t mMinEpochIterations{1}; // NOLINT(misc-non-private-member-variables-in-classes) + // If not 0, run *exactly* these number of iterations per epoch. + uint64_t mEpochIterations{0}; // NOLINT(misc-non-private-member-variables-in-classes) + uint64_t mWarmup = 0; // NOLINT(misc-non-private-member-variables-in-classes) + std::ostream* mOut = nullptr; // NOLINT(misc-non-private-member-variables-in-classes) + std::chrono::duration mTimeUnit = std::chrono::nanoseconds{1}; // NOLINT(misc-non-private-member-variables-in-classes) + std::string mTimeUnitName = "ns"; // NOLINT(misc-non-private-member-variables-in-classes) + bool mShowPerformanceCounters = true; // NOLINT(misc-non-private-member-variables-in-classes) + bool mIsRelative = false; // NOLINT(misc-non-private-member-variables-in-classes) + std::unordered_map mContext{}; // NOLINT(misc-non-private-member-variables-in-classes) Config(); ~Config(); - Config& operator=(Config const&); - Config& operator=(Config&&); - Config(Config const&); - Config(Config&&) noexcept; + Config& operator=(Config const& other); + Config& operator=(Config&& other) noexcept; + Config(Config const& other); + Config(Config&& other) noexcept; }; ANKERL_NANOBENCH(IGNORE_PADDED_POP) @@ -421,13 +426,13 @@ class Result { _size }; - explicit Result(Config const& benchmarkConfig); + explicit Result(Config benchmarkConfig); ~Result(); - Result& operator=(Result const&); - Result& operator=(Result&&); - Result(Result const&); - Result(Result&&) noexcept; + Result& operator=(Result const& other); + Result& operator=(Result&& other) noexcept; + Result(Result const& other); + Result(Result&& other) noexcept; // adds new measurement results // all values are scaled by iters (except iters...) @@ -442,6 +447,8 @@ class Result { ANKERL_NANOBENCH(NODISCARD) double sumProduct(Measure m1, Measure m2) const noexcept; ANKERL_NANOBENCH(NODISCARD) double minimum(Measure m) const noexcept; ANKERL_NANOBENCH(NODISCARD) double maximum(Measure m) const noexcept; + ANKERL_NANOBENCH(NODISCARD) std::string const& context(char const* variableName) const; + ANKERL_NANOBENCH(NODISCARD) std::string const& context(std::string const& variableName) const; ANKERL_NANOBENCH(NODISCARD) bool has(Measure m) const noexcept; ANKERL_NANOBENCH(NODISCARD) double get(size_t idx, Measure m) const; @@ -485,9 +492,9 @@ class Rng final { static constexpr uint64_t(max)(); /** - * As a safety precausion, we don't allow copying. Copying a PRNG would mean you would have two random generators that produce the + * As a safety precaution, we don't allow copying. Copying a PRNG would mean you would have two random generators that produce the * same sequence, which is generally not what one wants. Instead create a new rng with the default constructor Rng(), which is - * automatically seeded from `std::random_device`. If you really need a copy, use copy(). + * automatically seeded from `std::random_device`. If you really need a copy, use `copy()`. */ Rng(Rng const&) = delete; @@ -528,7 +535,7 @@ class Rng final { */ explicit Rng(uint64_t seed) noexcept; Rng(uint64_t x, uint64_t y) noexcept; - Rng(std::vector const& data); + explicit Rng(std::vector const& data); /** * Creates a copy of the Rng, thus the copy provides exactly the same random sequence as the original. @@ -620,8 +627,8 @@ class Bench { */ Bench(); - Bench(Bench&& other); - Bench& operator=(Bench&& other); + Bench(Bench&& other) noexcept; + Bench& operator=(Bench&& other) noexcept; Bench(Bench const& other); Bench& operator=(Bench const& other); ~Bench() noexcept; @@ -667,6 +674,10 @@ class Bench { */ Bench& title(char const* benchmarkTitle); Bench& title(std::string const& benchmarkTitle); + + /** + * @brief Gets the title of the benchmark + */ ANKERL_NANOBENCH(NODISCARD) std::string const& title() const noexcept; /// Name of the benchmark, will be shown in the table row. @@ -674,6 +685,31 @@ class Bench { Bench& name(std::string const& benchmarkName); ANKERL_NANOBENCH(NODISCARD) std::string const& name() const noexcept; + /** + * @brief Set context information. + * + * The information can be accessed using custom render templates via `{{context(variableName)}}`. + * Trying to render a variable that hasn't been set before raises an exception. + * Not included in (default) markdown table. + * + * @see clearContext, render + * + * @param variableName The name of the context variable. + * @param variableValue The value of the context variable. + */ + Bench& context(char const* variableName, char const* variableValue); + Bench& context(std::string const& variableName, std::string const& variableValue); + + /** + * @brief Reset context information. + * + * This may improve efficiency when using many context entries, + * or improve robustness by removing spurious context entries. + * + * @see context + */ + Bench& clearContext(); + /** * @brief Sets the batch size. * @@ -754,9 +790,9 @@ class Bench { * representation of the benchmarked code's runtime stability. * * Choose the value wisely. In practice, 11 has been shown to be a reasonable choice between runtime performance and accuracy. - * This setting goes hand in hand with minEpocIterations() (or minEpochTime()). If you are more interested in *median* runtime, you - * might want to increase epochs(). If you are more interested in *mean* runtime, you might want to increase minEpochIterations() - * instead. + * This setting goes hand in hand with minEpochIterations() (or minEpochTime()). If you are more interested in *median* runtime, + * you might want to increase epochs(). If you are more interested in *mean* runtime, you might want to increase + * minEpochIterations() instead. * * @param numEpochs Number of epochs. */ @@ -766,10 +802,10 @@ class Bench { /** * @brief Upper limit for the runtime of each epoch. * - * As a safety precausion if the clock is not very accurate, we can set an upper limit for the maximum evaluation time per + * As a safety precaution if the clock is not very accurate, we can set an upper limit for the maximum evaluation time per * epoch. Default is 100ms. At least a single evaluation of the benchmark is performed. * - * @see minEpochTime(), minEpochIterations() + * @see minEpochTime, minEpochIterations * * @param t Maximum target runtime for a single epoch. */ @@ -782,7 +818,7 @@ class Bench { * Default is zero, so we are fully relying on clockResolutionMultiple(). In most cases this is exactly what you want. If you see * that the evaluation is unreliable with a high `err%`, you can increase either minEpochTime() or minEpochIterations(). * - * @see maxEpochTime(), minEpochIterations() + * @see maxEpochTim), minEpochIterations * * @param t Minimum time each epoch should take. */ @@ -793,9 +829,9 @@ class Bench { * @brief Sets the minimum number of iterations each epoch should take. * * Default is 1, and we rely on clockResolutionMultiple(). If the `err%` is high and you want a more smooth result, you might want - * to increase the minimum number or iterations, or increase the minEpochTime(). + * to increase the minimum number of iterations, or increase the minEpochTime(). * - * @see minEpochTime(), maxEpochTime(), minEpochIterations() + * @see minEpochTime, maxEpochTime, minEpochIterations * * @param numIters Minimum number of iterations per epoch. */ @@ -886,10 +922,10 @@ class Bench { @endverbatim @tparam T Any type is cast to `double`. - @param b Length of N for the next benchmark run, so it is possible to calculate `bigO`. + @param n Length of N for the next benchmark run, so it is possible to calculate `bigO`. */ template - Bench& complexityN(T b) noexcept; + Bench& complexityN(T n) noexcept; ANKERL_NANOBENCH(NODISCARD) double complexityN() const noexcept; /*! @@ -993,7 +1029,7 @@ void doNotOptimizeAway(T const& val); #else // These assembly magic is directly from what Google Benchmark is doing. I have previously used what facebook's folly was doing, but -// this seemd to have compilation problems in some cases. Google Benchmark seemed to be the most well tested anyways. +// this seemed to have compilation problems in some cases. Google Benchmark seemed to be the most well tested anyways. // see https://github.com/google/benchmark/blob/master/include/benchmark/benchmark.h#L307 template void doNotOptimizeAway(T const& val) { @@ -1019,7 +1055,11 @@ void doNotOptimizeAway(T& val) { ANKERL_NANOBENCH(IGNORE_EFFCPP_PUSH) class IterationLogic { public: - explicit IterationLogic(Bench const& config) noexcept; + explicit IterationLogic(Bench const& bench); + IterationLogic(IterationLogic&&) = delete; + IterationLogic& operator=(IterationLogic&&) = delete; + IterationLogic(IterationLogic const&) = delete; + IterationLogic& operator=(IterationLogic const&) = delete; ~IterationLogic(); ANKERL_NANOBENCH(NODISCARD) uint64_t numIters() const noexcept; @@ -1036,7 +1076,9 @@ ANKERL_NANOBENCH(IGNORE_PADDED_PUSH) class PerformanceCounters { public: PerformanceCounters(PerformanceCounters const&) = delete; + PerformanceCounters(PerformanceCounters&&) = delete; PerformanceCounters& operator=(PerformanceCounters const&) = delete; + PerformanceCounters& operator=(PerformanceCounters&&) = delete; PerformanceCounters(); ~PerformanceCounters(); @@ -1081,11 +1123,11 @@ class BigO { : BigO(bigOName, mapRangeMeasure(rangeMeasure, rangeToN)) {} template - BigO(std::string const& bigOName, RangeMeasure const& rangeMeasure, Op rangeToN) - : BigO(bigOName, mapRangeMeasure(rangeMeasure, rangeToN)) {} + BigO(std::string bigOName, RangeMeasure const& rangeMeasure, Op rangeToN) + : BigO(std::move(bigOName), mapRangeMeasure(rangeMeasure, rangeToN)) {} BigO(char const* bigOName, RangeMeasure const& scaledRangeMeasure); - BigO(std::string const& bigOName, RangeMeasure const& scaledRangeMeasure); + BigO(std::string bigOName, RangeMeasure const& scaledRangeMeasure); ANKERL_NANOBENCH(NODISCARD) std::string const& name() const noexcept; ANKERL_NANOBENCH(NODISCARD) double constant() const noexcept; ANKERL_NANOBENCH(NODISCARD) double normalizedRootMeanSquare() const noexcept; @@ -1127,7 +1169,7 @@ uint64_t Rng::operator()() noexcept { ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") uint32_t Rng::bounded(uint32_t range) noexcept { - uint64_t r32 = static_cast(operator()()); + uint64_t const r32 = static_cast(operator()()); auto multiresult = r32 * range; return static_cast(multiresult >> 32U); } @@ -1136,18 +1178,23 @@ double Rng::uniform01() noexcept { auto i = (UINT64_C(0x3ff) << 52U) | (operator()() >> 12U); // can't use union in c++ here for type puning, it's undefined behavior. // std::memcpy is optimized anyways. - double d; + double d{}; std::memcpy(&d, &i, sizeof(double)); return d - 1.0; } template void Rng::shuffle(Container& container) noexcept { - auto size = static_cast(container.size()); - for (auto i = size; i > 1U; --i) { + auto i = container.size(); + while (i > 1U) { using std::swap; - auto p = bounded(i); // number in [0, i) - swap(container[i - 1], container[p]); + auto n = operator()(); + // using decltype(i) instead of size_t to be compatible to containers with 32bit index (see #80) + auto b1 = static_cast((static_cast(n) * static_cast(i)) >> 32U); + swap(container[--i], container[b1]); + + auto b2 = static_cast(((n >> 32U) * static_cast(i)) >> 32U); + swap(container[--i], container[b2]); } } @@ -1165,11 +1212,11 @@ Bench& Bench::run(Op&& op) { while (auto n = iterationLogic.numIters()) { pc.beginMeasure(); - Clock::time_point before = Clock::now(); + Clock::time_point const before = Clock::now(); while (n-- > 0) { op(); } - Clock::time_point after = Clock::now(); + Clock::time_point const after = Clock::now(); pc.endMeasure(); pc.updateResults(iterationLogic.numIters()); iterationLogic.add(after - before, pc); @@ -1270,7 +1317,6 @@ void doNotOptimizeAway(T const& val) { # include # include # include -# include # endif // declarations /////////////////////////////////////////////////////////////////////////////////// @@ -1436,31 +1482,37 @@ struct Node { template // NOLINTNEXTLINE(hicpp-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) bool operator==(char const (&str)[N]) const noexcept { + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay) return static_cast(std::distance(begin, end) + 1) == N && 0 == strncmp(str, begin, N - 1); } }; ANKERL_NANOBENCH(IGNORE_PADDED_POP) +// NOLINTNEXTLINE(misc-no-recursion) static std::vector parseMustacheTemplate(char const** tpl) { std::vector nodes; while (true) { - auto begin = std::strstr(*tpl, "{{"); - auto end = begin; + auto const* begin = std::strstr(*tpl, "{{"); + auto const* end = begin; if (begin != nullptr) { + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) begin += 2; end = std::strstr(begin, "}}"); } if (begin == nullptr || end == nullptr) { // nothing found, finish node + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) nodes.emplace_back(Node{*tpl, *tpl + std::strlen(*tpl), std::vector{}, Node::Type::content}); return nodes; } + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) nodes.emplace_back(Node{*tpl, begin - 2, std::vector{}, Node::Type::content}); // we found a tag + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) *tpl = end + 2; switch (*begin) { case '/': @@ -1468,10 +1520,12 @@ static std::vector parseMustacheTemplate(char const** tpl) { return nodes; case '#': + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) nodes.emplace_back(Node{begin + 1, end, parseMustacheTemplate(tpl), Node::Type::section}); break; case '^': + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) nodes.emplace_back(Node{begin + 1, end, parseMustacheTemplate(tpl), Node::Type::inverted_section}); break; @@ -1484,8 +1538,8 @@ static std::vector parseMustacheTemplate(char const** tpl) { static bool generateFirstLast(Node const& n, size_t idx, size_t size, std::ostream& out) { ANKERL_NANOBENCH_LOG("n.type=" << static_cast(n.type)); - bool matchFirst = n == "-first"; - bool matchLast = n == "-last"; + bool const matchFirst = n == "-first"; + bool const matchLast = n == "-last"; if (!matchFirst && !matchLast) { return false; } @@ -1518,7 +1572,7 @@ static bool matchCmdArgs(std::string const& str, std::vector& match matchResult.emplace_back(str.substr(0, idxOpen)); // split by comma - matchResult.emplace_back(std::string{}); + matchResult.emplace_back(); for (size_t i = idxOpen + 1; i != idxClose; ++i) { if (str[i] == ' ' || str[i] == '\t') { // skip whitespace @@ -1526,7 +1580,7 @@ static bool matchCmdArgs(std::string const& str, std::vector& match } if (str[i] == ',') { // got a comma => new string - matchResult.emplace_back(std::string{}); + matchResult.emplace_back(); continue; } // no whitespace no comma, append @@ -1541,49 +1595,63 @@ static bool generateConfigTag(Node const& n, Config const& config, std::ostream& if (n == "title") { out << config.mBenchmarkTitle; return true; - } else if (n == "name") { + } + if (n == "name") { out << config.mBenchmarkName; return true; - } else if (n == "unit") { + } + if (n == "unit") { out << config.mUnit; return true; - } else if (n == "batch") { + } + if (n == "batch") { out << config.mBatch; return true; - } else if (n == "complexityN") { + } + if (n == "complexityN") { out << config.mComplexityN; return true; - } else if (n == "epochs") { + } + if (n == "epochs") { out << config.mNumEpochs; return true; - } else if (n == "clockResolution") { + } + if (n == "clockResolution") { out << d(detail::clockResolution()); return true; - } else if (n == "clockResolutionMultiple") { + } + if (n == "clockResolutionMultiple") { out << config.mClockResolutionMultiple; return true; - } else if (n == "maxEpochTime") { + } + if (n == "maxEpochTime") { out << d(config.mMaxEpochTime); return true; - } else if (n == "minEpochTime") { + } + if (n == "minEpochTime") { out << d(config.mMinEpochTime); return true; - } else if (n == "minEpochIterations") { + } + if (n == "minEpochIterations") { out << config.mMinEpochIterations; return true; - } else if (n == "epochIterations") { + } + if (n == "epochIterations") { out << config.mEpochIterations; return true; - } else if (n == "warmup") { + } + if (n == "warmup") { out << config.mWarmup; return true; - } else if (n == "relative") { + } + if (n == "relative") { out << config.mIsRelative; return true; } return false; } +// NOLINTNEXTLINE(readability-function-cognitive-complexity) static std::ostream& generateResultTag(Node const& n, Result const& r, std::ostream& out) { if (generateConfigTag(n, r.config(), out)) { return out; @@ -1596,6 +1664,10 @@ static std::ostream& generateResultTag(Node const& n, Result const& r, std::ostr std::vector matchResult; if (matchCmdArgs(std::string(n.begin, n.end), matchResult)) { if (matchResult.size() == 2) { + if (matchResult[0] == "context") { + return out << r.context(matchResult[1]); + } + auto m = Result::fromString(matchResult[1]); if (m == Result::Measure::_size) { return out << 0.0; @@ -1712,7 +1784,7 @@ template T parseFile(std::string const& filename); void gatherStabilityInformation(std::vector& warnings, std::vector& recommendations); -void printStabilityInformationOnce(std::ostream* os); +void printStabilityInformationOnce(std::ostream* outStream); // remembers the last table settings used. When it changes, a new table header is automatically written for the new entry. uint64_t& singletonHeaderHash() noexcept; @@ -1779,13 +1851,13 @@ class Number { }; // helper replacement for std::to_string of signed/unsigned numbers so we are locale independent -std::string to_s(uint64_t s); +std::string to_s(uint64_t n); std::ostream& operator<<(std::ostream& os, Number const& n); class MarkDownColumn { public: - MarkDownColumn(int w, int prec, std::string const& tit, std::string const& suff, double val); + MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val); std::string title() const; std::string separator() const; std::string invalid() const; @@ -1823,8 +1895,9 @@ std::ostream& operator<<(std::ostream& os, MarkDownCode const& mdCode); namespace ankerl { namespace nanobench { +// NOLINTNEXTLINE(readability-function-cognitive-complexity) void render(char const* mustacheTemplate, std::vector const& results, std::ostream& out) { - detail::fmt::StreamStateRestorer restorer(out); + detail::fmt::StreamStateRestorer const restorer(out); out.precision(std::numeric_limits::digits10); auto nodes = templates::parseMustacheTemplate(&mustacheTemplate); @@ -1914,7 +1987,7 @@ void doNotOptimizeAwaySink(void const*) {} template T parseFile(std::string const& filename) { - std::ifstream fin(filename); + std::ifstream fin(filename); // NOLINT(misc-const-correctness) T num{}; fin >> num; return num; @@ -1925,20 +1998,20 @@ char const* getEnv(char const* name) { # pragma warning(push) # pragma warning(disable : 4996) // getenv': This function or variable may be unsafe. # endif - return std::getenv(name); + return std::getenv(name); // NOLINT(concurrency-mt-unsafe) # if defined(_MSC_VER) # pragma warning(pop) # endif } bool isEndlessRunning(std::string const& name) { - auto endless = getEnv("NANOBENCH_ENDLESS"); + auto const* const endless = getEnv("NANOBENCH_ENDLESS"); return nullptr != endless && endless == name; } // True when environment variable NANOBENCH_SUPPRESS_WARNINGS is either not set at all, or set to "0" bool isWarningsEnabled() { - auto suppression = getEnv("NANOBENCH_SUPPRESS_WARNINGS"); + auto const* const suppression = getEnv("NANOBENCH_SUPPRESS_WARNINGS"); return nullptr == suppression || suppression == std::string("0"); } @@ -1946,11 +2019,11 @@ void gatherStabilityInformation(std::vector& warnings, std::vector< warnings.clear(); recommendations.clear(); - bool recommendCheckFlags = false; - # if defined(DEBUG) warnings.emplace_back("DEBUG defined"); - recommendCheckFlags = true; + bool const recommendCheckFlags = true; +# else + bool const recommendCheckFlags = false; # endif bool recommendPyPerf = false; @@ -2000,7 +2073,7 @@ void gatherStabilityInformation(std::vector& warnings, std::vector< void printStabilityInformationOnce(std::ostream* outStream) { static bool shouldPrint = true; - if (shouldPrint && outStream && isWarningsEnabled()) { + if (shouldPrint && (nullptr != outStream) && isWarningsEnabled()) { auto& os = *outStream; shouldPrint = false; std::vector warnings; @@ -2050,7 +2123,7 @@ Clock::duration calcClockResolution(size_t numEvaluations) noexcept { // Calculates clock resolution once, and remembers the result Clock::duration clockResolution() noexcept { - static Clock::duration sResolution = calcClockResolution(20); + static Clock::duration const sResolution = calcClockResolution(20); return sResolution; } @@ -2183,6 +2256,7 @@ struct IterationLogic::Impl { << ", mState=" << static_cast(mState)); } + // NOLINTNEXTLINE(readability-function-cognitive-complexity) void showResult(std::string const& errorMessage) const { ANKERL_NANOBENCH_LOG(errorMessage); @@ -2208,7 +2282,7 @@ struct IterationLogic::Impl { rMedian / (mBench.timeUnit().count() * mBench.batch())); columns.emplace_back(22, 2, mBench.unit() + "/s", "", rMedian <= 0.0 ? 0.0 : mBench.batch() / rMedian); - double rErrorMedian = mResult.medianAbsolutePercentError(Result::Measure::elapsed); + double const rErrorMedian = mResult.medianAbsolutePercentError(Result::Measure::elapsed); columns.emplace_back(10, 1, "err%", "%", rErrorMedian * 100.0); double rInsMedian = -1.0; @@ -2226,7 +2300,7 @@ struct IterationLogic::Impl { columns.emplace_back(9, 3, "IPC", "", rCycMedian <= 0.0 ? 0.0 : rInsMedian / rCycMedian); } if (mBench.performanceCounters() && mResult.has(Result::Measure::branchinstructions)) { - double rBraMedian = mResult.median(Result::Measure::branchinstructions); + double const rBraMedian = mResult.median(Result::Measure::branchinstructions); columns.emplace_back(17, 2, "bra/" + mBench.unit(), "", rBraMedian / mBench.batch()); if (mResult.has(Result::Measure::branchmisses)) { double p = 0.0; @@ -2299,25 +2373,22 @@ struct IterationLogic::Impl { return elapsed * 3 >= mTargetRuntimePerEpoch * 2; } - uint64_t mNumIters = 1; - Bench const& mBench; - std::chrono::nanoseconds mTargetRuntimePerEpoch{}; - Result mResult; - Rng mRng{123}; - std::chrono::nanoseconds mTotalElapsed{}; - uint64_t mTotalNumIters = 0; - - State mState = State::upscaling_runtime; + uint64_t mNumIters = 1; // NOLINT(misc-non-private-member-variables-in-classes) + Bench const& mBench; // NOLINT(misc-non-private-member-variables-in-classes) + std::chrono::nanoseconds mTargetRuntimePerEpoch{}; // NOLINT(misc-non-private-member-variables-in-classes) + Result mResult; // NOLINT(misc-non-private-member-variables-in-classes) + Rng mRng{123}; // NOLINT(misc-non-private-member-variables-in-classes) + std::chrono::nanoseconds mTotalElapsed{}; // NOLINT(misc-non-private-member-variables-in-classes) + uint64_t mTotalNumIters = 0; // NOLINT(misc-non-private-member-variables-in-classes) + State mState = State::upscaling_runtime; // NOLINT(misc-non-private-member-variables-in-classes) }; ANKERL_NANOBENCH(IGNORE_PADDED_POP) -IterationLogic::IterationLogic(Bench const& bench) noexcept +IterationLogic::IterationLogic(Bench const& bench) : mPimpl(new Impl(bench)) {} IterationLogic::~IterationLogic() { - if (mPimpl) { - delete mPimpl; - } + delete mPimpl; } uint64_t IterationLogic::numIters() const noexcept { @@ -2344,11 +2415,16 @@ class LinuxPerformanceCounters { , correctMeasuringOverhead(correctMeasuringOverhead_) , correctLoopOverhead(correctLoopOverhead_) {} - uint64_t* targetValue{}; - bool correctMeasuringOverhead{}; - bool correctLoopOverhead{}; + uint64_t* targetValue{}; // NOLINT(misc-non-private-member-variables-in-classes) + bool correctMeasuringOverhead{}; // NOLINT(misc-non-private-member-variables-in-classes) + bool correctLoopOverhead{}; // NOLINT(misc-non-private-member-variables-in-classes) }; + LinuxPerformanceCounters() = default; + LinuxPerformanceCounters(LinuxPerformanceCounters const&) = delete; + LinuxPerformanceCounters(LinuxPerformanceCounters&&) = delete; + LinuxPerformanceCounters& operator=(LinuxPerformanceCounters const&) = delete; + LinuxPerformanceCounters& operator=(LinuxPerformanceCounters&&) = delete; ~LinuxPerformanceCounters(); // quick operation @@ -2370,13 +2446,13 @@ class LinuxPerformanceCounters { return; } - // NOLINTNEXTLINE(hicpp-signed-bitwise) + // NOLINTNEXTLINE(hicpp-signed-bitwise,cppcoreguidelines-pro-type-vararg) mHasError = -1 == ioctl(mFd, PERF_EVENT_IOC_RESET, PERF_IOC_FLAG_GROUP); if (mHasError) { return; } - // NOLINTNEXTLINE(hicpp-signed-bitwise) + // NOLINTNEXTLINE(hicpp-signed-bitwise,cppcoreguidelines-pro-type-vararg) mHasError = -1 == ioctl(mFd, PERF_EVENT_IOC_ENABLE, PERF_IOC_FLAG_GROUP); } @@ -2385,7 +2461,7 @@ class LinuxPerformanceCounters { return; } - // NOLINTNEXTLINE(hicpp-signed-bitwise) + // NOLINTNEXTLINE(hicpp-signed-bitwise,cppcoreguidelines-pro-type-vararg) mHasError = (-1 == ioctl(mFd, PERF_EVENT_IOC_DISABLE, PERF_IOC_FLAG_GROUP)); if (mHasError) { return; @@ -2406,9 +2482,9 @@ class LinuxPerformanceCounters { ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") static inline uint32_t mix(uint32_t x) noexcept { - x ^= x << 13; - x ^= x >> 17; - x ^= x << 5; + x ^= x << 13U; + x ^= x >> 17U; + x ^= x << 5U; return x; } @@ -2448,7 +2524,7 @@ class LinuxPerformanceCounters { // marsaglia's xorshift: mov, sal/shr, xor. Times 3. // This has the nice property that the compiler doesn't seem to be able to optimize multiple calls any further. // see https://godbolt.org/z/49RVQ5 - uint64_t const numIters = 100000U + (std::random_device{}() & 3); + uint64_t const numIters = 100000U + (std::random_device{}() & 3U); uint64_t n = numIters; uint32_t x = 1234567; @@ -2582,6 +2658,7 @@ bool LinuxPerformanceCounters::monitor(uint32_t type, uint64_t eventid, Target t const unsigned long flags = 0; # endif + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) auto fd = static_cast(syscall(__NR_perf_event_open, &pea, pid, cpu, mFd, flags)); if (-1 == fd) { return false; @@ -2591,7 +2668,7 @@ bool LinuxPerformanceCounters::monitor(uint32_t type, uint64_t eventid, Target t mFd = fd; } uint64_t id = 0; - // NOLINTNEXTLINE(hicpp-signed-bitwise) + // NOLINTNEXTLINE(hicpp-signed-bitwise,cppcoreguidelines-pro-type-vararg) if (-1 == ioctl(fd, PERF_EVENT_IOC_ID, &id)) { // couldn't get id return false; @@ -2639,9 +2716,8 @@ PerformanceCounters::PerformanceCounters() } PerformanceCounters::~PerformanceCounters() { - if (nullptr != mPc) { - delete mPc; - } + // no need to check for nullptr, delete nullptr has no effect + delete mPc; } void PerformanceCounters::beginMeasure() { @@ -2721,7 +2797,7 @@ Number::Number(int width, int precision, double value) , mValue(value) {} std::ostream& Number::write(std::ostream& os) const { - StreamStateRestorer restorer(os); + StreamStateRestorer const restorer(os); os.imbue(std::locale(os.getloc(), new NumSep(','))); os << std::setw(mWidth) << std::setprecision(mPrecision) << std::fixed << mValue; return os; @@ -2747,11 +2823,11 @@ std::ostream& operator<<(std::ostream& os, Number const& n) { return n.write(os); } -MarkDownColumn::MarkDownColumn(int w, int prec, std::string const& tit, std::string const& suff, double val) +MarkDownColumn::MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val) : mWidth(w) , mPrecision(prec) - , mTitle(tit) - , mSuffix(suff) + , mTitle(std::move(tit)) + , mSuffix(std::move(suff)) , mValue(val) {} std::string MarkDownColumn::title() const { @@ -2785,7 +2861,7 @@ std::string MarkDownColumn::value() const { MarkDownCode::MarkDownCode(std::string const& what) { mWhat.reserve(what.size() + 2); mWhat.push_back('`'); - for (char c : what) { + for (char const c : what) { mWhat.push_back(c); if ('`' == c) { mWhat.push_back('`'); @@ -2808,14 +2884,14 @@ std::ostream& operator<<(std::ostream& os, MarkDownCode const& mdCode) { Config::Config() = default; Config::~Config() = default; Config& Config::operator=(Config const&) = default; -Config& Config::operator=(Config&&) = default; +Config& Config::operator=(Config&&) noexcept = default; Config::Config(Config const&) = default; Config::Config(Config&&) noexcept = default; // provide implementation here so it's only generated once Result::~Result() = default; Result& Result::operator=(Result const&) = default; -Result& Result::operator=(Result&&) = default; +Result& Result::operator=(Result&&) noexcept = default; Result::Result(Result const&) = default; Result::Result(Result&&) noexcept = default; @@ -2827,15 +2903,15 @@ inline constexpr typename std::underlying_type::type u(T val) noexcept { } // namespace detail // Result returned after a benchmark has finished. Can be used as a baseline for relative(). -Result::Result(Config const& benchmarkConfig) - : mConfig(benchmarkConfig) +Result::Result(Config benchmarkConfig) + : mConfig(std::move(benchmarkConfig)) , mNameToMeasurements{detail::u(Result::Measure::_size)} {} void Result::add(Clock::duration totalElapsed, uint64_t iters, detail::PerformanceCounters const& pc) { using detail::d; using detail::u; - double dIters = d(iters); + double const dIters = d(iters); mNameToMeasurements[u(Result::Measure::iterations)].push_back(dIters); mNameToMeasurements[u(Result::Measure::elapsed)].push_back(d(totalElapsed) / dIters); @@ -2987,27 +3063,41 @@ double Result::maximum(Measure m) const noexcept { return *std::max_element(data.begin(), data.end()); } +std::string const& Result::context(char const* variableName) const { + return mConfig.mContext.at(variableName); +} + +std::string const& Result::context(std::string const& variableName) const { + return mConfig.mContext.at(variableName); +} + Result::Measure Result::fromString(std::string const& str) { if (str == "elapsed") { return Measure::elapsed; - } else if (str == "iterations") { + } + if (str == "iterations") { return Measure::iterations; - } else if (str == "pagefaults") { + } + if (str == "pagefaults") { return Measure::pagefaults; - } else if (str == "cpucycles") { + } + if (str == "cpucycles") { return Measure::cpucycles; - } else if (str == "contextswitches") { + } + if (str == "contextswitches") { return Measure::contextswitches; - } else if (str == "instructions") { + } + if (str == "instructions") { return Measure::instructions; - } else if (str == "branchinstructions") { + } + if (str == "branchinstructions") { return Measure::branchinstructions; - } else if (str == "branchmisses") { + } + if (str == "branchmisses") { return Measure::branchmisses; - } else { - // not found, return _size - return Measure::_size; } + // not found, return _size + return Measure::_size; } // Configuration of a microbenchmark. @@ -3015,8 +3105,8 @@ Bench::Bench() { mConfig.mOut = &std::cout; } -Bench::Bench(Bench&&) = default; -Bench& Bench::operator=(Bench&&) = default; +Bench::Bench(Bench&&) noexcept = default; +Bench& Bench::operator=(Bench&&) noexcept = default; Bench::Bench(Bench const&) = default; Bench& Bench::operator=(Bench const&) = default; Bench::~Bench() noexcept = default; @@ -3114,6 +3204,21 @@ std::string const& Bench::name() const noexcept { return mConfig.mBenchmarkName; } +Bench& Bench::context(char const* variableName, char const* variableValue) { + mConfig.mContext[variableName] = variableValue; + return *this; +} + +Bench& Bench::context(std::string const& variableName, std::string const& variableValue) { + mConfig.mContext[variableName] = variableValue; + return *this; +} + +Bench& Bench::clearContext() { + mConfig.mContext.clear(); + return *this; +} + // Number of epochs to evaluate. The reported result will be the median of evaluation of each epoch. Bench& Bench::epochs(size_t numEpochs) noexcept { mConfig.mNumEpochs = numEpochs; @@ -3295,27 +3400,27 @@ BigO::RangeMeasure BigO::collectRangeMeasure(std::vector const& results) return rangeMeasure; } -BigO::BigO(std::string const& bigOName, RangeMeasure const& rangeMeasure) - : mName(bigOName) { +BigO::BigO(std::string bigOName, RangeMeasure const& rangeMeasure) + : mName(std::move(bigOName)) { // estimate the constant factor double sumRangeMeasure = 0.0; double sumRangeRange = 0.0; - for (size_t i = 0; i < rangeMeasure.size(); ++i) { - sumRangeMeasure += rangeMeasure[i].first * rangeMeasure[i].second; - sumRangeRange += rangeMeasure[i].first * rangeMeasure[i].first; + for (const auto& rm : rangeMeasure) { + sumRangeMeasure += rm.first * rm.second; + sumRangeRange += rm.first * rm.first; } mConstant = sumRangeMeasure / sumRangeRange; // calculate root mean square double err = 0.0; double sumMeasure = 0.0; - for (size_t i = 0; i < rangeMeasure.size(); ++i) { - auto diff = mConstant * rangeMeasure[i].first - rangeMeasure[i].second; + for (const auto& rm : rangeMeasure) { + auto diff = mConstant * rm.first - rm.second; err += diff * diff; - sumMeasure += rangeMeasure[i].second; + sumMeasure += rm.second; } auto n = static_cast(rangeMeasure.size()); @@ -3347,7 +3452,7 @@ std::ostream& operator<<(std::ostream& os, BigO const& bigO) { } std::ostream& operator<<(std::ostream& os, std::vector const& bigOs) { - detail::fmt::StreamStateRestorer restorer(os); + detail::fmt::StreamStateRestorer const restorer(os); os << std::endl << "| coefficient | err% | complexity" << std::endl << "|--------------:|-------:|------------" << std::endl; for (auto const& bigO : bigOs) { os << "|" << std::setw(14) << std::setprecision(7) << std::scientific << bigO.constant() << " "; diff --git a/src/bench/peer_eviction.cpp b/src/bench/peer_eviction.cpp index d83342be93bc..e04f3c403c19 100644 --- a/src/bench/peer_eviction.cpp +++ b/src/bench/peer_eviction.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2021 The Bitcoin Core developers +// Copyright (c) 2021-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/poly1305.cpp b/src/bench/poly1305.cpp index ad5a72ffde57..f7d17dfa961f 100644 --- a/src/bench/poly1305.cpp +++ b/src/bench/poly1305.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2020 The Bitcoin Core developers +// Copyright (c) 2019-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/prevector.cpp b/src/bench/prevector.cpp index 9e2e7d11c422..59c4af086e31 100644 --- a/src/bench/prevector.cpp +++ b/src/bench/prevector.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2021 The Bitcoin Core developers +// Copyright (c) 2015-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -61,7 +61,7 @@ static void PrevectorResize(benchmark::Bench& bench) template static void PrevectorDeserialize(benchmark::Bench& bench) { - CDataStream s0(SER_NETWORK, 0); + DataStream s0{}; prevector<28, T> t0; t0.resize(28); for (auto x = 0; x < 900; ++x) { diff --git a/src/bench/rollingbloom.cpp b/src/bench/rollingbloom.cpp index 865d99f9e84f..de76a872784c 100644 --- a/src/bench/rollingbloom.cpp +++ b/src/bench/rollingbloom.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2021 The Bitcoin Core developers +// Copyright (c) 2016-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index 5a178f308a39..f68b6acb5b6f 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2021 The Bitcoin Core developers +// Copyright (c) 2016-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp index 6bac6419e569..e3e1a07c8331 100644 --- a/src/bench/rpc_mempool.cpp +++ b/src/bench/rpc_mempool.cpp @@ -1,9 +1,10 @@ -// Copyright (c) 2011-2021 The Bitcoin Core developers +// Copyright (c) 2011-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include +#include #include #include #include diff --git a/src/bench/util_time.cpp b/src/bench/util_time.cpp index 8256b5f7baf0..8dbbdec28ccc 100644 --- a/src/bench/util_time.cpp +++ b/src/bench/util_time.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2020 The Bitcoin Core developers +// Copyright (c) 2019-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index f0e9db8ba1a1..757094167a56 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2021 The Bitcoin Core developers +// Copyright (c) 2016-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,7 +18,6 @@ // modified to measure performance of other types of scripts. static void VerifyScriptBench(benchmark::Bench& bench) { - const ECCVerifyHandle verify_handle; ECC_Start(); const uint32_t flags{SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH}; diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index 5a52774a8eb0..d5d057e96dcd 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2021 The Bitcoin Core developers +// Copyright (c) 2012-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -28,7 +28,7 @@ static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const b const auto& ADDRESS_WATCHONLY = ADDRESS_BCRT1_UNSPENDABLE; - CWallet wallet{test_setup->m_node.chain.get(), "", gArgs, CreateMockWalletDatabase()}; + CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); diff --git a/src/bench/wallet_create_tx.cpp b/src/bench/wallet_create_tx.cpp index 3cbf2c90081f..cb31421598c6 100644 --- a/src/bench/wallet_create_tx.cpp +++ b/src/bench/wallet_create_tx.cpp @@ -83,7 +83,7 @@ static void WalletCreateTx(benchmark::Bench& bench, const OutputType output_type { const auto test_setup = MakeNoLogFileContext(); - CWallet wallet{test_setup->m_node.chain.get(), "", gArgs, CreateMockWalletDatabase()}; + CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -102,7 +102,7 @@ static void WalletCreateTx(benchmark::Bench& bench, const OutputType output_type } // Check available balance - auto bal = wallet::GetAvailableBalance(wallet); // Cache + auto bal = WITH_LOCK(wallet.cs_wallet, return wallet::AvailableCoins(wallet).GetTotalAmount()); // Cache assert(bal == 50 * COIN * (chain_size - COINBASE_MATURITY)); wallet::CCoinControl coin_control; @@ -133,11 +133,52 @@ static void WalletCreateTx(benchmark::Bench& bench, const OutputType output_type }); } +static void AvailableCoins(benchmark::Bench& bench, const std::vector& output_type) +{ + const auto test_setup = MakeNoLogFileContext(); + CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; + { + LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet.SetupDescriptorScriptPubKeyMans(); + if (wallet.LoadWallet() != DBErrors::LOAD_OK) assert(false); + } + + // Generate destinations + std::vector dest_wallet; + dest_wallet.reserve(output_type.size()); + for (auto type : output_type) { + dest_wallet.emplace_back(GetScriptForDestination(getNewDestination(wallet, type))); + } + + // Generate chain; each coinbase will have two outputs to fill-up the wallet + const auto& params = Params(); + unsigned int chain_size = 1000; + for (unsigned int i = 0; i < chain_size / dest_wallet.size(); ++i) { + for (const auto& dest : dest_wallet) { + generateFakeBlock(params, test_setup->m_node, wallet, dest); + } + } + + // Check available balance + auto bal = WITH_LOCK(wallet.cs_wallet, return wallet::AvailableCoins(wallet).GetTotalAmount()); // Cache + assert(bal == 50 * COIN * (chain_size - COINBASE_MATURITY)); + + bench.epochIterations(2).run([&] { + LOCK(wallet.cs_wallet); + const auto& res = wallet::AvailableCoins(wallet); + assert(res.All().size() == (chain_size - COINBASE_MATURITY) * 2); + }); +} + static void WalletCreateTxUseOnlyPresetInputs(benchmark::Bench& bench) { WalletCreateTx(bench, OutputType::BECH32, /*allow_other_inputs=*/false, {{/*num_of_internal_inputs=*/4}}); } static void WalletCreateTxUsePresetInputsAndCoinSelection(benchmark::Bench& bench) { WalletCreateTx(bench, OutputType::BECH32, /*allow_other_inputs=*/true, {{/*num_of_internal_inputs=*/4}}); } +static void WalletAvailableCoins(benchmark::Bench& bench) { AvailableCoins(bench, {OutputType::BECH32M}); } + BENCHMARK(WalletCreateTxUseOnlyPresetInputs, benchmark::PriorityLevel::LOW) BENCHMARK(WalletCreateTxUsePresetInputsAndCoinSelection, benchmark::PriorityLevel::LOW) +BENCHMARK(WalletAvailableCoins, benchmark::PriorityLevel::LOW); \ No newline at end of file diff --git a/src/bench/wallet_loading.cpp b/src/bench/wallet_loading.cpp index 2f7dc53b0c76..6b09adcc9df4 100644 --- a/src/bench/wallet_loading.cpp +++ b/src/bench/wallet_loading.cpp @@ -24,7 +24,7 @@ using wallet::WALLET_FLAG_DESCRIPTORS; using wallet::WalletContext; using wallet::WalletDatabase; -static const std::shared_ptr BenchLoadWallet(std::unique_ptr database, WalletContext& context, DatabaseOptions& options) +static std::shared_ptr BenchLoadWallet(std::unique_ptr database, WalletContext& context, DatabaseOptions& options) { bilingual_str error; std::vector warnings; diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index d972b71a65bf..abe3af70c394 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -11,6 +11,7 @@ // // It is part of the libbitcoinkernel project. +#include #include #include #include @@ -52,7 +53,7 @@ int main(int argc, char* argv[]) // SETUP: Misc Globals SelectParams(CBaseChainParams::MAIN); - const CChainParams& chainparams = Params(); + auto chainparams = CChainParams::Main(); kernel::Context kernel_context{}; // We can't use a goto here, but we can use an assert since none of the @@ -81,10 +82,11 @@ int main(int argc, char* argv[]) // SETUP: Chainstate const ChainstateManager::Options chainman_opts{ - .chainparams = chainparams, + .chainparams = *chainparams, + .datadir = gArgs.GetDataDirNet(), .adjusted_time_callback = NodeClock::now, }; - ChainstateManager chainman{chainman_opts}; + ChainstateManager chainman{chainman_opts, {}}; node::CacheSizes cache_sizes; cache_sizes.block_tree_db = 2 << 20; diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 55fa3116ac20..94ed765455d1 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2021 The Bitcoin Core developers +// Copyright (c) 2009-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -55,7 +56,10 @@ static constexpr int DEFAULT_WAIT_CLIENT_TIMEOUT = 0; static const bool DEFAULT_NAMED=false; static const int CONTINUE_EXECUTION=-1; static constexpr int8_t UNKNOWN_NETWORK{-1}; -static constexpr std::array NETWORKS{"ipv4", "ipv6", "onion", "i2p", "cjdns"}; +// See GetNetworkName() in netbase.cpp +static constexpr std::array NETWORKS{"not_publicly_routable", "ipv4", "ipv6", "onion", "i2p", "cjdns", "internal"}; +static constexpr std::array NETWORK_SHORT_NAMES{"npr", "ipv4", "ipv6", "onion", "i2p", "cjdns", "int"}; +static constexpr std::array UNREACHABLE_NETWORK_IDS{/*not_publicly_routable*/0, /*internal*/6}; /** Default number of blocks to generate for RPC generatetoaddress. */ static const std::string DEFAULT_NBLOCKS = "1"; @@ -158,7 +162,7 @@ static int AppInitRPC(int argc, char* argv[]) } return EXIT_SUCCESS; } - if (!CheckDataDirOption()) { + if (!CheckDataDirOption(gArgs)) { tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "")); return EXIT_FAILURE; } @@ -289,7 +293,7 @@ class AddrinfoRequestHandler : public BaseRequestHandler // Prepare result to return to user. UniValue result{UniValue::VOBJ}, addresses{UniValue::VOBJ}; uint64_t total{0}; // Total address count - for (size_t i = 0; i < NETWORKS.size(); ++i) { + for (size_t i = 1; i < NETWORKS.size() - 1; ++i) { addresses.pushKV(NETWORKS[i], counts.at(i)); total += counts.at(i); } @@ -506,7 +510,7 @@ class NetinfoRequestHandler : public BaseRequestHandler const bool is_addr_relay_enabled{peer["addr_relay_enabled"].isNull() ? false : peer["addr_relay_enabled"].get_bool()}; const bool is_bip152_hb_from{peer["bip152_hb_from"].get_bool()}; const bool is_bip152_hb_to{peer["bip152_hb_to"].get_bool()}; - m_peers.push_back({addr, sub_version, conn_type, network, age, min_ping, ping, addr_processed, addr_rate_limited, last_blck, last_recv, last_send, last_trxn, peer_id, mapped_as, version, is_addr_relay_enabled, is_bip152_hb_from, is_bip152_hb_to, is_outbound, is_tx_relay}); + m_peers.push_back({addr, sub_version, conn_type, NETWORK_SHORT_NAMES[network_id], age, min_ping, ping, addr_processed, addr_rate_limited, last_blck, last_recv, last_send, last_trxn, peer_id, mapped_as, version, is_addr_relay_enabled, is_bip152_hb_from, is_bip152_hb_to, is_outbound, is_tx_relay}); m_max_addr_length = std::max(addr.length() + 1, m_max_addr_length); m_max_addr_processed_length = std::max(ToString(addr_processed).length(), m_max_addr_processed_length); m_max_addr_rate_limited_length = std::max(ToString(addr_rate_limited).length(), m_max_addr_rate_limited_length); @@ -571,6 +575,13 @@ class NetinfoRequestHandler : public BaseRequestHandler reachable_networks.push_back(network_id); } }; + + for (const size_t network_id : UNREACHABLE_NETWORK_IDS) { + if (m_counts.at(2).at(network_id) == 0) continue; + result += strprintf("%8s", NETWORK_SHORT_NAMES.at(network_id)); // column header + reachable_networks.push_back(network_id); + } + result += " total block"; if (m_manual_peers_count) result += " manual"; @@ -636,13 +647,13 @@ class NetinfoRequestHandler : public BaseRequestHandler " \"manual\" - peer we manually added using RPC addnode or the -addnode/-connect config options\n" " \"feeler\" - short-lived connection for testing addresses\n" " \"addr\" - address fetch; short-lived connection for requesting addresses\n" - " net Network the peer connected through (\"ipv4\", \"ipv6\", \"onion\", \"i2p\", or \"cjdns\")\n" + " net Network the peer connected through (\"ipv4\", \"ipv6\", \"onion\", \"i2p\", \"cjdns\", or \"npr\" (not publicly routable))\n" " mping Minimum observed ping time, in milliseconds (ms)\n" " ping Last observed ping time, in milliseconds (ms)\n" " send Time since last message sent to the peer, in seconds\n" " recv Time since last message received from the peer, in seconds\n" " txn Time since last novel transaction received from the peer and accepted into our mempool, in minutes\n" - " \"*\" - whether we relay transactions to this peer (relaytxes is false)\n" + " \"*\" - we do not relay transactions to this peer (relaytxes is false)\n" " blk Time since last novel block passing initial validity checks received from the peer, in minutes\n" " hb High-bandwidth BIP152 compact block relay\n" " \".\" (to) - we selected the peer as a high-bandwidth peer\n" @@ -807,7 +818,7 @@ static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, co if (failedToGetAuthCookie) { throw std::runtime_error(strprintf( "Could not locate RPC credentials. No authentication cookie could be found, and RPC password is not set. See -rpcpassword and -stdinrpcpass. Configuration file: (%s)", - fs::PathToString(GetConfigFile(gArgs.GetPathArg("-conf", BITCOIN_CONF_FILENAME))))); + fs::PathToString(gArgs.GetConfigFilePath()))); } else { throw std::runtime_error("Authorization failed: Incorrect rpcuser or rpcpassword"); } @@ -822,7 +833,7 @@ static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, co UniValue valReply(UniValue::VSTR); if (!valReply.read(response.body)) throw std::runtime_error("couldn't parse reply from server"); - const UniValue reply = rh->ProcessReply(valReply); + UniValue reply = rh->ProcessReply(valReply); if (reply.empty()) throw std::runtime_error("expected reply to have result, error and id properties"); @@ -1021,6 +1032,7 @@ static void ParseGetInfoResult(UniValue& result) } std::vector formatted_proxies; + formatted_proxies.reserve(ordered_proxies.size()); for (const std::string& proxy : ordered_proxies) { formatted_proxies.emplace_back(strprintf("%s (%s)", proxy, Join(proxy_networks.find(proxy)->second, ", "))); } diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 010cac59204f..a05443a2e5a3 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2021 The Bitcoin Core developers +// Copyright (c) 2009-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,13 +13,14 @@ #include #include #include -#include #include #include #include