From 8db164783a42a69e0f537748436bd63e05e44de7 Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Wed, 19 Jun 2024 17:06:26 +0200 Subject: [PATCH 01/13] just change repo --- files/install-riak.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/files/install-riak.sh b/files/install-riak.sh index 5456b1f..6df45a1 100755 --- a/files/install-riak.sh +++ b/files/install-riak.sh @@ -3,10 +3,11 @@ set -e eu # Build riak mkdir -p /opt/riak && cd /opt/riak -curl -L https://github.com/basho/riak/archive/refs/tags/riak-${RIAK_VERSION}.tar.gz -o /opt/riak.tar.gz -tar zxf /opt/riak.tar.gz --strip-components 1 -patch -p0 < /riak.schema.patch -patch < /rebar.config.patch -patch < /rebar.lock.patch +# curl -L https://github.com/basho/riak/archive/refs/tags/riak-${RIAK_VERSION}.tar.gz -o /opt/riak.tar.gz +curl -L https://github.com/nhs-riak/riak/archive/refs/tags/riak-${RIAK_VERSION}-nhse.tar.gz -o /opt/riak.tar.gz -v +tar xvf /opt/riak.tar.gz --strip-components 1 +# patch -p0 < /riak.schema.patch +# patch < /rebar.config.patch +# patch < /rebar.lock.patch make all ./rebar3 as deb release --overlay_vars /vars.config From 91e5f56d65eb3fc3e27413a3c40defd15d9fb6f3 Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Wed, 19 Jun 2024 17:08:39 +0200 Subject: [PATCH 02/13] fix version --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index cca74d7..31be73e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -7,7 +7,7 @@ on: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} - RIAK_VERSION: 3.0.12 + RIAK_VERSION: 3.2.2 jobs: From 9735d909dde7115ba9f4330ae23dbd71ffb1a111 Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Wed, 19 Jun 2024 17:16:27 +0200 Subject: [PATCH 03/13] debug --- Dockerfile | 1 + files/install-riak.sh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4ff65ce..479cbf5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,7 @@ COPY files/poststart.d /tmp/portage-root/etc/riak/poststart.d # Install custom start script COPY files/riak-cluster.sh /tmp/portage-root/riak-cluster.sh + ##################################################################### # Riak image FROM erlang:22-slim diff --git a/files/install-riak.sh b/files/install-riak.sh index 6df45a1..11c98ad 100755 --- a/files/install-riak.sh +++ b/files/install-riak.sh @@ -4,10 +4,11 @@ set -e eu # Build riak mkdir -p /opt/riak && cd /opt/riak # curl -L https://github.com/basho/riak/archive/refs/tags/riak-${RIAK_VERSION}.tar.gz -o /opt/riak.tar.gz -curl -L https://github.com/nhs-riak/riak/archive/refs/tags/riak-${RIAK_VERSION}-nhse.tar.gz -o /opt/riak.tar.gz -v +curl -L https://github.com/nhs-riak/riak/archive/refs/tags/riak-${RIAK_VERSION}-nhse.tar.gz -o /opt/riak.tar.gz tar xvf /opt/riak.tar.gz --strip-components 1 # patch -p0 < /riak.schema.patch # patch < /rebar.config.patch # patch < /rebar.lock.patch make all ./rebar3 as deb release --overlay_vars /vars.config +ls -la /opt/riak/_build/deb/rel/riak From 2235ac9fe6101cf8056ff56a3cffeaa9ac7bc9e0 Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Wed, 19 Jun 2024 17:28:54 +0200 Subject: [PATCH 04/13] update erts --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 479cbf5..e722590 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ COPY --chown=102:102 --from=build /tmp/portage-root/ / COPY --from=build /opt/riak/_build/deb/rel/riak/lib /usr/lib/riak/lib COPY --from=build /opt/riak/_build/deb/rel/riak/share /usr/lib/riak/share COPY --from=build /opt/riak/_build/deb/rel/riak/releases /usr/lib/riak/releases -COPY --from=build /opt/riak/_build/deb/rel/riak/erts-10.7.2.18 /usr/lib/riak/erts-10.7.2.18 +COPY --from=build /opt/riak/_build/deb/rel/riak/erts-10.7.2.19 /usr/lib/riak/erts-10.7.2.19 COPY --from=build /opt/riak/_build/deb/rel/riak/bin /usr/lib/riak/bin COPY --chown=102:102 --from=build /opt/riak/_build/deb/rel/riak/etc/* /etc/riak/ COPY --chown=102:102 --from=build /opt/riak/_build/deb/rel/riak/data/* /var/lib/riak/data/ From 37a6c9ce7c606a30f8ff80f3b33883477be15f07 Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Thu, 20 Jun 2024 17:14:24 +0200 Subject: [PATCH 05/13] add sudo to image --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e722590..8f3b836 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,9 @@ FROM erlang:22-slim # Create riak user/group RUN adduser --uid 102 --gecos riak --disabled-password --home /var/lib/riak riak && \ mkdir -p /var/log/riak && \ - chown -R 102:102 /var/log/riak + chown -R 102:102 /var/log/riak && \ + apt-get update && \ + apt-get install -y sudo # Copy riak sources COPY --chown=102:102 --from=build /tmp/portage-root/ / From 3e93828cbe54eb4cef835c60ca65ecaa37d8233f Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Mon, 24 Jun 2024 13:26:49 +0300 Subject: [PATCH 06/13] Migrates to `logger` --- .github/workflows/docker-publish.yml | 2 +- files/rebar.config.patch | 31 +++----- files/rebar.lock.patch | 48 +++++++----- files/riak.schema.patch | 107 ++++++++++++++++++++++----- files/vars.config | 3 - 5 files changed, 132 insertions(+), 59 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index cca74d7..37d473f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -7,7 +7,7 @@ on: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} - RIAK_VERSION: 3.0.12 + RIAK_VERSION: 3.2.0 jobs: diff --git a/files/rebar.config.patch b/files/rebar.config.patch index 5a8b45b..e1a1219 100644 --- a/files/rebar.config.patch +++ b/files/rebar.config.patch @@ -1,20 +1,13 @@ ---- rebar.config.orig 2023-03-01 15:38:23.497209722 +0400 -+++ rebar.config 2023-03-01 15:42:59.569322130 +0400 -@@ -17,7 +17,8 @@ - - % {yokozuna, {git, "https://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - -- {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.12"}}} -+ {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.12"}}}, -+ {lager_logstash_formatter, ".*", {git, "https://github.com/rbkmoney/lager_logstash_formatter.git", {branch, "riak-2.9"}}} +--- a/rebar.config ++++ b/rebar.config +@@ -10,8 +10,9 @@ + {cluster_info, {git, "https://github.com/basho/cluster_info", {branch, "develop"}}}, + {riak_kv, {git, "https://github.com/basho/riak_kv", {branch, "develop"}}}, + {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods", {branch, "develop"}}}, +- {riak_repl, {git, "https://github.com/basho/riak_repl", {branch, "develop"}}} +- ++ {riak_repl, {git, "https://github.com/basho/riak_repl", {branch, "develop"}}}, ++ {logger_logstash_formatter, ++ {git, "https://github.com/valitydev/logger_logstash_formatter.git", {ref, "08a66a6"}}} + % {riaknostic, {git, "https://github.com/basho/riaknostic", {branch, "develop"}}} ]}. - - {project_plugins, [ -@@ -51,6 +52,7 @@ - stdlib, - lager, - lager_syslog, -+ lager_logstash_formatter, - sasl, - public_key, - ssl, diff --git a/files/rebar.lock.patch b/files/rebar.lock.patch index 8192d3c..8569963 100644 --- a/files/rebar.lock.patch +++ b/files/rebar.lock.patch @@ -1,24 +1,36 @@ ---- rebar.lock.orig 2023-03-01 15:38:30.045212676 +0400 -+++ rebar.lock 2023-03-01 15:44:57.753364158 +0400 -@@ -48,6 +48,10 @@ +--- a/rebar.lock ++++ b/rebar.lock +@@ -50,6 +50,7 @@ {git,"https://github.com/basho/ibrowse.git", {ref,"cdde6fe5e80ce71e7b5570e40a45d9f862721499"}}, 2}, -+ {<<"jsx">>, -+ {git,"https://github.com/talentdeficit/jsx.git", -+ {ref,"47ce39f040b5da98794008bf0126c8a68760d786"}}, -+ 1}, ++ {<<"jsx">>,{pkg,<<"jsx">>,<<"3.1.0">>},1}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"4cd9ab1e20772db36f593879272dff5f02fbbeda"}}, -@@ -56,6 +60,10 @@ - {git,"https://github.com/erlang-lager/lager.git", - {ref,"22e62f28e5afabe90a6f31bcde367a2b5799fc94"}}, - 1}, -+ {<<"lager_logstash_formatter">>, -+ {git,"https://github.com/rbkmoney/lager_logstash_formatter.git", -+ {ref,"687b556a05f86f0c1640db992611cec821451fa4"}}, + {ref,"493f94ba24028d823772665c244d33728eca2a6e"}}, +@@ -58,6 +59,10 @@ + {git,"https://github.com/martinsumner/leveled", + {ref,"a033e280e67931582cc9625993268db126abb4ff"}}, + 2}, ++ {<<"logger_logstash_formatter">>, ++ {git,"https://github.com/valitydev/logger_logstash_formatter.git", ++ {ref,"08a66a6001ad5db73a05b836e7a0c9edb192532a"}}, + 0}, - {<<"lager_syslog">>, - {git,"https://github.com/basho/lager_syslog.git", - {ref,"5c8eb6ec82fa45ff0f4e456747e095654e342746"}}, + {<<"lz4">>, + {git,"https://github.com/martinsumner/erlang-lz4", + {ref,"d407af5de71303b8e381df718d2e5d05eb775c59"}}, +@@ -153,12 +158,14 @@ + {<<"bear">>, <<"430419C1126B477686CDE843E88BA0F2C7DC5CDF0881C677500074F704339A99">>}, + {<<"folsom">>, <<"50ECC998D2149939F1D5E0AA3E32788F8ED16A58E390D81B5C0BE4CC4EF25589">>}, + {<<"hut">>, <<"71F2F054E657C03F959CF1ACC43F436EA87580696528CA2A55C8AFB1B06C85E7">>}, ++ {<<"jsx">>, <<"D12516BAA0BB23A59BB35DCCAF02A1BD08243FCBB9EFE24F2D9D056CCFF71268">>}, + {<<"parse_trans">>, <<"6E6AA8167CB44CC8F39441D05193BE6E6F4E7C2946CB2759F015F8C56B76E5FF">>}, + {<<"setup">>, <<"05F69185A5EB71474C9BC6BA892565651EC7507791F85632B7B914DBFE130510">>}]}, + {pkg_hash_ext,[ + {<<"bear">>, <<"157B67901ADF84FF0DA6EAE035CA1292A0AC18AA55148154D8C582B2C68959DB">>}, + {<<"folsom">>, <<"DD6AB97278E94F9E4CFC43E188224A7B8C7EAEC0DD2E935007005177F3EEBB0E">>}, + {<<"hut">>, <<"7E15D28555D8A1F2B5A3A931EC120AF0753E4853A4C66053DB354F35BF9AB563">>}, ++ {<<"jsx">>, <<"0C5CC8FDC11B53CC25CF65AC6705AD39E54ECC56D1C22E4ADB8F5A53FB9427F3">>}, + {<<"parse_trans">>, <<"620A406CE75DADA827B82E453C19CF06776BE266F5A67CFF34E1EF2CBB60E49A">>}, + {<<"setup">>, <<"EFD072578F0CF85BEA96CAAFFC7ADB0992398272522660A136E10567377071C5">>}]} + ]. diff --git a/files/riak.schema.patch b/files/riak.schema.patch index c291b41..e5aa67b 100644 --- a/files/riak.schema.patch +++ b/files/riak.schema.patch @@ -1,18 +1,89 @@ ---- priv/riak.schema.orig 2023-03-01 15:38:54.733223673 +0400 -+++ priv/riak.schema 2023-03-01 15:58:16.673602387 +0400 -@@ -74,6 +74,7 @@ - undefined -> []; - ErrorFilename -> [{lager_file_backend, [{file, ErrorFilename}, - {level, error}, -+ {formatter, lager_logstash_formatter}, - {size, 10485760}, - {date, "$D0"}, - {count, 5}]}] -@@ -85,6 +86,7 @@ - ConsoleHandler = {lager_console_backend, ConsoleLogLevel}, - ConsoleFileHandler = {lager_file_backend, [{file, ConsoleLogFile}, - {level, ConsoleLogLevel}, -+ {formatter, lager_logstash_formatter}, - {size, 10485760}, - {date, "$D0"}, - {count, 5}]}, +--- a/priv/riak.schema ++++ b/priv/riak.schema +@@ -6,63 +6,33 @@ + {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}} + ]}. + +-%% @doc Format string for the messages emitted to default log. The string is passed into +-%% the handler as a logger_formatter template, the format is a list containing strings +-%% and atoms. The atoms denote the keys for retrieving metadata from the logger events. +-%% More information on default metadata can be found here (https://www.erlang.org/doc/man/logger_formatter.html#type-template). +-{mapping, "logger.format", "kernel.logger", [ +- {default, "[time,\" [\",level,\"] \",pid,\"@\",mfa,\":\",line,\" \",msg,\"\\n\"]."} +-]}. +- +-%% @doc Filename to use for log files. +-{mapping, "logger.file", "kernel.logger", [ +- {default, "$(platform_log_dir)/console.log"}, +- {datatype, file} +-]}. +- +-%% @doc With log rotation enabled, this decides the maximum number of log +-%% files to store. +-{mapping, "logger.max_files", "kernel.logger", [ +- {default, 10}, +- {datatype, integer} +-]}. +- +-%% @doc The maximum size of a single log file. Total size used for log files will +-%% be max_file_size * max_files. +-{mapping, "logger.max_file_size", "kernel.logger", [ +- {default, "1MB"}, +- {datatype, bytesize} +-]}. +- + {translation, + "kernel.logger", + fun(Conf) -> +- LogFile = cuttlefish:conf_get("logger.file", Conf), +- MaxNumBytes = cuttlefish:conf_get("logger.max_file_size", Conf), +- MaxNumFiles = cuttlefish:conf_get("logger.max_files", Conf), +- +- DefaultFormatStr = cuttlefish:conf_get("logger.format", Conf), +- DefaultFormatTerm = +- case erl_scan:string(DefaultFormatStr) of +- {ok, DefaultTokens, _} -> +- case erl_parse:parse_term(DefaultTokens) of +- {ok, DefaultTerm} -> +- DefaultTerm; +- {error, {_, _, DefaultError}} -> +- cuttlefish:error(foo) +- end; +- {error, {_, _, DefaultScanError}} -> +- cuttlefish:error(foo) +- end, +- ConfigMap0 = #{config => #{file => LogFile, +- max_no_bytes => MaxNumBytes, +- max_no_files => MaxNumFiles}, +- formatter => {logger_formatter, +- #{template => DefaultFormatTerm} +- } +- }, ++ %% See https://www.erlang.org/doc/apps/kernel/logger_handler#t:olp_config/0 ++ LoggerConfig =#{type => standard_io, ++ sync_mode_qlen => 5000, ++ drop_mode_qlen => 5000, ++ flush_qlen => 10000, ++ %% We want almost all logs ++ burst_limit_enable => false}, ++ ++ FormatterConfig = #{ ++ log_level_map => #{emergency => 'ERROR', ++ alert => 'ERROR', ++ critical => 'ERROR', ++ error => 'ERROR', ++ warning => 'WARN', ++ notice => 'INFO', ++ info => 'INFO', ++ debug => 'DEBUG'} ++ }, ++ ++ %% See https://www.erlang.org/doc/apps/kernel/logger_std_h.html ++ ConfigMap0 = #{config => LoggerConfig, ++ %% See https://github.com/valitydev/logger_logstash_formatter ++ formatter => {logger_logstash_formatter, FormatterConfig}}, + + [{handler, default, logger_std_h, ConfigMap0}] +- + end + }. diff --git a/files/vars.config b/files/vars.config index 4404b6d..28e30f2 100644 --- a/files/vars.config +++ b/files/vars.config @@ -51,9 +51,6 @@ {yz_solr_port, 8093}. {yz_solr_jmx_port, 8985}. -%% lager -{console_log_default, file}. - %% %% etc/vm.args %% From 7c65d92e150fecb3a644247b6290064df41cba26 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Mon, 24 Jun 2024 13:46:07 +0300 Subject: [PATCH 07/13] Fixes patches headers --- files/rebar.config.patch | 4 ++-- files/rebar.lock.patch | 4 ++-- files/riak.schema.patch | 16 ++++++++-------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/files/rebar.config.patch b/files/rebar.config.patch index e1a1219..b591091 100644 --- a/files/rebar.config.patch +++ b/files/rebar.config.patch @@ -1,5 +1,5 @@ ---- a/rebar.config -+++ b/rebar.config +--- rebar.config ++++ rebar.config @@ -10,8 +10,9 @@ {cluster_info, {git, "https://github.com/basho/cluster_info", {branch, "develop"}}}, {riak_kv, {git, "https://github.com/basho/riak_kv", {branch, "develop"}}}, diff --git a/files/rebar.lock.patch b/files/rebar.lock.patch index 8569963..5bd2492 100644 --- a/files/rebar.lock.patch +++ b/files/rebar.lock.patch @@ -1,5 +1,5 @@ ---- a/rebar.lock -+++ b/rebar.lock +--- rebar.lock ++++ rebar.lock @@ -50,6 +50,7 @@ {git,"https://github.com/basho/ibrowse.git", {ref,"cdde6fe5e80ce71e7b5570e40a45d9f862721499"}}, diff --git a/files/riak.schema.patch b/files/riak.schema.patch index e5aa67b..79e08e4 100644 --- a/files/riak.schema.patch +++ b/files/riak.schema.patch @@ -1,9 +1,9 @@ ---- a/priv/riak.schema -+++ b/priv/riak.schema +--- priv/riak.schema ++++ priv/riak.schema @@ -6,63 +6,33 @@ {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}} ]}. - + -%% @doc Format string for the messages emitted to default log. The string is passed into -%% the handler as a logger_formatter template, the format is a list containing strings -%% and atoms. The atoms denote the keys for retrieving metadata from the logger events. @@ -18,14 +18,14 @@ - {datatype, file} -]}. - --%% @doc With log rotation enabled, this decides the maximum number of log +-%% @doc With log rotation enabled, this decides the maximum number of log -%% files to store. -{mapping, "logger.max_files", "kernel.logger", [ - {default, 10}, - {datatype, integer} -]}. - --%% @doc The maximum size of a single log file. Total size used for log files will +-%% @doc The maximum size of a single log file. Total size used for log files will -%% be max_file_size * max_files. -{mapping, "logger.max_file_size", "kernel.logger", [ - {default, "1MB"}, @@ -55,7 +55,7 @@ - ConfigMap0 = #{config => #{file => LogFile, - max_no_bytes => MaxNumBytes, - max_no_files => MaxNumFiles}, -- formatter => {logger_formatter, +- formatter => {logger_formatter, - #{template => DefaultFormatTerm} - } - }, @@ -82,8 +82,8 @@ + ConfigMap0 = #{config => LoggerConfig, + %% See https://github.com/valitydev/logger_logstash_formatter + formatter => {logger_logstash_formatter, FormatterConfig}}, - + [{handler, default, logger_std_h, ConfigMap0}] -- +- end }. From a6535f759d00c1eae3bbeac1a9080e46d21be10d Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Mon, 24 Jun 2024 13:51:14 +0300 Subject: [PATCH 08/13] Fixes minor version of erts in erlang:22 --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4ff65ce..b09a38f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends apt-utils \ curl apt-transport-https ca-certificates git \ - locales sudo patch make wget cmake g++ build-essential libpam0g-dev + locales sudo patch make wget cmake g++ build-essential libpam0g-dev ARG RIAK_VERSION @@ -26,14 +26,14 @@ FROM erlang:22-slim # Create riak user/group RUN adduser --uid 102 --gecos riak --disabled-password --home /var/lib/riak riak && \ mkdir -p /var/log/riak && \ - chown -R 102:102 /var/log/riak + chown -R 102:102 /var/log/riak # Copy riak sources COPY --chown=102:102 --from=build /tmp/portage-root/ / COPY --from=build /opt/riak/_build/deb/rel/riak/lib /usr/lib/riak/lib COPY --from=build /opt/riak/_build/deb/rel/riak/share /usr/lib/riak/share COPY --from=build /opt/riak/_build/deb/rel/riak/releases /usr/lib/riak/releases -COPY --from=build /opt/riak/_build/deb/rel/riak/erts-10.7.2.18 /usr/lib/riak/erts-10.7.2.18 +COPY --from=build /opt/riak/_build/deb/rel/riak/erts-10.7.2.19 /usr/lib/riak/erts-10.7.2.19 COPY --from=build /opt/riak/_build/deb/rel/riak/bin /usr/lib/riak/bin COPY --chown=102:102 --from=build /opt/riak/_build/deb/rel/riak/etc/* /etc/riak/ COPY --chown=102:102 --from=build /opt/riak/_build/deb/rel/riak/data/* /var/lib/riak/data/ From 9dc5e35af239e6d0edaa6ea29748cca2580ba5bf Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Mon, 24 Jun 2024 13:51:19 +0200 Subject: [PATCH 09/13] rebar conf for nhs --- files/rebar.config.patch | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/files/rebar.config.patch b/files/rebar.config.patch index b591091..b64fe5b 100644 --- a/files/rebar.config.patch +++ b/files/rebar.config.patch @@ -1,13 +1,13 @@ ---- rebar.config -+++ rebar.config -@@ -10,8 +10,9 @@ - {cluster_info, {git, "https://github.com/basho/cluster_info", {branch, "develop"}}}, - {riak_kv, {git, "https://github.com/basho/riak_kv", {branch, "develop"}}}, - {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods", {branch, "develop"}}}, -- {riak_repl, {git, "https://github.com/basho/riak_repl", {branch, "develop"}}} -- -+ {riak_repl, {git, "https://github.com/basho/riak_repl", {branch, "develop"}}}, +--- rebar.config.orig 2024-06-24 14:04:31 ++++ rebar.config 2024-06-24 14:05:05 +@@ -9,7 +9,9 @@ + {deps, [ + {riak_kv, {git, "https://github.com/nhs-riak/riak_kv", {branch, "nhse-develop"}}}, + {riak_auth_mods, {git, "https://github.com/nhs-riak/riak_auth_mods", {branch, "nhse-develop"}}}, +- {riak_repl, {git, "https://github.com/nhs-riak/riak_repl", {branch, "nhse-develop"}}} ++ {riak_repl, {git, "https://github.com/nhs-riak/riak_repl", {branch, "nhse-develop"}}}, + {logger_logstash_formatter, + {git, "https://github.com/valitydev/logger_logstash_formatter.git", {ref, "08a66a6"}}} - % {riaknostic, {git, "https://github.com/basho/riaknostic", {branch, "develop"}}} ]}. + + {project_plugins, [ From e1f22ef4f00889005c3943500da052b367b630d6 Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Mon, 24 Jun 2024 16:28:57 +0200 Subject: [PATCH 10/13] format fix --- files/riak.schema.patch | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/files/riak.schema.patch b/files/riak.schema.patch index 79e08e4..c865ce0 100644 --- a/files/riak.schema.patch +++ b/files/riak.schema.patch @@ -1,6 +1,6 @@ ---- priv/riak.schema -+++ priv/riak.schema -@@ -6,63 +6,33 @@ +--- priv/riak.schema.origin 2024-06-24 16:21:55 ++++ priv/riak.schema 2024-06-24 16:27:03 +@@ -6,64 +6,29 @@ {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}} ]}. @@ -38,7 +38,9 @@ - LogFile = cuttlefish:conf_get("logger.file", Conf), - MaxNumBytes = cuttlefish:conf_get("logger.max_file_size", Conf), - MaxNumFiles = cuttlefish:conf_get("logger.max_files", Conf), -- ++ %% See https://www.erlang.org/doc/apps/kernel/logger_handler#t:olp_config/0 ++ LoggerConfig =#{type => standard_io}, + - DefaultFormatStr = cuttlefish:conf_get("logger.format", Conf), - DefaultFormatTerm = - case erl_scan:string(DefaultFormatStr) of @@ -59,14 +61,6 @@ - #{template => DefaultFormatTerm} - } - }, -+ %% See https://www.erlang.org/doc/apps/kernel/logger_handler#t:olp_config/0 -+ LoggerConfig =#{type => standard_io, -+ sync_mode_qlen => 5000, -+ drop_mode_qlen => 5000, -+ flush_qlen => 10000, -+ %% We want almost all logs -+ burst_limit_enable => false}, -+ + FormatterConfig = #{ + log_level_map => #{emergency => 'ERROR', + alert => 'ERROR', @@ -77,13 +71,14 @@ + info => 'INFO', + debug => 'DEBUG'} + }, -+ + + %% See https://www.erlang.org/doc/apps/kernel/logger_std_h.html + ConfigMap0 = #{config => LoggerConfig, + %% See https://github.com/valitydev/logger_logstash_formatter + formatter => {logger_logstash_formatter, FormatterConfig}}, - ++ [{handler, default, logger_std_h, ConfigMap0}] - end }. + From 105debba88431b1880a9c6b4cca037c6985c109d Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Fri, 28 Jun 2024 13:16:13 +0300 Subject: [PATCH 11/13] Patches start script and riak schema --- files/riak-cluster.sh | 6 ++-- files/riak.schema.patch | 74 ++++++++++++++++++----------------------- 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/files/riak-cluster.sh b/files/riak-cluster.sh index d3e923f..f69b50e 100755 --- a/files/riak-cluster.sh +++ b/files/riak-cluster.sh @@ -39,7 +39,7 @@ done sleep 10 -$RIAK start & +$RIAK daemon & sleep 40 @@ -49,4 +49,6 @@ if [[ -z "$($RIAK_ADMIN cluster status | egrep $COORDINATOR_NODE)" && "$COORDINA riak admin cluster join $CLUSTER_NAME@$COORDINATOR_NODE riak admin cluster plan riak admin cluster commit -fi \ No newline at end of file +fi + +tail -n 1024 -F /var/log/riak/console.log diff --git a/files/riak.schema.patch b/files/riak.schema.patch index c865ce0..cbb3b11 100644 --- a/files/riak.schema.patch +++ b/files/riak.schema.patch @@ -1,6 +1,6 @@ --- priv/riak.schema.origin 2024-06-24 16:21:55 +++ priv/riak.schema 2024-06-24 16:27:03 -@@ -6,64 +6,29 @@ +@@ -6,28 +6,20 @@ {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}} ]}. @@ -12,34 +12,26 @@ - {default, "[time,\" [\",level,\"] \",pid,\"@\",mfa,\":\",line,\" \",msg,\"\\n\"]."} -]}. - --%% @doc Filename to use for log files. --{mapping, "logger.file", "kernel.logger", [ -- {default, "$(platform_log_dir)/console.log"}, -- {datatype, file} --]}. -- --%% @doc With log rotation enabled, this decides the maximum number of log --%% files to store. --{mapping, "logger.max_files", "kernel.logger", [ -- {default, 10}, -- {datatype, integer} --]}. -- --%% @doc The maximum size of a single log file. Total size used for log files will --%% be max_file_size * max_files. --{mapping, "logger.max_file_size", "kernel.logger", [ -- {default, "1MB"}, -- {datatype, bytesize} --]}. -- - {translation, - "kernel.logger", - fun(Conf) -> -- LogFile = cuttlefish:conf_get("logger.file", Conf), -- MaxNumBytes = cuttlefish:conf_get("logger.max_file_size", Conf), -- MaxNumFiles = cuttlefish:conf_get("logger.max_files", Conf), -+ %% See https://www.erlang.org/doc/apps/kernel/logger_handler#t:olp_config/0 -+ LoggerConfig =#{type => standard_io}, + %% @doc Filename to use for log files. + {mapping, "logger.file", "kernel.logger", [ + {default, "$(platform_log_dir)/console.log"}, + {datatype, file} + ]}. + + %% @doc With log rotation enabled, this decides the maximum number of log + %% files to store. + {mapping, "logger.max_files", "kernel.logger", [ + {default, 10}, + {datatype, integer} + ]}. + + %% @doc The maximum size of a single log file. Total size used for log files will + %% be max_file_size * max_files. + {mapping, "logger.max_file_size", "kernel.logger", [ + {default, "1MB"}, +@@ -41,29 +33,24 @@ + MaxNumBytes = cuttlefish:conf_get("logger.max_file_size", Conf), + MaxNumFiles = cuttlefish:conf_get("logger.max_files", Conf), - DefaultFormatStr = cuttlefish:conf_get("logger.format", Conf), - DefaultFormatTerm = @@ -54,13 +46,6 @@ - {error, {_, _, DefaultScanError}} -> - cuttlefish:error(foo) - end, -- ConfigMap0 = #{config => #{file => LogFile, -- max_no_bytes => MaxNumBytes, -- max_no_files => MaxNumFiles}, -- formatter => {logger_formatter, -- #{template => DefaultFormatTerm} -- } -- }, + FormatterConfig = #{ + log_level_map => #{emergency => 'ERROR', + alert => 'ERROR', @@ -71,14 +56,21 @@ + info => 'INFO', + debug => 'DEBUG'} + }, - -+ %% See https://www.erlang.org/doc/apps/kernel/logger_std_h.html -+ ConfigMap0 = #{config => LoggerConfig, -+ %% See https://github.com/valitydev/logger_logstash_formatter -+ formatter => {logger_logstash_formatter, FormatterConfig}}, + + ConfigMap0 = #{config => #{file => LogFile, +- max_no_bytes => MaxNumBytes, +- max_no_files => MaxNumFiles}, +- formatter => {logger_formatter, +- #{template => DefaultFormatTerm} +- } +- }, ++ max_no_bytes => MaxNumBytes, ++ max_no_files => MaxNumFiles}, ++ formatter => {logger_logstash_formatter, FormatterConfig}}}, + [{handler, default, logger_std_h, ConfigMap0}] - ++ end }. From dc6808878ff1e436545d808c53fc661ebd453ca6 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Fri, 28 Jun 2024 14:39:28 +0300 Subject: [PATCH 12/13] Fixes closing bracket and overlay_vars config file --- files/install-riak.sh | 4 ++-- files/rebar.config.patch | 17 +++++++++++++++++ files/riak.schema.patch | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/files/install-riak.sh b/files/install-riak.sh index d78b6af..2f68f79 100755 --- a/files/install-riak.sh +++ b/files/install-riak.sh @@ -3,11 +3,11 @@ set -e eu # Build riak mkdir -p /opt/riak && cd /opt/riak -curl -L https://github.com/nhs-riak/riak/archive/refs/tags/riak-${RIAK_VERSION}-nhse.tar.gz -o /opt/riak.tar.gz +curl -L https://github.com/nhs-riak/riak/archive/refs/tags/riak-${RIAK_VERSION}-nhse.tar.gz -o /opt/riak.tar.gz tar xvf /opt/riak.tar.gz --strip-components 1 patch -p0 < /riak.schema.patch patch < /rebar.config.patch patch < /rebar.lock.patch make all -./rebar3 as deb release --overlay_vars /vars.config +./rebar3 as deb release ls -la /opt/riak/_build/deb/rel/riak diff --git a/files/rebar.config.patch b/files/rebar.config.patch index b64fe5b..abb91a4 100644 --- a/files/rebar.config.patch +++ b/files/rebar.config.patch @@ -11,3 +11,20 @@ ]}. {project_plugins, [ +@@ -41,6 +43,7 @@ + {relx, [{release, {riak, "3.2.2-nhse"}, + [kernel, + stdlib, ++ logger_logstash_formatter, + public_key, + ssl, + exometer_core, +@@ -164,7 +167,7 @@ + + {deb, + [{relx, +- [{overlay_vars, "rel/pkg/deb/vars.config"}, ++ [{overlay_vars, "/vars.config"}, + {overlay, + [{template, "rel/files/riak", "usr/bin/riak"} + ]}, diff --git a/files/riak.schema.patch b/files/riak.schema.patch index cbb3b11..5eeb9be 100644 --- a/files/riak.schema.patch +++ b/files/riak.schema.patch @@ -66,7 +66,7 @@ - }, + max_no_bytes => MaxNumBytes, + max_no_files => MaxNumFiles}, -+ formatter => {logger_logstash_formatter, FormatterConfig}}}, ++ formatter => {logger_logstash_formatter, FormatterConfig}}, [{handler, default, logger_std_h, ConfigMap0}] - From 11f161c0c15b248e8bd941eb007046f6cbf6a640 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Fri, 28 Jun 2024 15:12:01 +0300 Subject: [PATCH 13/13] Fixes missing overlay vars --- files/install-riak.sh | 2 +- files/rebar.config.patch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/files/install-riak.sh b/files/install-riak.sh index 2f68f79..f6f733e 100755 --- a/files/install-riak.sh +++ b/files/install-riak.sh @@ -9,5 +9,5 @@ patch -p0 < /riak.schema.patch patch < /rebar.config.patch patch < /rebar.lock.patch make all -./rebar3 as deb release +./rebar3 as deb release --overlay_vars /vars.config ls -la /opt/riak/_build/deb/rel/riak diff --git a/files/rebar.config.patch b/files/rebar.config.patch index abb91a4..c3fcc9a 100644 --- a/files/rebar.config.patch +++ b/files/rebar.config.patch @@ -24,7 +24,7 @@ {deb, [{relx, - [{overlay_vars, "rel/pkg/deb/vars.config"}, -+ [{overlay_vars, "/vars.config"}, ++ [{overlay_vars, ["rel/pkg/deb/vars.config"]}, {overlay, [{template, "rel/files/riak", "usr/bin/riak"} ]},