From 8db164783a42a69e0f537748436bd63e05e44de7 Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Wed, 19 Jun 2024 17:06:26 +0200 Subject: [PATCH 01/15] just change repo --- files/install-riak.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/files/install-riak.sh b/files/install-riak.sh index 5456b1f..6df45a1 100755 --- a/files/install-riak.sh +++ b/files/install-riak.sh @@ -3,10 +3,11 @@ set -e eu # Build riak mkdir -p /opt/riak && cd /opt/riak -curl -L https://github.com/basho/riak/archive/refs/tags/riak-${RIAK_VERSION}.tar.gz -o /opt/riak.tar.gz -tar zxf /opt/riak.tar.gz --strip-components 1 -patch -p0 < /riak.schema.patch -patch < /rebar.config.patch -patch < /rebar.lock.patch +# curl -L https://github.com/basho/riak/archive/refs/tags/riak-${RIAK_VERSION}.tar.gz -o /opt/riak.tar.gz +curl -L https://github.com/nhs-riak/riak/archive/refs/tags/riak-${RIAK_VERSION}-nhse.tar.gz -o /opt/riak.tar.gz -v +tar xvf /opt/riak.tar.gz --strip-components 1 +# patch -p0 < /riak.schema.patch +# patch < /rebar.config.patch +# patch < /rebar.lock.patch make all ./rebar3 as deb release --overlay_vars /vars.config From 91e5f56d65eb3fc3e27413a3c40defd15d9fb6f3 Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Wed, 19 Jun 2024 17:08:39 +0200 Subject: [PATCH 02/15] fix version --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index cca74d7..31be73e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -7,7 +7,7 @@ on: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} - RIAK_VERSION: 3.0.12 + RIAK_VERSION: 3.2.2 jobs: From 9735d909dde7115ba9f4330ae23dbd71ffb1a111 Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Wed, 19 Jun 2024 17:16:27 +0200 Subject: [PATCH 03/15] debug --- Dockerfile | 1 + files/install-riak.sh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4ff65ce..479cbf5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,7 @@ COPY files/poststart.d /tmp/portage-root/etc/riak/poststart.d # Install custom start script COPY files/riak-cluster.sh /tmp/portage-root/riak-cluster.sh + ##################################################################### # Riak image FROM erlang:22-slim diff --git a/files/install-riak.sh b/files/install-riak.sh index 6df45a1..11c98ad 100755 --- a/files/install-riak.sh +++ b/files/install-riak.sh @@ -4,10 +4,11 @@ set -e eu # Build riak mkdir -p /opt/riak && cd /opt/riak # curl -L https://github.com/basho/riak/archive/refs/tags/riak-${RIAK_VERSION}.tar.gz -o /opt/riak.tar.gz -curl -L https://github.com/nhs-riak/riak/archive/refs/tags/riak-${RIAK_VERSION}-nhse.tar.gz -o /opt/riak.tar.gz -v +curl -L https://github.com/nhs-riak/riak/archive/refs/tags/riak-${RIAK_VERSION}-nhse.tar.gz -o /opt/riak.tar.gz tar xvf /opt/riak.tar.gz --strip-components 1 # patch -p0 < /riak.schema.patch # patch < /rebar.config.patch # patch < /rebar.lock.patch make all ./rebar3 as deb release --overlay_vars /vars.config +ls -la /opt/riak/_build/deb/rel/riak From 2235ac9fe6101cf8056ff56a3cffeaa9ac7bc9e0 Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Wed, 19 Jun 2024 17:28:54 +0200 Subject: [PATCH 04/15] update erts --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 479cbf5..e722590 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ COPY --chown=102:102 --from=build /tmp/portage-root/ / COPY --from=build /opt/riak/_build/deb/rel/riak/lib /usr/lib/riak/lib COPY --from=build /opt/riak/_build/deb/rel/riak/share /usr/lib/riak/share COPY --from=build /opt/riak/_build/deb/rel/riak/releases /usr/lib/riak/releases -COPY --from=build /opt/riak/_build/deb/rel/riak/erts-10.7.2.18 /usr/lib/riak/erts-10.7.2.18 +COPY --from=build /opt/riak/_build/deb/rel/riak/erts-10.7.2.19 /usr/lib/riak/erts-10.7.2.19 COPY --from=build /opt/riak/_build/deb/rel/riak/bin /usr/lib/riak/bin COPY --chown=102:102 --from=build /opt/riak/_build/deb/rel/riak/etc/* /etc/riak/ COPY --chown=102:102 --from=build /opt/riak/_build/deb/rel/riak/data/* /var/lib/riak/data/ From 37a6c9ce7c606a30f8ff80f3b33883477be15f07 Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Thu, 20 Jun 2024 17:14:24 +0200 Subject: [PATCH 05/15] add sudo to image --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e722590..8f3b836 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,9 @@ FROM erlang:22-slim # Create riak user/group RUN adduser --uid 102 --gecos riak --disabled-password --home /var/lib/riak riak && \ mkdir -p /var/log/riak && \ - chown -R 102:102 /var/log/riak + chown -R 102:102 /var/log/riak && \ + apt-get update && \ + apt-get install -y sudo # Copy riak sources COPY --chown=102:102 --from=build /tmp/portage-root/ / From 3e93828cbe54eb4cef835c60ca65ecaa37d8233f Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Mon, 24 Jun 2024 13:26:49 +0300 Subject: [PATCH 06/15] Migrates to `logger` --- .github/workflows/docker-publish.yml | 2 +- files/rebar.config.patch | 31 +++----- files/rebar.lock.patch | 48 +++++++----- files/riak.schema.patch | 107 ++++++++++++++++++++++----- files/vars.config | 3 - 5 files changed, 132 insertions(+), 59 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index cca74d7..37d473f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -7,7 +7,7 @@ on: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} - RIAK_VERSION: 3.0.12 + RIAK_VERSION: 3.2.0 jobs: diff --git a/files/rebar.config.patch b/files/rebar.config.patch index 5a8b45b..e1a1219 100644 --- a/files/rebar.config.patch +++ b/files/rebar.config.patch @@ -1,20 +1,13 @@ ---- rebar.config.orig 2023-03-01 15:38:23.497209722 +0400 -+++ rebar.config 2023-03-01 15:42:59.569322130 +0400 -@@ -17,7 +17,8 @@ - - % {yokozuna, {git, "https://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - -- {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.12"}}} -+ {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.12"}}}, -+ {lager_logstash_formatter, ".*", {git, "https://github.com/rbkmoney/lager_logstash_formatter.git", {branch, "riak-2.9"}}} +--- a/rebar.config ++++ b/rebar.config +@@ -10,8 +10,9 @@ + {cluster_info, {git, "https://github.com/basho/cluster_info", {branch, "develop"}}}, + {riak_kv, {git, "https://github.com/basho/riak_kv", {branch, "develop"}}}, + {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods", {branch, "develop"}}}, +- {riak_repl, {git, "https://github.com/basho/riak_repl", {branch, "develop"}}} +- ++ {riak_repl, {git, "https://github.com/basho/riak_repl", {branch, "develop"}}}, ++ {logger_logstash_formatter, ++ {git, "https://github.com/valitydev/logger_logstash_formatter.git", {ref, "08a66a6"}}} + % {riaknostic, {git, "https://github.com/basho/riaknostic", {branch, "develop"}}} ]}. - - {project_plugins, [ -@@ -51,6 +52,7 @@ - stdlib, - lager, - lager_syslog, -+ lager_logstash_formatter, - sasl, - public_key, - ssl, diff --git a/files/rebar.lock.patch b/files/rebar.lock.patch index 8192d3c..8569963 100644 --- a/files/rebar.lock.patch +++ b/files/rebar.lock.patch @@ -1,24 +1,36 @@ ---- rebar.lock.orig 2023-03-01 15:38:30.045212676 +0400 -+++ rebar.lock 2023-03-01 15:44:57.753364158 +0400 -@@ -48,6 +48,10 @@ +--- a/rebar.lock ++++ b/rebar.lock +@@ -50,6 +50,7 @@ {git,"https://github.com/basho/ibrowse.git", {ref,"cdde6fe5e80ce71e7b5570e40a45d9f862721499"}}, 2}, -+ {<<"jsx">>, -+ {git,"https://github.com/talentdeficit/jsx.git", -+ {ref,"47ce39f040b5da98794008bf0126c8a68760d786"}}, -+ 1}, ++ {<<"jsx">>,{pkg,<<"jsx">>,<<"3.1.0">>},1}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"4cd9ab1e20772db36f593879272dff5f02fbbeda"}}, -@@ -56,6 +60,10 @@ - {git,"https://github.com/erlang-lager/lager.git", - {ref,"22e62f28e5afabe90a6f31bcde367a2b5799fc94"}}, - 1}, -+ {<<"lager_logstash_formatter">>, -+ {git,"https://github.com/rbkmoney/lager_logstash_formatter.git", -+ {ref,"687b556a05f86f0c1640db992611cec821451fa4"}}, + {ref,"493f94ba24028d823772665c244d33728eca2a6e"}}, +@@ -58,6 +59,10 @@ + {git,"https://github.com/martinsumner/leveled", + {ref,"a033e280e67931582cc9625993268db126abb4ff"}}, + 2}, ++ {<<"logger_logstash_formatter">>, ++ {git,"https://github.com/valitydev/logger_logstash_formatter.git", ++ {ref,"08a66a6001ad5db73a05b836e7a0c9edb192532a"}}, + 0}, - {<<"lager_syslog">>, - {git,"https://github.com/basho/lager_syslog.git", - {ref,"5c8eb6ec82fa45ff0f4e456747e095654e342746"}}, + {<<"lz4">>, + {git,"https://github.com/martinsumner/erlang-lz4", + {ref,"d407af5de71303b8e381df718d2e5d05eb775c59"}}, +@@ -153,12 +158,14 @@ + {<<"bear">>, <<"430419C1126B477686CDE843E88BA0F2C7DC5CDF0881C677500074F704339A99">>}, + {<<"folsom">>, <<"50ECC998D2149939F1D5E0AA3E32788F8ED16A58E390D81B5C0BE4CC4EF25589">>}, + {<<"hut">>, <<"71F2F054E657C03F959CF1ACC43F436EA87580696528CA2A55C8AFB1B06C85E7">>}, ++ {<<"jsx">>, <<"D12516BAA0BB23A59BB35DCCAF02A1BD08243FCBB9EFE24F2D9D056CCFF71268">>}, + {<<"parse_trans">>, <<"6E6AA8167CB44CC8F39441D05193BE6E6F4E7C2946CB2759F015F8C56B76E5FF">>}, + {<<"setup">>, <<"05F69185A5EB71474C9BC6BA892565651EC7507791F85632B7B914DBFE130510">>}]}, + {pkg_hash_ext,[ + {<<"bear">>, <<"157B67901ADF84FF0DA6EAE035CA1292A0AC18AA55148154D8C582B2C68959DB">>}, + {<<"folsom">>, <<"DD6AB97278E94F9E4CFC43E188224A7B8C7EAEC0DD2E935007005177F3EEBB0E">>}, + {<<"hut">>, <<"7E15D28555D8A1F2B5A3A931EC120AF0753E4853A4C66053DB354F35BF9AB563">>}, ++ {<<"jsx">>, <<"0C5CC8FDC11B53CC25CF65AC6705AD39E54ECC56D1C22E4ADB8F5A53FB9427F3">>}, + {<<"parse_trans">>, <<"620A406CE75DADA827B82E453C19CF06776BE266F5A67CFF34E1EF2CBB60E49A">>}, + {<<"setup">>, <<"EFD072578F0CF85BEA96CAAFFC7ADB0992398272522660A136E10567377071C5">>}]} + ]. diff --git a/files/riak.schema.patch b/files/riak.schema.patch index c291b41..e5aa67b 100644 --- a/files/riak.schema.patch +++ b/files/riak.schema.patch @@ -1,18 +1,89 @@ ---- priv/riak.schema.orig 2023-03-01 15:38:54.733223673 +0400 -+++ priv/riak.schema 2023-03-01 15:58:16.673602387 +0400 -@@ -74,6 +74,7 @@ - undefined -> []; - ErrorFilename -> [{lager_file_backend, [{file, ErrorFilename}, - {level, error}, -+ {formatter, lager_logstash_formatter}, - {size, 10485760}, - {date, "$D0"}, - {count, 5}]}] -@@ -85,6 +86,7 @@ - ConsoleHandler = {lager_console_backend, ConsoleLogLevel}, - ConsoleFileHandler = {lager_file_backend, [{file, ConsoleLogFile}, - {level, ConsoleLogLevel}, -+ {formatter, lager_logstash_formatter}, - {size, 10485760}, - {date, "$D0"}, - {count, 5}]}, +--- a/priv/riak.schema ++++ b/priv/riak.schema +@@ -6,63 +6,33 @@ + {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}} + ]}. + +-%% @doc Format string for the messages emitted to default log. The string is passed into +-%% the handler as a logger_formatter template, the format is a list containing strings +-%% and atoms. The atoms denote the keys for retrieving metadata from the logger events. +-%% More information on default metadata can be found here (https://www.erlang.org/doc/man/logger_formatter.html#type-template). +-{mapping, "logger.format", "kernel.logger", [ +- {default, "[time,\" [\",level,\"] \",pid,\"@\",mfa,\":\",line,\" \",msg,\"\\n\"]."} +-]}. +- +-%% @doc Filename to use for log files. +-{mapping, "logger.file", "kernel.logger", [ +- {default, "$(platform_log_dir)/console.log"}, +- {datatype, file} +-]}. +- +-%% @doc With log rotation enabled, this decides the maximum number of log +-%% files to store. +-{mapping, "logger.max_files", "kernel.logger", [ +- {default, 10}, +- {datatype, integer} +-]}. +- +-%% @doc The maximum size of a single log file. Total size used for log files will +-%% be max_file_size * max_files. +-{mapping, "logger.max_file_size", "kernel.logger", [ +- {default, "1MB"}, +- {datatype, bytesize} +-]}. +- + {translation, + "kernel.logger", + fun(Conf) -> +- LogFile = cuttlefish:conf_get("logger.file", Conf), +- MaxNumBytes = cuttlefish:conf_get("logger.max_file_size", Conf), +- MaxNumFiles = cuttlefish:conf_get("logger.max_files", Conf), +- +- DefaultFormatStr = cuttlefish:conf_get("logger.format", Conf), +- DefaultFormatTerm = +- case erl_scan:string(DefaultFormatStr) of +- {ok, DefaultTokens, _} -> +- case erl_parse:parse_term(DefaultTokens) of +- {ok, DefaultTerm} -> +- DefaultTerm; +- {error, {_, _, DefaultError}} -> +- cuttlefish:error(foo) +- end; +- {error, {_, _, DefaultScanError}} -> +- cuttlefish:error(foo) +- end, +- ConfigMap0 = #{config => #{file => LogFile, +- max_no_bytes => MaxNumBytes, +- max_no_files => MaxNumFiles}, +- formatter => {logger_formatter, +- #{template => DefaultFormatTerm} +- } +- }, ++ %% See https://www.erlang.org/doc/apps/kernel/logger_handler#t:olp_config/0 ++ LoggerConfig =#{type => standard_io, ++ sync_mode_qlen => 5000, ++ drop_mode_qlen => 5000, ++ flush_qlen => 10000, ++ %% We want almost all logs ++ burst_limit_enable => false}, ++ ++ FormatterConfig = #{ ++ log_level_map => #{emergency => 'ERROR', ++ alert => 'ERROR', ++ critical => 'ERROR', ++ error => 'ERROR', ++ warning => 'WARN', ++ notice => 'INFO', ++ info => 'INFO', ++ debug => 'DEBUG'} ++ }, ++ ++ %% See https://www.erlang.org/doc/apps/kernel/logger_std_h.html ++ ConfigMap0 = #{config => LoggerConfig, ++ %% See https://github.com/valitydev/logger_logstash_formatter ++ formatter => {logger_logstash_formatter, FormatterConfig}}, + + [{handler, default, logger_std_h, ConfigMap0}] +- + end + }. diff --git a/files/vars.config b/files/vars.config index 4404b6d..28e30f2 100644 --- a/files/vars.config +++ b/files/vars.config @@ -51,9 +51,6 @@ {yz_solr_port, 8093}. {yz_solr_jmx_port, 8985}. -%% lager -{console_log_default, file}. - %% %% etc/vm.args %% From 7c65d92e150fecb3a644247b6290064df41cba26 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Mon, 24 Jun 2024 13:46:07 +0300 Subject: [PATCH 07/15] Fixes patches headers --- files/rebar.config.patch | 4 ++-- files/rebar.lock.patch | 4 ++-- files/riak.schema.patch | 16 ++++++++-------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/files/rebar.config.patch b/files/rebar.config.patch index e1a1219..b591091 100644 --- a/files/rebar.config.patch +++ b/files/rebar.config.patch @@ -1,5 +1,5 @@ ---- a/rebar.config -+++ b/rebar.config +--- rebar.config ++++ rebar.config @@ -10,8 +10,9 @@ {cluster_info, {git, "https://github.com/basho/cluster_info", {branch, "develop"}}}, {riak_kv, {git, "https://github.com/basho/riak_kv", {branch, "develop"}}}, diff --git a/files/rebar.lock.patch b/files/rebar.lock.patch index 8569963..5bd2492 100644 --- a/files/rebar.lock.patch +++ b/files/rebar.lock.patch @@ -1,5 +1,5 @@ ---- a/rebar.lock -+++ b/rebar.lock +--- rebar.lock ++++ rebar.lock @@ -50,6 +50,7 @@ {git,"https://github.com/basho/ibrowse.git", {ref,"cdde6fe5e80ce71e7b5570e40a45d9f862721499"}}, diff --git a/files/riak.schema.patch b/files/riak.schema.patch index e5aa67b..79e08e4 100644 --- a/files/riak.schema.patch +++ b/files/riak.schema.patch @@ -1,9 +1,9 @@ ---- a/priv/riak.schema -+++ b/priv/riak.schema +--- priv/riak.schema ++++ priv/riak.schema @@ -6,63 +6,33 @@ {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}} ]}. - + -%% @doc Format string for the messages emitted to default log. The string is passed into -%% the handler as a logger_formatter template, the format is a list containing strings -%% and atoms. The atoms denote the keys for retrieving metadata from the logger events. @@ -18,14 +18,14 @@ - {datatype, file} -]}. - --%% @doc With log rotation enabled, this decides the maximum number of log +-%% @doc With log rotation enabled, this decides the maximum number of log -%% files to store. -{mapping, "logger.max_files", "kernel.logger", [ - {default, 10}, - {datatype, integer} -]}. - --%% @doc The maximum size of a single log file. Total size used for log files will +-%% @doc The maximum size of a single log file. Total size used for log files will -%% be max_file_size * max_files. -{mapping, "logger.max_file_size", "kernel.logger", [ - {default, "1MB"}, @@ -55,7 +55,7 @@ - ConfigMap0 = #{config => #{file => LogFile, - max_no_bytes => MaxNumBytes, - max_no_files => MaxNumFiles}, -- formatter => {logger_formatter, +- formatter => {logger_formatter, - #{template => DefaultFormatTerm} - } - }, @@ -82,8 +82,8 @@ + ConfigMap0 = #{config => LoggerConfig, + %% See https://github.com/valitydev/logger_logstash_formatter + formatter => {logger_logstash_formatter, FormatterConfig}}, - + [{handler, default, logger_std_h, ConfigMap0}] -- +- end }. From 9dc5e35af239e6d0edaa6ea29748cca2580ba5bf Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Mon, 24 Jun 2024 13:51:19 +0200 Subject: [PATCH 08/15] rebar conf for nhs --- files/rebar.config.patch | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/files/rebar.config.patch b/files/rebar.config.patch index b591091..b64fe5b 100644 --- a/files/rebar.config.patch +++ b/files/rebar.config.patch @@ -1,13 +1,13 @@ ---- rebar.config -+++ rebar.config -@@ -10,8 +10,9 @@ - {cluster_info, {git, "https://github.com/basho/cluster_info", {branch, "develop"}}}, - {riak_kv, {git, "https://github.com/basho/riak_kv", {branch, "develop"}}}, - {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods", {branch, "develop"}}}, -- {riak_repl, {git, "https://github.com/basho/riak_repl", {branch, "develop"}}} -- -+ {riak_repl, {git, "https://github.com/basho/riak_repl", {branch, "develop"}}}, +--- rebar.config.orig 2024-06-24 14:04:31 ++++ rebar.config 2024-06-24 14:05:05 +@@ -9,7 +9,9 @@ + {deps, [ + {riak_kv, {git, "https://github.com/nhs-riak/riak_kv", {branch, "nhse-develop"}}}, + {riak_auth_mods, {git, "https://github.com/nhs-riak/riak_auth_mods", {branch, "nhse-develop"}}}, +- {riak_repl, {git, "https://github.com/nhs-riak/riak_repl", {branch, "nhse-develop"}}} ++ {riak_repl, {git, "https://github.com/nhs-riak/riak_repl", {branch, "nhse-develop"}}}, + {logger_logstash_formatter, + {git, "https://github.com/valitydev/logger_logstash_formatter.git", {ref, "08a66a6"}}} - % {riaknostic, {git, "https://github.com/basho/riaknostic", {branch, "develop"}}} ]}. + + {project_plugins, [ From e1f22ef4f00889005c3943500da052b367b630d6 Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Mon, 24 Jun 2024 16:28:57 +0200 Subject: [PATCH 09/15] format fix --- files/riak.schema.patch | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/files/riak.schema.patch b/files/riak.schema.patch index 79e08e4..c865ce0 100644 --- a/files/riak.schema.patch +++ b/files/riak.schema.patch @@ -1,6 +1,6 @@ ---- priv/riak.schema -+++ priv/riak.schema -@@ -6,63 +6,33 @@ +--- priv/riak.schema.origin 2024-06-24 16:21:55 ++++ priv/riak.schema 2024-06-24 16:27:03 +@@ -6,64 +6,29 @@ {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}} ]}. @@ -38,7 +38,9 @@ - LogFile = cuttlefish:conf_get("logger.file", Conf), - MaxNumBytes = cuttlefish:conf_get("logger.max_file_size", Conf), - MaxNumFiles = cuttlefish:conf_get("logger.max_files", Conf), -- ++ %% See https://www.erlang.org/doc/apps/kernel/logger_handler#t:olp_config/0 ++ LoggerConfig =#{type => standard_io}, + - DefaultFormatStr = cuttlefish:conf_get("logger.format", Conf), - DefaultFormatTerm = - case erl_scan:string(DefaultFormatStr) of @@ -59,14 +61,6 @@ - #{template => DefaultFormatTerm} - } - }, -+ %% See https://www.erlang.org/doc/apps/kernel/logger_handler#t:olp_config/0 -+ LoggerConfig =#{type => standard_io, -+ sync_mode_qlen => 5000, -+ drop_mode_qlen => 5000, -+ flush_qlen => 10000, -+ %% We want almost all logs -+ burst_limit_enable => false}, -+ + FormatterConfig = #{ + log_level_map => #{emergency => 'ERROR', + alert => 'ERROR', @@ -77,13 +71,14 @@ + info => 'INFO', + debug => 'DEBUG'} + }, -+ + + %% See https://www.erlang.org/doc/apps/kernel/logger_std_h.html + ConfigMap0 = #{config => LoggerConfig, + %% See https://github.com/valitydev/logger_logstash_formatter + formatter => {logger_logstash_formatter, FormatterConfig}}, - ++ [{handler, default, logger_std_h, ConfigMap0}] - end }. + From 3d602850cfe65e1eaa7dc0e0bb275cfbc9b34aa0 Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Fri, 28 Jun 2024 12:54:10 +0200 Subject: [PATCH 10/15] try new log format --- files/riak.schema.patch | 100 +++++++--------------------------------- 1 file changed, 17 insertions(+), 83 deletions(-) diff --git a/files/riak.schema.patch b/files/riak.schema.patch index c865ce0..9d02275 100644 --- a/files/riak.schema.patch +++ b/files/riak.schema.patch @@ -1,84 +1,18 @@ ---- priv/riak.schema.origin 2024-06-24 16:21:55 -+++ priv/riak.schema 2024-06-24 16:27:03 -@@ -6,64 +6,29 @@ - {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}} - ]}. - --%% @doc Format string for the messages emitted to default log. The string is passed into --%% the handler as a logger_formatter template, the format is a list containing strings --%% and atoms. The atoms denote the keys for retrieving metadata from the logger events. --%% More information on default metadata can be found here (https://www.erlang.org/doc/man/logger_formatter.html#type-template). --{mapping, "logger.format", "kernel.logger", [ +--- riak.schema.origin 2024-06-28 12:50:20 ++++ riak.schema 2024-06-28 12:50:20 +@@ -11,7 +11,14 @@ + %% and atoms. The atoms denote the keys for retrieving metadata from the logger events. + %% More information on default metadata can be found here (https://www.erlang.org/doc/man/logger_formatter.html#type-template). + {mapping, "logger.format", "kernel.logger", [ - {default, "[time,\" [\",level,\"] \",pid,\"@\",mfa,\":\",line,\" \",msg,\"\\n\"]."} --]}. -- --%% @doc Filename to use for log files. --{mapping, "logger.file", "kernel.logger", [ -- {default, "$(platform_log_dir)/console.log"}, -- {datatype, file} --]}. -- --%% @doc With log rotation enabled, this decides the maximum number of log --%% files to store. --{mapping, "logger.max_files", "kernel.logger", [ -- {default, 10}, -- {datatype, integer} --]}. -- --%% @doc The maximum size of a single log file. Total size used for log files will --%% be max_file_size * max_files. --{mapping, "logger.max_file_size", "kernel.logger", [ -- {default, "1MB"}, -- {datatype, bytesize} --]}. -- - {translation, - "kernel.logger", - fun(Conf) -> -- LogFile = cuttlefish:conf_get("logger.file", Conf), -- MaxNumBytes = cuttlefish:conf_get("logger.max_file_size", Conf), -- MaxNumFiles = cuttlefish:conf_get("logger.max_files", Conf), -+ %% See https://www.erlang.org/doc/apps/kernel/logger_handler#t:olp_config/0 -+ LoggerConfig =#{type => standard_io}, - -- DefaultFormatStr = cuttlefish:conf_get("logger.format", Conf), -- DefaultFormatTerm = -- case erl_scan:string(DefaultFormatStr) of -- {ok, DefaultTokens, _} -> -- case erl_parse:parse_term(DefaultTokens) of -- {ok, DefaultTerm} -> -- DefaultTerm; -- {error, {_, _, DefaultError}} -> -- cuttlefish:error(foo) -- end; -- {error, {_, _, DefaultScanError}} -> -- cuttlefish:error(foo) -- end, -- ConfigMap0 = #{config => #{file => LogFile, -- max_no_bytes => MaxNumBytes, -- max_no_files => MaxNumFiles}, -- formatter => {logger_formatter, -- #{template => DefaultFormatTerm} -- } -- }, -+ FormatterConfig = #{ -+ log_level_map => #{emergency => 'ERROR', -+ alert => 'ERROR', -+ critical => 'ERROR', -+ error => 'ERROR', -+ warning => 'WARN', -+ notice => 'INFO', -+ info => 'INFO', -+ debug => 'DEBUG'} -+ }, - -+ %% See https://www.erlang.org/doc/apps/kernel/logger_std_h.html -+ ConfigMap0 = #{config => LoggerConfig, -+ %% See https://github.com/valitydev/logger_logstash_formatter -+ formatter => {logger_logstash_formatter, FormatterConfig}}, -+ - [{handler, default, logger_std_h, ConfigMap0}] -- - end - }. - ++ {default, [ ++ "{\"time\": \"", time, "\", ", ++ "\"level\": \"", level, "\", ", ++ "\"pid\": \"", pid, "\", ", ++ "\"mfa\": \"", mfa, "\", ", ++ "\"line\": ", line, ", ", ++ "\"msg\": \"", msg, "\"}\n" ++ ]} + ]}. + + %% @doc Filename to use for log files. From d3af941244ce828a3646f67370ff745ca47d0846 Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Fri, 28 Jun 2024 13:04:26 +0200 Subject: [PATCH 11/15] fix --- files/riak.schema.patch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/files/riak.schema.patch b/files/riak.schema.patch index 9d02275..2c13887 100644 --- a/files/riak.schema.patch +++ b/files/riak.schema.patch @@ -1,5 +1,5 @@ ---- riak.schema.origin 2024-06-28 12:50:20 -+++ riak.schema 2024-06-28 12:50:20 ++++ riak.schema.origin 2024-06-28 12:50:20 +--- riak.schema 2024-06-28 12:50:20 @@ -11,7 +11,14 @@ %% and atoms. The atoms denote the keys for retrieving metadata from the logger events. %% More information on default metadata can be found here (https://www.erlang.org/doc/man/logger_formatter.html#type-template). From e5c84c5a508c3153bc8aa59ee10fa499c4c5e448 Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Fri, 28 Jun 2024 13:19:28 +0200 Subject: [PATCH 12/15] fix patch --- files/riak.schema.patch | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/files/riak.schema.patch b/files/riak.schema.patch index 2c13887..1abcf02 100644 --- a/files/riak.schema.patch +++ b/files/riak.schema.patch @@ -1,5 +1,5 @@ -+++ riak.schema.origin 2024-06-28 12:50:20 ---- riak.schema 2024-06-28 12:50:20 +--- priv/riak.schema.origin 2024-06-28 13:14:00 ++++ priv/riak.schema 2024-06-28 13:14:25 @@ -11,7 +11,14 @@ %% and atoms. The atoms denote the keys for retrieving metadata from the logger events. %% More information on default metadata can be found here (https://www.erlang.org/doc/man/logger_formatter.html#type-template). @@ -15,4 +15,4 @@ + ]} ]}. - %% @doc Filename to use for log files. + %% @doc Filename to use for log files. \ No newline at end of file From 90c5951255481159dea6e82abe5614432deef954 Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Fri, 28 Jun 2024 13:27:09 +0200 Subject: [PATCH 13/15] fix patch --- files/install-riak.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/files/install-riak.sh b/files/install-riak.sh index d78b6af..99ab424 100755 --- a/files/install-riak.sh +++ b/files/install-riak.sh @@ -3,11 +3,11 @@ set -e eu # Build riak mkdir -p /opt/riak && cd /opt/riak -curl -L https://github.com/nhs-riak/riak/archive/refs/tags/riak-${RIAK_VERSION}-nhse.tar.gz -o /opt/riak.tar.gz +curl -L https://github.com/nhs-riak/riak/archive/refs/tags/riak-${RIAK_VERSION}-nhse.tar.gz -o /opt/riak.tar.gz tar xvf /opt/riak.tar.gz --strip-components 1 patch -p0 < /riak.schema.patch -patch < /rebar.config.patch -patch < /rebar.lock.patch +# patch < /rebar.config.patch +# patch < /rebar.lock.patch make all ./rebar3 as deb release --overlay_vars /vars.config ls -la /opt/riak/_build/deb/rel/riak From d5fc201678a4510b19891b1ca602ee6156c6327a Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Fri, 28 Jun 2024 13:48:44 +0200 Subject: [PATCH 14/15] just check --- files/install-riak.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/install-riak.sh b/files/install-riak.sh index 99ab424..dcfc12c 100755 --- a/files/install-riak.sh +++ b/files/install-riak.sh @@ -5,7 +5,7 @@ set -e eu mkdir -p /opt/riak && cd /opt/riak curl -L https://github.com/nhs-riak/riak/archive/refs/tags/riak-${RIAK_VERSION}-nhse.tar.gz -o /opt/riak.tar.gz tar xvf /opt/riak.tar.gz --strip-components 1 -patch -p0 < /riak.schema.patch +# patch -p0 < /riak.schema.patch # patch < /rebar.config.patch # patch < /rebar.lock.patch make all From a03e5cd9e7f2d59dda0fbd9b5d2cb5c3b3dcd03c Mon Sep 17 00:00:00 2001 From: Ian Bodrievskii Date: Fri, 28 Jun 2024 14:58:03 +0200 Subject: [PATCH 15/15] pam pam --- files/riak.schema.patch | 57 ++++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/files/riak.schema.patch b/files/riak.schema.patch index 1abcf02..60bf1bb 100644 --- a/files/riak.schema.patch +++ b/files/riak.schema.patch @@ -1,18 +1,51 @@ ---- priv/riak.schema.origin 2024-06-28 13:14:00 -+++ priv/riak.schema 2024-06-28 13:14:25 -@@ -11,7 +11,14 @@ +--- priv/riak.schema.origin 2024-06-28 14:44:19 ++++ priv/riak.schema 2024-06-28 14:54:04 +@@ -11,9 +11,11 @@ %% and atoms. The atoms denote the keys for retrieving metadata from the logger events. %% More information on default metadata can be found here (https://www.erlang.org/doc/man/logger_formatter.html#type-template). {mapping, "logger.format", "kernel.logger", [ - {default, "[time,\" [\",level,\"] \",pid,\"@\",mfa,\":\",line,\" \",msg,\"\\n\"]."} -+ {default, [ -+ "{\"time\": \"", time, "\", ", -+ "\"level\": \"", level, "\", ", -+ "\"pid\": \"", pid, "\", ", -+ "\"mfa\": \"", mfa, "\", ", -+ "\"line\": ", line, ", ", -+ "\"msg\": \"", msg, "\"}\n" -+ ]} ++ {default, "{\"time\": \"~s\", \"level\": \"~s\", \"pid\": \"~p\", \"mfa\": \"~p\", \"line\": \"~p\", \"msg\": \"~s\"}"}, ++ {datatype, string} ]}. - %% @doc Filename to use for log files. \ No newline at end of file ++ + %% @doc Filename to use for log files. + {mapping, "logger.file", "kernel.logger", [ + {default, "$(platform_log_dir)/console.log"}, +\ No newline at end of file +@@ -42,31 +44,19 @@ + MaxNumFiles = cuttlefish:conf_get("logger.max_files", Conf), + + DefaultFormatStr = cuttlefish:conf_get("logger.format", Conf), +- DefaultFormatTerm = +- case erl_scan:string(DefaultFormatStr) of +- {ok, DefaultTokens, _} -> +- case erl_parse:parse_term(DefaultTokens) of +- {ok, DefaultTerm} -> +- DefaultTerm; +- {error, {_, _, DefaultError}} -> +- cuttlefish:error(foo) +- end; +- {error, {_, _, DefaultScanError}} -> +- cuttlefish:error(foo) +- end, + ConfigMap0 = #{config => #{file => LogFile, + max_no_bytes => MaxNumBytes, + max_no_files => MaxNumFiles}, + formatter => {logger_formatter, +- #{template => DefaultFormatTerm} ++ #{template => DefaultFormatStr} + } + }, + + [{handler, default, logger_std_h, ConfigMap0}] +- + end + }. + ++ + %% @doc Cookie for distributed node communication. All nodes in the + %% same cluster should use the same cookie or they will not be able to + %% communicate. +\ No newline at end of file