From c94fb2f7b639305e4acd600973ae87048b1ca1ce Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Tue, 13 Jun 2017 13:29:51 +0200 Subject: [PATCH 1/7] First stab at #808: accept unwrapped config parts, replace ml-config.xml with ml-config/ --- deploy/lib/server_config.rb | 21 +- deploy/lib/xquery/setup.xqy | 122 +++++++-- deploy/sample/build.sample.properties | 2 +- .../ml-config.sample/databases/content-db.xml | 231 ++++++++++++++++++ .../ml-config.sample/databases/modules-db.xml | 32 +++ .../databases/rest-modules-db.xml | 1 + .../ml-config.sample/databases/schemas-db.xml | 1 + .../databases/test-content-db.xml | 1 + .../databases/test-modules-db.xml | 2 + .../databases/triggers-db.xml | 2 + .../ml-config.sample/forests/forests.xml | 17 ++ .../ml-config.sample/groups/default-group.xml | 13 + .../ml-config.sample/hosts/bootstrap-host.xml | 4 + .../security/amps/sample-amp.xml | 12 + .../security/certificates/ssl-certificate.xml | 6 + .../ml-config.sample/security/credentials.xml | 4 + .../external-securities/sample-security.xml | 15 ++ .../security/mimetypes/sample-mimetype.xml | 8 + .../security/privileges/sample-exec-priv.xml | 8 + .../security/privileges/sample-uri-priv.xml | 8 + .../security/roles/app-role.xml | 51 ++++ .../security/roles/unit-test-role.xml | 44 ++++ .../security/users/default-user.xml | 10 + .../security/users/test-user.xml | 1 + .../ml-config.sample/servers/app-server.xml | 21 ++ .../ml-config.sample/servers/odbc-server.xml | 1 + .../ml-config.sample/servers/rest-server.xml | 1 + .../ml-config.sample/servers/test-server.xml | 1 + .../ml-config.sample/servers/xdbc-xerver.xml | 1 + .../sample/ml-config.sample/tasks/tasks.xml | 92 +++++++ 30 files changed, 710 insertions(+), 23 deletions(-) create mode 100644 deploy/sample/ml-config.sample/databases/content-db.xml create mode 100644 deploy/sample/ml-config.sample/databases/modules-db.xml create mode 100644 deploy/sample/ml-config.sample/databases/rest-modules-db.xml create mode 100644 deploy/sample/ml-config.sample/databases/schemas-db.xml create mode 100644 deploy/sample/ml-config.sample/databases/test-content-db.xml create mode 100644 deploy/sample/ml-config.sample/databases/test-modules-db.xml create mode 100644 deploy/sample/ml-config.sample/databases/triggers-db.xml create mode 100644 deploy/sample/ml-config.sample/forests/forests.xml create mode 100644 deploy/sample/ml-config.sample/groups/default-group.xml create mode 100644 deploy/sample/ml-config.sample/hosts/bootstrap-host.xml create mode 100644 deploy/sample/ml-config.sample/security/amps/sample-amp.xml create mode 100644 deploy/sample/ml-config.sample/security/certificates/ssl-certificate.xml create mode 100644 deploy/sample/ml-config.sample/security/credentials.xml create mode 100644 deploy/sample/ml-config.sample/security/external-securities/sample-security.xml create mode 100644 deploy/sample/ml-config.sample/security/mimetypes/sample-mimetype.xml create mode 100644 deploy/sample/ml-config.sample/security/privileges/sample-exec-priv.xml create mode 100644 deploy/sample/ml-config.sample/security/privileges/sample-uri-priv.xml create mode 100644 deploy/sample/ml-config.sample/security/roles/app-role.xml create mode 100644 deploy/sample/ml-config.sample/security/roles/unit-test-role.xml create mode 100644 deploy/sample/ml-config.sample/security/users/default-user.xml create mode 100644 deploy/sample/ml-config.sample/security/users/test-user.xml create mode 100644 deploy/sample/ml-config.sample/servers/app-server.xml create mode 100644 deploy/sample/ml-config.sample/servers/odbc-server.xml create mode 100644 deploy/sample/ml-config.sample/servers/rest-server.xml create mode 100644 deploy/sample/ml-config.sample/servers/test-server.xml create mode 100644 deploy/sample/ml-config.sample/servers/xdbc-xerver.xml create mode 100644 deploy/sample/ml-config.sample/tasks/tasks.xml diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index 02dc13cd..eb8bf2eb 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -324,8 +324,8 @@ def self.init if !force && !force_config && File.exists?(target_config) error_msg << "ml-config.xml has already been created." else - #create clean marklogic configuration file - copy_file sample_config, target_config + #create clean marklogic configuration file(s) + FileUtils.cp_r sample_config, target_config end raise HelpException.new("init", error_msg.join("\n")) if error_msg.length > 0 @@ -2977,9 +2977,22 @@ def ssl_certificate_xml } end - def build_config(config_files) + def build_config(config_paths) + config_files = [] + config_paths.split(",").each do |config_path| + if File.directory?(config_path) + Dir.glob(File.join(config_path, '**', '*')).reject { + |p| File.directory? p + }.each do |file| + config_files.push file + end + else + config_files.push config_path + end + end + configs = [] - config_files.split(",").each do |config_file| + config_files.each do |config_file| config = File.read(config_file) # Build the triggers db if it is provided diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 5f955d5e..b57476f6 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -571,6 +571,67 @@ declare function setup:unique-attributes($attrs) { return map:keys($result) ! map:get($result, .) }; +declare function setup:wrap-config-fragments($fragments) { + if (fn:exists($fragments)) then + + { + setup:unique-attributes($fragments/self::ho:hosts/@*), + $fragments/self::ho:hosts/*, + $fragments/self::ho:host + } + { + setup:unique-attributes($fragments/self::as:assignments/@*), + $fragments/self::as:assignments/*, + $fragments/self::as:assignment + } + { + setup:unique-attributes($fragments/self::db:databases/@*), + $fragments/self::db:databases/*, + $fragments/self::db:database + } + { + setup:unique-attributes($fragments/self::pki:certificates/@*), + $fragments/self::pki:certificates/*, + $fragments/self::pki:certificate + } + { + setup:unique-attributes($fragments/self::sec:roles/@*), + $fragments/self::sec:roles/*, + $fragments/self::sec:role + } + { + setup:unique-attributes($fragments/self::sec:users/@*), + $fragments/self::sec:users/*, + $fragments/self::sec:user + } + { + setup:unique-attributes($fragments/self::sec:amps/@*), + $fragments/self::sec:amps/*, + $fragments/self::sec:amp + } + { + setup:unique-attributes($fragments/self::sec:privileges/@*), + $fragments/self::sec:privileges/*, + $fragments/self::sec:privilege + } + { + setup:unique-attributes($fragments/self::mt:mimetypes/@*), + $fragments/self::mt:mimetypes/*, + $fragments/self::mt:mimetype + } + { + setup:unique-attributes($fragments/self::sec:external-securities/@*), + $fragments/self::sec:external-securities/*, + $fragments/self::sec:external-security + } + { + setup:unique-attributes($fragments/self::sec:credentials/@*), + $fragments/self::sec:credentials/* + } + /* + else () +}; + (: for backwards-compatibility :) declare function setup:rewrite-config($import-configs as node()+, $properties as map:map) as element(configuration) { @@ -586,28 +647,43 @@ declare function setup:rewrite-config($import-configs as node()+, $properties as { let $import-configs := setup:process-conditionals($import-configs, $properties) let $config := - element { fn:node-name($import-configs[1]) } { - setup:unique-attributes($import-configs/@*), + element configuration { + setup:unique-attributes($import-configs/self::configuration/@*), (: capture comments before gr:groups, and its older counterparts :) - $import-configs/( + $import-configs/self::configuration/( gr:groups, gr:http-servers, gr:xdbc-servers, gr:odbc-servers, gr:task-server )/preceding-sibling::node(), { setup:unique-attributes($import-configs/gr:groups/@*), - let $default-group := ($import-configs/@default-group, "Default")[1] - for $group in fn:distinct-values( - ($import-configs/gr:groups/gr:group/gr:group-name, $import-configs/(gr:http-servers/gr:http-server, gr:xdbc-servers/gr:xdbc-server, - gr:odbc-servers/gr:odbc-server, gr:task-server, db:databases/db:database)/@group, $default-group)) - let $http-servers := $import-configs/gr:http-servers/gr:http-server[@group = $group or ($group = $default-group and fn:empty(@group))] - let $xdbc-servers := $import-configs/gr:xdbc-servers/gr:xdbc-server[@group = $group or ($group = $default-group and fn:empty(@group))] - let $odbc-servers := $import-configs/gr:odbc-servers/gr:odbc-server[@group = $group or ($group = $default-group and fn:empty(@group))] - let $task-server := $import-configs/gr:task-server[@group = $group or ($group = $default-group and fn:empty(@group))] + let $default-group := ($import-configs/self::*:configuration/@default-group, "Default")[1] + for $group in fn:distinct-values(( + $import-configs/descendant-or-self::gr:group/gr:group-name, + $import-configs/descendant-or-self::*/( + self::gr:http-server, self::gr:xdbc-server, + self::gr:odbc-server, self::gr:task-server, self::db:database + )/@group, + $default-group + )) + let $http-servers := $import-configs/descendant-or-self::gr:http-server[ + @group = $group or ( $group = $default-group and fn:empty(@group) ) + ] + let $xdbc-servers := $import-configs/descendant-or-self::gr:xdbc-server[ + @group = $group or ( $group = $default-group and fn:empty(@group) ) + ] + let $odbc-servers := $import-configs/descendant-or-self::gr:odbc-server[ + @group = $group or ( $group = $default-group and fn:empty(@group) ) + ] + let $task-server := $import-configs/descendant-or-self::gr:task-server[ + @group = $group or ( $group = $default-group and fn:empty(@group) ) + ] let $servers := ($http-servers, $xdbc-servers, $odbc-servers, $task-server) - let $databases := $import-configs/db:databases/db:database[@group = $group or ($group = $default-group and fn:empty(@group))] - let $group-config := $import-configs/gr:groups/gr:group[gr:group-name = $group] + let $databases := $import-configs/descendant-or-self::db:database[ + @group = $group or ( $group = $default-group and fn:empty(@group) ) + ] + let $group-config := $import-configs/descendant-or-self::gr:group[gr:group-name = $group] where fn:exists($servers | $databases | $group-config) return @@ -632,14 +708,24 @@ declare function setup:rewrite-config($import-configs as node()+, $properties as }, (: capture anything following gr:groups, and its older counterparts :) - $import-configs/( + $import-configs/self::*:configuration/( gr:groups, gr:http-servers, gr:xdbc-servers, gr:odbc-servers, gr:task-server )/following-sibling::node(), - (: other fragments with configuration as root :) - $import-configs[fn:empty(( - gr:groups, gr:http-servers, gr:xdbc-servers, gr:odbc-servers, gr:task-server - ))]/node() + (: in case of config fragments, merge and wrap them :) + setup:wrap-config-fragments(( + (: fragments with configuration as root :) + $import-configs/self::*:configuration[fn:empty(( + gr:groups, gr:http-servers, gr:xdbc-servers, gr:odbc-servers, gr:task-server + ))]/node(), + + (: other fragments :) + $import-configs[fn:not(self::*:configuration)]/(self::node() except ( + self::gr:groups, self::gr:group, self::gr:http-servers, self::gr:http-server, + self::gr:xdbc-servers, self::gr:xdbc-server, self::gr:odbc-servers, self::gr:odbc-server, + self::gr:task-server + )) + )) } (: Check config on group consistency! :) diff --git a/deploy/sample/build.sample.properties b/deploy/sample/build.sample.properties index 01803a47..0bbd7d04 100644 --- a/deploy/sample/build.sample.properties +++ b/deploy/sample/build.sample.properties @@ -31,7 +31,7 @@ modules-prefix=/ # # the location of your marklogic configuration file # -# config.file=${basedir}/deploy/ml-config.xml +config.file=${basedir}/deploy/ml-config/ # # the location of your triggers configuration file diff --git a/deploy/sample/ml-config.sample/databases/content-db.xml b/deploy/sample/ml-config.sample/databases/content-db.xml new file mode 100644 index 00000000..adeec9ce --- /dev/null +++ b/deploy/sample/ml-config.sample/databases/content-db.xml @@ -0,0 +1,231 @@ + + + ${content-db} + ${content-forests-per-host} + @ml.schemas-mapping + @ml.triggers-mapping + + + @ml.forest-data-dir-xml + + + + + true + true + manual + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/deploy/sample/ml-config.sample/databases/modules-db.xml b/deploy/sample/ml-config.sample/databases/modules-db.xml new file mode 100644 index 00000000..0a0e55c7 --- /dev/null +++ b/deploy/sample/ml-config.sample/databases/modules-db.xml @@ -0,0 +1,32 @@ + + + ${modules-db} + + + + off + false + false + false + false + false + false + false + false + false + false + false + false + false + false + false + false + false + + false + false + true + false + automatic + false + diff --git a/deploy/sample/ml-config.sample/databases/rest-modules-db.xml b/deploy/sample/ml-config.sample/databases/rest-modules-db.xml new file mode 100644 index 00000000..12a0c513 --- /dev/null +++ b/deploy/sample/ml-config.sample/databases/rest-modules-db.xml @@ -0,0 +1 @@ + @ml.rest-modules-db-xml diff --git a/deploy/sample/ml-config.sample/databases/schemas-db.xml b/deploy/sample/ml-config.sample/databases/schemas-db.xml new file mode 100644 index 00000000..d7b802ab --- /dev/null +++ b/deploy/sample/ml-config.sample/databases/schemas-db.xml @@ -0,0 +1 @@ + @ml.schemas-db-xml diff --git a/deploy/sample/ml-config.sample/databases/test-content-db.xml b/deploy/sample/ml-config.sample/databases/test-content-db.xml new file mode 100644 index 00000000..383c8688 --- /dev/null +++ b/deploy/sample/ml-config.sample/databases/test-content-db.xml @@ -0,0 +1 @@ + @ml.test-content-db-xml diff --git a/deploy/sample/ml-config.sample/databases/test-modules-db.xml b/deploy/sample/ml-config.sample/databases/test-modules-db.xml new file mode 100644 index 00000000..71f032b5 --- /dev/null +++ b/deploy/sample/ml-config.sample/databases/test-modules-db.xml @@ -0,0 +1,2 @@ + + @ml.test-modules-db-xml diff --git a/deploy/sample/ml-config.sample/databases/triggers-db.xml b/deploy/sample/ml-config.sample/databases/triggers-db.xml new file mode 100644 index 00000000..d3d13571 --- /dev/null +++ b/deploy/sample/ml-config.sample/databases/triggers-db.xml @@ -0,0 +1,2 @@ + + @ml.triggers-db-xml diff --git a/deploy/sample/ml-config.sample/forests/forests.xml b/deploy/sample/ml-config.sample/forests/forests.xml new file mode 100644 index 00000000..e5c80d87 --- /dev/null +++ b/deploy/sample/ml-config.sample/forests/forests.xml @@ -0,0 +1,17 @@ + + + + + ${content-db} + @ml.forest-data-dir-xml + + @ml.test-content-db-assignment + @ml.test-modules-db-assignment + @ml.rest-modules-db-assignment + + ${modules-db} + + @ml.schemas-assignment + @ml.triggers-assignment + + diff --git a/deploy/sample/ml-config.sample/groups/default-group.xml b/deploy/sample/ml-config.sample/groups/default-group.xml new file mode 100644 index 00000000..df2633ec --- /dev/null +++ b/deploy/sample/ml-config.sample/groups/default-group.xml @@ -0,0 +1,13 @@ + + + + + ${group} + + + diff --git a/deploy/sample/ml-config.sample/hosts/bootstrap-host.xml b/deploy/sample/ml-config.sample/hosts/bootstrap-host.xml new file mode 100644 index 00000000..0af885fd --- /dev/null +++ b/deploy/sample/ml-config.sample/hosts/bootstrap-host.xml @@ -0,0 +1,4 @@ + + @ml.server-name + + diff --git a/deploy/sample/ml-config.sample/security/amps/sample-amp.xml b/deploy/sample/ml-config.sample/security/amps/sample-amp.xml new file mode 100644 index 00000000..18860b8a --- /dev/null +++ b/deploy/sample/ml-config.sample/security/amps/sample-amp.xml @@ -0,0 +1,12 @@ + + + http://marklogic.com/roxy + sample + /app/models/sample.xqy + ${modules-db} + a-privileged-role + diff --git a/deploy/sample/ml-config.sample/security/certificates/ssl-certificate.xml b/deploy/sample/ml-config.sample/security/certificates/ssl-certificate.xml new file mode 100644 index 00000000..be6e1a80 --- /dev/null +++ b/deploy/sample/ml-config.sample/security/certificates/ssl-certificate.xml @@ -0,0 +1,6 @@ + + + @ml.ssl-certificate-xml + diff --git a/deploy/sample/ml-config.sample/security/credentials.xml b/deploy/sample/ml-config.sample/security/credentials.xml new file mode 100644 index 00000000..012637dd --- /dev/null +++ b/deploy/sample/ml-config.sample/security/credentials.xml @@ -0,0 +1,4 @@ + + 1234abc + 1234abc + diff --git a/deploy/sample/ml-config.sample/security/external-securities/sample-security.xml b/deploy/sample/ml-config.sample/security/external-securities/sample-security.xml new file mode 100644 index 00000000..6f500f06 --- /dev/null +++ b/deploy/sample/ml-config.sample/security/external-securities/sample-security.xml @@ -0,0 +1,15 @@ + + + test + a big test + ldap + 300 + ldap + ldap://dc1.mltest1.local:389 + CN=Users,DC=MLTEST1,DC=LOCAL + sAMAccountName + + + diff --git a/deploy/sample/ml-config.sample/security/mimetypes/sample-mimetype.xml b/deploy/sample/ml-config.sample/security/mimetypes/sample-mimetype.xml new file mode 100644 index 00000000..9a16af51 --- /dev/null +++ b/deploy/sample/ml-config.sample/security/mimetypes/sample-mimetype.xml @@ -0,0 +1,8 @@ + + + application/crazy + crazy stuff + text + diff --git a/deploy/sample/ml-config.sample/security/privileges/sample-exec-priv.xml b/deploy/sample/ml-config.sample/security/privileges/sample-exec-priv.xml new file mode 100644 index 00000000..05797c7d --- /dev/null +++ b/deploy/sample/ml-config.sample/security/privileges/sample-exec-priv.xml @@ -0,0 +1,8 @@ + + + my-action + http://marklogic.com/custom/privilege/my-action + execute + diff --git a/deploy/sample/ml-config.sample/security/privileges/sample-uri-priv.xml b/deploy/sample/ml-config.sample/security/privileges/sample-uri-priv.xml new file mode 100644 index 00000000..dbcdc275 --- /dev/null +++ b/deploy/sample/ml-config.sample/security/privileges/sample-uri-priv.xml @@ -0,0 +1,8 @@ + + + users-uri + /users/ + uri + diff --git a/deploy/sample/ml-config.sample/security/roles/app-role.xml b/deploy/sample/ml-config.sample/security/roles/app-role.xml new file mode 100644 index 00000000..69a277d7 --- /dev/null +++ b/deploy/sample/ml-config.sample/security/roles/app-role.xml @@ -0,0 +1,51 @@ + + ${app-role} + A role for users of the ${app-name} application + + + + + execute + ${app-role} + + + update + ${app-role} + + + insert + ${app-role} + + + read + ${app-role} + + + + + + + rest-reader + + + xdmp:value + + + xdmp:add-response-header + + + xdmp:invoke + + + xdmp:with-namespaces + + + + xdmp:get-server-field + + + xdmp:set-server-field + + + + diff --git a/deploy/sample/ml-config.sample/security/roles/unit-test-role.xml b/deploy/sample/ml-config.sample/security/roles/unit-test-role.xml new file mode 100644 index 00000000..0b25cd8c --- /dev/null +++ b/deploy/sample/ml-config.sample/security/roles/unit-test-role.xml @@ -0,0 +1,44 @@ + + ${app-role}-unit-test + A role for unit testing the ${app-name} application + + ${app-role} + + + + + xdmp:document-get + + + xdmp:filesystem-directory + + + xdmp:save + + + + any-uri + + + xdmp:eval + + + xdmp:eval-in + + + xdmp:http-delete + + + xdmp:http-get + + + xdmp:http-head + + + xdmp:http-post + + + xdmp:http-put + + + diff --git a/deploy/sample/ml-config.sample/security/users/default-user.xml b/deploy/sample/ml-config.sample/security/users/default-user.xml new file mode 100644 index 00000000..187a1038 --- /dev/null +++ b/deploy/sample/ml-config.sample/security/users/default-user.xml @@ -0,0 +1,10 @@ + + ${default-user} + A user for the ${app-name} application + ${appuser-password} + + ${app-role} + + + + diff --git a/deploy/sample/ml-config.sample/security/users/test-user.xml b/deploy/sample/ml-config.sample/security/users/test-user.xml new file mode 100644 index 00000000..5122d34a --- /dev/null +++ b/deploy/sample/ml-config.sample/security/users/test-user.xml @@ -0,0 +1 @@ +@ml.test-user-xml diff --git a/deploy/sample/ml-config.sample/servers/app-server.xml b/deploy/sample/ml-config.sample/servers/app-server.xml new file mode 100644 index 00000000..c821a582 --- /dev/null +++ b/deploy/sample/ml-config.sample/servers/app-server.xml @@ -0,0 +1,21 @@ + + + ${app-name} + ${app-port} + + + ${modules-root} + ${authentication-method} + + ${url-rewriter} + ${error-handler} + @ml.rewrite-resolves-globally + + + + diff --git a/deploy/sample/ml-config.sample/servers/odbc-server.xml b/deploy/sample/ml-config.sample/servers/odbc-server.xml new file mode 100644 index 00000000..2219255a --- /dev/null +++ b/deploy/sample/ml-config.sample/servers/odbc-server.xml @@ -0,0 +1 @@ +@ml.odbc-server diff --git a/deploy/sample/ml-config.sample/servers/rest-server.xml b/deploy/sample/ml-config.sample/servers/rest-server.xml new file mode 100644 index 00000000..5ef1f153 --- /dev/null +++ b/deploy/sample/ml-config.sample/servers/rest-server.xml @@ -0,0 +1 @@ +@ml.rest-appserver diff --git a/deploy/sample/ml-config.sample/servers/test-server.xml b/deploy/sample/ml-config.sample/servers/test-server.xml new file mode 100644 index 00000000..a85bf2f9 --- /dev/null +++ b/deploy/sample/ml-config.sample/servers/test-server.xml @@ -0,0 +1 @@ +@ml.test-appserver diff --git a/deploy/sample/ml-config.sample/servers/xdbc-xerver.xml b/deploy/sample/ml-config.sample/servers/xdbc-xerver.xml new file mode 100644 index 00000000..e97f0f1d --- /dev/null +++ b/deploy/sample/ml-config.sample/servers/xdbc-xerver.xml @@ -0,0 +1 @@ +@ml.xdbc-server diff --git a/deploy/sample/ml-config.sample/tasks/tasks.xml b/deploy/sample/ml-config.sample/tasks/tasks.xml new file mode 100644 index 00000000..76417a73 --- /dev/null +++ b/deploy/sample/ml-config.sample/tasks/tasks.xml @@ -0,0 +1,92 @@ + + + + ${group} + + + + + + /some/daily-task.xqy + / + daily + 2 + 13:00:00-05:00 + + + + + + /some/hourly-task.xqy + / + hourly + 2 + 15 + + + + + + /some/minutely-task.xqy + / + minutely + 3 + + + + + + /some/monthly-task.xqy + / + monthly + 1 + 15 + 13:00:00-05:00 + + + + + + /some/once-task.xqy + / + once + 1 + 2019-01-01T13:00:00-05:00 + + + + + + /some/weekly-task.xqy + / + weekly + 1 + + monday + wednesday + friday + + 13:00:00-05:00 + + + + + + + + + From 189785b0b72a1803703257df0652a9af9afa9a3f Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Tue, 13 Jun 2017 19:49:30 +0200 Subject: [PATCH 2/7] #808: a few small changes to make it actually work --- deploy/lib/server_config.rb | 32 +++++++++---------- .../security/amps/sample-amp.xml | 2 +- .../sample/ml-config.sample/tasks/tasks.xml | 13 ++++---- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index eb8bf2eb..cc0819be 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -2778,7 +2778,7 @@ def conditional_prop(prop, default_prop) def triggers_db_xml %Q{ - + @ml.triggers-db @@ -2789,7 +2789,7 @@ def triggers_db_xml def triggers_assignment %Q{ - + @ml.triggers-db } @@ -2798,7 +2798,7 @@ def triggers_assignment def xdbc_server xdbc_auth_method = conditional_prop('ml.xdbc-authentication-method', 'ml.authentication-method') %Q{ - + @ml.app-name-xcc @ml.xcc-port @@ -2811,7 +2811,7 @@ def xdbc_server def odbc_server odbc_auth_method = conditional_prop('ml.odbc-authentication-method', 'ml.authentication-method') %Q{ - + @ml.app-name-odbc @ml.odbc-port @@ -2823,7 +2823,7 @@ def odbc_server def schemas_db_xml %Q{ - + @ml.schemas-db @@ -2834,7 +2834,7 @@ def schemas_db_xml def schemas_assignment %Q{ - + @ml.schemas-db } @@ -2842,7 +2842,7 @@ def schemas_assignment def test_content_db_xml %Q{ - + @ml.test-content-db @@ -2853,7 +2853,7 @@ def test_content_db_xml def test_content_db_assignment %Q{ - + @ml.test-content-db } @@ -2866,7 +2866,7 @@ def test_appserver test_default_user = conditional_prop('ml.test-default-user', 'ml.default-user') %Q{ - + @ml.app-name-test @ml.test-port @@ -2879,7 +2879,7 @@ def test_appserver def test_modules_db_xml %Q{ - + @ml.test-modules-db @@ -2890,7 +2890,7 @@ def test_modules_db_xml def test_user_xml %Q{ - + ${test-user} A user for the ${app-name} unit tests ${test-user-password} @@ -2905,7 +2905,7 @@ def test_user_xml def test_modules_db_assignment %Q{ - + @ml.test-modules-db } @@ -2926,7 +2926,7 @@ def rest_appserver end %Q{ - + @ml.app-name-rest @ml.rest-port @@ -2944,7 +2944,7 @@ def rest_modules_db_xml rest_modules_db = conditional_prop('ml.rest-modules-db', 'ml.modules-db') %Q{ - + #{rest_modules_db} @@ -2957,7 +2957,7 @@ def rest_modules_db_assignment rest_modules_db = conditional_prop('ml.rest-modules-db', 'ml.modules-db') %Q{ - + #{rest_modules_db} } @@ -2965,7 +2965,7 @@ def rest_modules_db_assignment def ssl_certificate_xml %Q{ - + @ml.ssl-certificate-template @ml.ssl-certificate-countryName @ml.ssl-certificate-stateOrProvinceName diff --git a/deploy/sample/ml-config.sample/security/amps/sample-amp.xml b/deploy/sample/ml-config.sample/security/amps/sample-amp.xml index 18860b8a..14eb42b5 100644 --- a/deploy/sample/ml-config.sample/security/amps/sample-amp.xml +++ b/deploy/sample/ml-config.sample/security/amps/sample-amp.xml @@ -8,5 +8,5 @@ sample /app/models/sample.xqy ${modules-db} - a-privileged-role + ${app-role} diff --git a/deploy/sample/ml-config.sample/tasks/tasks.xml b/deploy/sample/ml-config.sample/tasks/tasks.xml index 76417a73..a561bdc3 100644 --- a/deploy/sample/ml-config.sample/tasks/tasks.xml +++ b/deploy/sample/ml-config.sample/tasks/tasks.xml @@ -28,7 +28,7 @@ 13:00:00-05:00 - + /some/hourly-task.xqy @@ -38,7 +38,7 @@ 15 - + /some/minutely-task.xqy @@ -47,7 +47,7 @@ 3 - + /some/monthly-task.xqy @@ -58,17 +58,16 @@ 13:00:00-05:00 - + /some/once-task.xqy / once - 1 2019-01-01T13:00:00-05:00 - + /some/weekly-task.xqy @@ -83,7 +82,7 @@ 13:00:00-05:00 - + From d6c64d0ab860cab6bcc8d1b6054b967af3bf0521 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 15 Jun 2017 16:51:15 +0200 Subject: [PATCH 3/7] #808: added initial stab at export_config to split existing ml-config.xml into parts --- deploy/lib/server_config.rb | 54 +++++++++++++++++-- deploy/lib/xquery/setup.xqy | 43 +++++++++++++-- .../{odbc-server.xml => app-name-odbc.xml} | 0 .../{rest-server.xml => app-name-rest.xml} | 0 .../{xdbc-xerver.xml => app-name-xcc.xml} | 0 .../servers/{app-server.xml => app-name.xml} | 0 6 files changed, 90 insertions(+), 7 deletions(-) rename deploy/sample/ml-config.sample/servers/{odbc-server.xml => app-name-odbc.xml} (100%) rename deploy/sample/ml-config.sample/servers/{rest-server.xml => app-name-rest.xml} (100%) rename deploy/sample/ml-config.sample/servers/{xdbc-xerver.xml => app-name-xcc.xml} (100%) rename deploy/sample/ml-config.sample/servers/{app-server.xml => app-name.xml} (100%) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index cc0819be..a9bbf653 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -1220,6 +1220,50 @@ def deploy return true end + def export_config + setup = File.read ServerConfig.expand_path("#{@@path}/lib/xquery/setup.xqy") + query = %Q{ + #{setup} + try { + for $part in setup:split-config( + setup:rewrite-config(#{get_config(true)}, (), fn:true()), + "#{@properties["ml.app-name"]}" + ) + return + xdmp:quote( + $part, + + yes + yes + + ) + } catch($ex) { + xdmp:log($ex), + fn:concat($ex/err:format-string/text(), ' See MarkLogic Server error log for more details.') + } + } + logger.debug query + r = execute_query query + logger.debug "code: #{r.code.to_i}" + + r.body = parse_body(r.body) + logger.info r.body + return true + end + + def export + what = ARGV.shift + raise HelpException.new("export", "Missing WHAT") unless what + + case what + when 'config' + export_config + else + raise HelpException.new("export", "Invalid WHAT") + end + return true + end + def load dir = ARGV.shift db = find_arg(['--db']) || @properties['ml.content-db'] @@ -2457,7 +2501,7 @@ def mlRest end end - def get_config + def get_config(preserve_props = false) if @server_version > 7 && @properties["ml.app-type"] == 'rest' && @properties["ml.url-rewriter"] == "/MarkLogic/rest-api/rewriter.xqy" @logger.info "WARN: XQuery REST rewriter has been deprecated since MarkLogic 8" @properties["ml.url-rewriter"] = "/MarkLogic/rest-api/rewriter.xml" @@ -2473,7 +2517,7 @@ def get_config @logger.info " See https://github.com/marklogic/roxy/issues/416 for details." end - @config ||= build_config(@options[:config_file]) + @config ||= build_config(@options[:config_file], preserve_props) end def execute_query_4(query, properties) @@ -2977,7 +3021,7 @@ def ssl_certificate_xml } end - def build_config(config_paths) + def build_config(config_paths, preserve_props = false) config_files = [] config_paths.split(",").each do |config_path| if File.directory?(config_path) @@ -3135,7 +3179,9 @@ def build_config(config_paths) config.gsub!("@ml.ssl-certificate-xml", "") end - replace_properties(config, File.basename(config_file), true) + if ! preserve_props + replace_properties(config, File.basename(config_file), true) + end # escape unresolved braces, they have special meaning in XQuery config.gsub!("{", "{{") diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index b57476f6..812ab78e 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -660,7 +660,7 @@ declare function setup:rewrite-config($import-configs as node()+, $properties as let $default-group := ($import-configs/self::*:configuration/@default-group, "Default")[1] for $group in fn:distinct-values(( - $import-configs/descendant-or-self::gr:group/gr:group-name, + $import-configs/descendant-or-self::gr:group/gr:group-name/fn:string(), $import-configs/descendant-or-self::*/( self::gr:http-server, self::gr:xdbc-server, self::gr:odbc-server, self::gr:task-server, self::db:database @@ -683,7 +683,7 @@ declare function setup:rewrite-config($import-configs as node()+, $properties as let $databases := $import-configs/descendant-or-self::db:database[ @group = $group or ( $group = $default-group and fn:empty(@group) ) ] - let $group-config := $import-configs/descendant-or-self::gr:group[gr:group-name = $group] + let $group-config := $import-configs/descendant-or-self::gr:group[gr:group-name/fn:string() = $group] where fn:exists($servers | $databases | $group-config) return @@ -732,7 +732,7 @@ declare function setup:rewrite-config($import-configs as node()+, $properties as let $_ := if ($silent) then () else - for $group in $config/gr:groups/gr:group/gr:group-name + for $group in $config/gr:groups/gr:group/gr:group-name/fn:string() let $hosts := ($config/ho:hosts/ho:host[ho:group/@name = $group], try { xdmp:group-hosts(xdmp:group($group)) } catch ($ignore) {}) where fn:empty($hosts) return @@ -745,6 +745,43 @@ declare function setup:rewrite-config($import-configs as node()+, $properties as return if ($keep-comments) then $config else setup:suppress-comments($config) }; +declare function setup:split-config($config as element(configuration), $app-name as xs:string) as node()* { + for $part in ( + $config/*/*, + $config/gr:groups/gr:group/( + (gr:http-servers, gr:xdbc-servers, gr:odbc-servers)/*, + gr:task-server + ) + ) + let $type := fn:local-name($part) + + let $name := + if ($part instance of element(gr:task-server)) then + "TaskServer" + else + $part/*[local-name() = ("name", "forest-name", "local-name", concat($type, "-name"))][1] + /fn:replace(fn:replace(fn:string(), "^(.*/)?([^/]+)", "$2"), "^\$\{group\}$", "default-group") + + let $path := fn:replace(fn:replace($type, "(http|xdbc|odbc|task)-", "") || "s", "ys$", "ies") + let $path := + if (fn:contains(fn:namespace-uri($part), "security")) then + "security/" || $path + else + fn:replace($path, "assignments/", "forests/") + + let $file := fn:replace($name, "(@ml.|[${}])", "") || ".xml" + + return ( + comment { "SAVE-PART-AS:" || $path || "/" || $file }, + typeswitch ($part) + case element(gr:group) + return element gr:group { + $part/@*, + $part/(node() except (gr:http-servers, gr:xdbc-servers, gr:odbc-servers, gr:task-server)) + } + default return $part + ) +}; (: base-name : Original forest base name - this should be the name from the config. diff --git a/deploy/sample/ml-config.sample/servers/odbc-server.xml b/deploy/sample/ml-config.sample/servers/app-name-odbc.xml similarity index 100% rename from deploy/sample/ml-config.sample/servers/odbc-server.xml rename to deploy/sample/ml-config.sample/servers/app-name-odbc.xml diff --git a/deploy/sample/ml-config.sample/servers/rest-server.xml b/deploy/sample/ml-config.sample/servers/app-name-rest.xml similarity index 100% rename from deploy/sample/ml-config.sample/servers/rest-server.xml rename to deploy/sample/ml-config.sample/servers/app-name-rest.xml diff --git a/deploy/sample/ml-config.sample/servers/xdbc-xerver.xml b/deploy/sample/ml-config.sample/servers/app-name-xcc.xml similarity index 100% rename from deploy/sample/ml-config.sample/servers/xdbc-xerver.xml rename to deploy/sample/ml-config.sample/servers/app-name-xcc.xml diff --git a/deploy/sample/ml-config.sample/servers/app-server.xml b/deploy/sample/ml-config.sample/servers/app-name.xml similarity index 100% rename from deploy/sample/ml-config.sample/servers/app-server.xml rename to deploy/sample/ml-config.sample/servers/app-name.xml From 3fd8f0539c629ba84b228cbb400dc8560f84e48d Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Fri, 16 Jun 2017 17:10:29 +0200 Subject: [PATCH 4/7] #808: extended upon #692 to be able to expose decision logic within ml-config --- deploy/lib/server_config.rb | 224 +++++++++++++++++++----------------- deploy/lib/xquery/setup.xqy | 159 +++++++++++++++---------- 2 files changed, 215 insertions(+), 168 deletions(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index a9bbf653..e4b14281 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -786,11 +786,12 @@ def properties_map def config setup = File.read ServerConfig.expand_path("#{@@path}/lib/xquery/setup.xqy") + unresolved = find_arg(['--unresolved']).present? query = %Q{ #{setup} try { xdmp:quote( - setup:rewrite-config(#{get_config}, #{properties_map}, (), fn:true()), + setup:rewrite-config(#{get_config(unresolved)}, #{properties_map}, (), fn:#{unresolved}()), yes yes @@ -2822,7 +2823,7 @@ def conditional_prop(prop, default_prop) def triggers_db_xml %Q{ - + @ml.triggers-db @@ -2833,7 +2834,7 @@ def triggers_db_xml def triggers_assignment %Q{ - + @ml.triggers-db } @@ -2842,7 +2843,7 @@ def triggers_assignment def xdbc_server xdbc_auth_method = conditional_prop('ml.xdbc-authentication-method', 'ml.authentication-method') %Q{ - + @ml.app-name-xcc @ml.xcc-port @@ -2855,7 +2856,7 @@ def xdbc_server def odbc_server odbc_auth_method = conditional_prop('ml.odbc-authentication-method', 'ml.authentication-method') %Q{ - + @ml.app-name-odbc @ml.odbc-port @@ -2867,7 +2868,7 @@ def odbc_server def schemas_db_xml %Q{ - + @ml.schemas-db @@ -2878,7 +2879,7 @@ def schemas_db_xml def schemas_assignment %Q{ - + @ml.schemas-db } @@ -2886,7 +2887,7 @@ def schemas_assignment def test_content_db_xml %Q{ - + @ml.test-content-db @@ -2897,7 +2898,7 @@ def test_content_db_xml def test_content_db_assignment %Q{ - + @ml.test-content-db } @@ -2910,7 +2911,7 @@ def test_appserver test_default_user = conditional_prop('ml.test-default-user', 'ml.default-user') %Q{ - + @ml.app-name-test @ml.test-port @@ -2923,7 +2924,7 @@ def test_appserver def test_modules_db_xml %Q{ - + @ml.test-modules-db @@ -2932,9 +2933,17 @@ def test_modules_db_xml } end + def test_modules_db_assignment + %Q{ + + @ml.test-modules-db + + } + end + def test_user_xml %Q{ - + ${test-user} A user for the ${app-name} unit tests ${test-user-password} @@ -2947,14 +2956,6 @@ def test_user_xml } end - def test_modules_db_assignment - %Q{ - - @ml.test-modules-db - - } - end - def rest_appserver rest_modules_db = conditional_prop('ml.rest-modules-db', 'ml.modules-db') rest_auth_method = conditional_prop('ml.rest-authentication-method', 'ml.authentication-method') @@ -2970,7 +2971,7 @@ def rest_appserver end %Q{ - + @ml.app-name-rest @ml.rest-port @@ -2988,7 +2989,7 @@ def rest_modules_db_xml rest_modules_db = conditional_prop('ml.rest-modules-db', 'ml.modules-db') %Q{ - + #{rest_modules_db} @@ -3001,7 +3002,7 @@ def rest_modules_db_assignment rest_modules_db = conditional_prop('ml.rest-modules-db', 'ml.modules-db') %Q{ - + #{rest_modules_db} } @@ -3009,7 +3010,7 @@ def rest_modules_db_assignment def ssl_certificate_xml %Q{ - + @ml.ssl-certificate-template @ml.ssl-certificate-countryName @ml.ssl-certificate-stateOrProvinceName @@ -3022,6 +3023,12 @@ def ssl_certificate_xml end def build_config(config_paths, preserve_props = false) + # some extra assertions + raise ExitException.new("You must use different numbers for app-port and rest-port.") if @properties["ml.app-port"] == @properties["ml.rest-port"] + raise ExitException.new("You must use different numbers for app-port and odbc-port.") if @properties["ml.app-port"] == @properties["ml.odbc-port"] + raise ExitException.new("You must use different numbers for app-port and xdbc-port.") if @properties["ml.app-port"] == @properties["ml.xdbc-port"] + raise ExitException.new("You must use different numbers for app-port and test-port.") if @properties["ml.app-port"] == @properties["ml.test-port"] + config_files = [] config_paths.split(",").each do |config_path| if File.directory?(config_path) @@ -3040,144 +3047,145 @@ def build_config(config_paths, preserve_props = false) config = File.read(config_file) # Build the triggers db if it is provided - if @properties['ml.triggers-db'].present? + #if @properties['ml.triggers-db'].present? - if @properties['ml.triggers-db'] != @properties['ml.modules-db'] + #if @properties['ml.triggers-db'] != @properties['ml.modules-db'] config.gsub!("@ml.triggers-db-xml", triggers_db_xml) config.gsub!("@ml.triggers-assignment", triggers_assignment) - else - config.gsub!("@ml.triggers-db-xml", "") - config.gsub!("@ml.triggers-assignment", "") - end + #else + # config.gsub!("@ml.triggers-db-xml", "") + # config.gsub!("@ml.triggers-assignment", "") + #end config.gsub!("@ml.triggers-mapping", %Q{ - + }) - else - config.gsub!("@ml.triggers-db-xml", "") - config.gsub!("@ml.triggers-assignment", "") - config.gsub!("@ml.triggers-mapping", "") - end + #else + # config.gsub!("@ml.triggers-db-xml", "") + # config.gsub!("@ml.triggers-assignment", "") + # config.gsub!("@ml.triggers-mapping", "") + #end - if @properties['ml.xcc-port'].present? and @properties['ml.install-xcc'] != 'false' + #if @properties['ml.xcc-port'].present? and @properties['ml.install-xcc'] != 'false' config.gsub!("@ml.xdbc-server", xdbc_server) - else - config.gsub!("@ml.xdbc-server", "") - end + #else + # config.gsub!("@ml.xdbc-server", "") + #end - if @properties['ml.odbc-port'].present? + #if @properties['ml.odbc-port'].present? config.gsub!("@ml.odbc-server", odbc_server) - else - config.gsub!("@ml.odbc-server", "") - end + #else + # config.gsub!("@ml.odbc-server", "") + #end # Build the schemas db if it is provided - if @properties['ml.schemas-db'].present? + #if @properties['ml.schemas-db'].present? - if @properties['ml.schemas-db'] != @properties['ml.modules-db'] + #if @properties['ml.schemas-db'] != @properties['ml.modules-db'] config.gsub!("@ml.schemas-db-xml", schemas_db_xml) config.gsub!("@ml.schemas-assignment", schemas_assignment) - else - config.gsub!("@ml.schemas-db-xml", "") - config.gsub!("@ml.schemas-assignment", "") - end + #else + # config.gsub!("@ml.schemas-db-xml", "") + # config.gsub!("@ml.schemas-assignment", "") + #end config.gsub!("@ml.schemas-mapping", %Q{ - + }) - else - config.gsub!("@ml.schemas-db-xml", "") - config.gsub!("@ml.schemas-assignment", "") - config.gsub!("@ml.schemas-mapping", "") - end + #else + # config.gsub!("@ml.schemas-db-xml", "") + # config.gsub!("@ml.schemas-assignment", "") + # config.gsub!("@ml.schemas-mapping", "") + #end # Build the test appserver and db if it is provided - if @properties['ml.test-content-db'].present? && - @properties['ml.test-port'].present? && - @environment != "prod" + #if @properties['ml.test-content-db'].present? && + # @properties['ml.test-port'].present? && + # @environment != "prod" config.gsub!("@ml.test-content-db-xml", test_content_db_xml) config.gsub!("@ml.test-content-db-assignment", test_content_db_assignment) config.gsub!("@ml.test-appserver", test_appserver) - else - config.gsub!("@ml.test-content-db-xml", "") - config.gsub!("@ml.test-content-db-assignment", "") - config.gsub!("@ml.test-appserver", "") - end + #else + # config.gsub!("@ml.test-content-db-xml", "") + # config.gsub!("@ml.test-content-db-assignment", "") + # config.gsub!("@ml.test-appserver", "") + #end # Build the test modules db if it is different from the app modules db - if @properties['ml.test-modules-db'].present? && - @properties['ml.test-modules-db'] != @properties['ml.modules-db'] + #if @properties['ml.test-modules-db'].present? && + # @properties['ml.test-modules-db'] != @properties['ml.modules-db'] config.gsub!("@ml.test-modules-db-xml", test_modules_db_xml) config.gsub!("@ml.test-modules-db-assignment", test_modules_db_assignment) - else - config.gsub!("@ml.test-modules-db-xml", "") - config.gsub!("@ml.test-modules-db-assignment", "") - end + #else + # config.gsub!("@ml.test-modules-db-xml", "") + # config.gsub!("@ml.test-modules-db-assignment", "") + #end - if @properties['ml.test-user'].present? + #if @properties['ml.test-user'].present? config.gsub!("@ml.test-user-xml", test_user_xml) - else - config.gsub!("@ml.test-user-xml", "") - end + #else + # config.gsub!("@ml.test-user-xml", "") + #end - if @properties['ml.rest-port'].present? + #if @properties['ml.rest-port'].present? # Set up a REST API app server, distinct from the main application. config.gsub!("@ml.rest-appserver", rest_appserver) - if @properties['ml.rest-modules-db'].present? && - @properties['ml.rest-modules-db'] != @properties['ml.modules-db'] + #if @properties['ml.rest-modules-db'].present? && + # @properties['ml.rest-modules-db'] != @properties['ml.modules-db'] config.gsub!("@ml.rest-modules-db-xml", rest_modules_db_xml) config.gsub!("@ml.rest-modules-db-assignment", rest_modules_db_assignment) - else - config.gsub!("@ml.rest-modules-db-xml", "") - config.gsub!("@ml.rest-modules-db-assignment", "") - end - - else - config.gsub!("@ml.rest-appserver", "") - config.gsub!("@ml.rest-modules-db-xml", "") - config.gsub!("@ml.rest-modules-db-assignment", "") - end - - if @properties['ml.forest-data-dir'].present? + #else + # config.gsub!("@ml.rest-modules-db-xml", "") + # config.gsub!("@ml.rest-modules-db-assignment", "") + #end + + #else + # config.gsub!("@ml.rest-appserver", "") + # config.gsub!("@ml.rest-modules-db-xml", "") + # config.gsub!("@ml.rest-modules-db-assignment", "") + #end + + #if @properties['ml.forest-data-dir'].present? config.gsub!("@ml.forest-data-dir-xml", %Q{ - @ml.forest-data-dir + @ml.forest-data-dir }) - else - config.gsub!("@ml.forest-data-dir-xml", "") - end + #else + # config.gsub!("@ml.forest-data-dir-xml", "") + #end - if !@properties['ml.rewrite-resolves-globally'].nil? - config.gsub!("@ml.rewrite-resolves-globally", - %Q{ - #{@properties['ml.rewrite-resolves-globally']} - }) - elsif ['rest', 'hybrid'].include?(@properties["ml.app-type"]) + #if !@properties['ml.rewrite-resolves-globally'].nil? config.gsub!("@ml.rewrite-resolves-globally", %Q{ - true + @ml.rewrite-resolves-globally + true }) - else - config.gsub!("@ml.rewrite-resolves-globally", "") - end - - if @properties['ml.ssl-certificate-template'].present? + #elsif ['rest', 'hybrid'].include?(@properties["ml.app-type"]) + # config.gsub!("@ml.rewrite-resolves-globally", + # %Q{ + # true + # }) + #else + # config.gsub!("@ml.rewrite-resolves-globally", "") + #end + + #if @properties['ml.ssl-certificate-template'].present? config.gsub!("@ml.ssl-certificate-xml", ssl_certificate_xml) - else - config.gsub!("@ml.ssl-certificate-xml", "") - end + #else + # config.gsub!("@ml.ssl-certificate-xml", "") + #end if ! preserve_props replace_properties(config, File.basename(config_file), true) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 812ab78e..6613017a 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -434,7 +434,7 @@ declare variable $cts:parse := fn:function-lookup(xs:QName("cts:pars declare variable $if-parser := (); declare function setup:get-if-parser($properties as map:map) { - if ($if-parser) then + if (fn:exists($if-parser)) then $if-parser else let $parser := function($query) { @@ -469,44 +469,88 @@ declare function setup:get-if-parser($properties as map:map) { }; declare function setup:eval-query($query as cts:query, $properties as map:map) { - typeswitch ($query) - case cts:and-query return fn:not( - (cts:and-query-queries($query) ! setup:eval-query(., $properties)) = fn:false() - ) - case cts:or-query return ( - (cts:or-query-queries($query) ! setup:eval-query(., $properties)) = fn:true() - ) - case cts:not-query return fn:not( - cts:not-query-query($query) ! setup:eval-query(., $properties) - ) - case cts:element-value-query return ( - let $property := fn:string(cts:element-value-query-element-name($query)) - let $operator := '=' - let $values := cts:element-value-query-text($query) - return map:get($properties, $property) = $values - ) - case cts:element-word-query return ( - let $property := fn:string(cts:element-word-query-element-name($query)) - let $operator := '=' - let $values := cts:element-word-query-text($query) - return map:get($properties, $property) = $values - ) - case cts:element-range-query return ( - let $property := fn:string(cts:element-range-query-element-name($query)) - let $operator := cts:element-range-query-operator($query) - let $values := cts:element-range-query-value($query) + typeswitch ($query) + case cts:and-query return fn:not( + (cts:and-query-queries($query) ! setup:eval-query(., $properties)) = fn:false() + ) + case cts:or-query return ( + (cts:or-query-queries($query) ! setup:eval-query(., $properties)) = fn:true() + ) + case cts:not-query return fn:not( + cts:not-query-query($query) ! setup:eval-query(., $properties) + ) + case cts:element-value-query return ( + let $property := fn:string(cts:element-value-query-element-name($query)) + let $operator := '=' + let $values := cts:element-value-query-text($query) + return map:get($properties, $property) = $values + ) + case cts:element-word-query return ( + let $property := fn:string(cts:element-word-query-element-name($query)) + let $operator := '=' + let $values := cts:element-word-query-text($query) + return map:get($properties, $property) = $values + ) + case cts:element-range-query return ( + let $property := fn:string(cts:element-range-query-element-name($query)) + let $operator := cts:element-range-query-operator($query) + let $values := cts:element-range-query-value($query) + return + if ($operator = ('=', '!=', '>', '>=', '<', '<=')) then + xdmp:value("map:get($properties, $property) " || $operator || " $values") + else + fn:error(xs:QName("UNSUPPORTED"), "Unsupported operator " || $operator) + ) + case cts:word-query return ( + fn:error(xs:QName("SYNTAX"), "Syntax error near " || cts:word-query-text($query)) + ) + default return ( + fn:error(xs:QName("UNSUPPORTED"), "Cannot parse " || fn:upper-case(fn:replace(fn:string(xdmp:type($query)), "-query$", ""))) + ) +}; + +declare function setup:eval-conditionals($attrs, $properties) { + (: process conditional attrs in doc order, and stop at first failure :) + let $attr := fn:head($attrs) + let $remainder := fn:tail($attrs) + let $res := + typeswitch($attr) + case attribute(if) return + let $parser := setup:get-if-parser($properties) + let $expression := fn:string($attr) + return try { + let $query := $parser($expression)[1] + return try { + setup:eval-query($query, $properties) + } catch ($e) { + fn:error(xs:QName("IF-PARSE-ERROR"), + "Unable to evauluate the expression '" || $expression || "': " || $e/error:format-string/fn:string()) + } + } catch ($e) { + fn:error(xs:QName("IF-PARSE-ERROR"), + "Unable to parse the expression '" || $expression || "': " || $e/error:format-string/fn:string()) + } + case attribute(if-exists) return fn:not( + for $prop in fn:tokenize(fn:string($attr), "\s+AND\s+") return - if ($operator = ('=', '!=', '>', '>=', '<', '<=')) then - xdmp:value("map:get($properties, $property) " || $operator || " $values") - else - fn:error(xs:QName("UNSUPPORTED"), "Unsupported operator " || $operator) + fn:exists(map:get($properties, "ml." || $prop)[. != '']) + = fn:false() ) - case cts:word-query return ( - fn:error(xs:QName("SYNTAX"), "Syntax error near " || cts:word-query-text($query)) - ) - default return ( - fn:error(xs:QName("UNSUPPORTED"), "Cannot parse " || fn:upper-case(fn:replace(fn:string(xdmp:type($query)), "-query$", ""))) + case attribute(if-not-exists) return fn:not( + for $prop in fn:tokenize(fn:string($attr), "\s+AND\s+") + return + fn:empty(map:get($properties, "ml." || $prop)[. != '']) + = fn:false() ) + default return + fn:true() + return + if (fn:empty($remainder)) then + $res + else if ($res) then + setup:eval-conditionals($remainder, $properties) + else + fn:false() }; declare function setup:process-conditionals($nodes, $properties) { @@ -515,29 +559,20 @@ declare function setup:process-conditionals($nodes, $properties) { typeswitch ($node) case element() return let $if-valid := - if (fn:exists($node/@if)) then - let $parser := setup:get-if-parser($properties) - let $expression := string($node/@if) - return try { - let $query := $parser($expression)[1] - return try { - setup:eval-query($query, $properties) - } catch ($e) { - fn:error(xs:QName("IF-PARSE-ERROR"), - "Unable to evauluate the expression '" || $expression || "': " || $e/error:format-string/data()) - } - } catch ($e) { - fn:error(xs:QName("IF-PARSE-ERROR"), - "Unable to parse the expression '" || $expression || "': " || $e/error:format-string/data()) - } + if (fn:exists($node/(@if-exists, @if-not-exists, @if))) then + setup:eval-conditionals($node/(@if-exists, @if-not-exists, @if), $properties) else fn:true() where $if-valid - return element { fn:node-name($node) } { - $node/@*, - setup:process-conditionals($node/node(), $properties) - } - case comment() return () + return + if ($node/self::*:if) then + (: unwrap `if` elements :) + setup:process-conditionals($node/node(), $properties) + else + element { fn:node-name($node) } { + $node/(@* except (@if, @if-exists, @if-not-exists)), + setup:process-conditionals($node/node(), $properties) + } default return $node }; @@ -643,9 +678,13 @@ declare function setup:rewrite-config($import-configs as node()+, $properties as setup:rewrite-config($import-configs, $properties, $silent, ()) }; -declare function setup:rewrite-config($import-configs as node()+, $properties as map:map, $silent as xs:boolean?, $keep-comments as xs:boolean?) as element(configuration) +declare function setup:rewrite-config($import-configs as node()+, $properties as map:map, $silent as xs:boolean?, $unresolved as xs:boolean?) as element(configuration) { - let $import-configs := setup:process-conditionals($import-configs, $properties) + let $import-configs := + if ($unresolved) then + $import-configs + else + setup:process-conditionals($import-configs, $properties) let $config := element configuration { setup:unique-attributes($import-configs/self::configuration/@*), @@ -730,7 +769,7 @@ declare function setup:rewrite-config($import-configs as node()+, $properties as (: Check config on group consistency! :) let $_ := - if ($silent) then () + if ($silent or $unresolved) then () else for $group in $config/gr:groups/gr:group/gr:group-name/fn:string() let $hosts := ($config/ho:hosts/ho:host[ho:group/@name = $group], try { xdmp:group-hosts(xdmp:group($group)) } catch ($ignore) {}) @@ -742,7 +781,7 @@ declare function setup:rewrite-config($import-configs as node()+, $properties as ) (: all good :) - return if ($keep-comments) then $config else setup:suppress-comments($config) + return if ($unresolved) then $config else setup:suppress-comments($config) }; declare function setup:split-config($config as element(configuration), $app-name as xs:string) as node()* { From 2bdae6d6c0bb610f49d34deb12352fa3f3feb963 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 11 Jan 2018 11:19:33 +0100 Subject: [PATCH 5/7] Undoing the default config.file change for now --- deploy/sample/build.sample.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/sample/build.sample.properties b/deploy/sample/build.sample.properties index 0bbd7d04..7f1f8fc1 100644 --- a/deploy/sample/build.sample.properties +++ b/deploy/sample/build.sample.properties @@ -31,7 +31,7 @@ modules-prefix=/ # # the location of your marklogic configuration file # -config.file=${basedir}/deploy/ml-config/ +config.file=${basedir}/deploy/ml-config.xml # # the location of your triggers configuration file From bb14dad5e23ae42480f6f8ccbfa3cf538cfdc08d Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 11 Jan 2018 11:24:47 +0100 Subject: [PATCH 6/7] Undoing the default config.file change for now --- deploy/sample/build.sample.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/sample/build.sample.properties b/deploy/sample/build.sample.properties index 7f1f8fc1..01803a47 100644 --- a/deploy/sample/build.sample.properties +++ b/deploy/sample/build.sample.properties @@ -31,7 +31,7 @@ modules-prefix=/ # # the location of your marklogic configuration file # -config.file=${basedir}/deploy/ml-config.xml +# config.file=${basedir}/deploy/ml-config.xml # # the location of your triggers configuration file From 0a542893d0483d7d70bb9b238bf4bd2c254dc715 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 11 Jan 2018 21:10:23 +0100 Subject: [PATCH 7/7] Fixing self-test for ML7 and ML8 --- deploy/lib/server_config.rb | 2 +- deploy/lib/xquery/setup.xqy | 74 ++++++++++++++++++++----------------- 2 files changed, 41 insertions(+), 35 deletions(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index e4b14281..934a7add 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -1227,7 +1227,7 @@ def export_config #{setup} try { for $part in setup:split-config( - setup:rewrite-config(#{get_config(true)}, (), fn:true()), + setup:rewrite-config(#{get_config(true)}, map:new(), (), fn:true()), "#{@properties["ml.app-name"]}" ) return diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 6613017a..be4c5faa 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -3210,19 +3210,21 @@ declare function setup:add-geospatial-path-indexes( $database as xs:unsignedLong, $db-config as element(db:database)) as element(configuration) { - admin:database-add-geospatial-path-index( - $admin-config, - $database, - for $index in $db-config/db:geospatial-path-indexes/db:geospatial-path-index - return - if (setup:at-least-version("8.0-0")) then - xdmp:value(" - admin:database-geospatial-path-index( - $index/db:path-expression, - $index/db:coordinate-system, - $index/db:range-value-positions, - ($index/db:point-format, ""point"")[1], - ($index/db:invalid-values, $default-invalid-values)[1] + if (exists($db-config/db:geospatial-path-indexes/db:geospatial-path-index)) then + if (setup:at-least-version("8.0-0")) then + xdmp:value(" + admin:database-add-geospatial-path-index( + $admin-config, + $database, + for $index in $db-config/db:geospatial-path-indexes/db:geospatial-path-index + return + admin:database-geospatial-path-index( + $index/db:path-expression, + $index/db:coordinate-system, + $index/db:range-value-positions, + ($index/db:point-format, ""point"")[1], + ($index/db:invalid-values, $default-invalid-values)[1] + ) ) ") else @@ -3230,7 +3232,8 @@ declare function setup:add-geospatial-path-indexes( xs:QName("VERSION_NOT_SUPPORTED"), "Roxy does not support geospatial path indexes for this version of MarkLogic. Use 8.0-0 or later." ) - ) + else + $admin-config }; declare function setup:validate-geospatial-path-indexes( @@ -3271,27 +3274,30 @@ declare function setup:add-geospatial-region-path-indexes( $database as xs:unsignedLong, $db-config as element(db:database)) as element(configuration) { - admin:database-add-geospatial-region-path-index( - $admin-config, - $database, - for $index in $db-config/db:geospatial-region-path-indexes/db:geospatial-region-path-index - return - if (setup:at-least-version("9.0-0")) then - xdmp:value(" - admin:database-geospatial-region-path-index( - $index/db:path-expression, - $index/db:coordinate-system, - $index/db:geohash-precision, - ($index/db:invalid-values, $default-invalid-values)[1], - ($index/db:units, ""miles"")[1] - ) - ") - else - fn:error( - xs:QName("VERSION_NOT_SUPPORTED"), - "Roxy does not support geospatial region path indexes for this version of MarkLogic. Use 9.0-0 or later." + if (exists($db-config/db:geospatial-region-path-indexes/db:geospatial-region-path-index)) then + if (setup:at-least-version("9.0-0")) then + xdmp:value(" + admin:database-add-geospatial-region-path-index( + $admin-config, + $database, + for $index in $db-config/db:geospatial-region-path-indexes/db:geospatial-region-path-index + return + admin:database-geospatial-region-path-index( + $index/db:path-expression, + $index/db:coordinate-system, + $index/db:geohash-precision, + ($index/db:invalid-values, $default-invalid-values)[1], + ($index/db:units, ""miles"")[1] + ) ) - ) + ") + else + fn:error( + xs:QName("VERSION_NOT_SUPPORTED"), + "Roxy does not support geospatial region path indexes for this version of MarkLogic. Use 9.0-0 or later." + ) + else + $admin-config }; declare function setup:validate-geospatial-region-path-indexes(