From 1e5503baf741186c283b1cd45a7c7205ad866fde Mon Sep 17 00:00:00 2001 From: fzbm <52308785+fzbm@users.noreply.github.com> Date: Mon, 7 Oct 2024 16:08:03 +0200 Subject: [PATCH 1/4] Load only normal term groups if specified It is now possible to specify which term groups should be loaded when initializing the token parser. By default all groups including site collection specific ones are loaded. If disabled, only normal term groups will be loaded, which reduces the amount of data to retrieve from SharePoint. --- ...ProvisioningTemplateApplyingInformation.cs | 15 ++++++- .../ObjectHandlers/TokenParser.cs | 41 ++++++++++++++----- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ProvisioningTemplateApplyingInformation.cs b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ProvisioningTemplateApplyingInformation.cs index 0ae8393d1..99e3a821a 100644 --- a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ProvisioningTemplateApplyingInformation.cs +++ b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ProvisioningTemplateApplyingInformation.cs @@ -34,16 +34,21 @@ public partial class ProvisioningTemplateApplyingInformation { private Handlers handlersToProcess = Handlers.All; private List extensibilityHandlers = new List(); + private Dictionary _accessTokens; public ProvisioningProgressDelegate ProgressDelegate { get; set; } + public ProvisioningMessagesDelegate MessagesDelegate { get; set; } + public ProvisioningSiteProvisionedDelegate SiteProvisionedDelegate { get; set; } internal ApplyConfiguration ApplyConfiguration { get; set; } + /// /// If true then persists template information /// public bool PersistTemplateInfo { get; set; } = true; + /// /// If true, system propertybag entries that start with _, vti_, dlc_ etc. will be overwritten if overwrite = true on the PropertyBagEntry. If not true those keys will be skipped, regardless of the overwrite property of the entry. /// @@ -53,6 +58,7 @@ public partial class ProvisioningTemplateApplyingInformation /// If true, existing navigation nodes of the site (where applicable) will be cleared out before applying the navigation nodes from the template (if any). This setting will override any settings made in the template. /// public bool ClearNavigation { get; set; } + /// /// If true then duplicate id errors when the same importing datarows simply generate a warning don't stop the engine. Reason for this is being able to apply the same template multiple times (Delta handling) /// without that failing cause the same record is being added twice @@ -69,6 +75,13 @@ public partial class ProvisioningTemplateApplyingInformation /// public bool ProvisionFieldsToSubWebs { get; set; } + /// + /// Specifies whether to also load site collection term groups when initializing the . If + /// false, only normal term groups will be loaded. This does not affect loading the site collection term group + /// when one of the sitecollectionterm tokens was found. + /// + public bool LoadSiteCollectionTermGroups { get; set; } = true; + /// /// Lists of Handlers to process /// @@ -100,8 +113,6 @@ public List ExtensibilityHandlers } } - private Dictionary _accessTokens; - /// /// Allows to provide a dictionary of custom OAuth access tokens /// when working across different URLs during provisioning and diff --git a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/TokenParser.cs b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/TokenParser.cs index 7bfe81a48..2f4dd5848 100644 --- a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/TokenParser.cs +++ b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/TokenParser.cs @@ -15,6 +15,7 @@ using PnP.Framework.Provisioning.Model; using PnP.Framework.Provisioning.ObjectHandlers.TokenDefinitions; using PnP.Framework.Utilities; +using TermGroup = Microsoft.SharePoint.Client.Taxonomy.TermGroup; namespace PnP.Framework.Provisioning.ObjectHandlers { @@ -327,7 +328,7 @@ public TokenParser(Web web, ProvisioningTemplate template, ProvisioningTemplateA AddPropertyBagTokens(web); // TermStore related tokens - AddTermStoreTokens(web, tokenIds); + AddTermStoreTokens(web, tokenIds, applyingInformation?.LoadSiteCollectionTermGroups ?? true); CalculateTokenCount(_tokens, out int cacheableCount, out int nonCacheableCount); BuildTokenCache(cacheableCount, nonCacheableCount); @@ -552,7 +553,7 @@ private void AddGroupTokens(Web web) } } - private void AddTermStoreTokens(Web web, List tokenIds) + private void AddTermStoreTokens(Web web, List tokenIds, bool loadSiteCollectionTermGroups) { if (!tokenIds.Contains("termstoreid") && !tokenIds.Contains("termsetid") @@ -582,16 +583,36 @@ private void AddTermStoreTokens(Web web, List tokenIds) { if (!termStore.ServerObjectIsNull.Value) { - web.Context.Load(termStore.Groups, - g => g.Include( - tg => tg.Name, - tg => tg.TermSets.Include( - ts => ts.Name, - ts => ts.Id) - )); + IEnumerable termGroups; + + if (loadSiteCollectionTermGroups) + { + termGroups = termStore.Groups; + + web.Context.Load( + termStore.Groups, + groups => groups.Include( + group => group.Name, + group => group.TermSets.Include( + termSet => termSet.Name, + termSet => termSet.Id) + )); + } + else + { + termGroups = web.Context.LoadQuery(termStore + .Groups + .Where(group => !group.IsSiteCollectionGroup) + .Include( + group => group.Name, + group => group.TermSets.Include( + termSet => termSet.Name, + termSet => termSet.Id))); + } + web.Context.ExecuteQueryRetry(); - foreach (var termGroup in termStore.Groups) + foreach (var termGroup in termGroups) { foreach (var termSet in termGroup.TermSets) { From 63d4e0f04e5dbc753237d1a21dc459c203d81b59 Mon Sep 17 00:00:00 2001 From: fzbm <52308785+fzbm@users.noreply.github.com> Date: Mon, 7 Oct 2024 19:46:42 +0200 Subject: [PATCH 2/4] Extended term group loading behavior A "LoadSiteCollectionTermGroups" property has been added to the "ApplyConfiguration" class. Additionally various usages of the "TokenParser" class have been updated to now also provide the template applying information. --- .../Model/Configuration/ApplyConfiguration.cs | 26 ++++-- .../ObjectHandlers/ObjectComposedLook.cs | 2 +- .../ObjectHierarchySequenceSites.cs | 4 +- ...ProvisioningTemplateCreationInformation.cs | 7 ++ .../SiteToTemplateConversion.cs | 4 +- .../ObjectHandlers/TokenParser.cs | 81 ++++++++++--------- 6 files changed, 72 insertions(+), 52 deletions(-) diff --git a/src/lib/PnP.Framework/Provisioning/Model/Configuration/ApplyConfiguration.cs b/src/lib/PnP.Framework/Provisioning/Model/Configuration/ApplyConfiguration.cs index fcdce51b0..e3c3c34ab 100644 --- a/src/lib/PnP.Framework/Provisioning/Model/Configuration/ApplyConfiguration.cs +++ b/src/lib/PnP.Framework/Provisioning/Model/Configuration/ApplyConfiguration.cs @@ -49,6 +49,7 @@ public Dictionary AccessTokens [JsonPropertyName("parameters")] public Dictionary Parameters { get; set; } = new Dictionary(); + /// /// Defines Tenant Extraction Settings /// @@ -73,11 +74,25 @@ public Dictionary AccessTokens [JsonPropertyName("extensibility")] public Extensibility.ApplyExtensibilityConfiguration Extensibility { get; set; } = new Extensibility.ApplyExtensibilityConfiguration(); + /// + /// Specifies whether to also load site collection term groups when initializing the . If + /// false, only normal term groups will be loaded. This does not affect loading the site collection term group + /// when one of the sitecollectionterm tokens was found. + /// + [JsonPropertyName("loadSiteCollectionTermGroups")] + public bool LoadSiteCollectionTermGroups { get; set; } = true; + public ProvisioningTemplateApplyingInformation ToApplyingInformation() { var ai = new ProvisioningTemplateApplyingInformation { - ApplyConfiguration = this + ApplyConfiguration = this, + ProvisionContentTypesToSubWebs = this.ContentTypes.ProvisionContentTypesToSubWebs, + OverwriteSystemPropertyBagValues = this.PropertyBag.OverwriteSystemValues, + IgnoreDuplicateDataRowErrors = this.Lists.IgnoreDuplicateDataRowErrors, + ClearNavigation = this.Navigation.ClearNavigation, + ProvisionFieldsToSubWebs = this.Fields.ProvisionFieldsToSubWebs, + LoadSiteCollectionTermGroups = LoadSiteCollectionTermGroups }; if (this.AccessTokens != null && this.AccessTokens.Any()) @@ -85,12 +100,6 @@ public ProvisioningTemplateApplyingInformation ToApplyingInformation() ai.AccessTokens = this.AccessTokens; } - ai.ProvisionContentTypesToSubWebs = this.ContentTypes.ProvisionContentTypesToSubWebs; - ai.OverwriteSystemPropertyBagValues = this.PropertyBag.OverwriteSystemValues; - ai.IgnoreDuplicateDataRowErrors = this.Lists.IgnoreDuplicateDataRowErrors; - ai.ClearNavigation = this.Navigation.ClearNavigation; - ai.ProvisionFieldsToSubWebs = this.Fields.ProvisionFieldsToSubWebs; - if (Handlers.Any()) { ai.HandlersToProcess = Model.Handlers.None; @@ -178,12 +187,13 @@ public static ApplyConfiguration FromApplyingInformation(ProvisioningTemplateApp config.ContentTypes.ProvisionContentTypesToSubWebs = information.ProvisionContentTypesToSubWebs; config.Fields.ProvisionFieldsToSubWebs = information.ProvisionFieldsToSubWebs; config.SiteProvisionedDelegate = information.SiteProvisionedDelegate; + config.LoadSiteCollectionTermGroups = information.LoadSiteCollectionTermGroups; + return config; } public static ApplyConfiguration FromString(string input) { - return JsonSerializer.Deserialize(input); } } diff --git a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectComposedLook.cs b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectComposedLook.cs index 2d747c859..c62faa989 100644 --- a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectComposedLook.cs +++ b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectComposedLook.cs @@ -128,7 +128,7 @@ public override ProvisioningTemplate ExtractObjects(Web web, ProvisioningTemplat { scope.LogDebug(CoreResources.Provisioning_ObjectHandlers_ComposedLooks_ExtractObjects_Creating_SharePointConnector); // Let's create a SharePoint connector since our files anyhow are in SharePoint at this moment - TokenParser parser = new TokenParser(web, template); + var parser = new TokenParser(web, template, new ProvisioningTemplateApplyingInformation { LoadSiteCollectionTermGroups = creationInfo.LoadSiteCollectionTermGroups }); DownLoadFile(spConnector, spConnectorRoot, creationInfo.FileConnector, web.Url, parser.ParseString(composedLook.BackgroundFile), scope); DownLoadFile(spConnector, spConnectorRoot, creationInfo.FileConnector, web.Url, parser.ParseString(composedLook.ColorFile), scope); DownLoadFile(spConnector, spConnectorRoot, creationInfo.FileConnector, web.Url, parser.ParseString(composedLook.FontFile), scope); diff --git a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectHierarchySequenceSites.cs b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectHierarchySequenceSites.cs index caf4c39c9..ebe612329 100644 --- a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectHierarchySequenceSites.cs +++ b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectHierarchySequenceSites.cs @@ -724,7 +724,7 @@ public override TokenParser ProvisionObjects(Tenant tenant, Model.ProvisioningHi //} //else //{ - // siteTokenParser.Rebase(web, provisioningTemplate); + // siteTokenParser.Rebase(web, provisioningTemplate, provisioningTemplateApplyingInformation); //} WriteMessage($"Applying Template", ProvisioningMessageType.Progress); new SiteToTemplateConversion().ApplyRemoteTemplate(web, provisioningTemplate, provisioningTemplateApplyingInformation, true, siteTokenParser); @@ -871,7 +871,7 @@ private TokenParser ApplySubSiteTemplates(ProvisioningHierarchy hierarchy, Token provisioningTemplate.Connector = hierarchy.Connector; if (tokenParser == null) { - tokenParser = new TokenParser(subweb, provisioningTemplate); + tokenParser = new TokenParser(subweb, provisioningTemplate, provisioningTemplateApplyingInformation); } else { diff --git a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ProvisioningTemplateCreationInformation.cs b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ProvisioningTemplateCreationInformation.cs index b5443c4ec..bc97e7da2 100644 --- a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ProvisioningTemplateCreationInformation.cs +++ b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ProvisioningTemplateCreationInformation.cs @@ -333,6 +333,13 @@ public bool IncludeHiddenLists /// public List ListsToExtract { get; set; } = new List(); + /// + /// Specifies whether to also load site collection term groups when initializing the . If + /// false, only normal term groups will be loaded. This does not affect loading the site collection term group + /// when one of the sitecollectionterm tokens was found. + /// + public bool LoadSiteCollectionTermGroups { get; set; } = true; + /// /// List which contains information about resource tokens used and/or created during the extraction of a template. /// diff --git a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/SiteToTemplateConversion.cs b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/SiteToTemplateConversion.cs index 7cb583418..1d7988749 100644 --- a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/SiteToTemplateConversion.cs +++ b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/SiteToTemplateConversion.cs @@ -184,7 +184,7 @@ internal void ApplyTenantTemplate(Tenant tenant, PnP.Framework.Provisioning.Mode int step = 2; - TokenParser sequenceTokenParser = new TokenParser(tenant, hierarchy); + TokenParser sequenceTokenParser = new TokenParser(tenant, hierarchy, new ProvisioningTemplateApplyingInformation { LoadSiteCollectionTermGroups = configuration.LoadSiteCollectionTermGroups }); CallWebHooks(hierarchy.Templates.FirstOrDefault(), sequenceTokenParser, ProvisioningTemplateWebhookKind.ProvisioningStarted); @@ -414,7 +414,7 @@ internal void ApplyRemoteTemplate(Web web, ProvisioningTemplate template, Provis progressDelegate?.Invoke("Initializing engine", 1, count); // handlers + initializing message) if (tokenParser == null) { - tokenParser = new TokenParser(web, template); + tokenParser = new TokenParser(web, template, provisioningInfo); } if (provisioningInfo.HandlersToProcess.HasFlag(Handlers.ExtensibilityProviders)) { diff --git a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/TokenParser.cs b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/TokenParser.cs index 2f4dd5848..10268fae4 100644 --- a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/TokenParser.cs +++ b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/TokenParser.cs @@ -29,8 +29,9 @@ public class TokenParser : ICloneable private Dictionary _tokenDictionary; private Dictionary _nonCacheableTokenDictionary; private Dictionary _listTokenDictionary; - private readonly Dictionary _listsTitles = new Dictionary(StringComparer.OrdinalIgnoreCase); + private bool _loadSiteCollectionTermGroups; + private readonly Dictionary _listsTitles = new Dictionary(StringComparer.OrdinalIgnoreCase); private readonly bool _initializedFromHierarchy; private readonly Action _addTokenWithCacheUpdateDelegate; private readonly Action _addTokenToListDelegate; @@ -79,6 +80,7 @@ public void Rebase(Web web, ProvisioningTemplate template, ProvisioningTemplateA { var tokenIds = ParseTemplate(template); _web = web; + _loadSiteCollectionTermGroups = applyingInformation?.LoadSiteCollectionTermGroups ?? true; foreach (var token in _tokens.OfType()) { @@ -132,12 +134,14 @@ public void Rebase(Web web, ProvisioningTemplate template, ProvisioningTemplateA /// /// The list with TokenDefinitions to copy over /// The Web context to copy over - private TokenParser(Web web, List tokens) + /// Specifies if site collection term groups should be loaded. + private TokenParser(Web web, List tokens, bool loadSiteCollectionTermGroups) { _web = web; _tokens = tokens; _addTokenWithCacheUpdateDelegate = AddTokenWithCacheUpdate; _addTokenToListDelegate = AddTokenToList; + _loadSiteCollectionTermGroups = loadSiteCollectionTermGroups; CalculateTokenCount(_tokens, out int cacheableCount, out int nonCacheableCount); BuildTokenCache(cacheableCount, nonCacheableCount); @@ -152,6 +156,7 @@ public TokenParser(Tenant tenant, ProvisioningHierarchy hierarchy, ProvisioningT { _addTokenWithCacheUpdateDelegate = AddTokenWithCacheUpdate; _addTokenToListDelegate = AddTokenToList; + _loadSiteCollectionTermGroups = applyingInformation?.LoadSiteCollectionTermGroups ?? true; // CHANGED: To avoid issues with low privilege users Web web; @@ -218,6 +223,7 @@ public TokenParser(Web web, ProvisioningTemplate template, ProvisioningTemplateA _tokens = new List(); _addTokenWithCacheUpdateDelegate = AddTokenWithCacheUpdate; _addTokenToListDelegate = AddTokenToList; + _loadSiteCollectionTermGroups = applyingInformation?.LoadSiteCollectionTermGroups ?? true; if (tokenIds.Contains("sitecollection")) _tokens.Add(new SiteCollectionToken(web)); @@ -328,7 +334,7 @@ public TokenParser(Web web, ProvisioningTemplate template, ProvisioningTemplateA AddPropertyBagTokens(web); // TermStore related tokens - AddTermStoreTokens(web, tokenIds, applyingInformation?.LoadSiteCollectionTermGroups ?? true); + AddTermStoreTokens(web, tokenIds); CalculateTokenCount(_tokens, out int cacheableCount, out int nonCacheableCount); BuildTokenCache(cacheableCount, nonCacheableCount); @@ -553,7 +559,7 @@ private void AddGroupTokens(Web web) } } - private void AddTermStoreTokens(Web web, List tokenIds, bool loadSiteCollectionTermGroups) + private void AddTermStoreTokens(Web web, List tokenIds) { if (!tokenIds.Contains("termstoreid") && !tokenIds.Contains("termsetid") @@ -579,45 +585,42 @@ private void AddTermStoreTokens(Web web, List tokenIds, bool loadSiteCol web.Context.Load(termStore); web.Context.ExecuteQueryRetry(); - if (tokenIds.Contains("termsetid")) + if (tokenIds.Contains("termsetid") && !termStore.ServerObjectIsNull.Value) { - if (!termStore.ServerObjectIsNull.Value) - { - IEnumerable termGroups; + IEnumerable termGroups; - if (loadSiteCollectionTermGroups) - { - termGroups = termStore.Groups; - - web.Context.Load( - termStore.Groups, - groups => groups.Include( - group => group.Name, - group => group.TermSets.Include( - termSet => termSet.Name, - termSet => termSet.Id) - )); - } - else - { - termGroups = web.Context.LoadQuery(termStore - .Groups - .Where(group => !group.IsSiteCollectionGroup) - .Include( - group => group.Name, - group => group.TermSets.Include( - termSet => termSet.Name, - termSet => termSet.Id))); - } + if (_loadSiteCollectionTermGroups) + { + termGroups = termStore.Groups; + + web.Context.Load( + termStore.Groups, + groups => groups.Include( + group => group.Name, + group => group.TermSets.Include( + termSet => termSet.Name, + termSet => termSet.Id) + )); + } + else + { + termGroups = web.Context.LoadQuery(termStore + .Groups + .Where(group => !group.IsSiteCollectionGroup) + .Include( + group => group.Name, + group => group.TermSets.Include( + termSet => termSet.Name, + termSet => termSet.Id))); + } - web.Context.ExecuteQueryRetry(); + web.Context.ExecuteQueryRetry(); - foreach (var termGroup in termGroups) + foreach (var termGroup in termGroups) + { + foreach (var termSet in termGroup.TermSets) { - foreach (var termSet in termGroup.TermSets) - { - _tokens.Add(new TermSetIdToken(web, termGroup.Name, termSet.Name, termSet.Id)); - } + _tokens.Add(new TermSetIdToken(web, termGroup.Name, termSet.Name, termSet.Id)); } } } @@ -1437,7 +1440,7 @@ private void AddTokenToList(TokenDefinition tokenDefinition) /// New cloned instance of the TokenParser public object Clone() { - return new TokenParser(_web, _tokens); + return new TokenParser(_web, _tokens, _loadSiteCollectionTermGroups); } } } From 7f5ff944acdaf62ace89db8bdea3769bc84f141f Mon Sep 17 00:00:00 2001 From: fzbm <52308785+fzbm@users.noreply.github.com> Date: Mon, 7 Oct 2024 20:59:26 +0200 Subject: [PATCH 3/4] Extended term group loading behavior "ObjectTermGroups" and "ObjectHierarchySequenceTermGroups" now also respect "LoadSiteCollectionTermGroups" when loading term groups. --- .../ObjectHierarchySequenceTermGroups.cs | 53 ++++++++++++----- .../ObjectHandlers/ObjectTermGroups.cs | 58 ++++++++++++------- .../Utilities/TermGroupHelper.cs | 6 +- 3 files changed, 78 insertions(+), 39 deletions(-) diff --git a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectHierarchySequenceTermGroups.cs b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectHierarchySequenceTermGroups.cs index 3061ca57f..7618a746d 100644 --- a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectHierarchySequenceTermGroups.cs +++ b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectHierarchySequenceTermGroups.cs @@ -1,14 +1,15 @@ -using Microsoft.Online.SharePoint.TenantAdministration; +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Online.SharePoint.TenantAdministration; using Microsoft.SharePoint.Client; using Microsoft.SharePoint.Client.Taxonomy; using PnP.Framework.Diagnostics; using PnP.Framework.Provisioning.Model; using PnP.Framework.Provisioning.Model.Configuration; using PnP.Framework.Provisioning.ObjectHandlers.Utilities; -using System; -using System.Collections.Generic; -using System.Linq; using Term = Microsoft.SharePoint.Client.Taxonomy.Term; +using TermGroup = Microsoft.SharePoint.Client.Taxonomy.TermGroup; namespace PnP.Framework.Provisioning.ObjectHandlers { @@ -25,26 +26,46 @@ public override TokenParser ProvisionObjects(Tenant tenant, Model.ProvisioningHi { foreach (var sequence in hierarchy.Sequences) { - this.reusedTerms = new List(); var context = tenant.Context as ClientContext; TaxonomySession taxSession = TaxonomySession.GetTaxonomySession(context); - TermStore termStore = null; + TermStore termStore; + IEnumerable termGroups; try { termStore = taxSession.GetDefaultKeywordsTermStore(); - context.Load(termStore, - ts => ts.Languages, - ts => ts.DefaultLanguage, - ts => ts.Groups.Include( - tg => tg.Name, - tg => tg.Id, - tg => tg.TermSets.Include( - tset => tset.Name, - tset => tset.Id))); + context.Load(termStore, ts => ts.Languages, ts => ts.DefaultLanguage); + + if (configuration.LoadSiteCollectionTermGroups) + { + termGroups = termStore.Groups; + + context.Load( + termStore.Groups, + groups => groups.Include( + group => group.Name, + group => group.Id, + group => group.TermSets.Include( + termSet => termSet.Name, + termSet => termSet.Id) + )); + } + else + { + termGroups = context.LoadQuery(termStore + .Groups + .Where(group => !group.IsSiteCollectionGroup) + .Include( + group => group.Name, + group => group.Id, + group => group.TermSets.Include( + termSet => termSet.Name, + termSet => termSet.Id))); + } + context.ExecuteQueryRetry(); } catch (ServerException) @@ -56,7 +77,7 @@ public override TokenParser ProvisionObjects(Tenant tenant, Model.ProvisioningHi foreach (var modelTermGroup in sequence.TermStore.TermGroups) { - this.reusedTerms.AddRange(TermGroupHelper.ProcessGroup(context, taxSession, termStore, modelTermGroup, null, parser, scope)); + this.reusedTerms.AddRange(TermGroupHelper.ProcessGroup(context, taxSession, termStore, termGroups, modelTermGroup, null, parser, scope)); } foreach (var reusedTerm in this.reusedTerms) diff --git a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectTermGroups.cs b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectTermGroups.cs index 07e4e23d5..8ec1df815 100644 --- a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectTermGroups.cs +++ b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectTermGroups.cs @@ -1,11 +1,10 @@ -using Microsoft.SharePoint.Client; +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.SharePoint.Client; using Microsoft.SharePoint.Client.Taxonomy; using PnP.Framework.Diagnostics; -using PnP.Framework.Provisioning.ObjectHandlers.TokenDefinitions; using PnP.Framework.Provisioning.ObjectHandlers.Utilities; -using System; -using System.Collections.Generic; -using System.Linq; namespace PnP.Framework.Provisioning.ObjectHandlers { @@ -24,22 +23,44 @@ public override TokenParser ProvisionObjects(Web web, Model.ProvisioningTemplate this.reusedTerms = new List(); TaxonomySession taxSession = TaxonomySession.GetTaxonomySession(web.Context); - TermStore termStore = null; - TermGroup siteCollectionTermGroup = null; + TermStore termStore; + TermGroup siteCollectionTermGroup; + IEnumerable termGroups; try { termStore = taxSession.GetDefaultKeywordsTermStore(); - web.Context.Load(termStore, - ts => ts.Languages, - ts => ts.DefaultLanguage, - ts => ts.Groups.Include( - tg => tg.Name, - tg => tg.Id, - tg => tg.TermSets.Include( - tset => tset.Name, - tset => tset.Id))); + + web.Context.Load(termStore, ts => ts.Languages, ts => ts.DefaultLanguage); siteCollectionTermGroup = termStore.GetSiteCollectionGroup((web.Context as ClientContext).Site, false); + + if (applyingInformation.LoadSiteCollectionTermGroups) + { + termGroups = termStore.Groups; + + web.Context.Load( + termStore.Groups, + groups => groups.Include( + group => group.Name, + group => group.Id, + group => group.TermSets.Include( + termSet => termSet.Name, + termSet => termSet.Id) + )); + } + else + { + termGroups = web.Context.LoadQuery(termStore + .Groups + .Where(group => !group.IsSiteCollectionGroup) + .Include( + group => group.Name, + group => group.Id, + group => group.TermSets.Include( + termSet => termSet.Name, + termSet => termSet.Id))); + } + web.Context.Load(siteCollectionTermGroup); web.Context.ExecuteQueryRetry(); } @@ -52,12 +73,9 @@ public override TokenParser ProvisionObjects(Web web, Model.ProvisioningTemplate return parser; } - SiteCollectionTermGroupNameToken siteCollectionTermGroupNameToken = - new SiteCollectionTermGroupNameToken(web); - foreach (var modelTermGroup in template.TermGroups) { - this.reusedTerms.AddRange(TermGroupHelper.ProcessGroup(web.Context as ClientContext, taxSession, termStore, modelTermGroup, siteCollectionTermGroup, parser, scope)); + this.reusedTerms.AddRange(TermGroupHelper.ProcessGroup(web.Context as ClientContext, taxSession, termStore, termGroups, modelTermGroup, siteCollectionTermGroup, parser, scope)); } foreach (var reusedTerm in this.reusedTerms) diff --git a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/Utilities/TermGroupHelper.cs b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/Utilities/TermGroupHelper.cs index b10b32fc2..869142b84 100644 --- a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/Utilities/TermGroupHelper.cs +++ b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/Utilities/TermGroupHelper.cs @@ -10,7 +10,7 @@ namespace PnP.Framework.Provisioning.ObjectHandlers.Utilities { internal static class TermGroupHelper { - internal static List ProcessGroup(ClientContext context, TaxonomySession session, TermStore termStore, Model.TermGroup modelTermGroup, TermGroup siteCollectionTermGroup, TokenParser parser, PnPMonitoredScope scope) + internal static List ProcessGroup(ClientContext context, TaxonomySession session, TermStore termStore, IEnumerable termGroups, Model.TermGroup modelTermGroup, TermGroup siteCollectionTermGroup, TokenParser parser, PnPMonitoredScope scope) { List reusedTerms = new List(); @@ -26,7 +26,7 @@ internal static List ProcessGroup(ClientContext context, TaxonomySes var normalizedGroupName = TaxonomyItem.NormalizeName(context, modelGroupName); context.ExecuteQueryRetry(); - TermGroup group = termStore.Groups.FirstOrDefault( + TermGroup group = termGroups.FirstOrDefault( g => g.Id == modelTermGroup.Id || g.Name == normalizedGroupName.Value); if (group == null) { @@ -45,7 +45,7 @@ internal static List ProcessGroup(ClientContext context, TaxonomySes } else { - group = termStore.Groups.FirstOrDefault(g => g.Name == normalizedGroupName.Value); + group = termGroups.FirstOrDefault(g => g.Name == normalizedGroupName.Value); if (group == null) { From 4bef96c5c63919395cad3a47dc44cf9411bbe113 Mon Sep 17 00:00:00 2001 From: fzbm <52308785+fzbm@users.noreply.github.com> Date: Wed, 5 Mar 2025 12:12:53 +0100 Subject: [PATCH 4/4] Explicitly load the site collection term group The previous change exclude site collection term groups from being loaded by the token parser and the object handler for term groups when specified. This introduced an issue where "termsetid" token, which reference a site-specific term set, could no longer be resolved because the group was excluded. To remedy this, the site collection term group now gets explicitly processed in both the token parser and the object handler. The token parser will add additional "termsetid" token if required and the object handler includes the term group again in the list groups to process. (cherry picked from commit 9993f2f1c053ba54a2187a71d60d6b80732d7495) --- .../ObjectHandlers/ObjectTermGroups.cs | 55 +++++++++++++------ .../ObjectHandlers/TokenParser.cs | 42 +++++++++++--- 2 files changed, 71 insertions(+), 26 deletions(-) diff --git a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectTermGroups.cs b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectTermGroups.cs index 8ec1df815..30c9644cb 100644 --- a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectTermGroups.cs +++ b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectTermGroups.cs @@ -15,12 +15,13 @@ internal class ObjectTermGroups : ObjectHandlerBase public override string Name => "Term Groups"; public override string InternalName => "TermGroups"; + public override TokenParser ProvisionObjects(Web web, Model.ProvisioningTemplate template, TokenParser parser, ProvisioningTemplateApplyingInformation applyingInformation) { - using (var scope = new PnPMonitoredScope(this.Name)) + using (var scope = new PnPMonitoredScope(Name)) { - this.reusedTerms = new List(); + reusedTerms = new List(); TaxonomySession taxSession = TaxonomySession.GetTaxonomySession(web.Context); TermStore termStore; @@ -32,7 +33,7 @@ public override TokenParser ProvisionObjects(Web web, Model.ProvisioningTemplate termStore = taxSession.GetDefaultKeywordsTermStore(); web.Context.Load(termStore, ts => ts.Languages, ts => ts.DefaultLanguage); - siteCollectionTermGroup = termStore.GetSiteCollectionGroup((web.Context as ClientContext).Site, false); + siteCollectionTermGroup = termStore.GetSiteCollectionGroup(((ClientContext)web.Context).Site, createIfMissing: false); if (applyingInformation.LoadSiteCollectionTermGroups) { @@ -47,22 +48,40 @@ public override TokenParser ProvisionObjects(Web web, Model.ProvisioningTemplate termSet => termSet.Name, termSet => termSet.Id) )); + + web.Context.Load(siteCollectionTermGroup); + web.Context.ExecuteQueryRetry(); } else { - termGroups = web.Context.LoadQuery(termStore - .Groups - .Where(group => !group.IsSiteCollectionGroup) - .Include( - group => group.Name, - group => group.Id, - group => group.TermSets.Include( - termSet => termSet.Name, - termSet => termSet.Id))); - } + IEnumerable groups = web + .Context + .LoadQuery(termStore + .Groups + .Where(group => !group.IsSiteCollectionGroup) + .Include( + group => group.Name, + group => group.Id, + group => group.TermSets.Include( + termSet => termSet.Name, + termSet => termSet.Id))); + web.Context.ExecuteQueryRetry(); - web.Context.Load(siteCollectionTermGroup); - web.Context.ExecuteQueryRetry(); + // Convert the loaded term groups to a list and add the site collection one. + List loadedTermGroups = groups.ToList(); + loadedTermGroups.Add(siteCollectionTermGroup); + + web.Context.Load( + siteCollectionTermGroup, + group => group.Name, + group => group.Id, + group => group.TermSets.Include( + termSet => termSet.Name, + termSet => termSet.Id)); + web.Context.ExecuteQueryRetry(); + + termGroups = loadedTermGroups; + } } catch (ServerException) { @@ -75,10 +94,10 @@ public override TokenParser ProvisionObjects(Web web, Model.ProvisioningTemplate foreach (var modelTermGroup in template.TermGroups) { - this.reusedTerms.AddRange(TermGroupHelper.ProcessGroup(web.Context as ClientContext, taxSession, termStore, termGroups, modelTermGroup, siteCollectionTermGroup, parser, scope)); + reusedTerms.AddRange(TermGroupHelper.ProcessGroup(web.Context as ClientContext, taxSession, termStore, termGroups, modelTermGroup, siteCollectionTermGroup, parser, scope)); } - foreach (var reusedTerm in this.reusedTerms) + foreach (var reusedTerm in reusedTerms) { TermGroupHelper.TryReuseTerm(web.Context as ClientContext, reusedTerm.ModelTerm, reusedTerm.Parent, reusedTerm.TermStore, parser, scope); } @@ -94,7 +113,7 @@ private class TryReuseTermResult public override Model.ProvisioningTemplate ExtractObjects(Web web, Model.ProvisioningTemplate template, ProvisioningTemplateCreationInformation creationInfo) { - using (var scope = new PnPMonitoredScope(this.Name)) + using (var scope = new PnPMonitoredScope(Name)) { if (creationInfo.IncludeSiteCollectionTermGroup || creationInfo.IncludeAllTermGroups) { diff --git a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/TokenParser.cs b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/TokenParser.cs index 10268fae4..84379b711 100644 --- a/src/lib/PnP.Framework/Provisioning/ObjectHandlers/TokenParser.cs +++ b/src/lib/PnP.Framework/Provisioning/ObjectHandlers/TokenParser.cs @@ -375,7 +375,7 @@ private void AddResourceTokens(Web web, LocalizationCollection localizations, Fi continue; } - // Use raw XML approach as the .Net Framework resxreader seems to choke on some resx files + // Use raw XML approach as the .Net Framework resxreader seems to choke on some resx files // TODO: research this! var xElement = XElement.Load(stream); @@ -561,11 +561,13 @@ private void AddGroupTokens(Web web) private void AddTermStoreTokens(Web web, List tokenIds) { + bool siteCollectionTermSetIdTokenFound = tokenIds.Contains("sitecollectiontermsetid"); + if (!tokenIds.Contains("termstoreid") && !tokenIds.Contains("termsetid") && !tokenIds.Contains("sitecollectiontermgroupid") && !tokenIds.Contains("sitecollectiontermgroupname") - && !tokenIds.Contains("sitecollectiontermsetid")) + && !siteCollectionTermSetIdTokenFound) { return; } @@ -631,23 +633,47 @@ private void AddTermStoreTokens(Web web, List tokenIds) if (tokenIds.Contains("sitecollectiontermgroupname")) _tokens.Add(new SiteCollectionTermGroupNameToken(web)); - if (!tokenIds.Contains("sitecollectiontermsetid")) + // We can exit the method here, if we already loaded all term groups or the template does not contain at least + // one "sitecollectiontermsetid" token. Otherwise, we want to explicitly load the site collection term group + // and its term sets in order to support "termsetid" token referencing a site-specific term set. + if (_loadSiteCollectionTermGroups && !siteCollectionTermSetIdTokenFound) { return; } - var site = (web.Context as ClientContext).Site; + // Load the site collection term group and its term sets. + var site = ((ClientContext)web.Context).Site; var siteCollectionTermGroup = termStore.GetSiteCollectionGroup(site, true); web.Context.Load(siteCollectionTermGroup); try { web.Context.ExecuteQueryRetry(); - if (null != siteCollectionTermGroup && !siteCollectionTermGroup.ServerObjectIsNull.Value) + + if (siteCollectionTermGroup.ServerObjectIsNull()) + { + return; + } + + web.Context.Load( + siteCollectionTermGroup, + group => group.Name, + group => group.TermSets.Include( + ts => ts.Name, + ts => ts.Id)); + web.Context.ExecuteQueryRetry(); + + foreach (var termSet in siteCollectionTermGroup.TermSets) { - web.Context.Load(siteCollectionTermGroup, group => group.TermSets.Include(ts => ts.Name, ts => ts.Id)); - web.Context.ExecuteQueryRetry(); - foreach (var termSet in siteCollectionTermGroup.TermSets) + // Add a normal "termsetid" token if not all term groups were loaded. There might be token which + // reference a site-specific term set by not using the "sitecollectiontermsetid" token. + if (!_loadSiteCollectionTermGroups) + { + _tokens.Add(new TermSetIdToken(web, siteCollectionTermGroup.Name, termSet.Name, termSet.Id)); + } + + // Add a "sitecollectiontermsetid" token if at least one of those were found. + if (siteCollectionTermSetIdTokenFound) { _tokens.Add(new SiteCollectionTermSetIdToken(web, termSet.Name, termSet.Id)); }