Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handling deeper level (terms children) Reused Child Terms hierarchy in TermGroups Object Provisioning (only TermSet reused terms direct children hierarchies were being provisioned). #801

Open
wants to merge 3 commits into
base: dev
Choose a base branch
from
Open

Handling deeper level (terms children) Reused Child Terms hierarchy in TermGroups Object Provisioning (only TermSet reused terms direct children hierarchies were being provisioned). #801

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d53c1c9
Handling depeer level (terms children) Reused Child Terms hierarchy i…
Autophanous Dec 2, 2022
7998295
Handling depeer level (terms children) Reused Child Terms hierarchy i…
Autophanous Dec 2, 2022
b867796
Merge branch 'HandleNestedReusedTerms' of https://github.com/Autophan…
Feb 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectHierarchySequenceTermGroups.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,18 @@ public override TokenParser ProvisionObjects(Tenant tenant, Model.ProvisioningHi
this.reusedTerms.AddRange(TermGroupHelper.ProcessGroup(context, taxSession, termStore, modelTermGroup, null, parser, scope));
}

foreach (var reusedTerm in this.reusedTerms)
// process the pending Reused Terms list:
TermGroupHelper.ReusedTerm reusedTerm;
List<TermGroupHelper.ReusedTerm> triedReusedTerms = new List<TermGroupHelper.ReusedTerm>();
while (this.reusedTerms.Count > 0 && (reusedTerm = this.reusedTerms[0]) != null)
{
TermGroupHelper.TryReuseTerm(context, reusedTerm.ModelTerm, reusedTerm.Parent, reusedTerm.TermStore, parser, scope);
if (!this.reusedTerms.Except(triedReusedTerms).Any())
{
Log.Warning(Constants.LOGGING_SOURCE, "We were not able to resolve {0} reused Terms in the processing scope to their source terms which are not in the processed scope or present in the TermStore (maybe they are provisioned in a different scope)! We'll need to give up! The following Reused Terms hierarchies were not created: {1}", this.reusedTerms.Count, string.Join(";", this.reusedTerms.Select(x => string.Concat(x.ModelTerm.Id, ":", x.ModelTerm.Name)).ToArray()));
break;
}
var reuseTermTrialResult = TermGroupHelper.TryReuseTerm(context, reusedTerm.ModelTerm, reusedTerm.Parent, this.reusedTerms, reusedTerm.TermStore, parser, scope);
triedReusedTerms.Add(reusedTerm);
}
}
}
Expand Down
13 changes: 11 additions & 2 deletions src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectTermGroups.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,18 @@ public override TokenParser ProvisionObjects(Web web, Model.ProvisioningTemplate
this.reusedTerms.AddRange(TermGroupHelper.ProcessGroup(web.Context as ClientContext, taxSession, termStore, modelTermGroup, siteCollectionTermGroup, parser, scope));
}

foreach (var reusedTerm in this.reusedTerms)
// process the pending Reused Terms list:
TermGroupHelper.ReusedTerm reusedTerm;
List<TermGroupHelper.ReusedTerm> triedReusedTerms = new List<TermGroupHelper.ReusedTerm>();
while (this.reusedTerms.Count > 0 && (reusedTerm = this.reusedTerms[0]) != null)
{
TermGroupHelper.TryReuseTerm(web.Context as ClientContext, reusedTerm.ModelTerm, reusedTerm.Parent, reusedTerm.TermStore, parser, scope);
if(!this.reusedTerms.Except(triedReusedTerms).Any())
{
Log.Warning(Constants.LOGGING_SOURCE, "We were not able to resolve {0} reused Terms in the processing scope to their source terms which are not in the processed scope or present in the TermStore (maybe they are provisioned in a different scope)! We'll need to give up! The following Reused Terms hierarchies were not created: {1}", this.reusedTerms.Count, string.Join(";", this.reusedTerms.Select(x => string.Concat(x.ModelTerm.Id, ":", x.ModelTerm.Name)).ToArray()));
break;
}
var reuseTermTrialResult = TermGroupHelper.TryReuseTerm(web.Context as ClientContext, reusedTerm.ModelTerm, reusedTerm.Parent, this.reusedTerms, reusedTerm.TermStore, parser, scope);
triedReusedTerms.Add(reusedTerm);
}
}
return parser;
Expand Down
127 changes: 99 additions & 28 deletions src/lib/PnP.Framework/Provisioning/ObjectHandlers/Utilities/TermGroupHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,8 +172,8 @@ internal static List<ReusedTerm> ProcessGroup(ClientContext context, TaxonomySes
{
modelTerm.Id = returnTuple.Item1;
parser = returnTuple.Item2;
reusedTerms.AddRange(returnTuple.Item3); // add any potentially existing reused terms to the list to be processed later!
}
reusedTerms.AddRange(returnTuple.Item3);
}
else
{
Expand All @@ -189,7 +189,7 @@ internal static List<ReusedTerm> ProcessGroup(ClientContext context, TaxonomySes

if (term != null)
{
CheckChildTerms(context, modelTerm, term, termStore, parser, scope);
CheckChildTerms(context, modelTerm, term, reusedTerms, termStore, parser, scope);
}
}
else
Expand All @@ -199,8 +199,8 @@ internal static List<ReusedTerm> ProcessGroup(ClientContext context, TaxonomySes
{
modelTerm.Id = returnTuple.Item1;
parser = returnTuple.Item2;
reusedTerms.AddRange(returnTuple.Item3); // add any potentially existing reused terms to the list to be processed later!
}
reusedTerms.AddRange(returnTuple.Item3);
}
}
else
Expand All @@ -210,8 +210,8 @@ internal static List<ReusedTerm> ProcessGroup(ClientContext context, TaxonomySes
{
modelTerm.Id = returnTuple.Item1;
parser = returnTuple.Item2;
reusedTerms.AddRange(returnTuple.Item3); // add any potentially existing reused terms to the list to be processed later!
}
reusedTerms.AddRange(returnTuple.Item3);
}
}

Expand Down Expand Up @@ -276,7 +276,7 @@ internal static Tuple<Guid, TokenParser, List<ReusedTerm>> CreateTerm(ClientCont
});
return Tuple.Create(modelTerm.Id, parser, reusedTerms);
}

// Create new term
Term term;
if (modelTerm.Id == Guid.Empty)
Expand Down Expand Up @@ -334,10 +334,30 @@ internal static Tuple<Guid, TokenParser, List<ReusedTerm>> CreateTerm(ClientCont
}
}

termStore.CommitAll();
try
{
termStore.CommitAll();

context.Load(term);
context.ExecuteQueryRetry();
context.Load(term);
context.ExecuteQueryRetry();
}
catch (ServerException ex)
{
// This is a really sucky (temporary) way to handle these transient Server Exceptions (I found no other solution for now)!
// Maybe Server Exceptions should be included/handled in the Microsoft.SharePoint.Client.ClientContextExtensions.ExecuteQueryRetry retry logic (currently it's just throwing the exception)
// ref: https://techcommunity.microsoft.com/t5/sharepoint-developer/inconsistent-behavior-in-using-taxonomy-api/m-p/184696
if (ex.Message.StartsWith("Taxonomy item instantiation failed."))
{
Log.Debug(Constants.LOGGING_SOURCE, ex, "Possibly transient error on Term: '{0};{1}' creation! Retrying once in 10 sec ...", parent.Name, modelTerm.Name);
System.Threading.Thread.Sleep(10000);
context.Load(term);
context.ExecuteQueryRetry();
}
else // pass trought all other exceptions!
{
throw;
}
}

// Deprecate term if needed
if (modelTerm.IsDeprecated != term.IsDeprecated)
Expand All @@ -347,7 +367,7 @@ internal static Tuple<Guid, TokenParser, List<ReusedTerm>> CreateTerm(ClientCont
}


parser = CreateChildTerms(context, modelTerm, term, termStore, parser, scope);
parser = CreateChildTerms(context, modelTerm, term, reusedTerms, termStore, parser, scope);
return Tuple.Create(modelTerm.Id, parser, reusedTerms);
}

Expand Down Expand Up @@ -389,18 +409,41 @@ private static void SetTermLocalCustomProperties(Model.Term modelTerm, TokenPars
/// <param name="context"></param>
/// <param name="modelTerm"></param>
/// <param name="term"></param>
/// <param name="reusedTerms"></param>
/// <param name="termStore"></param>
/// <param name="parser"></param>
/// <param name="scope"></param>
/// <returns>Updated parser object</returns>
private static TokenParser CreateChildTerms(ClientContext context, Model.Term modelTerm, Term term, TermStore termStore, TokenParser parser, PnPMonitoredScope scope)
private static TokenParser CreateChildTerms(ClientContext context, Model.Term modelTerm, Term term, List<ReusedTerm> reusedTerms, TermStore termStore, TokenParser parser, PnPMonitoredScope scope)
{
if (modelTerm.Terms.Any())
{
foreach (var modelTermTerm in modelTerm.Terms)
{
context.Load(term.Terms);
context.ExecuteQueryRetry();
try
{
context.Load(term);
context.Load(term.Terms);
context.ExecuteQueryRetry();
}
catch (ServerException ex)
{
// This is a really sucky (temporary) way to handle these transient Server Exceptions (I found no other solution for now)!
// Maybe Server Exceptions should be included/handled in the Microsoft.SharePoint.Client.ClientContextExtensions.ExecuteQueryRetry retry logic (currently it's just throwing the exception)
// ref: https://techcommunity.microsoft.com/t5/sharepoint-developer/inconsistent-behavior-in-using-taxonomy-api/m-p/184696
if (ex.Message.StartsWith("Taxonomy item instantiation failed."))
{
Log.Debug(Constants.LOGGING_SOURCE, ex, "Possibly transient error on Term: '{0}' loading! Retrying once in 10 sec ...", term.PathOfTerm);
System.Threading.Thread.Sleep(10000);
context.Load(term);
context.Load(term.Terms);
context.ExecuteQueryRetry();
}
else // pass trought all other exceptions!
{
throw;
}
}
var termTerms = term.Terms;
if (termTerms.Any())
{
Expand All @@ -415,6 +458,7 @@ private static TokenParser CreateChildTerms(ClientContext context, Model.Term mo
{
modelTermTerm.Id = returnTuple.Item1;
parser = returnTuple.Item2;
reusedTerms.AddRange(returnTuple.Item3); // add any potentially existing reused terms to the list to be processed later!
}
}
else
Expand All @@ -434,6 +478,7 @@ private static TokenParser CreateChildTerms(ClientContext context, Model.Term mo
{
modelTermTerm.Id = returnTuple.Item1;
parser = returnTuple.Item2;
reusedTerms.AddRange(returnTuple.Item3); // add any potentially existing reused terms to the list to be processed later!
}
}
}
Expand All @@ -456,35 +501,41 @@ private static TokenParser CreateChildTerms(ClientContext context, Model.Term mo
/// Attempts to reuse the model term. If the term does not yet exists it will return
/// false for the first part of the the return tuple. this will notify the system
/// that the term should be created instead of re-used.
/// Any additional child reused terms are added to the reused terms list to process.
/// </summary>
/// <param name="context"></param>
/// <param name="modelTerm"></param>
/// <param name="parent"></param>
/// <param name="reusedTerms"></param>
/// <param name="termStore"></param>
/// <param name="parser"></param>
/// <param name="scope"></param>
/// <returns></returns>
internal static TryReuseTermResult TryReuseTerm(ClientContext context, Model.Term modelTerm, TaxonomyItem parent, TermStore termStore, TokenParser parser, PnPMonitoredScope scope)
internal static TryReuseTermResult TryReuseTerm(ClientContext context, Model.Term modelTerm, TaxonomyItem parent, List<ReusedTerm> reusedTerms, TermStore termStore, TokenParser parser, PnPMonitoredScope scope)
{
// mark the reused term being processed:
var reusedTerm = reusedTerms.Find(rt => rt.ModelTerm == modelTerm); // get the reference of the reused term being processed
reusedTerms.Remove(reusedTerm); // assume we processed the reused term (remove it from the pending reused terms list)!

if (!modelTerm.IsReused) return new TryReuseTermResult() { Success = false, UpdatedParser = parser };
if (modelTerm.Id == Guid.Empty) return new TryReuseTermResult() { Success = false, UpdatedParser = parser };

// Since we're reusing terms ensure the previous terms are committed
termStore.CommitAll();
context.ExecuteQueryRetry();

// Try to retrieve a matching term from the website also marked from re-use.
var taxonomySession = TaxonomySession.GetTaxonomySession(context);
context.Load(taxonomySession);
context.ExecuteQueryRetry();

if (taxonomySession.ServerObjectIsNull())
{
return new TryReuseTermResult() { Success = false, UpdatedParser = parser };
}
//// Try to retrieve a matching term from the website also marked from re-use.
//var taxonomySession = TaxonomySession.GetTaxonomySession(context);
//context.Load(taxonomySession);
//context.ExecuteQueryRetry();

var freshTermStore = taxonomySession.GetDefaultKeywordsTermStore();
Term preExistingTerm = freshTermStore.GetTerm(modelTerm.Id);
//if (taxonomySession.ServerObjectIsNull())
//{
// return new TryReuseTermResult() { Success = false, UpdatedParser = parser };
//}
//
//var freshTermStore = taxonomySession.GetDefaultKeywordsTermStore();
Term preExistingTerm = termStore.GetTerm(modelTerm.Id);

try
{
Expand All @@ -504,6 +555,7 @@ internal static TryReuseTermResult TryReuseTerm(ClientContext context, Model.Ter
// If the matching term is not found, return false... we can't re-use just yet
if (preExistingTerm == null)
{
reusedTerms.Add(reusedTerm); // re-add it to the reusedTerms to process!
return new TryReuseTermResult() { Success = false, UpdatedParser = parser };
}
// if the matching term is found re-use, create child terms, and return true
Expand Down Expand Up @@ -546,17 +598,34 @@ internal static TryReuseTermResult TryReuseTerm(ClientContext context, Model.Ter

termStore.CommitAll();
context.Load(createdTerm);
context.ExecuteQueryRetry();
try
{
context.ExecuteQueryRetry();
}
catch (ServerException ex)
{
// Handle cases where terms were reused and then deprecated at the source which will generate this server exception at the target!
// We'll ignore it and inform the user so that he change the source if needed!
// Maybe Server Exceptions should be included/handled in the Microsoft.SharePoint.Client.ClientContextExtensions.ExecuteQueryRetry retry logic (currently it's just throwing the exception)
if (ex.Message.Contains("Reusing a deprecated Term is disallowed"))
{
Log.Info(Constants.LOGGING_SOURCE, "Attempting to reuse the deprecated term: '{0}:{1}' which is not allowed! We'll ignore it! Please change the source if needed ...", preExistingTerm.Id, preExistingTerm.PathOfTerm);
}
else // pass trought all other exceptions!
{
throw;
}
}

// Create any child terms
parser = CreateChildTerms(context, modelTerm, createdTerm, termStore, parser, scope);
parser = CreateChildTerms(context, modelTerm, createdTerm, reusedTerms, termStore, parser, scope);

// Return true, because our TryReuseTerm attempt succeeded!
return new TryReuseTermResult() { Success = true, UpdatedParser = parser };
}
}

private static TokenParser CheckChildTerms(ClientContext context, Model.Term modelTerm, Term parentTerm, TermStore termStore, TokenParser parser, PnPMonitoredScope scope)
private static TokenParser CheckChildTerms(ClientContext context, Model.Term modelTerm, Term parentTerm, List<ReusedTerm> reusedTerms, TermStore termStore, TokenParser parser, PnPMonitoredScope scope)
{
if (modelTerm.Terms.Any())
{
Expand All @@ -583,6 +652,7 @@ private static TokenParser CheckChildTerms(ClientContext context, Model.Term mod
{
childTerm.Id = returnTuple.Item1;
parser = returnTuple.Item2;
reusedTerms.AddRange(returnTuple.Item3); // add any potentially existing reused terms to the list to be processed later!
}
}
else
Expand All @@ -597,7 +667,7 @@ private static TokenParser CheckChildTerms(ClientContext context, Model.Term mod

if (term != null)
{
parser = CheckChildTerms(context, childTerm, term, termStore, parser, scope);
parser = CheckChildTerms(context, childTerm, term, reusedTerms, termStore, parser, scope);
}
}
else
Expand All @@ -607,6 +677,7 @@ private static TokenParser CheckChildTerms(ClientContext context, Model.Term mod
{
childTerm.Id = returnTuple.Item1;
parser = returnTuple.Item2;
reusedTerms.AddRange(returnTuple.Item3); // add any potentially existing reused terms to the list to be processed later!
}
}
}
Expand Down