Three Tangent Arc/Circle

I'm pretty sure that the points you pass in to the arc builder just need to be somewhat close. I think that they are used to help calculate the result. That said, I'd try passing in the mid point on each of the three curves. Something like the following (the journal won't run for you as-is because it is tied to the objects in my test file, but you can see how the mid-point of the curve is found).

Option Strict Off
Imports System
Imports NXOpen

Module NXJournal
Sub Main (ByVal args() As String)

Dim theSession As Session = Session.GetSession()
Dim workPart As Part = theSession.Parts.Work

Dim displayPart As Part = theSession.Parts.Display

' ----------------------------------------------
' Menu: Insert->Curve->Lines and Arcs->Arc Tangent-Tangent-Tangent...
' ----------------------------------------------
Dim markId1 As Session.UndoMarkId
markId1 = theSession.SetUndoMark(Session.MarkVisibility.Visible, "Arc Tangent-Tangent-Tangent")

Dim nullFeatures_AssociativeArc As Features.AssociativeArc = Nothing

Dim associativeArcBuilder1 As Features.AssociativeArcBuilder
associativeArcBuilder1 = workPart.BaseFeatures.CreateAssociativeArcBuilder(nullFeatures_AssociativeArc)

associativeArcBuilder1.Limits.StartLimit.LimitOption = GeometricUtilities.CurveExtendData.LimitOptions.AtPoint

associativeArcBuilder1.Limits.EndLimit.LimitOption = GeometricUtilities.CurveExtendData.LimitOptions.AtPoint

associativeArcBuilder1.StartPointOptions = Features.AssociativeArcBuilder.StartOption.Tangent

Dim studioSpline1 As Features.StudioSpline = CType(workPart.Features.FindObject("SPLINE(1)"), Features.StudioSpline)

Dim spline1 As Spline = CType(studioSpline1.FindObject("CURVE 1"), Spline)

associativeArcBuilder1.StartTangent.Value = spline1

associativeArcBuilder1.EndPointOptions = Features.AssociativeArcBuilder.EndOption.Tangent

Dim studioSpline2 As Features.StudioSpline = CType(workPart.Features.FindObject("SPLINE(2)"), Features.StudioSpline)

Dim spline2 As Spline = CType(studioSpline2.FindObject("CURVE 1"), Spline)

associativeArcBuilder1.EndTangent.Value = spline2

associativeArcBuilder1.MidPointOptions = Features.AssociativeArcBuilder.MidOption.Tangent

associativeArcBuilder1.Limits.FullCircle = False

associativeArcBuilder1.Type = Features.AssociativeArcBuilder.Types.ThreePointArc

Dim arc1 As Arc = CType(workPart.Arcs.FindObject("ENTITY 5 1 1"), Arc)

associativeArcBuilder1.MidTangent.Value = arc1

Dim nullView As View = Nothing

Dim scalar1 As Scalar
scalar1 = workPart.Scalars.CreateScalar(50.0, Scalar.DimensionalityType.None, SmartObject.UpdateOption.AfterModeling)
Dim point1 As Point
point1 = workPart.Points.CreatePoint(spline1, scalar1, PointCollection.PointOnCurveLocationOption.PercentParameter, SmartObject.UpdateOption.AfterModeling)
workPart.Points.RemoveParameters(point1)

associativeArcBuilder1.StartTangent.SetValue(spline1, nullView, point1.Coordinates)

Dim scalar2 As Scalar
scalar2 = workPart.Scalars.CreateScalar(50.0, Scalar.DimensionalityType.None, SmartObject.UpdateOption.AfterModeling)
Dim point2 As Point
point2 = workPart.Points.CreatePoint(spline2, scalar2, PointCollection.PointOnCurveLocationOption.PercentParameter, SmartObject.UpdateOption.AfterModeling)
workPart.Points.RemoveParameters(point2)

associativeArcBuilder1.EndTangent.SetValue(spline2, nullView, point2.Coordinates)

Dim scalar3 As Scalar
scalar3 = workPart.Scalars.CreateScalar(50.0, Scalar.DimensionalityType.None, SmartObject.UpdateOption.AfterModeling)
Dim point3 As Point
point3 = workPart.Points.CreatePoint(arc1, scalar3, PointCollection.PointOnCurveLocationOption.PercentParameter, SmartObject.UpdateOption.AfterModeling)
workPart.Points.RemoveParameters(point3)

associativeArcBuilder1.MidTangent.SetValue(arc1, nullView, point3.Coordinates)

associativeArcBuilder1.ZonePoint = point3.Coordinates

associativeArcBuilder1.Associative = True

Dim nXObject1 As NXObject
nXObject1 = associativeArcBuilder1.Commit()

associativeArcBuilder1.Destroy()

End Sub
End Module