I sent an email to SW API Support on this problem and below is the code that was sent back. It appears to work every time but I have a lot to learn on how to find some of the commands that were used.
'------------------------------------------------
Option Explicit
Public Enum swMateType_e
swMateCOINCIDENT = 0
swMateCONCENTRIC = 1
swMatePERPENDICULAR = 2
swMatePARALLEL = 3
swMateTANGENT = 4
swMateDISTANCE = 5
swMateANGLE = 6
swMateUNKNOWN = 7
swMateSYMMETRIC = 8
swMateCAMFOLLOWER = 9
swMateGEAR = 10
End Enum
Public Enum swMateAlign_e
swMateAlignALIGNED = 0
swMateAlignANTI_ALIGNED = 1
swMateAlignCLOSEST = 2
End Enum
Public Enum swMateEntity2ReferenceType_e
swMateEntity2ReferenceType_Point = 0
swMateEntity2ReferenceType_Line = 1
swMateEntity2ReferenceType_Circle = 2
swMateEntity2ReferenceType_Plane = 3
swMateEntity2ReferenceType_Cylinder = 4
swMateEntity2ReferenceType_Sphere = 5
swMateEntity2ReferenceType_Set = 6
swMateEntity2ReferenceType_Cone = 7
swMateEntity2ReferenceType_SweptSurface = 8
swMateEntity2ReferenceType_MultipleSurface = 9
swMateEntity2ReferenceType_GenSurface = 10
swMateEntity2ReferenceType_Ellipse = 11
swMateEntity2ReferenceType_GeneralCurve = 12
swMateEntity2ReferenceType_UNKNOWN = 13
End Enum
Public Enum swAddMateError_e
swAddMateError_ErrorUknown = 0
swAddMateError_NoError = 1
swAddMateError_IncorrectMateType = 2
swAddMateError_IncorrectAlignment = 3
swAddMateError_IncorrectSelections = 4
swAddMateError_OverDefinedAssembly = 5
End Enum
Function SelectMateEntity _
( _
swApp As SldWorks.SldWorks, _
swModel As SldWorks.ModelDoc2, _
swMateEnt As SldWorks.MateEntity2, _
nMark As Long _
) As Boolean
Dim swEnt As SldWorks.entity
Dim swSelMgr As SldWorks.SelectionMgr
Dim swSelData As SldWorks.SelectData
Dim bRet As Boolean
Select Case swMateEnt.ReferenceType
Case swMateEntity2ReferenceType_Point, _
swMateEntity2ReferenceType_Line, _
swMateEntity2ReferenceType_Circle, _
swMateEntity2ReferenceType_Plane, _
swMateEntity2ReferenceType_Cylinder, _
swMateEntity2ReferenceType_Sphere, _
swMateEntity2ReferenceType_Cone, _
swMateEntity2ReferenceType_SweptSurface
Set swSelMgr = swModel.SelectionManager
Set swSelData = swSelMgr.CreateSelectData
Set swEnt = swMateEnt.reference: Debug.Assert Not swEnt Is Nothing
swSelData.mark = nMark
bRet = swEnt.Select4(True, swSelData): Debug.Assert bRet
SelectMateEntity = bRet
Exit Function
Case swMateEntity2ReferenceType_Set, _
swMateEntity2ReferenceType_MultipleSurface, _
swMateEntity2ReferenceType_GenSurface, _
swMateEntity2ReferenceType_Ellipse, _
swMateEntity2ReferenceType_GeneralCurve, _
swMateEntity2ReferenceType_UNKNOWN
Debug.Assert False
Case Else
Debug.Assert False
End Select
SelectMateEntity = False
End Function
Sub main()
Dim swApp As SldWorks.SldWorks
Dim swModel As SldWorks.ModelDoc2
Dim swAssy As SldWorks.AssemblyDoc
Dim swSelMgr As SldWorks.SelectionMgr
Dim swFeat As SldWorks.feature
Dim swMate As SldWorks.Mate2
Dim swDispDim As SldWorks.DisplayDimension
Dim swDim As SldWorks.Dimension
Dim sVarType As String
Dim nVarFactor As Double
Dim nMateDist As Double
Dim nNumMateEnt As Long
Dim swMateEnt() As SldWorks.MateEntity2
Dim vMateEntPar As Variant
Dim swComp As SldWorks.Component2
Dim nNewMateAlign As Long
Dim nRetVal As Long
Dim i As Long
Dim bRet As Boolean
Set swApp = Application.SldWorks
Set swModel = swApp.ActiveDoc
Set swAssy = swModel
Set swSelMgr = swModel.SelectionManager
Set swFeat = swSelMgr.GetSelectedObject5(1)
Set swMate = swFeat.GetSpecificFeature2
Set swDispDim = swMate.DisplayDimension
Debug.Print "File = " & swModel.GetPathName
Debug.Print " " & swFeat.Name
Debug.Print " Type = " & swMate.Type
Debug.Print " Alignment = " & swMate.alignment
Debug.Print " CanBeFlipped = " & swMate.CanBeFlipped
Select Case swMate.Type
Case swMateANGLE
sVarType = " deg"
nVarFactor = 57.3
Case swMateDISTANCE
sVarType = " mm"
nVarFactor = 1000#
Case swMateGEAR
sVarType = " ratio"
nVarFactor = 1#
End Select
If swMateANGLE = swMate.Type Or swMateDISTANCE = swMate.Type Then
Debug.Print " MaxVar = " & swMate.MaximumVariation * nVarFactor & sVarType
Debug.Print " MinVar = " & swMate.MinimumVariation * nVarFactor & sVarType
End If
If Not swDispDim Is Nothing Then
Set swDim = swDispDim.GetDimension
nMateDist = swDim.GetSystemValue2(""
' broken:
' SPR 200810 - "Mate2:

isplayDimension returns incorrect value for a swMateGEAR mate if first ratio is not unity"
Debug.Print " Dim Value = " & nMateDist * nVarFactor & sVarType
End If
nNumMateEnt = swMate.GetMateEntityCount
ReDim swMateEnt(nNumMateEnt)
For i = 0 To nNumMateEnt - 1
Set swMateEnt(i) = swMate.MateEntity(i)
Set swComp = swMateEnt(i).ReferenceComponent
vMateEntPar = swMateEnt(i).EntityParams
Debug.Print " RefType(" & i & "

= " & swMateEnt(i).ReferenceType
Debug.Print " Component = " & swComp.Name2 & " (" & swComp.ReferencedConfiguration & "

--> " & swComp.GetPathName
Debug.Print " Point = (" & vMateEntPar(0) * 1000# & ", " & vMateEntPar(1) * 1000# & ", " & vMateEntPar(2) * 1000# & "

mm"
Debug.Print " Vector = (" & vMateEntPar(3) & ", " & vMateEntPar(4) & ", " & vMateEntPar(5) & "

"
Debug.Print " Radius 1 = " & vMateEntPar(6) * 1000# & " mm"
Debug.Print " Radius 2 = " & vMateEntPar(7) * 1000# & " mm"
Next i
Select Case swMate.Type
' cannot change alignment on these mate types
Case swMateGEAR
Exit Sub
End Select
If swMateAlignALIGNED = swMate.alignment Then
nNewMateAlign = swMateAlignANTI_ALIGNED
Else
If swMateAlignANTI_ALIGNED = swMate.alignment Then
nNewMateAlign = swMateAlignALIGNED
Else
' closest alignment, so changing alignment does not make sense
Debug.Assert swMateAlignCLOSEST = swMate.alignment
Exit Sub
End If
End If
swModel.ClearSelection2 True
For i = 0 To nNumMateEnt - 1
bRet = SelectMateEntity(swApp, swModel, swMateEnt(i), 1): Debug.Assert bRet
Next i
' AssemblyDoc::EditMate2 requires mate feature to be last selected object
bRet = swFeat.Select2(True, 0): Debug.Assert bRet
' broken:
' SPR 200809 - "AssemblyDoc::EditMate2 returns swAddMateError_ErrorUknown when successful"
swAssy.EditMate2 _
swMate.Type, _
nNewMateAlign, _
True, _
nMateDist, _
swMate.MaximumVariation, _
swMate.MinimumVariation, _
0#, _
0#, _
nMateDist, _
swMate.MaximumVariation, _
swMate.MinimumVariation, nRetVal
' do not assert since may overdefine assy or other error
'Debug.Assert swAddMateError_NoError = nRetVal
' do not assert since assy may have rebuild errors due to changing mate alignment
bRet = swModel.EditRebuild3
'Debug.Assert bRet
End Sub
'------------------------------------------------