A vCenter Single Sign-On endpoint certificate validation error has occurred.
Resolution
Ensure that the endpoint service registrations in vmdir match their corresponding machine SSL certificates in VECS. For more information, see Knowledge Base article KB 2121701.
I could not find the file to fix the issue, however eventually did find the file however not on the Broadcom website 🙁 however it did fix the error i was having on the upgrade.
Use the code below to create the file ls_ssltrust_fixer.py. Winscp that to the VCSA appliance
Then run the commands python ls_ssltrust_fixer.py -f scan & python ls_ssltrust_fixer.py -f fix
#!/usr/bin/env python
'''
Program: ls_ssltrust_fixer
Attempt to automate https://kb.vmware.com/s/article/2121701
with intentions to scan against certificate mismatch on service registrations and fixing the mismatch
Scope: Scan for mismatch, fix the mismatch based on scan result as second step
Authors: Jishnu Surendran Thankamani (jishnut@vmware.com), Ramprasad K.S. (ramprasad@vmware.com)
Copyright: 2017 Vmware Inc
'''
import lstoolutil
import ssl
import socket
import re
import hashlib
import base64
import logging
import os,errno
import argparse
import getpass
import sys
import subprocess
ConnectFailure_ct=0
ConnectFailure_nodes=[]
certcache = {}
logger = None
logdir=os.environ["VMWARE_LOG_DIR"]+ os.path.sep +"ls_ssltrust_fixer" + os.path.sep
#Move lstool_communicate function here to control logging
def lstoolcommunicate(argv, stdout=subprocess.PIPE):
"""
Lookup service client tool
"""
log4jcfile = logdir + 'log4j.conf'
with open(log4jcfile, "w") as log4jcfile_fh:
log4jcfile_fh.write("log4j.rootLogger=OFF")
java = lstoolutil._get_java()
javacpath = lstoolutil._get_classpath()
javasec = lstoolutil._get_java_security_properties()
cmd = [java,
"-Djava.security.properties=%s" % javasec,
"-cp",
javacpath,
"-Dlog4j.configuration=file:%s" % log4jcfile]
cmd.append("com.vmware.vim.lookup.client.tool.LsTool")
cmd += argv
process = subprocess.Popen(cmd, stdout=stdout)
stdout, _ = process.communicate(None)
return process.returncode, stdout
def right_psc(id,psctosite,idtosite, psc_blacklist):
thesite = None
for site in psctosite:
if ((psctosite[site] == idtosite.get(id)) and (site not in psc_blacklist)):
thesite = site
break
return thesite
def read_topology(lsout):
psctosite={}
idtosite={}
id = site = ""
for line in lsout.splitlines():
if "Service ID" in line:
(dummy, id) = line.split(': ', 1)
elif "Site ID" in line:
(dummy, site) = line.split(': ', 1)
idtosite[id] = site
elif (("URL" in line) and ("sso-adminserver" in line)):
(dummy, url) = line.split(': ', 1)
url = url.split('//', 2)[1].split('/')[0].split(':')[0]
psctosite[url] = site
return(psctosite,idtosite)
def _findFirstMatch(lines, pat):
idx = 0
for line in lines:
if re.match(pat, line):
return (line, idx)
break
idx = idx + 1
return (None, -1)
def _modify_ep_certs(oldspec, newspec, newCert):
update_ct = 0
ssltrust_ct = 0
oldlines = []
newlines = []
with open(oldspec,"r") as oldspec_fh:
lines = oldspec_fh.read().splitlines()
for line in lines:
if (line.find('ssltrust') == -1):
newlines.append(line)
else:
(key,oldcert) = line.split('=', 1)
newlines.append('{0}={1}'.format(key, newCert.replace('\\', '\\\\')))
update_ct = update_ct + 1
with open(newspec,"w") as newspec_fh:
newspec_fh.write("\n".join(newlines))
return update_ct
def parseopts(args):
'''Parse the command line options'''
parser = argparse.ArgumentParser()
required_set = parser.add_argument_group('required')
required_set.add_argument('-f', '--function', dest='function', help='scan or fix', default = '', required=True)
return parser.parse_args(args)
def get_cur_cert(spec):
global ConnectFailure_ct
global ConnectFailure_nodes
newcert = None
with open(spec, "r") as spec_fh:
for line in spec_fh.read().splitlines():
if "endpoint0.url" in line:
url = line.split('=', 1)[1].split('//', 2)[1].split('/')[0].split(':')[0]
if (url=="localhost"):
endpointurl="endpoint1.url"
else:
endpointurl="endpoint0.url"
with open(spec, "r") as spec_fh:
for line in spec_fh.read().splitlines():
if endpointurl in line:
url = line.split('=', 1)[1].split('//', 2)[1].split('/')[0].split(':')[0]
print("FQDN used to retrieve current certificate:"+url)
if (url=="localhost"):
print("First and second end points found to be using localhost as fqdn - manually new cert and update .NewCert file before fix")
break
port = 443
endpoint = "{0}:{1}".format(url, port)
if (endpoint in certcache):
logger.debug("Using cached certificate for %s", endpoint)
newcert = certcache[endpoint]
else:
logger.debug("Retreiving certificate for %s", endpoint)
conn = None
newcert = None
try:
conn = socket.create_connection((url, port), timeout=5)
sock = ssl.wrap_socket(conn)
current_cert = sock.getpeercert(True)
newcert = (ssl.DER_cert_to_PEM_cert(current_cert))
certcache[endpoint] = newcert
except Exception:
logger.error("**Failed to get in use certificate from node %s:%s**", url,port)
ConnectFailure_ct = ConnectFailure_ct + 1
if url not in ConnectFailure_nodes: ConnectFailure_nodes.append(url)
finally:
if conn is not None:
socket.SHUT_RDWR
conn.close()
break
return newcert
def read_pem_cert(cert):
pat = "-----BEGIN CERTIFICATE-----([a-zA-Z0-9/+=\r\n]+)-----END CERTIFICATE-----"
m = re.match(pat, cert)
if not m:
raise Exception("Failed to parse cert")
return m.group(1).replace("\n", "").replace("\r", "")
def _setupLogging():
try:
os.makedirs(logdir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
loghandle = logging.getLogger('ls_ssltrus_fixer')
fileformatter = logging.Formatter('%(asctime)s %(name)s %(levelname)s %(message)s')
consoleformatter = logging.Formatter('%(message)s')
loghandle.setLevel(logging.DEBUG)
fh = logging.FileHandler(logdir+os.path.sep+'ls_ssltrust_fixer.log')
fh.setLevel(logging.DEBUG)
fh.setFormatter(fileformatter)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
ch.setFormatter(consoleformatter)
loghandle.addHandler(fh)
loghandle.addHandler(ch)
return loghandle
def get_filename_from_id(id):
temp = id
specfile = "{0}{1}".format(logdir, id.decode('utf-8').replace(":","%"))
certfile = "{0}.newcert".format(specfile)
return(specfile, certfile)
def _doScan():
mismatchedIDs=[]
matchedIDs=[]
lsUrl="https://localhost/lookupservice/sdk"
mismatchlistinput = "{0}mismatchIDs".format(logdir)
logger.info("Scan Phase1: Getting service IDs")
rc, ids = lstoolcommunicate(["list","--no-check-cert","--url",lsUrl,"--id-only"])
if (rc != 0):
raise Exception("'lstool get' failed: %d" % rc)
ids = ids.splitlines()
logger.info("Found %d service IDs", len(ids) - 1)
logger.info("Scan Phase2: Getting spec and verifying certicate/trust")
for id in ids:
if not id:
continue
if "_com" in str(id):
print("skipping validation as external solution on:"+str(id))
continue
logger.info("Processing ID: %s", id)
logger.debug("Calling get for ID: %s", id)
rc, oldSpec = lstoolcommunicate(["get","--no-check-cert","--url",lsUrl,"--id",id.decode('utf-8'),"--as-spec",])
if (rc != 0):
logger.error("'lstool get' failed for ID: %s", id)
continue
(specfile, certfile) = get_filename_from_id(id)
logger.debug("Creating spec file %s", specfile)
with open(specfile,"w") as specfile_fh:
specfile_fh.write(oldSpec.decode('utf-8'))
logger.debug("Created spec file %s", specfile)
logger.debug("Creating cert file %s", certfile)
logger.debug("Getting certificate for ID: %s", id)
cert=get_cur_cert(specfile)
if(cert):
with open(certfile, "w") as certfile_fh:
certfile_fh.write(cert)
logger.debug("Created cert file %s", certfile)
newcert_parsed=read_pem_cert(cert)
oldcert = ""
for line in oldSpec.splitlines():
if "ssltrust0".encode('utf-8') in line:
(key, oldcert) = line.decode('utf-8').split("=", 1)
break
o = hashlib.sha1()
o.update(base64.decodestring(oldcert.encode('utf-8')))
n = hashlib.sha1()
n.update(base64.decodestring(newcert_parsed.encode('utf-8')))
othumb = o.hexdigest().lower()
nthumb = n.hexdigest().lower()
logger.debug("ID: %s Old thumbprint: %s new thumbprint %s", id, othumb, nthumb)
if (othumb == nthumb):
matchedIDs.append(id)
logger.debug("Trust matches the current certificate. Added %s to matchedIDs", id)
else:
mismatchedIDs.append(id)
logger.debug("***Trust DOES NOT match the current certificate***. Added %s to mismatchedIDs", id)
logger.info("")
if len(matchedIDs) !=0:
for id in matchedIDs:
(specfile, certfile) = get_filename_from_id(id)
logger.debug("Matched: id: %s spec: %s cert in use: %s", id, specfile, certfile)
if len(mismatchedIDs) !=0:
logger.warn("***WARNING*** %d Mismatched ID(s) found", len(mismatchedIDs))
with open(mismatchlistinput,"w") as mismatchIDstoFile:
mismatchIDstoFile.write(b"\n".join(mismatchedIDs).decode())
logger.info("Written mismatched IDs to %s",mismatchlistinput )
logger.info("List of registrations with cert mismatch")
logger.info("****************************************")
for id in mismatchedIDs:
(specfile, certfile) = get_filename_from_id(id)
logger.info("ID: %s\n spec: %s\n cert in use: %s\n", id, specfile, certfile)
logger.warn("Please DOUBLE CHECK the detection before running 'fix'")
logger.warn("NOTE: Partial upgrade state of 5.5 to 6.x is unsupported for this tool- 5.5 web client registration might change")
logger.info("")
else:
mismatchIDstoFile = open(mismatchlistinput,"w")
mismatchIDstoFile.close()
if ConnectFailure_ct!=0:
logger.info("")
logger.info("***WARNING*** %s ID(s) skipped comparison due to connect failure, ignore if node is dead, use KB:2121701 for manual update procedure. Note: Port 443 is hardcoded", str(ConnectFailure_ct))
logger.info("List of node(s) with connect failure")
logger.info("************************************")
for entry in ConnectFailure_nodes:
logger.info(entry)
def _doFix():
update_idct=0
updated_endpoint = 0
psc_Blacklist = []
lstooloutfile = logdir + 'lstooloutput'
lsUrl="https://localhost/lookupservice/sdk"
mismatchlistfile = "{0}mismatchIDs".format(logdir)
logger.info("Fix phase 1: Reading IDs with incorrect certificate from scan results")
logger.info("Using mismatch ID list from: %s", mismatchlistfile)
try:
mismatchlist_fh=open(mismatchlistfile,"r")
mismatchlist=mismatchlist_fh.read().splitlines()
mismatchlist_fh.close()
except:
logger.error("Mismatch ID list file does not exist, Please run tool with 'scan' function")
return
if not mismatchlist:
logger.info("Mismatch ID list file is empty, no registrations to fix")
return
user=input("SSO administrator user (Default:Administrator@vsphere.local):") or "Administrator@vsphere.local"
passwd=getpass.getpass("Password for "+ user + ":")
logger.info("Fix phase 2: Collecting site topology information")
rc, lsoutput = lstoolcommunicate(["list","--no-check-cert","--url",lsUrl])
if (rc != 0):
raise Exception("'lstool get' failed: %d" % rc)
with open(lstooloutfile, "w") as lstool_fh:
print(type(lsoutput))
lstool_fh.write(lsoutput.decode('utf-8'))
psctosite,idtosite = read_topology(lsoutput.decode('utf-8'))
logger.info("Fix Phase 3: creating new spec file with new ssltrust values and register")
for id in mismatchlist:
logger.info("\nFixing ID: %s",id)
(specfile, certfile) = get_filename_from_id(id.encode('utf-8'))
newspecfile = specfile + ".newspec"
newcert_parsed = cert = None
#cert=get_cur_cert(specfile) #Use this for production
with open(certfile, "r") as certfile_fh: #Debug only
cert = certfile_fh.read()
newcert_parsed=read_pem_cert(cert)
if(cert):
updated=_modify_ep_certs(specfile, newspecfile, newcert_parsed)
logger.info("Updated %d End points with new cert for ID: %s", updated, id)
if updated != 0:
site = right_psc(id, psctosite, idtosite, psc_Blacklist)
rc = -1
while (site):
lsUrl="https://"+site+"/lookupservice/sdk"
logger.info("Re-registering ID: %s using lsURL: %s", id, lsUrl)
try:
rc, _ = lstoolcommunicate(["reregister", "--no-check-cert",
"--url", lsUrl,
"--id", id,
"--spec", newspecfile,
"--user", user,
"--password", passwd,
])
if rc==0:
update_idct = update_idct + 1
site = None
except:
pass
if (rc != 0):
psc_Blacklist.append(site)
logger.info("Blacklisted PSC at %s as connecting failed", site)
site = right_psc(id, psctosite, idtosite, psc_Blacklist)
if (rc != 0):
logger.error("'lstool reregister' failed for ID: %s with error %d", id, rc)
else:
updated_endpoint = updated_endpoint + updated
logger.info("Fixing ID: %s completed\n",id)
logger.info("*** %d endpoints for %d service IDs updated with current cetificates and trust ***", updated_endpoint, update_idct)
def main():
global ConnectFailure_ct
global ConnectFailure_nodes
opts = parseopts(sys.argv[1:])
if opts.function=="scan":
logger.info("Running function 'scan'")
_doScan()
logger.info("Completed running function 'scan'")
elif opts.function=="fix":
logger.info("Running function 'fix'")
_doFix()
logger.info("Completed running function 'fix'")
else:
logger.error("Unknown Function '%s'. Choose scan/fix", opts.function)
sys.exit()
if __name__ == '__main__':
logger = _setupLogging()
main()
Leave a Reply