SOAP service with WS-Security (WSS with X.509 certificates)

[Davide Angeli 44]

Hi all,

 

I'm really becaming crazy trying to do this with Delphi... I've to interact with a SOAP service that implements WSS security with X.509 certificate so:

• they give me the WSDL file 
• they give me a Java Keystore (jks file) containg a certificate, a public key and a private key password protected
• the SOAP request must be signed and crypted with the jks data
• the SOAP response must be decrypted and signed verified

 

a request like this one:

<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:fut="....." xmlns:ass="...." xmlns:com="....">
   <soapenv:Header/>
   <soapenv:Body>
      <fut:FutureConvocationsRequest>
         <associativeServicesGenericRequest>
            <ass:locale>
               <com:language>it</com:language>
            	<com:country>IT</com:country>
            </ass:locale>
            <ass:consumer>Me</ass:consumer>
            <ass:associationCode>aaaa</ass:associationCode>
            <ass:sectionCode>bbbbb</ass:sectionCode>
            <ass:startDate></ass:startDate>
            <ass:endDate></ass:endDate>
         </associativeServicesGenericRequest>
      </fut:FutureConvocationsRequest>
   </soapenv:Body>
</soapenv:Envelope>

 

must become like this one (grabbed it from SoapUI logs that interact well with this service):

<soapenv:Envelope xmlns:ass="..." xmlns:com="..." xmlns:fut="..." xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/">
   <soapenv:Header>
      <wsse:Security xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd" xmlns:wsu="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd">
         <xenc:EncryptedKey Id="EK-6085B5AE3B1746C574167481306190992" xmlns:xenc="http://www.w3.org/2001/04/xmlenc#">
            <xenc:EncryptionMethod Algorithm="http://www.w3.org/2001/04/xmlenc#rsa-oaep-mgf1p"/>
            <ds:KeyInfo xmlns:ds="http://www.w3.org/2000/09/xmldsig#">
               <wsse:SecurityTokenReference>
                  <ds:X509Data>
                     <ds:X509IssuerSerial>
                        <ds:X509IssuerName>....</ds:X509IssuerName>
                        <ds:X509SerialNumber>....</ds:X509SerialNumber>
                     </ds:X509IssuerSerial>
                  </ds:X509Data>
               </wsse:SecurityTokenReference>
            </ds:KeyInfo>
            <xenc:CipherData>
               <xenc:CipherValue>....</xenc:CipherValue>
            </xenc:CipherData>
            <xenc:ReferenceList>
               <xenc:DataReference URI="#ED-6085B5AE3B1746C574167481306190993"/>
            </xenc:ReferenceList>
         </xenc:EncryptedKey>
         <wsse:BinarySecurityToken EncodingType="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-soap-message-security-1.0#Base64Binary" ValueType="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-x509-token-profile-1.0#X509v3" wsu:Id="X509-6085B5AE3B1746C574167481306190587">....</wsse:BinarySecurityToken>
         <wsu:Timestamp wsu:Id="TS-6085B5AE3B1746C574167481306190486">
            <wsu:Created>2023-01-27T09:51:01Z</wsu:Created>
            <wsu:Expires>2023-01-27T09:56:01Z</wsu:Expires>
         </wsu:Timestamp>
         <ds:Signature Id="SIG-6085B5AE3B1746C574167481306190591" xmlns:ds="http://www.w3.org/2000/09/xmldsig#">
            <ds:SignedInfo>
               <ds:CanonicalizationMethod Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#">
                  <ec:InclusiveNamespaces PrefixList="ass com fut soapenv" xmlns:ec="http://www.w3.org/2001/10/xml-exc-c14n#"/>
               </ds:CanonicalizationMethod>
               <ds:SignatureMethod Algorithm="http://www.w3.org/2000/09/xmldsig#rsa-sha1"/>
               <ds:Reference URI="#id-6085B5AE3B1746C574167481306190590">
                  <ds:Transforms>
                     <ds:Transform Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#">
                        <ec:InclusiveNamespaces PrefixList="ass com fut" xmlns:ec="http://www.w3.org/2001/10/xml-exc-c14n#"/>
                     </ds:Transform>
                  </ds:Transforms>
                  <ds:DigestMethod Algorithm="http://www.w3.org/2000/09/xmldsig#sha1"/>
                  <ds:DigestValue>7APwAAzaHndcDEHSBYlA6b/rihY=</ds:DigestValue>
               </ds:Reference>
            </ds:SignedInfo>
            <ds:SignatureValue>....</ds:SignatureValue>
            <ds:KeyInfo Id="KI-6085B5AE3B1746C574167481306190588">
               <wsse:SecurityTokenReference wsu:Id="STR-6085B5AE3B1746C574167481306190589">
                  <wsse:Reference URI="#X509-6085B5AE3B1746C574167481306190587" ValueType="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-x509-token-profile-1.0#X509v3"/>
               </wsse:SecurityTokenReference>
            </ds:KeyInfo>
         </ds:Signature>
      </wsse:Security>
   </soapenv:Header>
   <soapenv:Body wsu:Id="id-6085B5AE3B1746C574167481306190590" xmlns:wsu="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd">
      <xenc:EncryptedData Id="ED-6085B5AE3B1746C574167481306190993" Type="http://www.w3.org/2001/04/xmlenc#Content" xmlns:xenc="http://www.w3.org/2001/04/xmlenc#">
         <xenc:EncryptionMethod Algorithm="http://www.w3.org/2001/04/xmlenc#aes128-cbc"/>
         <ds:KeyInfo xmlns:ds="http://www.w3.org/2000/09/xmldsig#">
            <wsse:SecurityTokenReference wsse11:TokenType="http://docs.oasis-open.org/wss/oasis-wss-soap-message-security-1.1#EncryptedKey" xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd" xmlns:wsse11="http://docs.oasis-open.org/wss/oasis-wss-wssecurity-secext-1.1.xsd">
               <wsse:Reference URI="#EK-6085B5AE3B1746C574167481306190992"/>
            </wsse:SecurityTokenReference>
         </ds:KeyInfo>
         <xenc:CipherData>
            <xenc:CipherValue>....</xenc:CipherValue>
         </xenc:CipherData>
      </xenc:EncryptedData>
   </soapenv:Body>
</soapenv:Envelope>

 

I've used several SOAP service but never with this kind of security so til now I've always used default THTTPRio with WSDL importer. I don't know if in this case this is the best approach but I've imported the WSDL file in Delphi creating all SOAP objects to use with THTTPRIO . Then I've coded the THTTPRio.OnBeforeExecute to inject the necessary data and trasform the request as needed. I've a regular license of SecureBlackBox components so I can easily read the certificate data from Keystore and using the SOAPSigner + XMLEncryptor components (with some post processing to adjust some Secureblackbox issues), I'm capable to transform the request as needed but when I send it to the service I got always an error like "The signature or decryption was invalid" (Unfortunately I can't get quick answers from the service provider because it is a government agency and not very responsive...). I suppose that could be an encryption problem because the XML soap request just signed (not encrypted) seems ok (just tested with SecureBlackbox SOAPVerifier).

 

If there is here some SOAP Delphi guru my questions are:

- do you know if exists a good tool to verify if the request is well formed? If the encryption is well done? If the sign is correct?

- SecureBlackBox suite is very complex to configure... do you know if exists some other Delphi solution to obtain the result easier?

 

[mjustin 23]

Have you compared the requests from SoapUI with the requests sent from your code?

[Davide Angeli 44]

On 1/28/2023 at 9:55 AM, mjustin said:

Have you compared the requests from SoapUI with the requests sent from your code?

Yes, with SBB is not possibile to reach the same request without doing some further manipulation of the xml and it misses also the reference list for decryption. I'm working on that and another problem I think should be the order of doing operations: I'm doing crypt+sign but I'm investigating  if service aspect sign and then crypt.  I will do theese changes but I could not test because of server down for maintenance. I'll report the results.

Edited January 31, 2023 by Davide Angeli

[Davide Angeli 44]

I solved this thanks to the kind collaboration of the nSoftware assistance service (SecureBlackBox's developer). 

 

There were many challenges to solve to achieve the goal and very little can be found online. It is my understanding that this type of security is not used much in the Delphi environment. However I solved by combining several components provided with the SecurityBlackBox suite:

• TsbxCertificateStorage to read JKS file to reach public key (used to encrypt the request and verify the signed response) and private key (used to sign the request and decrypt the response)

• TsbxSOAPSigner to sign the SOAP request (to produce a WSSSignature)
• TsbxXMLEncryptor to embed in the request the encryption key (transported with RSA-OAEP sha1) and to crypt the SOAP request body (with AES128 algorithm)
• TsbxXMLDecryptor to decrypt the response (two items encrypted in the response: the sign and the body)
• TsbxSOAPVerifier to verifiy the sign in the response

I've injected this encryption/decryption stuff in THTTPRio events, somthing like this;

procedure TMultyAssociativeServiceWrapper.SOAPOnBeforeExecute(const MethodName: String; SOAPRequest: TStream);
begin
  // save the original request (created by THTTPRio) for debug purpose
  SOAPRequest.Seek(0,0);
  (SOAPRequest as TMemoryStream).SaveToFile(FFileRequestOriginal);

  // sign and encrypt the original request
  SBBSoapSignAndEncryptWithJKS(FFileRequestOriginal, FFileRequestToSend, FCertificateFileJKS, OnPasswordNeeded);
  
  // ... do some other stuff ?

  // update the outcoming request with the signed crypted version
  SOAPRequest.Seek(0,0);
  (SOAPRequest as TMemoryStream).LoadFromFile(FFileRequestToSend);
end;

procedure TMultyAssociativeServiceWrapper.SOAPOnAfterExecute(const MethodName: String; SOAPResponse: TStream);
begin
  // save te orginal response arrived from service for debug purpose
  SOAPResponse.Seek(0,0);
  (SOAPResponse as TMemoryStream).SaveToFile(FFileResponseOriginal);

  // decrypt and verify sign of response
  SBBSoapDecryptAndVerifyWithJKS(FFileResponseOriginal, FFileResponseDecoded, FCertificateFileJKS,  OnPasswordNeeded);
  
  //... do some other stuff?

  // update the incoming response with the decrypted data
  SOAPResponse.Seek(0,0);
  (SOAPResponse as TMemoryStream).LoadFromFile(FFileResponseDecoded);
end;

 

SBBSoapSignAndEncryptWithJKS does sign + encryption stuff.

SBBSoapDecryptAndVerifyWithJKS does decrypt and verify sign stuff.

 

I hope this can be helpful for someone else who runs into this need. If you need the code of the two SBB procedures described above please ask me.

 

I've used SecureBlackBox components but I think that same stuff could be reached also with ChilKat library. I did some tests with the demo library, it works at a lower level of SBB but there are several examples online that could be combined to reach the goal.

[juanluanaya 0]

Hi Davide, Could you help me out with the code of SBBSoapSignAndEncryptWithJKS?

 

Thank in advanced.

[Davide Angeli 44]

5 hours ago, juanluanaya said:

Hi Davide, Could you help me out with the code of SBBSoapSignAndEncryptWithJKS?

Hi, this is what is working for me. You'll probably need to adapt it because it is built based on my needs, but I hope it can be a good starting point for you.

 

// Returns 0 = ok (sign and encrypt is fine)
//         1 = failed open keystore to sign/encrypt
//         2 = failed to load sing/encryption certificate
function SBBSoapSignAndEncryptJKS(const InputFilename, OutputFilename, JKSFileName : String; OnPasswordNeeded : TPasswordNeededEvent; out Error : String; DebugSign : Boolean = FALSE) : Integer;
var
  XMLEncryptor : TsbxXMLEncryptor;
  SOAPSigner : TsbxSOAPSigner;
  NSItem : TsbxXMLNamespace;
  sCSerialNumber,sIssuer : String;

begin
  Result:=0;
  Error:='';

  // calculate some IDs and variables for debug purpose
  var BinarySecurityTokenID:='X509-'+FormatDateTime('yyyymmddhhnnsszzz',Now);
  var BodyReference:='ID-'+FormatDateTime('yyyymmddhhnnsszzz',Now);
  var EncryptedDataID:='ED-'+FormatDateTime('yyyymmddhhnnsszzz',Now);
  var EncryptedKeyID:='EK-'+FormatDateTime('yyyymmddhhnnsszzz',Now);

  // some temporary variables to handle debug filenames
  var BasePath:=ExtractFilePath(OutputFileName);
  var BaseFileName:=ExtractFileName(InputFileName);
  BaseFileName:=BaseFileName.SubString(0,BaseFileName.IndexOf('.'));

  XMLEncryptor := TsbxXMLEncryptor.Create(nil);
  SOAPSigner := TsbxSOAPSigner.Create(nil);
  try
    /// Step 1
    /// Signing body element
    var CertificateStorage := TsbxCertificateStorage.Create(Nil);
    try
      try
        CertificateStorage.OnPasswordNeeded:=OnPasswordNeeded;
        CertificateStorage.Open('java://'+JKSFileName);
        if CertificateStorage.Opened then begin
           CertificateStorage.Select('*',True,1);
           SOAPSigner.SigningCertificate:=CertificateStorage.SelectedCertificates[0];
        end else begin
           Result:=1; Error:='Failed to open keystore to sign';
           Exit;
        end;
      except
        on E: Exception do
        begin
          Result:=2; Error:='Failed to signing load certificate: ' + E.Message;
          Exit;
        end;
      end;
    finally
      FreeAndNil(CertificateStorage);
    end;

    SOAPSigner.InputFile := InputFilename;
    SOAPSigner.InputBytes := SOAPSigner.OutputBytes;

    SOAPSigner.NewSignature.SignatureType := sstWSSSignature;
    SOAPSigner.NewSignature.HashAlgorithm := 'SHA1';
    SOAPSigner.NewSignature.XAdES := false;
    SOAPSigner.SecurityHeaderIndex := -1;

    var k := SOAPSigner.AddBodyReference(BodyReference, false);
    SOAPSigner.References[k].InclusiveNamespacesPrefixList := 'xsd xsi';
    SOAPSigner.Config('KeyInfoID=KI-'+FormatDateTime('yyyymmddhhnnsszzz',Now));
    SOAPSigner.Config('SecurityTokenReferenceID=STR-'+FormatDateTime('yyyymmddhhnnsszzz',Now));
    SOAPSigner.Config('BinarySecurityTokenID='+BinarySecurityTokenID);
    SOAPSigner.Config('InclusiveNamespacesPrefixList=SOAP-ENV xsd xsi');
    SOAPSigner.Config('PrependCustomXML=<wsu:Timestamp xmlns:wsu="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd" wsu:Id="TS-'+FormatDateTime('yyyymmddhhnnsszzz',Now)+'">' +
                                         '<wsu:Created>'+DateToISO8601(Now,False)+'</wsu:Created>' +
                                         '<wsu:Expires>'+DateToISO8601(IncMinute(Now,50),False)+'</wsu:Expires>' +
                                        '</wsu:Timestamp>');
    SOAPSigner.Sign;

    // - just for debug purpose save the signed result
    if DebugSign then
       TFile.WriteAllBytes(BasePath+BaseFileName+'signed.xml', SOAPSigner.OutputBytes);

    ///  Step 2
    ///  Encrypting body element
    CertificateStorage := TsbxCertificateStorage.Create(Nil);
    try
      try
        CertificateStorage.OnPasswordNeeded:=OnPasswordNeeded;
        CertificateStorage.Open('java://'+JKSFileName);
        if CertificateStorage.Opened then begin
           XMLEncryptor.KeyEncryptionCertificate:=CertificateStorage.Certificates[1];
           sCSerialNumber:=TBytesSerialToInt64(XMLEncryptor.KeyEncryptionCertificate.SerialNumber).ToString;
           sIssuer:=XMLEncryptor.KeyEncryptionCertificate.IssuerRDN;

           var a:=sIssuer.Split(['/']);
           Delete(a,0,1);
           sIssuer:='';
           for var J:=High(a) downto Low(a) do begin
               if Not sIssuer.IsEmpty then sIssuer:=sIssuer+',';
               sIssuer:=sIssuer+a[J];
           end;
        end else begin
          Result:=1; Error:='Failed to open keystore to encrypt';
          Exit;
        end;
      except
        on E: Exception do
        begin
          Result:=2; Error:='Failed to load encrypt certificate: ' + E.Message;
          Exit;
        end;
      end;
    finally
      FreeAndNil(CertificateStorage);
    end;

    XMLEncryptor.InputBytes := SOAPSigner.OutputBytes;

    XMLEncryptor.XMLNode := '/SOAP-ENV:Envelope/SOAP-ENV:Body';
    XMLEncryptor.EncryptedDataType := cxedtContent;
    XMLEncryptor.EncryptionMethod := 'AES128';
    XMLEncryptor.EncryptKey := true;
    XMLEncryptor.KeyEncryptionType := cxetKeyTransport;
    XMLEncryptor.KeyTransportMethod := cxktRSAOAEP;

    XMLEncryptor.Config('EncryptedKeyID='+EncryptedKeyID);
    XMLEncryptor.Config('EncryptedDataID='+EncryptedDataID);
    XMLEncryptor.Config('EncryptedKeyXMLElement=/SOAP-ENV:Envelope/SOAP-ENV:Header/wsse:Security');
    XMLEncryptor.Config('KeyInfoCustomXML=<wsse:SecurityTokenReference wsse11:TokenType="http://docs.oasis-open.org/wss/oasis-wss-soap-message-security-1.1#EncryptedKey"'+
                                                                     ' xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd"' +
                                                                     ' xmlns:wsse11="http://docs.oasis-open.org/wss/oasis-wss-wssecurity-secext-1.1.xsd">' +
                                           '<wsse:Reference URI="#'+EncryptedKeyID+'"/>' +
                                          '</wsse:SecurityTokenReference>');

    XMLEncryptor.Config('EncryptedKeyInfoCustomXML=<wsse:SecurityTokenReference xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd">' +
                                                   '<ds:X509Data xmlns:ds="http://www.w3.org/2000/09/xmldsig#">' +
                                                    '<ds:X509IssuerSerial>' +
                                                     '<ds:X509IssuerName>'{'CN='}+sIssuer+'</ds:X509IssuerName>' +
                                                     '<ds:X509SerialNumber>'+sCSerialNumber+'</ds:X509SerialNumber>' +
                                                    '</ds:X509IssuerSerial>' +
                                                   '</ds:X509Data>' +
                                                  '</wsse:SecurityTokenReference>');

    XMLEncryptor.Encrypt;

    ///  Step 3
    SOAPSigner.InputBytes := XMLEncryptor.OutputBytes;
    SOAPSigner.InputFile := '';
    SOAPSigner.OutputFile := OutputFilename;
    SOAPSigner.Open;

    // for XPath expressions we need to define prefixes used
    // prefixes: SOAP-ENV, xenc, wsu, ds - are already known by the component
    // wsse prefix better to define explicitly, since it is SOAP version specific
    NSItem := TsbxXMLNamespace.Create;
    try
      NSItem.Prefix := 'wsse';
      NSItem.URI := 'http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd';
      SOAPSigner.XPathNamespaces.Add(NSItem);
    finally
      FreeAndNil(NSItem);
    end;

    SOAPSigner.SetInnerXML('/SOAP-ENV:Envelope/SOAP-ENV:Header/wsse:Security/xenc:EncryptedKey',
      SOAPSigner.GetInnerXML('/SOAP-ENV:Envelope/SOAP-ENV:Header/wsse:Security/xenc:EncryptedKey') +
       '<xenc:ReferenceList>' +
        '<xenc:DataReference URI="#'+EncryptedDataID+'"/>' +
       '</xenc:ReferenceList>');

    SOAPSigner.SetInnerXML('/SOAP-ENV:Envelope/SOAP-ENV:Header/wsse:Security',
      SOAPSigner.GetOuterXML('/SOAP-ENV:Envelope/SOAP-ENV:Header/wsse:Security/xenc:EncryptedKey') +
      SOAPSigner.GetOuterXML('/SOAP-ENV:Envelope/SOAP-ENV:Header/wsse:Security/wsse:BinarySecurityToken') +
      SOAPSigner.GetOuterXML('/SOAP-ENV:Envelope/SOAP-ENV:Header/wsse:Security/wsu:Timestamp') +
      SOAPSigner.GetOuterXML('/SOAP-ENV:Envelope/SOAP-ENV:Header/wsse:Security/ds:Signature')
    );

    SOAPSigner.Close(true);

  finally
    FreeAndNil(XMLEncryptor);
    FreeAndNil(SOAPSigner);
  end;
end;

 

[juanluanaya 0]

Thanks you for your help.