Authenticate from Curl into SharePoint Online with Modern Authentication

Code-snippet for interoperability from Curl context - for example, could be from a Linux or MacOS workstation / server -, to Office 365 SharePoint Online; with service-based authentication by applying Active / Modern Authentication protocol handling:

#General variables
ProxyAccount="sa-curlAccount"
ProxyPassword="******************"
ProxyProtocol="http"
ProxyServer="xxx.xxx.xxx.xxx"
ProxyPort="8080"
SharePointCurlAccount="sa-curlAccount"
SharePointOnlineTenant="<URL of SharePoint Online tenant>"
UploadFile="<file to upload>"
UploadLocation="<URL of SharePoint Document Library>"

#Fixed variables
OUTPUT=${HOME}/Interop/output
TMP=${HOME}/Interop/tmp/spo

#the following steps are required to upload data from Curl context to SharePoint Online:
#
#1. Retrieve an authentication cookie to Office 365 through invocation of webservices
#1.a. (Optional) Step 0: determine the URL of the custom Security Token Service (STS) to next
#     request a SAML:assertion for account identified by credentials
#1.b. Step 1: request SAML:assertion from the identified custom STS for account identified by
#     credentials
#1.c. Step 2: use the SAML:assertion to request binary security token from Office 365
#1.d. Step 3: use the binary security token to retrieve the authentication cookie
#2. Step 4: Use that Office 365 authentication cookie in subsequent webservice requests to
#   SharePoint Online REST API
 
#1.a. (Optional) Step 0: determine the URL of the custom Security Token Service (STS) to next
#     request a SAML:assertion for account identified by credentials (outside datacenter, with proxy)
curl -U ${ProxyAccount}:${ProxyPassword} -k -x ${ProxyProtocol}://${ProxyServer}:${ProxyPort} -X POST -H "Content-Type: application/x-www-form-urlencoded" -d "login=${SharePointCurlAccount}&xml=1" https://login.microsoftonline.com/GetUserRealm.srf -w "\n" > ${TMP}/O365_response_step_0

#Extract requested STSAuthURL from response step 1
STSURL=`sed -n 's:.*<STSAuthURL>\(.*\)</STSAuthURL>.*:\1:p' ${TMP}/O365_response_step_0`

#Create input for step 1

File: O365_request_step_1-1

<?xml version="1.0" encoding="UTF-8"?>
<s:Envelope
    xmlns:s="http://www.w3.org/2003/05/soap-envelope"
    xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd"
    xmlns:saml="urn:oasis:names:tc:SAML:1.0:assertion"
    xmlns:wsp="http://schemas.xmlsoap.org/ws/2004/09/policy"
    xmlns:wsu="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd"
    xmlns:wsa="http://www.w3.org/2005/08/addressing"
    xmlns:wssc="http://schemas.xmlsoap.org/ws/2005/02/sc"
    xmlns:wst="http://schemas.xmlsoap.org/ws/2005/02/trust">
    <s:Header>
        <wsa:Action s:mustUnderstand="1">http://schemas.xmlsoap.org/ws/2005/02/trust/RST/Issue</wsa:Action>
        <wsa:To s:mustUnderstand="1">https://sts.<tenant>.com/adfs/services/trust/2005/usernamemixed</wsa:To>
        <wsa:MessageID>b07da3ec-9824-46a5-a102-2329e0c5f63f</wsa:MessageID>
        <ps:AuthInfo
            xmlns:ps="http://schemas.microsoft.com/Passport/SoapServices/PPCRL" Id="PPAuthInfo">
            <ps:HostingApp>Managed IDCRL</ps:HostingApp>
            <ps:BinaryVersion>6</ps:BinaryVersion>
            <ps:UIVersion>1</ps:UIVersion>
            <ps:Cookies></ps:Cookies>
            <ps:RequestParams>AQAAAAIAAABsYwQAAAAxMDMz</ps:RequestParams>
        </ps:AuthInfo>
        <wsse:Security>
            <wsse:UsernameToken wsu:Id="user">
                <wsse:Username>sa-curlAccount@<tenant>.com</wsse:Username>
                <wsse:Password>*************</wsse:Password>
            </wsse:UsernameToken>
            <wsu:Timestamp Id="Timestamp">

File: O365_request_step_1-2

            </wsu:Timestamp>
        </wsse:Security>
    </s:Header>
    <s:Body>
        <wst:RequestSecurityToken Id="RST0">
            <wst:RequestType>http://schemas.xmlsoap.org/ws/2005/02/trust/Issue</wst:RequestType>
            <wsp:AppliesTo>
                <wsa:EndpointReference>
                    <wsa:Address>urn:federation:MicrosoftOnline</wsa:Address>
                </wsa:EndpointReference>
            </wsp:AppliesTo>
            <wst:KeyType>http://schemas.xmlsoap.org/ws/2005/05/identity/NoProofKey</wst:KeyType>
        </wst:RequestSecurityToken>
    </s:Body>
</s:Envelope>

cat ${TMP}/O365_request_step_1-1 > ${TMP}/O365_request_step_1
echo "<wsu:Created>`date -u +'%Y-%m-%dT%H:%M:%SZ'`</wsu:Created>" >> ${TMP}/O365_request_step_1
echo "<wsu:Expires>`date -u +'%Y-%m-%dT%H:%M:%SZ' --date='-15 minutes ago'`</wsu:Expires>" >> ${TMP}/O365_request_step_1
cat ${TMP}/O365_request_step_1-2 >> ${TMP}/O365_request_step_1

#1.b. Step 1: request SAML:assertion from the identified custom STS for account identified by
#     credentials (internal datacenter, without webproxy to outside)
curl -X POST -H "Content-Type: application/soap+xml; charset=utf-8" -d "@${TMP}/O365_request_step_1" ${STSURL} -w "\n" > ${TMP}/O365_response_step_1

#Extract requested SAML:assertion from response step 1
sed 's/^.*\(<saml:Assertion.*saml:Assertion>\).*$/\1/' ${TMP}/O365_response_step_1 > ${TMP}/O365_response_step_1.tmp

#Create input for step 2

File: O365_request_step_2-1

<?xml version="1.0" encoding="UTF-8"?>
<S:Envelope
    xmlns:S="http://www.w3.org/2003/05/soap-envelope"
    xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd"
    xmlns:wsp="http://schemas.xmlsoap.org/ws/2004/09/policy"
    xmlns:wsu="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd"
    xmlns:wsa="http://www.w3.org/2005/08/addressing"
    xmlns:wst="http://schemas.xmlsoap.org/ws/2005/02/trust">
    <S:Header>
        <wsa:Action S:mustUnderstand="1">http://schemas.xmlsoap.org/ws/2005/02/trust/RST/Issue</wsa:Action>
        <wsa:To S:mustUnderstand="1">https://login.microsoftonline.com/rst2.srf</wsa:To>
        <ps:AuthInfo
            xmlns:ps="http://schemas.microsoft.com/LiveID/SoapServices/v1" Id="PPAuthInfo">
            <ps:BinaryVersion>5</ps:BinaryVersion>
            <ps:HostingApp>Managed IDCRL</ps:HostingApp>
        </ps:AuthInfo>
        <wsse:Security>

File: O365_request_step_2-2

        </wsse:Security>
    </S:Header>
    <S:Body>
        <wst:RequestSecurityToken xmlns:wst="http://schemas.xmlsoap.org/ws/2005/02/trust" Id="RST0">
            <wst:RequestType>http://schemas.xmlsoap.org/ws/2005/02/trust/Issue</wst:RequestType>
            <wsp:AppliesTo>
                <wsa:EndpointReference>
                    <wsa:Address>sharepoint.com</wsa:Address>
                </wsa:EndpointReference>
            </wsp:AppliesTo>
            <wsp:PolicyReference URI="MBI"></wsp:PolicyReference>
        </wst:RequestSecurityToken>
    </S:Body>
</S:Envelope>

cat ${TMP}/O365_request_step_2-1 > ${TMP}/O365_request_step_2
cat ${TMP}/O365_response_step_1.tmp >> ${TMP}/O365_request_step_2
cat ${TMP}/O365_request_step_2-2 >> ${TMP}/O365_request_step_2

rm ${TMP}/O365_response_step_1.tmp

#1.c. Step 2: use the SAML:assertion to request binary security token from Office 365
#     (outside datacenter, with proxy)
curl -U ${ProxyAccount}:${ProxyPassword} -k -x ${ProxyProtocol}://${ProxyServer}:${ProxyPort} -X POST -H "Content-Type: application/soap+xml; charset=utf-8" -d "@${TMP}/O365_request_step_2" https://login.microsoftonline.com/RST2.srf -w "\n" > ${TMP}/O365_response_step_2

#Extract requested binary security token from response step 2
sed 's/^.*\(<wsse:BinarySecurityToken.*wsse:BinarySecurityToken>\).*$/\1/' ${TMP}/O365_response_step_2 > ${TMP}/O365_response_step_2.tmp

#Create input for step 3

cat ${TMP}/O365_response_step_2.tmp | cut -d'>' -f2 | cut -d'<' -f1 > ${TMP}/O365_request_step_3
BinarySecurityToken=`cat ${TMP}/O365_request_step_3`

rm ${TMP}/O365_response_step_2.tmp

#1.d. Step 3: use the binary security token to retrieve the authentication cookie (outside
#     datacenter, need to pass webproxy)
curl -v -U ${ProxyAccount}:${ProxyPassword} -k -x ${ProxyProtocol}://${ProxyServer}:${ProxyPort} -X GET -H "Authorization: BPOSIDCRL ${BinarySecurityToken}" -H "X-IDCRL_ACCEPTED: t" -H "User-Agent:" ${SharePointOnlineTenant}/_vti_bin/idcrl.svc/ > ${TMP}/O365_response_step_3 2>&1

#Remove DOS ^M from response step 3
cat ${TMP}/O365_response_step_3 | sed 's/^M//' > ${TMP}/O365_response_step_3.tmp

#Extract requested authentication cookie from response step 3 and create input for step 4
echo "Set-Cookie: SPOIDCRL=`cat ${TMP}/O365_response_step_3.tmp | grep Set-Cookie | awk -F'SPOIDCRL=' '{print $2}'`" > ${TMP}/O365_request_step_4

rm ${TMP}/O365_response_step_3.tmp

#2. Step 4: Use that Office 365 authentication cookie in subsequent webservice requests to
#   SharePoint Online REST API (outside datacenter, with proxy)
curl -U ${ProxyAccount}:${ProxyPassword} -k -x ${ProxyProtocol}://${ProxyServer}:${ProxyPort} -b ${TMP}/O365_request_step_4 -T "{${OUTPUT}/${UploadFile}}" ${UploadLocation}

exit 0

Alternative for the upload handling; interoperation via SharePoint API / webservice:

curl -U ${ProxyAccount}:${ProxyPassword} -k -x ${ProxyProtocol}://${ProxyServer}:${ProxyPort} -X POST -H "Accept: application/json;odata=verbose" -d "" ${SharePointOnlineTenant}/_api/contextinfo > ${TMP}/O365_response_step_4_tmp

FormDigest=`sed -n 's:.*FormDigestvalue:\(.*\),.*:\1:p' ${TMP}/O365_response_step_4_tmp`
rm ${TMP}/O365_response_step_4.tmp

curl -U ${ProxyAccount}:${ProxyPassword} -k -x ${ProxyProtocol}://${ProxyServer}:${ProxyPort} -X POST -H "X-RequestDigest: @${FormDigest}; X-HTTP-Method: PUT” --data-binary  "{${OUTPUT}/${UploadFile}}"  ${SharePointOnlineTenant}/teams/siteX/_api/web/GetFileByServerRelativeUrl('Shared%20Documents/SubFolder/${UploadFile}')/Files/$value