SQL Loop Salesforce Download

70 views Asked by At

I've got a loop which downloads data from Salesforce, based on a query parameter

This can download large amounts of data; is there anyways to make it more efficient?

It is a stored procedure that gets as much json as possible (2000) then combines them all and passes it out of the procedure for interpreting and sticking into a table

Code below

SET @url = @baseUrl + '/services/data/v58.0/query/?q=' + @query
SET @authHeader = CONCAT('Bearer ', (SELECT TOP 1 whatVal FROM TKTable))

WHILE @responseText NOT LIKE '%,"done":true,%' AND @loopNo<=300
BEGIN

    PRINT ''
    PRINT 'LOOP NUMBER: ' + CONVERT(varchar(3),@loopNo)
    PRINT ''

    PRINT ''
    PRINT 'URL: ' +@url
    PRINT ''

    IF @responseText = 'First Loop' EXEC sp_OACreate 'MSXML2.XMLHttp.6.0', @internalToken OUT;

    EXEC sp_OAMethod @internalToken, 'open', NULL, 'get', @url, 'false'
    EXEC sp_OAMethod @internalToken, 'setRequestHeader', NULL, 'Authorization', @authHeader
    EXEC sp_OAMethod @internalToken, 'send', NULL, ''

    DELETE FROM @json
    INSERT into @json (response) exec sp_OAGetProperty @internalToken, 'responseText'

    SET @responseText=(SELECT TOP 1 response FROM @json)
    PRINT ''
    PRINT 'DOWNLOADED: ' + @responseText

    SET @url = @baseURL +
    (SELECT nextRecordsUrl FROM openjson(@responseText) WITH
    (
        nextRecordsUrl varchar(255) '$.nextRecordsUrl'
    )
    )

    SET @jsonString = ISNULL(@jsonString,'') + (SELECT STRING_AGG([value],',') FROM openjson(@responseText, '$.records')) + ','
    SET @loopNo = @loopNo+1

END

SET @jsonString = LEFT(@jsonString,LEN(@jsonString)-1) --Remove extra comma
SET @outputJson = '[ ' + @jsonString + ']'

--Close the connection.
EXEC sp_OADestroy @internalToken;

Some ideas I've tried and failed with :

-Reusing @internalToken -Grabbing @responseText without putting it in a table

0

There are 0 answers