I've got a loop which downloads data from Salesforce, based on a query parameter
This can download large amounts of data; is there anyways to make it more efficient?
It is a stored procedure that gets as much json as possible (2000) then combines them all and passes it out of the procedure for interpreting and sticking into a table
Code below
SET @url = @baseUrl + '/services/data/v58.0/query/?q=' + @query
SET @authHeader = CONCAT('Bearer ', (SELECT TOP 1 whatVal FROM TKTable))
WHILE @responseText NOT LIKE '%,"done":true,%' AND @loopNo<=300
BEGIN
PRINT ''
PRINT 'LOOP NUMBER: ' + CONVERT(varchar(3),@loopNo)
PRINT ''
PRINT ''
PRINT 'URL: ' +@url
PRINT ''
IF @responseText = 'First Loop' EXEC sp_OACreate 'MSXML2.XMLHttp.6.0', @internalToken OUT;
EXEC sp_OAMethod @internalToken, 'open', NULL, 'get', @url, 'false'
EXEC sp_OAMethod @internalToken, 'setRequestHeader', NULL, 'Authorization', @authHeader
EXEC sp_OAMethod @internalToken, 'send', NULL, ''
DELETE FROM @json
INSERT into @json (response) exec sp_OAGetProperty @internalToken, 'responseText'
SET @responseText=(SELECT TOP 1 response FROM @json)
PRINT ''
PRINT 'DOWNLOADED: ' + @responseText
SET @url = @baseURL +
(SELECT nextRecordsUrl FROM openjson(@responseText) WITH
(
nextRecordsUrl varchar(255) '$.nextRecordsUrl'
)
)
SET @jsonString = ISNULL(@jsonString,'') + (SELECT STRING_AGG([value],',') FROM openjson(@responseText, '$.records')) + ','
SET @loopNo = @loopNo+1
END
SET @jsonString = LEFT(@jsonString,LEN(@jsonString)-1) --Remove extra comma
SET @outputJson = '[ ' + @jsonString + ']'
--Close the connection.
EXEC sp_OADestroy @internalToken;
Some ideas I've tried and failed with :
-Reusing @internalToken -Grabbing @responseText without putting it in a table