When sending a request to the google CSE API, I end up getting multiple variations of the same image.
const pagesToDownload = 1;
let pagesDownloaded = 0;
(async () => {
const { google } = require('googleapis');
const search = google.customsearch('v1');
const query = 'pasta';
let erroring = false;
const nextPage = () => {
if (pagesDownloaded != pagesToDownload && !erroring) {
search.cse.list({
q: query,
auth: process.env['CX_KEY'],
cx: "410a480db45d74a71",
searchType: 'image',
start: 1 + (pagesDownloaded * 10),
fileType: "jpeg,jpg,png,gif,ico,bmp"
}, async (err, res) => {
if (err) {
console.log("REQUEST FAILED: " + err);
erroring = true;
} else {
console.log(res);
await downloadImagesFromSearchResult(res);
pagesDownloaded++;
nextPage();
}
});
} else {
console.log("Successfully downloaded %d images", pagesDownloaded * 10);
}
}
nextPage();
})();
I've tried turning on and off the duplication filter, but neither make a difference. Here is an example of three pictures returned in the same request:
How could I go about removing duplicates?


