Importing data Through Scheduled Job

[SomashreeD]

We have created Scheduled job to import data from third party into ServiceNow , but we are only receiving 10k records , I have used pagination then also same thing is receiving, can anyone help me to understand if any issues there with my code.

Sample code

var pagesize = 10000;

var pagestofetch = 10;

var pageoffset = 0;

var j = 0;

var allResults = [];

while (j <= pagestofetch) {

    gs.log('test' + j);

    try {

        var r = new sn_ws.RESTMessageV2('testing, 'testing234');

        r.setStringParameterNoEscape('sysparm_offset', pageoffset);

        r.setStringParameterNoEscape('sysparm_limit', pagesize);

        var response = r.execute();

        var responseBody = response.getBody();

        var httpStatus = response.getStatusCode();

        var parser = new JSONParser();

        var parsed = parser.parse(responseBody);

        gs.info("endpoint"+ r.getEndpoint());

        gs.info("requestBody"+ r.getRequestBody());

       // allResults.push(parsed);

        gs.log('testallresult' + JSON.stringify(allResults));

        gs.log('testpagesize' + pagesize);

        gs.log('testpageoffset' + pageoffset);

        pageoffset += pagesize;

        j++;

    } catch (ex) {

        var message = ex.getMessage();

    }

}

//Create a new import set

var crImpSet = new GlideRecord('sys_import_set');

crImpSet.initialize();

crImpSet.mode = 'asynchronous';

crImpSet.table_name = 'test'; //Set the extended importset table

crImpSet.state = 'loading';

crImpSet.insert();

// var locs = allResults.value;

var locs = parsed.value;

var costCenter;

var restGR = new GlideRecord(test'); //Query extended import set rows table

// //Loop through the json results until 0 remain creating import set rows to be transformed

for (var i = 0; i < locs.length; i++) {

    restGR.insert();

}

crImpSet.state = 'loaded';

crImpSet.update();

[Ankur Bawiskar]

@SomashreeD 

so what's your concern?

1) API is bringing always same 10k records?

OR
2) API is bringing 10k records in each call but import set is only loading and transforming the 1st 10k records which were fetched?

If my response helped please mark it correct and close the thread so that it benefits future readers.

[SomashreeD]

Hello @Ankur Bawiskar , yeah my query is API always bringing the 1st 10k records but the API has more than 30k response.

[Ankur Bawiskar]

@SomashreeD 

try this and see what came in logs

var pagesize = 10000;
var pagestofetch = 10;
var pageoffset = 0;
var j = 0;
var allResults = [];

while (j < pagestofetch) {
    gs.log('Fetching page ' + (j + 1));
    try {
        var r = new sn_ws.RESTMessageV2('testing', 'testing234');
        r.setStringParameterNoEscape('sysparm_offset', pageoffset);
        r.setStringParameterNoEscape('sysparm_limit', pagesize);
        var response = r.execute();
        var responseBody = response.getBody();
        var httpStatus = response.getStatusCode();
        var parser = new JSONParser();
        var parsed = parser.parse(responseBody);

        gs.info("Endpoint: " + r.getEndpoint());
        gs.info("Request Body: " + r.getRequestBody());
        gs.info("Response Body: " + responseBody);
        gs.info("HTTP Status: " + httpStatus);

        if (parsed.length === 0) {
            gs.log('No more records to fetch.');
            break;
        }

        allResults.push(parsed);
        gs.log('Fetched ' + parsed.length + ' records.');

        pageoffset += pagesize;
        j++;
    } catch (ex) {
        var message = ex.getMessage();
        gs.error('Error fetching records: ' + message);
        break;
    }
}

gs.log('Total records fetched: ' + allResults.length);

If my response helped please mark it correct and close the thread so that it benefits future readers.