Importing data Through Scheduled Job

[SomashreeD]

We have created Scheduled job to import data from third party into ServiceNow , but we are only receiving 10k records , I have used pagination then also same thing is receiving, can anyone help me to understand if any issues there with my code.

Sample code

var pagesize = 10000;

var pagestofetch = 10;

var pageoffset = 0;

var j = 0;

var allResults = [];

while (j <= pagestofetch) {

    gs.log('test' + j);

    try {

        var r = new sn_ws.RESTMessageV2('testing, 'testing234');

        r.setStringParameterNoEscape('sysparm_offset', pageoffset);

        r.setStringParameterNoEscape('sysparm_limit', pagesize);

        var response = r.execute();

        var responseBody = response.getBody();

        var httpStatus = response.getStatusCode();

        var parser = new JSONParser();

        var parsed = parser.parse(responseBody);

        gs.info("endpoint"+ r.getEndpoint());

        gs.info("requestBody"+ r.getRequestBody());

       // allResults.push(parsed);

        gs.log('testallresult' + JSON.stringify(allResults));

        gs.log('testpagesize' + pagesize);

        gs.log('testpageoffset' + pageoffset);

        pageoffset += pagesize;

        j++;

    } catch (ex) {

        var message = ex.getMessage();

    }

}

//Create a new import set

var crImpSet = new GlideRecord('sys_import_set');

crImpSet.initialize();

crImpSet.mode = 'asynchronous';

crImpSet.table_name = 'test'; //Set the extended importset table

crImpSet.state = 'loading';

crImpSet.insert();

// var locs = allResults.value;

var locs = parsed.value;

var costCenter;

var restGR = new GlideRecord(test'); //Query extended import set rows table

// //Loop through the json results until 0 remain creating import set rows to be transformed

for (var i = 0; i < locs.length; i++) {

    restGR.insert();

}

crImpSet.state = 'loaded';

crImpSet.update();

[Vasantharajan N]

@SomashreeD  - Hope you are trying to fetch the data from another ServiceNow instance. Please check the below script i've made few changes that includes converting the import set data load as a function which will be processed once after the while loop is completed.

Note: You need to add your logic for the import set insert

var pagesize = 10000;
var pagestofetch = 10;
var pageoffset = 0;
var j = 0;
var allResults = [];
while (j <= pagestofetch) {
    gs.log('test' + j);
    try {
        var r = new sn_ws.RESTMessageV2('testing', "testing234");
        r.setStringParameterNoEscape('sysparm_offset', pageoffset);
        r.setStringParameterNoEscape('sysparm_limit', pagesize);
        var response = r.execute();
        var responseBody = response.getBody();
        var httpStatus = response.getStatusCode();
        var parser = new JSONParser();
        var parsed = parser.parse(responseBody);
        gs.info("endpoint" + r.getEndpoint());
        gs.info("requestBody" + r.getRequestBody());
        allResults.push(parsed);
        gs.log('testallresult' + JSON.stringify(allResults));
        gs.log('testpagesize' + pagesize);
        gs.log('testpageoffset' + pageoffset);
        pageoffset += pagesize;
        j++;

    } catch (ex) {
        var message = ex.getMessage();
    }

}

loadData(allResults)

function loadData(restResultsArray) {

    //Create a new import set
    var crImpSet = new GlideRecord('sys_import_set');
    crImpSet.initialize();
    crImpSet.mode = 'asynchronous';
    crImpSet.table_name = 'test'; //Set the extended importset table
    crImpSet.state = 'loading';
    crImpSet.insert();

	// Process the response result array where you stored all your response per API call
    for (var res = 0; res < restResultsArray.length; res++) {
        // var locs = allResults.value;
        var parsed = restResultsArray[res];
        var locs = parsed.value;
        var costCenter;
        var restGR = new GlideRecord('test'); //Query extended import set rows table
        // //Loop through the json results until 0 remain creating import set rows to be transformed
        for (var i = 0; i < locs.length; i++) {
            // logic to set the values to restGr
            restGR.insert();
        }
    }
    crImpSet.state = 'loaded';
    crImpSet.update();
}

[SomashreeD]

Hi @Vasantharajan N ,

It's going into infinite loop it seems and in import sets no data is coming.

Is the code above provided by you is working well for you.

[Vasantharajan N]

@SomashreeD - As you increment the j variable value it will exit the while loop when it reach the value of 11. Not sure how you are saying this is going in infinite loop. 

for testing purpose limit the pagesize value to 10 or 100 and check and share your findings

[SomashreeD]

I checked the value of j putting logs it seems for evrytime it's 0 only the value not increasing till 11 .