Process REST Payload and Pagination

[Community Alums]

Hello Everyone,

I am working on integration where ServiceNow makes call to 3rd party tool using REST Message and we have a GET method defined bring incident data in ServiceNow. We have established a connection and we are getting the data in below format. Now the problem is, 1) The field value (key) contains spaces in it, how to process that data and insert into ServiceNow incident table 2) We have 60000 records to be processed, how all can be processed using pagination, looking for script.

Note: there are fields with spaces: ex. "Assignee Login ID" and some field has colon in it "Test: "

Payload received (receiving the data with offset=2000)

{"entries":[

{
"values": {
"Entry ID": "INC000000000623",
"Submitter": "gash.te.net",
"Ebond_ValidateFlag": null,
"Submit Date": "2015-11-03T19:36:06.000+0000",
"Ebond_WorkLogID": null,
"Assignee Login ID": "gash.te.net",
"z1D_SV_IA_Organization": null,
"Last Modified By": "AR_ESCALATOR",
"Last Modified Date": "2024-09-20T02:40:37.000+0000",
"Status": "Closed",
"Short Description": ".",
"z1D_AssigneeManagerLogin": null,
"z1D_SV_IA_Region": null,
"Notifier Listening": "Not Listening",
"z1D_SV_IA_Site": null,
"z1D_DefaultCompanyAction": null,
"z1D_SkipIfSmartITInstalled": null,
"z1D Status Integer Parm02": null,
"Vendor Name": null,
"z1D Status Integer Parm01": null,
"z1D_SV_IA_Department": null,
"z1D_SV_IsFormDirty": null,
"z1D_AssociationType02": null,
"z1D_SV_IA_Company": null,
"Owner Support Company": "IT Security",
"Owner Group ID": "SGP000000001025",
"z1D_SV_IA_SiteGroup": null,
"z1d_SV_AIOPs_NotImpacted": null,
"Owner Group": "Security",
"Impact_OR_Root": null,
"z2AF_Act_Attachment_2": null,
"z1D_googleMapApiKey": null,
"Test:NTT:SNOW_SYSID": null,
"Test:NTT:SNOW_FLAG": null,
"z2AF_SV_DD_Attachment": null,
"z1D_AssigneeManager": null,
"Assignee Groups": "1000000020;",
"z1D_DefaultCompanyInteger": null,
"SV_Password_MCSM": "***",
"z1D Char16": null,
"z1D Date01": null,
"z1D_SV_SetDetailedDescription_Assignment_Activity": null,
"z1D_SV_SetDescription_Assignment_Activity": null,
"z1D_SV_SetMessageID_Assignment_Activity": null,
"z1D_PreviousAssignedCompany": "Test
"z2AF_Act_Attachment_1": null,
"status_reason2": null,
"Assigned Group Shift Name": null,
"Shifts Flag": null,
"InstanceId": "IDHAA5V0G7RG3ANXJH5H1BB94O5537",
"z1D_Assignee_Email": null}

]}}],

"_links": {
"self": [
{
"href": "/api/arsys/v1.0/entry/HPD:Help%20Desk/INC000000000623"
}
]
}
}

[Shivalika · Accepted solution]

Hello @Community Alums 

Please refer below code - 

// Base URL
var baseUrl = "";

// Step 1: Get Auth Token
var r = new sn_ws.RESTMessageV2('Test Integration', 'get token');
var TOKEN_SYS_PROP_NAME = '';
var authBodyContent = 'dXNlcm5hbWU9YWtob3Nobm9vZEBhY29yaW8uY29tJnBhc3N3b3JkPTEyMzQ=';

r.setStringParameterNoEscape('url', baseUrl);

var response = r.execute();
var responseBody = response.getBody();
var httpStatus = response.getStatusCode();

if (httpStatus !== 200) {
gs.error("Failed to fetch token. HTTP Status: " + httpStatus);
return;
}

TOKEN_SYS_PROP_NAME = "AR-JWT " + responseBody;
gs.log("Auth Token: " + TOKEN_SYS_PROP_NAME);

// Step 2: Fetch Incidents
var rm = new sn_ws.RESTMessageV2('Test Integration', 'get test incidents');
rm.setStringParameterNoEscape('authToken', TOKEN_SYS_PROP_NAME);
rm.setStringParameterNoEscape('url', baseUrl);

var response1 = rm.execute();
var responseBody1 = response1.getBody();
var parsedResponse = JSON.parse(responseBody1);
var httpStatus1 = response1.getStatusCode();

if (httpStatus1 !== 200) {
gs.error("Failed to fetch incidents. HTTP Status: " + httpStatus1);
return;
}

// Step 3: Process Incident Records
function processRecords(entries) {
for (var i = 0; i < entries.length; i++) {
var values = entries[i].values;

var gr = new GlideRecord('sn_customerservice_case'); // Replace with target table
gr.initialize();
gr.setValue('u_ticket_type', 'Incident');
gr.setValue('u_status', values["Status"] || "");
gr.setValue('u_priority', values["Priority"] || "");
gr.setValue('u_assigned_group', values["Assignment Group"] || "");
gr.setValue('u_impact', values["Impact"] || "");
gr.setValue('u_urgency', values["Urgency"] || "");
gr.setValue('u_incident_number', values["Incident Number"] || "");
gr.setValue('short_description', values["Description"] || "");
gr.insert();

gs.print("Inserted Record: " + values["Incident Number"]);
}
}

if (parsedResponse.entries && parsedResponse.entries.length > 0) {
processRecords(parsedResponse.entries);
}

// Step 4: Handle Pagination
if (parsedResponse._links && parsedResponse._links.next) {
var nextPageUrl = JSON.stringify(parsedResponse._links.next);
gs.print("Fetching next page: " + nextPageUrl);
}

gs.print("Process completed.");

You can add more fields if you want in the process of modification , I could do this much. 

Kindly mark my answer as helpful and accept solution if it helped you in anyway. This will help me be recognized for the efforts and also move this questions from unsolved to solved bucket. 

Regards,

Shivalika 

My LinkedIn - https://www.linkedin.com/in/shivalika-gupta-540346194

My youtube - https://youtube.com/playlist?list=PLsHuNzTdkE5Cn4PyS7HdV0Vg8JsfdgQlA&si=0WynLcOwNeEISQCY

[Community Alums]

Hello @Shivalika,

Needed one additional help on the same code, now code is returning the data. However it's lopping through 1st page and not going to subsequent pages and additionally, some values are showing as undefined. Could you please have look.

var baseUrl = "....";

function fetchRecords(url) {
var r = new sn_ws.RESTMessageV2('global.Helix Integration', 'get token');
var TOKEN_SYS_PROP_NAME = '';
var authBodyContent = 'dXNlcm5hbWU9YWtob3Nobm9vZEBhY29yaW8uY29tJnBhc3N3b3JkPTEyMzQ=';
r.setStringParameterNoEscape('url', ''); //
var authBodyContentDecoded = GlideStringUtil.base64Decode(authBodyContent); //
r.setRequestBody(authBodyContentDecoded);

var response = r.execute();
var responseBody = response.getBody();
var httpStatus = response.getStatusCode();
TOKEN_SYS_PROP_NAME = ("AR-JWT " + responseBody + '');
gs.print(TOKEN_SYS_PROP_NAME);

rm = new sn_ws.RESTMessageV2('global.Helix Integration', 'get helix incidents');
rm.setStringParameterNoEscape('authToken', TOKEN_SYS_PROP_NAME);
rm.setStringParameterNoEscape('url', baseUrl);

var response1 = rm.execute();
var responseBody1 = response1.getBody();
var parsedResponse = JSON.parse(responseBody1);
//var parser = new JSONParser();
//var result = parsedResponse;
var httpStatus1 = response.getStatusCode();
//gs.print(result);

function processRecords(entries) {
for (var i = 0; i < entries.length; i++) {
var num = entries[i].values["Incident Number"];
var submitter = entries[i].values["Submitter"];
var sub_date = entries[i].values["Submit Date"];
var cat = entries[i].values["Category"];
var ass_id = entries[i].values["Assignee Id"];
var last_mod_by = entries[i].values["Last Modified By"];
var last_mod_date = entries[i].values["Last Modified Date"];
var status = entries[i].values["Status"];
var desc = entries[i].values["Description"];
var created_by = entries[i].values["Created_By"];
var res_cat = entries[i].values["Resolution Category"];
var site = entries[i].values["Site"];
var res_cat_tier_3 = entries[i].values["Resolution Category Tier 3"];
var res_cat_tier_2 = entries[i].values["Resolution Category Tier 2"];
var resolution = entries[i].values["Resolution"];
var urgency = entries[i].values["Urgency"];
var impact = entries[i].values["Impact"];
var incident_num = entries[i].values["Incident Number"];
var priority = entries[i].values["Priority"];
var assign_group = entries[i].values["Assigned Group"];
var rep_source = entries[i].values["Reported Source"];
var detailed_desc = entries[i].values["Detailed Description"];
var ph_number = entries[i].values["Phone Number"];
var product_name = entries[i].values["Product Name"];
var cat_tier_1 = entries[i].values["Categorization Tier 1"];
var last_res_date = entries[i].values["Last Resolved Date"];
var closed_date = entries[i].values["Closed Date"];
var srattach = entries[i].values["SRAttachment/Attachment"];
var cat_tier_2 = entries[i].values["Categorization Tier 2"];
var cat_tier_3 = entries[i].values["Categorization Tier 3"];
var pro_cat_1 = entries[i].values["Product Categorization Tier 1"];
var pro_cat_3 = entries[i].values["Product Categorization Tier 3"];
var pro_cat_2 = entries[i].values["Product Categorization Tier 2"];
var req_id = entries[i].values["SRID"];

var inc = new GlideRecord('sn_customerservice_case');
inc.addEncodedQuery('number=' + num);
inc.query;

if (!inc.next()) {
var grUser = new GlideRecord('sys_user');
grUser.addQuery('email', submitter); // Replace with the actual email address
grUser.query();

if (grUser.next()) {
var sys_id = grUser.getUniqueValue(); // Or grUser.sys_id

var grGroup = new GlideRecord('sys_user_group');
grGroup.addQuery('name', assign_group); // Replace with the actual email address
grGroup.query();

var group_sys_id = grGroup.getUniqueValue();

var gr = new GlideRecord('sn_customerservice_case'); // Replace with target table name
gr.initialize();
gr.setValue('u_ticket_type', 'Incident'); // Replace FIELD_1 and KEY_1
gr.setValue('number', num);
gr.setValue('description', req_id);
gr.setValue('internal_user', sys_id);
gr.setValue('category', cat);
gr.setValue('assignment_group', group_sys_id);
gr.setValue('u_this_issue_is_for', 'Employee(s)');
gr.setValue('sys_created_on', sub_date);
gr.setValue('assigned_to', ass_id);
gr.setValue('sys_updated_by', last_mod_by);
gr.setValue('sys_updated_on', last_mod_date);
gr.setValue('state', status);
gr.setValue('state', status);
gr.setValue('short_description', desc);
gr.setWorkflow(false);
gr.insertWithReferences();
}

} else {
gs.print(+num + "Record already exists in the table");
}

}

}
if (parsedResponse.entries && parsedResponse.entries.length > 0) {
processRecords(parsedResponse.entries);
}

if (parsedResponse._links) {
fetchRecords(JSON.stringify(parsedResponse._links.next));
}
}
fetchRecords(baseUrl);

[Community Alums]

Shivalika,

One request, could you please re-arrange the code once, below is my code so far

var baseUrl = "https://test/api/arsys/v1.0/entry/HPD:Help%20Desk?fields=values(Incident%20Number)&RecordInstanceDat...";

var r = new sn_ws.RESTMessageV2('Test Integration', 'get token');

var TOKEN_SYS_PROP_NAME = '';

var authBodyContent = 'dXNlcm5hbWU9YWtob3Nobm9vZEBhY29yaW8uY29tJnBhc3N3b3JkPTEyMzQ=';

r.setStringParameterNoEscape('url', 'https://test/api/jwt/login'); //

var authBodyContentDecoded = GlideStringUtil.base64Decode(authBodyContent); //

r.setRequestBody(authBodyContentDecoded);

var output = [];

var response = r.execute();

var responseBody = response.getBody();

var httpStatus = response.getStatusCode();

//gs.setProperty(this.TOKEN_SYS_PROP_NAME, "AR-JWT " + responseBody + '');

//this.debugLog('getToken:value: ' + JSON.stringify(responseBody));

gs.log(JSON.stringify(responseBody));

TOKEN_SYS_PROP_NAME = ("AR-JWT " + responseBody + '');

gs.log(TOKEN_SYS_PROP_NAME);

rm = new sn_ws.RESTMessageV2('Test Integration', 'get test incidents');

rm.setStringParameterNoEscape('authToken', TOKEN_SYS_PROP_NAME);

//r.setStringParameterNoEscape('incidentNumber', responseInc);

rm.setStringParameterNoEscape('url', baseUrl);

//rm.setStringParameterNoEscape('field', "values(Submitter, Submit Date, Assignee Id, Last Modified By, Last Modified, Date, Status, Description, Created_By, Resolution Category, Site, Resolution, Category Tier 3, Resolution Category Tier 2, Resolution, Urgency, Impact, Incident Number, Priority, Assignment Group,, Reported Source, Detailed Description, Phone Number, Product Name, Categorization Tier 1, Last Resolved Date, Closed Date, SRAttachment/Attachment, Categorization, Tier 2, Categorization Tier 3, Product Categorization Tier 1, Product Categorization Tier 3, Product Categorization Tier 2)");

var response1 = rm.execute();

var responseBody1 = response1.getBody();

var parsedResponse = JSON.parse(responseBody1);

//var parser = new JSONParser();

var result = parsedResponse;

var httpStatus1 = response.getStatusCode();

// if (parsedResponse.entries && parsedResponse.entries.length > 0) {

//     processRecords(parsedResponse.entries);

// }

function processRecords(entries) {

    for (var i = 0; i < parsedResponse.entries.length; i++) {

        var submitter = parsedResponse.entries[i].values["Incident Number"];

        var sub_date = parsedResponse.entries[i].values["Submit Date"];

        var ass_id = parsedResponse.entries[i].values["Assignee Id"];

        var last_mod_by = parsedResponse.entries[i].values["Last Modified By"];

        var last_mod_date = parsedResponse.entries[i].values["Last Modified Date"]

        var status = parsedResponse.entries[i].values["Status"];

        var desc = parsedResponse.entries[i].values["Description"];

        var created_by = parsedResponse.entries[i].values["Created_By"];

        var res_cat = parsedResponse.entries[i].values["Resolution Category"];

        var site = parsedResponse.entries[i].values["Site"];

        var res_cat_tier_3 = parsedResponse.entries[i].values["Resolution Category Tier 3"];

        var res_cat_tier_2 = parsedResponse.entries[i].values["Resolution Category Tier 2"];

        var resolution = parsedResponse.entries[i].values["Resolution"];

        var urgency = parsedResponse.entries[i].values["Urgency"];

        var impact = parsedResponse.entries[i].values["Impact"];

        var incident_num = parsedResponse.entries[i].values["Incident Number"];

        var priority = parsedResponse.entries[i].values["Priority"];

        var assign_group = parsedResponse.entries[i].values["Assignment Group"];

        var rep_source = parsedResponse.entries[i].values["Reported Source"];

        var detailed_desc = parsedResponse.entries[i].values["Detailed Description"];

        var ph_number = parsedResponse.entries[i].values["Phone Number"];

        var product_name = parsedResponse.entries[i].values["Product Name"];

        var cat_tier_1 = parsedResponse.entries[i].values["Categorization Tier 1"];

        var last_res_date = parsedResponse.entries[i].values["Last Resolved Date"];

        var closed_date = parsedResponse.entries[i].values["Closed Date"];

        var srattach = parsedResponse.entries[i].values["SRAttachment/Attachment"];

        var cat_tier_2 = parsedResponse.entries[i].values["Categorization Tier 2"];

        var cat_tier_3 = parsedResponse.entries[i].values["Categorization Tier 3"];

        var pro_cat_1 = parsedResponse.entries[i].values["Product Categorization Tier 1"];

        var pro_cat_3 = parsedResponse.entries[i].values["Product Categorization Tier 3"];

        var pro_cat_2 = parsedResponse.entries[i].values["Product Categorization Tier 2"];

    }

    var gr = new GlideRecord('sn_customerservice_case'); // Replace with target table name

    gr.initialize();

    gr.setValue('u_ticket_type', 'Incident'); // Replace FIELD_1 and KEY_1

    gr.setValue('<FIELD_2>', recordData["<KEY_2>"]); // Replace FIELD_2 and KEY_2

    gr.insert();

}

if (parsedResponse._links) {

    var url = JSON.stringify(parsedResponse._links.next);

    gs.print(url);

}

//gs.print(desc);