Uploading data in Batches to Github

[rohanbhatia]

var githubToken = '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'; // Replace with your GitHub token
var repository = 'rohanb459/sn_power_bi_poc'; // Format: username/repository-name
var filePath = 'data.json'; // Path in the repo (subdirectories allowed)

var BATCH_SIZE = 10;
var offset = 0;


// checking if the file already exists 
function getFileContent() {
    var request = new sn_ws.RESTMessageV2();
    request.setHttpMethod('GET');
    request.setRequestHeader('Authorization', 'Bearer ' + githubToken);
    request.setEndpoint('https://api.github.com/repos/' + repository + '/contents/' + filePath);

    var response = request.execute();
    var responseBody = response.getBody();
    var httpStatus = response.getStatusCode();

    if (httpStatus == 200) {
        var fileDetails = JSON.parse(responseBody);
        var contentDecoded = GlideStringUtil.base64Decode(fileDetails.content);
        return {
            sha: fileDetails.sha,
            content: JSON.parse(contentDecoded) // Convert existing JSON to an object
        };
    } else if (httpStatus == 404) {
        return { sha: null, content: [] }; // File does not exist, start fresh
    } else {
        gs.print('Error retrieving file: ' + httpStatus);
        return null;
    }
}

var sha = getFileContent().sha;
var initialRequestBody;

if (sha) {
    // If file exists, get the sha from the response
    initialRequestBody = {
        "message": "Updated incident data", // Commit message
        "content": GlideStringUtil.base64Encode("[]"), // Base64 encoded content
        "sha": sha // Pass the sha for updating the file
    };  
    
} else {
    // If the file doesn't exist, create a new file
    initialRequestBody = {
        "message": "Initial commit with incident data", // Commit message
        "content": GlideStringUtil.base64Encode("[]") // Base64 encoded content
    };

}


var initialRequest = new sn_ws.RESTMessageV2();
initialRequest.setHttpMethod('PUT');
initialRequest.setRequestHeader('Authorization', 'Bearer ' + githubToken);
initialRequest.setRequestHeader('Content-Type', 'application/json');
initialRequest.setEndpoint('https://api.github.com/repos/' + repository + '/contents/' + filePath);
initialRequest.setRequestBody(JSON.stringify(initialRequestBody));

var initialResponse = initialRequest.execute();
var initialResponseBody = initialResponse.getBody();
var initialHttpStatus = initialResponse.getStatusCode();
gs.print('Create HTTP Status: ' + initialHttpStatus);
gs.print('Create Response Body: ' + initialResponseBody);


var isNewRecordsToPush=true;

while(isNewRecordsToPush){
    var fileData = getFileContent();
    var fileExistingContent = fileData.content;
    var fileSha = fileData.sha;

    var gr = new GlideRecord("incident");
    gr.orderBy("sys_created_on");
    gr.setLimit(BATCH_SIZE);
    gr.setOffset(offset);
    gr.query();

    var recordsArray = [];
    while (gr.next()) {
        var record = {};
        var fields = gr.getFields();
        for (var i = 0; i < fields.size(); i++) {
            var fieldName = fields.get(i).getName();
            record[fieldName] = gr.getValue(fieldName);
        }
        recordsArray.push(record);
    }
    if (recordsArray.length === 0) {
        gs.print("No more records to process. Stopping...");
        break; // ✅ Exit when no records are left
    }

    fileExistingContent = fileExistingContent.concat(recordsArray);

    var updateRequestBody = {
        "message": "Appending new incident data",
        "content": GlideStringUtil.base64Encode(JSON.stringify(fileExistingContent, null, 2)), // Base64 encoded content
        "sha": fileSha // Required for updates
    };

    var updateRequest = new sn_ws.RESTMessageV2();
    updateRequest.setHttpMethod('PUT');
    updateRequest.setRequestHeader('Authorization', 'Bearer ' + githubToken);
    updateRequest.setRequestHeader('Content-Type', 'application/json');
    updateRequest.setEndpoint('https://api.github.com/repos/' + repository + '/contents/' + filePath);
    updateRequest.setRequestBody(JSON.stringify(updateRequestBody));

    var updateResponse = updateRequest.execute();
    gs.print('Batch uploaded, Offset: ' + offset);

    offset += BATCH_SIZE;
}

Above code is for Pushing data into  Batches of 10 records to a github repo. But some records got pushed multiple times. I am unable to find where it is going wrong, I guess there is some issue with setLimit and setOffset together. On output offset reaches to 360 but there is only 69 records present.