Remove Duplicate Objects

[Yuvan Raj Kuma1]

I really need some help to remove duplicate from the below object array, Assume key as unique identifier. Attaching the below code which i used to remove duplicate 

var inputData = [

{"records":[{"key":"BBC123","value":{"messageId":"1709222769268","messageTime":1709222769297,"messageType":"update","messageName":"INVENTORY.SPECIFIC.EVENT","pattern":"SPECIFIC.EVENT","version":"1","source":"dummy","uniqueId":"456789","correlationId":"2024-02-29T16:06:09.297Z","payload":{"callbackURL":" https://test123","productInstanceId":"bbnn","billingAccountNumber":"ABCDDG"}} }]},

{"records":[{"key":"BBC12358","value":{"messageId":"1709222769268","messageTime":1709222769297,"messageType":"update","messageName":"INVENTORY.SPECIFIC.EVENT","pattern":"SPECIFIC.EVENT","version":"1","source":"dummy","uniqueId":"456789","correlationId":"2024-02-29T16:06:09.297Z","payload":{"callbackURL":" https://test123","productInstanceId":"bbnn","billingAccountNumber":"ABCDDG"}} }]},

{"records":[{"key":"BBC123","value":{"messageId":"1709222769268","messageTime":1709222769297,"messageType":"update","messageName":"INVENTORY.SPECIFIC.EVENT","pattern":"SPECIFIC.EVENT","version":"1","source":"dummy","uniqueId":"456789","correlationId":"2024-02-29T16:06:09.297Z","payload":{"callbackURL":" https://test123","productInstanceId":"bbnn","billingAccountNumber":"ABCDDG"}} }]},

{"records":[{"key":"BBC123","value":{"messageId":"1709222769268","messageTime":1709222769297,"messageType":"update","messageName":"INVENTORY.SPECIFIC.EVENT","pattern":"SPECIFIC.EVENT","version":"1","source":"dummy","uniqueId":"456789","correlationId":"2024-02-29T16:06:09.297Z","payload":{"callbackURL":" https://test123","productInstanceId":"bbnn","billingAccountNumber":"ABCDDG"}} }]},

{"records":[{"key":"BBC1234","value":{"messageId":"1709222769268","messageTime":1709222769297,"messageType":"update","messageName":"INVENTORY.SPECIFIC.EVENT","pattern":"SPECIFIC.EVENT","version":"1","source":"dummy","uniqueId":"456789","correlationId":"2024-02-29T16:06:09.297Z","payload":{"callbackURL":" https://test123","productInstanceId":"bbnn","billingAccountNumber":"ABCDDG"}} }]},

{"records":[{"key":"BBC123023","value":{"messageId":"1709222769268","messageTime":1709222769297,"messageType":"update","messageName":"INVENTORY.SPECIFIC.EVENT","pattern":"SPECIFIC.EVENT","version":"1","source":"dummy","uniqueId":"456789","correlationId":"2024-02-29T16:06:09.297Z","payload":{"callbackURL":" https://test123","productInstanceId":"bbnn","billingAccountNumber":"ABCDDG"}} }]}

];

code 1 : 

var uniq = removeDupObj(inputData ,"records.key[]");

function removeDupObj(arr, prop) {

        var seen = {};

        var filteredArray = arr.filter(function(item) {

            var key = item[prop];

            return seen.hasOwnProperty(key) ? false : (seen[key] = true);

        });

        return filteredArray;

    }

Problem : Its giving me only one record not all the unique values

Code 2 : 

function removeDuplicateRecords(data) {

    var uniqueRecords = {};

    var result = [];

 gs.info(data)

    for (var i = 0; i < data.length; i++) {

        var key = data[i].records[0].key;

        gs.info(key);

        if (!uniqueRecords[key]) {

            uniqueRecords[key] = true;

            result.push(data[i]);

        }

    }

    return result;

}

var deduplicatedData = removeDuplicateRecords(JSON.stringify(productInvetoriesCDW));

gs.log(JSON.stringify(deduplicatedData, null, 2));

Problem getting 

Evaluator: com.glide.script.RhinoEcmaError: Cannot read property "0" from undefined
   script : Line(72) column(0)

[Sandeep Rajput · Accepted solution]

@Yuvan Raj Kuma1 I have fixed your code 1 in the following script and managed to get the unique results through it.

var inputData = [
{"records":[{"key":"BBC123","value":{"messageId":"1709222769268","messageTime":1709222769297,"messageType":"update","messageName":"INVENTORY.SPECIFIC.EVENT","pattern":"SPECIFIC.EVENT","version":"1","source":"dummy","uniqueId":"456789","correlationId":"2024-02-29T16:06:09.297Z","payload":{"callbackURL":" https://test123","productInstanceId":"bbnn","billingAccountNumber":"ABCDDG"}} }]},

{"records":[{"key":"BBC12358","value":{"messageId":"1709222769268","messageTime":1709222769297,"messageType":"update","messageName":"INVENTORY.SPECIFIC.EVENT","pattern":"SPECIFIC.EVENT","version":"1","source":"dummy","uniqueId":"456789","correlationId":"2024-02-29T16:06:09.297Z","payload":{"callbackURL":" https://test123","productInstanceId":"bbnn","billingAccountNumber":"ABCDDG"}} }]},

{"records":[{"key":"BBC123","value":{"messageId":"1709222769268","messageTime":1709222769297,"messageType":"update","messageName":"INVENTORY.SPECIFIC.EVENT","pattern":"SPECIFIC.EVENT","version":"1","source":"dummy","uniqueId":"456789","correlationId":"2024-02-29T16:06:09.297Z","payload":{"callbackURL":" https://test123","productInstanceId":"bbnn","billingAccountNumber":"ABCDDG"}} }]},

{"records":[{"key":"BBC123","value":{"messageId":"1709222769268","messageTime":1709222769297,"messageType":"update","messageName":"INVENTORY.SPECIFIC.EVENT","pattern":"SPECIFIC.EVENT","version":"1","source":"dummy","uniqueId":"456789","correlationId":"2024-02-29T16:06:09.297Z","payload":{"callbackURL":" https://test123","productInstanceId":"bbnn","billingAccountNumber":"ABCDDG"}} }]},

{"records":[{"key":"BBC1234","value":{"messageId":"1709222769268","messageTime":1709222769297,"messageType":"update","messageName":"INVENTORY.SPECIFIC.EVENT","pattern":"SPECIFIC.EVENT","version":"1","source":"dummy","uniqueId":"456789","correlationId":"2024-02-29T16:06:09.297Z","payload":{"callbackURL":" https://test123","productInstanceId":"bbnn","billingAccountNumber":"ABCDDG"}} }]},

{"records":[{"key":"BBC123023","value":{"messageId":"1709222769268","messageTime":1709222769297,"messageType":"update","messageName":"INVENTORY.SPECIFIC.EVENT","pattern":"SPECIFIC.EVENT","version":"1","source":"dummy","uniqueId":"456789","correlationId":"2024-02-29T16:06:09.297Z","payload":{"callbackURL":" https://test123","productInstanceId":"bbnn","billingAccountNumber":"ABCDDG"}} }]}

];

 var uniq = removeDupObj(inputData ,"records");


 gs.print(JSON.stringify(uniq));

function removeDupObj(arr, prop) {
        var seen = {};
        var i=0;
        var filteredArray = arr.filter(function(item) {
            var key = item[prop][0].key;            
            return seen.hasOwnProperty(key) ? false : (seen[key] = true);
        });
        return filteredArray;
    }

Here is the output

*** Script: [{"records":[{"key":"BBC123","value":{"messageId":"1709222769268","messageTime":1709222769297,"messageType":"update","messageName":"INVENTORY.SPECIFIC.EVENT","pattern":"SPECIFIC.EVENT","version":"1","source":"dummy","uniqueId":"456789","correlationId":"2024-02-29T16:06:09.297Z","payload":{"callbackURL":" https://test123","productInstanceId":"bbnn","billingAccountNumber":"ABCDDG"}}}]},{"records":[{"key":"BBC12358","value":{"messageId":"1709222769268","messageTime":1709222769297,"messageType":"update","messageName":"INVENTORY.SPECIFIC.EVENT","pattern":"SPECIFIC.EVENT","version":"1","source":"dummy","uniqueId":"456789","correlationId":"2024-02-29T16:06:09.297Z","payload":{"callbackURL":" https://test123","productInstanceId":"bbnn","billingAccountNumber":"ABCDDG"}}}]},{"records":[{"key":"BBC1234","value":{"messageId":"1709222769268","messageTime":1709222769297,"messageType":"update","messageName":"INVENTORY.SPECIFIC.EVENT","pattern":"SPECIFIC.EVENT","version":"1","source":"dummy","uniqueId":"456789","correlationId":"2024-02-29T16:06:09.297Z","payload":{"callbackURL":" https://test123","productInstanceId":"bbnn","billingAccountNumber":"ABCDDG"}}}]},{"records":[{"key":"BBC123023","value":{"messageId":"1709222769268","messageTime":1709222769297,"messageType":"update","messageName":"INVENTORY.SPECIFIC.EVENT","pattern":"SPECIFIC.EVENT","version":"1","source":"dummy","uniqueId":"456789","correlationId":"2024-02-29T16:06:09.297Z","payload":{"callbackURL":" https://test123","productInstanceId":"bbnn","billingAccountNumber":"ABCDDG"}}}]}]

Please mark the answer helpful and correct if it manages to address your issue.

[Yuvan Raj Kuma1]

@Sandeep Rajput Thats awesome thanks for your help and time!!!😊