Uh oh, your requirement was not clear initially.

now you need both script include,and  glideajax to prevent creation.

using your script itself:

make sure client callable is checked

var siteDetails = Class.create();
siteDetails.prototype = Object.extendsObject(AbstractAjaxProcessor, {
sitecheck: function(){
var ans= 'false';
var gr=new GlideRecord('u_cmdb_ci_sites_using');
gr.addQuery("name",this.getParameter('sysparam_id'));
gr.query();
gr.next();
if(gr.next())
{
ans='true';
}
return ans;
},
type: 'siteDetails'
});

Catalog client script(inside multiline variable set)

function onSubmit() {
var arr=[];
var loc=g_form.getValue('create_new_sites');// Internal name of multiline variable set
if(loc == '[]' ){
g_form.addErrorMessage('You must enter at least 1 row');
return false;
}
var data = JSON.parse(loc);
var ct= 0;
for(var i in data){
if (data.hasOwnProperty(i)) {
arr.push(data[i].name1.toString().replace(/\s/g, "")) ; //added a ) here
}
}

for(var j=0;j<arr.length;j++){

var ga=new GlideAjax('siteDetails');
ga.addParam('sysparm_name','sitecheck');
ga.addParam('sysparam_id',arr[j].toString());
ga.getXML(Process);
}
}

function Process(response)
{
var answer=response.responseXML.documentElement.getAttribute("answer");
if(answer=='false')
{
g_form.clearValue('name1');
alert("error");
return false;
}
}

these should work unless there are  other dependency with the namings