@satya 

Try this if you want to allow only excel files

function onSubmit() {

var sys_id =   g_form.getUniqueValue();
var gr = new GlideRecord('sys_attachment');
gr.addQuery('table_sys_id',sys_id);
gr.query();
if(gr.next()){
if(gr.getValue('file_name').indexOf('.xlsx') != -1 || gr.getValue('file_name').indexOf('.pdf') != -1){
return true;
}
else{
return false;

}
}

Thanks,

Murthy