Hi Vasu,

I ran into the same issue and made use of g_modal.showFrame which calls an UI Page configured by you with a small field, like in your case, assignment group and pulls out the sys_user_group with simple query attribute. 

You will of course, need to remove this part of your code:

{

            type: 'reference', // type of modal (ie text, reference, etc)
            name: 'assignment_group', // reference field on the current record used to search
            label: 'Assignment Group', // message to display above the search field
            mandatory: true, // sets the field to mandatory
            reference: 'sys_user_group', // table that the reference field in "name" refers to
            referringTable: 'incident', // table of the current record
            referringRecordId: g_form.getUniqueValue(), // sys_id of the current record
            sysparm_query: 'sysId1,sysId2'

        }

and add the following at the end. 

g_modal.showFrame({
     url:'/{name of ui page}?from_workspace=true&{any parameter that will help you achieve your filter}',
     title: 'string',
     size: 'sm, md or lg',
     height: in px,
     callback: function (ret,data) {
     if (ret)
          updateData(data);
     }
});			
function updateData(data){
	if(data.group !== ""){
	       g_form.setValue('assignment_group',data.group);
	       g_form.save();
         }
}

Here's also an example of an UI Page:

HTML:

<g:ui_form>
			
		<input type="hidden" id="support_lvl" name="support_lvl" value="${jvar_support_lvl}" />

		Assignment Group  <g:ui_reference name="group" id="group" table="sys_user_group" completer="AJAXTableCompleter" query="${jvar_query}" />
	
		<table width="100%" cellpadding="0" cellspacing="0">
		<tr>
			<td align="left" nowrap="true"><br />
				<g:dialog_buttons_ok_cancel ok="return ok()" cancel="return cancel()" ok_type="button" ok_style_class="btn btn-primary" cancel_type="button"/>
			</td>
		</tr>
		</table>

</g:ui_form>

Client: 

function cancel() {
    window.location.href = window.location.href;
}

function ok() {

    var x = document.getElementById("group").value;
    iframeMsgHelper.confirm({
        group: x
    });
    if (x == "") {
        if(confirm('Please provide data to submit the dialog.')){
			alert('You will need to restart the process');
		}
		
    }

}
var iframeMsgHelper = (function() {
    function createPayload(action, modalId, data) {
        return {
            messageType: 'IFRAME_MODAL_MESSAGE_TYPE',
            modalAction: action,
            modalId: modalId,
            data: (data ? data : {})
        };
    }

    function pm(window, payload) {
        if (window.parent === window) {
            console.warn('Parent is missing. Is this called inside an iFrame?');
            return;
        }
        window.parent.postMessage(payload, location.origin);
    }

    function IFrameMessagingHelper(window) {
        this.window = window;
        this.src = location.href;
        this.messageHandler = this.messageHandler.bind(this);
        this.window.addEventListener('message', this.messageHandler);
    }

    IFrameMessagingHelper.prototype.messageHandler = function(e) {
        if (e.data.messageType !== 'IFRAME_MODAL_MESSAGE_TYPE' || e.data.modalAction !== 'IFRAME_MODAL_ACTION_INIT') {
            return;
        }
        this.modalId = e.data.modalId;
    };

    IFrameMessagingHelper.prototype.confirm = function(data) {

        var payload = createPayload('IFRAME_MODAL_ACTION_CONFIRMED', this.modalId, data);
        pm(this.window, payload);
    };

    return new IFrameMessagingHelper(window);
}());