Displaying records from a Data Fabric table in Reference variable on a Catalog Item

[Mike Grimm]

I have a requirement to display records from a Data Fabric table in a Reference variable on a Record Producer based on the Account that is selected on the form. I created a script include and called it in the Reference qual on the Reference variable, and the logs show that the Account is passing and that the Data Fabric record sys_ids are being returned, but on the variable itself the records never actually show and it ends up freezing up the instance. If I do wait long enough, no values are ever displayed. In doing some research, I found that this might be a platform limitation and that I would not be able to show the records. The suggestions that were given were to either A) Make it a Lookup Select Box (not a viable option because I want to display additional columns and the platform says there's a limit of 10K records that can be returned) or B) Create native tables for each of the Data Fabric tables and create a scheduled job that pulls the information from the Data Fabric table to the native tables (there are over 100 million records on the table and it is growing). I'm not sure that the 2nd option will work either with the sheer number of records and we were originally using native tables and the load times for the data was very slow (in some instances 20-30 seconds for the data to load). Does anyone have any experience with this? Below is the code that I am using in the script include:

getRecords: function(acct) {
        var account = new GlideRecord('customer_account');
        account.get(acct);
        var acctID = account.identification_number;
        var encQuery;
        if (acctID == "" && account.account_parent == "") {
            encQuery = this._getCustomerIDEncQuery(acct);
        } else {
            encQuery = "u_client_id=" + acctID;
        }
 
        var recordIds = [];
        var dfRecords = new GlideRecord('u_data_fabric_table');
        dfRecords.addEncodedQuery(encQuery);
        dfRecords.query();

        while (dfRecords.next()) {
            recordIds.push(dfRecords.sys_id + '');
        }
        return "sys_idIN" + recordIds.toString();
    },

_getCustomerIDEncQuery: function(acct) {
        var accountIDs = [];
        var childAccounts = new GlideRecord('customer_account');
        childAccounts.addQuery('account_parent', acct);
        childAccounts.query();

        while (childAccounts.next()) {
            if (childAccounts.identification_number != '') {
                accountIDs.push(childAccounts.identification_number.toString());
            }
        }

        var encQuery;
        if (childAccounts.getRowCount() == 1) {
            encQuery = "u_client_id=" + accountIDs;
        } else {
            encQuery = "u_client_id=" + accountIDs.join("^ORu_client_id=");
        }
        return encQuery;
    },