API Token Refresh

[BVal]

We are attempting to grab logs from the "syslog_transaction", "sysevent", "sys_audit_delete", "sys_audit_role" tables and some of our customers have large datasets which require a long time to access and parse. During this parsing time, the access token in get_token() expires and we attempt to run update_token() 5 times max. We are able to get a new token and update the session headers successfully but the API is rejecting our new token. 

In the code: config is our credentials file that we pass through that has all the parameters required to access the target account.

Any ideas why this might be happening and how to resolve? 

import requests
import time
import json

from urllib.parse import urljoin
from datetime import datetime, timedelta


def get_token(config, session):
# Grab initial token to access the account
    token_url = f"{config.parameters.transport.instance}/oauth_token.do"
    headers = {
        "Content-Type": "application/x-www-form-urlencoded",
    }
    payload = {
        "grant_type": "password",
        "client_id": config.parameters.transport.client_id,
        "client_secret": config.parameters.transport.secrets.client_secret,
        "username": config.parameters.transport.secrets.username,
        "password": config.parameters.transport.secrets.password
    }
    response = session.post(token_url, headers=headers, data=payload, verify=False)
    if response.status_code == 200:
        time_now = datetime.utcnow()
        time_now_str = time_now.strftime("'%Y-%m-%d','%H:%M:%S'")
        token_expiration = time_now + timedelta(seconds=1799)
        response_json = response.json()
        access_token = response_json['access_token']
        refresh_token = response_json['refresh_token']
        headers = {
        "Accept": "application/json",
        "Content-Type": "application/json",
        "Authorization": f"Bearer {access_token}"
        }        
        session.headers.update(headers)
        return refresh_token, time_now_str, token_expiration
    else:
        sys.exit(1)

def update_token(config, session, refresh_token):
# attempt to refresh the token and uupdate the headers with the new token data.
    time_now = datetime.utcnow()
    token_expiration = time_now + timedelta(seconds=1799)
    token_url = f"{config.parameters.transport.instance}/oauth_token.do"
    headers = {
        "Content-Type": "application/x-www-form-urlencoded",
    }
    payload = {
        "grant_type": "refresh_token",
        "client_id": config.parameters.transport.client_id,
        "client_secret": config.parameters.transport.secrets.client_secret,
        "refresh_token": refresh_token
    }
    response = session.post(token_url, headers=headers, data=payload, verify=False)
    if response.status_code == 200:
        response_json = response.json()
        access_token = response_json['access_token']
        headers = {
        "Accept": "application/json",
        "Content-Type": "application/json",
        "Authorization": f"Bearer {access_token}"
        }
        session.headers.update(headers)
        return token_expiration
    else:
        return None
    
def process_events(config, session, outputs, refresh_token, time_now, token_expiration):
# main logic to grab and process events
    last_pull = (datetime.utcnow() - timedelta(days=1)).strftime("'%Y-%m-%d','%H:%M:%S'")
    tables = ["syslog_transaction", "sysevent", "sys_audit_delete", "sys_audit_role"]
    failed = []
    for table in tables:
        limit = 1000
        offset = 0
        while True:
            if datetime.utcnow() >= token_expiration:
                token_expiration = update_token(config, session, refresh_token)
                if token_expiration == None:
                    return time_now
            count = 0
            params = {
            "sysparm_limit": limit,
            "sysparm_offset": offset,
            "sysparm_query": f"sys_created_onBETWEENjavascript:gs.dateGenerate({last_pull})@javascript:gs.dateGenerate({time_now})"
            }
            base_uri = config.parameters.transport.instance
            endpoint = f"/api/now/table/{table}"
            uri = urljoin(base_uri, endpoint)
            log_response = session.get(uri, params=params, verify=False)
            retry_count = 0
            if log_response.status_code == 401 and retry_count < 5:
                retry_count += 1
                token_expiration = update_token(config, session, refresh_token)
                time.sleep(30)
                continue
            if log_response.status_code != 200:
                break
            log_response_json = log_response.json()
            events = log_response_json.get('result', [])
            if len(events) == 0:
                break
            for event in events:
                count += 1
                print (event)
            if count == limit:
                offset += count
            else:
                break
    return time_now

# config; our creds.ini file that is being passed through with all of the parameters required to access the target account.
with requests.Session() as session:
    refresh_token, time_now, token_expiration = get_token(config, session)
    time_now = process_events(config, session, refresh_token, time_now, token_expiration)