Node.js Promise resolution with Airtable API before completing all record pagination

[SilentSailing34]

I’m working with the Airtable API in Node.js and having trouble with async/await flow. My goal is to fetch all records from an Airtable base and only return a success value after every single record has been processed. The issue I’m facing is that the Promise resolves too early, before all pages of records have been retrieved. I need the function to wait until the complete dataset is processed before moving to the next step. Here’s my current implementation:

let statusFlag = 0;

async function MainProcess(){
    statusFlag = await dataProcessor.fetchAllRecords();
    console.log("Status flag value: ", statusFlag);
    if(statusFlag == 1){
        statusFlag = await qualityChecker.fetchAllRecords(); 
        console.log("Status flag value: ", statusFlag);
    }
}

And here’s my Airtable API function:

async function fetchAllRecords()
{
    return new Promise((resolve, reject) => {
        console.log("Starting data fetch process");
        var recordCount = 0;
        let query = database("User Data").select({
        }).eachPage(function processPage(recordSet, getNextPage) {
            recordSet.forEach(function(item) {
                let itemID = item.get("ID");
                let userEmail = item.get("User Email");
                ProcessUserData(itemID, userEmail);
                recordCount++;
            });
            getNextPage();
        }, function finished(error) {
            if (error) {
                console.error(error);
                reject("Operation failed");
            } else {
                resolve(1);
            }
        });
    });
}

How can I make sure the Promise only resolves after all records from every page have been successfully retrieved and processed?

[Nate_91Surf]

Your ProcessUserData function is probably async, but you’re not waiting for it to finish. When you call ProcessUserData(itemID, userEmail) without await, the code just keeps running without waiting for the processing to complete.

I hit this same issue working with large API datasets. You need to make ProcessUserData return a Promise and collect all those promises before moving on:

let allPromises = [];
recordSet.forEach(function(item) {
    let itemID = item.get("ID");
    let userEmail = item.get("User Email");
    allPromises.push(ProcessUserData(itemID, userEmail));
    recordCount++;
});
await Promise.all(allPromises);

Call getNextPage() after Promise.all finishes. This way each page waits for all records to process before moving to the next one. Your finished callback will only run after everything’s actually done.

[emparker]

The Airtable eachPage method handles pagination automatically, but your Promise resolves when the last page is reached - not when all processing finishes.

I hit this exact issue processing thousands of records from different APIs. The problem is eachPage runs synchronously through pages, but your ProcessUserData calls are probably async operations that don’t block the pagination.

Here’s how I’d fix it:

async function fetchAllRecords() {
    const allRecords = [];
    
    return new Promise((resolve, reject) => {
        database("User Data").select({}
        ).eachPage(function processPage(recordSet, getNextPage) {
            // Collect records instead of processing immediately
            recordSet.forEach(function(item) {
                allRecords.push({
                    id: item.get("ID"),
                    email: item.get("User Email")
                });
            });
            getNextPage();
        }, async function finished(error) {
            if (error) {
                reject("Operation failed");
                return;
            }
            
            // Now process all records after pagination is complete
            try {
                for (const record of allRecords) {
                    await ProcessUserData(record.id, record.email);
                }
                resolve(1);
            } catch (processError) {
                reject(processError);
            }
        });
    });
}

This separates data collection from processing. Collect all records first, then process them sequentially after pagination finishes. Way more predictable than trying to sync processing with pagination.

[jess_brown]

your eachPage callback is async but you’re not handling it right. wrap the whole thing in async/await instead of the Promise constructor - way cleaner and fewer bugs. also check if ProcessUserData actually needs awaiting because right now it’s just fire-and-forget.

[sofiag]

The problem is you’re mixing Promise constructors with async operations. I’ve hit similar pagination issues - it’s way cleaner to skip the manual Promise wrapper around eachPage. Here’s what works better: javascript async function fetchAllRecords() { const processedItems = []; await new Promise((resolve, reject) => { database("User Data").select({}) .eachPage(async function processPage(recordSet, getNextPage) { const pagePromises = recordSet.map(async (item) => { const itemID = item.get("ID"); const userEmail = item.get("User Email"); return await ProcessUserData(itemID, userEmail); }); await Promise.all(pagePromises); processedItems.push(...recordSet); getNextPage(); }, function finished(error) { if (error) reject(error); else resolve(); }); }); return 1; // Only reached after all pages processed } The key is awaiting Promise.all for each page before calling getNextPage. This makes sure each page finishes processing before moving on, so the function only returns when everything’s done.