Leveraging APIs like Notion with a nice caching system

import { AssetCache } from '@11ty/eleventy-fetch';

/**
 * Query an API to check for updates, and if the data is stale, query it and cache it, else, serve from the cache.
 * @param {object} settings Configuration object for the cacheable API calls.
 * @param {object} settings.label Unique name given to each this particular API in the context of the build. If already in use, it will collide with existing data and serve the wrong cache.
 * @param {boolean} [settings.skipLocalCache] Optional. Whether the cache will be skipped (this should be read from a dot-env value and passed here). Defaults to `false`.
 * @param {string[]} settings.infoDateMarkers List of possible properties from the database info query that will hold useful date information, in order of usefulness.
 * @param {() => Promise<object>} settings.getInfo The async function which queries and processes data from the API to check for cache staleness. Should return an object with at least one property date-marker.
 * @param {(dbInfo: object) => Promise<any>} settings.getData The async function which queries and processes data from the API when the cache is unusable. Receives the `getInfo` result as the sole argument.
 * @returns {any} Data retrived from the API of from the cache.
 */
export default async function (settings) {
	const apiLabel = settings.label;
	const infoApiLabel = `${apiLabel}_info`;
	const dataApiLabel = `${apiLabel}_data`;
	const skipLocalCache = settings.skipLocalCache || false;
	let infoDateMarkers = settings.infoDateMarkers;
	if (infoDateMarkers.length === 0) {
		throw new Error('The `infoDateMarkers` cannot be empty.');
	} else if (!Array.isArray(infoDateMarkers) && typeof infoDateMarkers !== 'string') {
		throw new Error('The `infoDateMarkers` must be a string or an array of strings.');
	} else if (!Array.isArray(infoDateMarkers) && typeof infoDateMarkers === 'string') {
		infoDateMarkers = [infoDateMarkers]; // We need this to be an array
	}
	const getInfo = settings.getInfo;
	const getData = settings.getData;
	const getInfoMarker = (info) => info[infoDateMarkers.find((m) => info.hasOwnProperty(m) && info[m])] || '';

	// Initialise the asset caches
	const dbInfoCache = new AssetCache(infoApiLabel);
	const dbDataCache = new AssetCache(dataApiLabel);

	// Local dev: allow complete bypass
	if (skipLocalCache) {
		console.log(apiLabel + ': Skipping data cache for local development.');
	}

	// Grab the database's latest info (not the content, just metadata about it)
	const dbInfo = await (!skipLocalCache && dbInfoCache.cachedObject ? dbInfoCache.getCachedContents('json') : getInfo());

	// Determine when the database was updated by looking for the first available property
	const dbLastEdit = getInfoMarker(dbInfo);

	// Check if there is a cache object for the value we're after
	const isCachePresent = !skipLocalCache && dbInfoCache.cachedObject && dbDataCache.cachedObject;

	// If we have cached data, check if it's not outdated
	if (isCachePresent) {
		// Get the last cached value for the database info
		const cacheLastEditInfo = (await dbInfoCache.getCachedContents('json')) || {};
		const cacheLastEdit = getInfoMarker(cacheLastEditInfo);

		// If the cached last edit matches the live last edit, return the cached database contents and stop here
		if (dbLastEdit === cacheLastEdit) {
			const dbCache = await dbDataCache.getCachedContents('json');
			console.log(apiLabel + ': Found and reused cached data.');
			return dbCache;
		}
	}

	// We don't have data we can use, call the full APIs and cache the data
	console.log(apiLabel + ': No cached data, fetching latest data.');
	const dbData = await getData(dbInfo);

	// If this did change, save the new last edit date value after we know that the database access was successful
	dbInfoCache.save(dbInfo, 'json');

	// Save the data in the cache
	dbDataCache.save(dbData, 'json');

	// And finally, return the data
	return dbData;
}

import 'dotenv/config';
import apiCache from './api-cache.js';
import { Client as NotionClient } from '@notionhq/client';

/**
 * Retrieve data from a Notion database.
 * @param {object} queryConfig The configuration for the database call and data processing.
 * @param {string} queryConfig.databaseId The unique ID for the database to query.
 * @param {string} [queryConfig.label] Optional. Custom label to assign to the query, used for cache key generation and identification in logs. Defaults to the database ID.
 * @param {string[]} [queryConfig.propsToUse] Optional. List of properties to retrieve for each item in the database. Defaults to `['Title']`.
 * @param {object} [queryConfig.filter] Optional. Structured filtering to restrict which results are returned from the database. Defaults to an empty object to load all items. @see <https://developers.notion.com/reference/post-database-query-filter>
 * @param {(data: any) => any} [queryConfig.dataPostProcess] Optional. Processing function to run on the fetched data before it is cached. Async-friendly. Defaults to unprocessed data.
 * @returns {object[]} List of results.
 */
export default async function (queryConfig) {
	const databaseId = queryConfig.databaseId;
	const label = queryConfig.label || databaseId;
	const propsToUse = queryConfig.propsToUse || ['Title'];
	const filter = queryConfig.filter || {};
	const dataPostProcess = queryConfig.dataPostProcess || null;

	// Set up Notion stuff
	const notionClient = new NotionClient({ auth: process.env.NOTION_BEARER_TOKEN });

	// Get the cache data, and if missing or stale, provide the complete data query logic
	const cachedData = apiCache({
		label: label,
		infoDateMarkers: ['last_edited_time', 'created_time'],
		getInfo: async () => notionClient.databases.retrieve({ database_id: databaseId }),
		getData: async function (dbInfo) {
			// Based on the database info, build a list of the IDs for the properties needed based on their name
			const databaseProps = dbInfo.properties;
			let propsById = [];
			for (let p in databaseProps) {
				if (propsToUse.includes(p)) {
					propsById.push(databaseProps[p].id);
				}
			}

			// Grab the results from the database
			let dbData = [];
			let data = { has_more: true, next_cursor: -1, results: [] };
			let queryObject = {
				database_id: databaseId,
				filter_properties: propsById,
				filter: filter,
			};

			// Loop as long as there are results
			while (data.has_more) {
				const queryCursor = data.next_cursor !== -1 ? { start_cursor: data.next_cursor } : {};
				const query = Object.assign(queryCursor, queryObject);
				data = await notionClient.databases.query(query);

				dbData = [...dbData, ...data.results];
			}

			// Only keep useful data
			const dbDataProcessed = typeof dataPostProcess === 'function' ? dataPostProcess(dbData) : dbData;

			// Return that sweet, sweet data
			return dbDataProcessed;
		},
	});

	return cachedData;
}

// The PSN API is still in CommonJS while I use ES Modules, so this little hack allows to use require()
import Module from 'node:module';
const require = Module.createRequire(import.meta.url);
import 'dotenv/config';
import apiCache from './api-cache.js';
const { exchangeNpssoForCode, exchangeCodeForAccessToken, getUserTitles } = require('psn-api');

const PSN_API_MAX_PAGE_SIZE = 800; // This is a hardcoded limit to the number of titles that can be returned in a single getUserTitles call
const psnApiPageSize = Math.min(500, PSN_API_MAX_PAGE_SIZE); // Preferred page size, but not more than the max

/** Retrieve trophy data from the PlayStation Network. */
export default async function () {
	// Set up access to the PSN API
	const accessCode = await exchangeNpssoForCode(process.env.PLAYSTATION_NPSSO_TOKEN);
	const authorization = await exchangeCodeForAccessToken(accessCode);

	// Get the cache data, and if missing or stale, provide the complete data query logic
	const cachedData = apiCache({
		label: 'psn-api.js',
		infoDateMarkers: ['lastUpdatedDateTime'],
		getInfo: async () => getUserTitles({ accessToken: authorization.accessToken }, 'me', { limit: 1 }).then((t) => t.trophyTitles[0]),
		getData: async function (dbInfo) {
			let trophyTitlesResponse = []; // Initialise array to hold all the values
			let nextOffset = 0; // Start at the beginning

			// Loop through the pages of the API
			while (nextOffset !== undefined) {
				// Get the slice of data based on the current page
				let responseSlice = await getUserTitles({ accessToken: authorization.accessToken }, 'me', { limit: psnApiPageSize, offset: nextOffset });

				// Append the data from this page
				trophyTitlesResponse = trophyTitlesResponse.concat(responseSlice.trophyTitles);

				// This value will be `undefined` if there is no more data, stopping the loop as a result
				nextOffset = responseSlice.nextOffset;
			}

			// Remove hidden titles
			const trophyTitles = trophyTitlesResponse.filter((t) => !t.hiddenFlag);

			// Return that sweet, sweet data
			return trophyTitles;
		},
	});

	return cachedData;
}