From 1335083bedd1dd563cfa9c54e69abc189ba3ec7b Mon Sep 17 00:00:00 2001 From: Emelia Smith Date: Thu, 18 Jan 2024 19:40:25 +0100 Subject: [PATCH] Streaming: replace npmlog with pino & pino-http (#27828) --- streaming/.eslintrc.js | 11 ++ streaming/index.js | 334 ++++++++++++++++++++---------------- streaming/logging.js | 119 +++++++++++++ streaming/package.json | 5 +- yarn.lock | 376 +++++++++++++++++++++++++++++------------ 5 files changed, 593 insertions(+), 252 deletions(-) create mode 100644 streaming/logging.js diff --git a/streaming/.eslintrc.js b/streaming/.eslintrc.js index 5e2d233c6..188ebb512 100644 --- a/streaming/.eslintrc.js +++ b/streaming/.eslintrc.js @@ -15,7 +15,18 @@ module.exports = defineConfig({ ecmaVersion: 2021, }, rules: { + // In the streaming server we need to delete some variables to ensure + // garbage collection takes place on the values referenced by those objects; + // The alternative is to declare the variable as nullable, but then we need + // to assert it's in existence before every use, which becomes much harder + // to maintain. + 'no-delete-var': 'off', + + // The streaming server is written in commonjs, not ESM for now: 'import/no-commonjs': 'off', + + // This overrides the base configuration for this rule to pick up + // dependencies for the streaming server from the correct package.json file. 'import/no-extraneous-dependencies': [ 'error', { diff --git a/streaming/index.js b/streaming/index.js index c8124fcc0..aa75a08b7 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -10,12 +10,11 @@ const dotenv = require('dotenv'); const express = require('express'); const Redis = require('ioredis'); const { JSDOM } = require('jsdom'); -const log = require('npmlog'); const pg = require('pg'); const dbUrlToConfig = require('pg-connection-string').parse; -const uuid = require('uuid'); const WebSocket = require('ws'); +const { logger, httpLogger, initializeLogLevel, attachWebsocketHttpLogger, createWebsocketLogger } = require('./logging'); const { setupMetrics } = require('./metrics'); const { isTruthy } = require("./utils"); @@ -28,15 +27,30 @@ dotenv.config({ path: path.resolve(__dirname, path.join('..', dotenvFile)) }); -log.level = process.env.LOG_LEVEL || 'verbose'; +initializeLogLevel(process.env, environment); + +/** + * Declares the result type for accountFromToken / accountFromRequest. + * + * Note: This is here because jsdoc doesn't like importing types that + * are nested in functions + * @typedef ResolvedAccount + * @property {string} accessTokenId + * @property {string[]} scopes + * @property {string} accountId + * @property {string[]} chosenLanguages + * @property {string} deviceId + */ /** * @param {Object.} config */ const createRedisClient = async (config) => { const { redisParams, redisUrl } = config; + // @ts-ignore const client = new Redis(redisUrl, redisParams); - client.on('error', (err) => log.error('Redis Client Error!', err)); + // @ts-ignore + client.on('error', (err) => logger.error({ err }, 'Redis Client Error!')); return client; }; @@ -61,12 +75,12 @@ const parseJSON = (json, req) => { */ if (req) { if (req.accountId) { - log.warn(req.requestId, `Error parsing message from user ${req.accountId}: ${err}`); + req.log.error({ err }, `Error parsing message from user ${req.accountId}`); } else { - log.silly(req.requestId, `Error parsing message from ${req.remoteAddress}: ${err}`); + req.log.error({ err }, `Error parsing message from ${req.remoteAddress}`); } } else { - log.warn(`Error parsing message from redis: ${err}`); + logger.error({ err }, `Error parsing message from redis`); } return null; } @@ -105,6 +119,7 @@ const pgConfigFromEnv = (env) => { baseConfig.password = env.DB_PASS; } } else { + // @ts-ignore baseConfig = pgConfigs[environment]; if (env.DB_SSLMODE) { @@ -149,6 +164,7 @@ const redisConfigFromEnv = (env) => { // redisParams.path takes precedence over host and port. if (env.REDIS_URL && env.REDIS_URL.startsWith('unix://')) { + // @ts-ignore redisParams.path = env.REDIS_URL.slice(7); } @@ -195,6 +211,7 @@ const startServer = async () => { app.set('trust proxy', process.env.TRUSTED_PROXY_IP ? process.env.TRUSTED_PROXY_IP.split(/(?:\s*,\s*|\s+)/) : 'loopback,uniquelocal'); + app.use(httpLogger); app.use(cors()); // Handle eventsource & other http requests: @@ -202,32 +219,37 @@ const startServer = async () => { // Handle upgrade requests: server.on('upgrade', async function handleUpgrade(request, socket, head) { + // Setup the HTTP logger, since websocket upgrades don't get the usual http + // logger. This decorates the `request` object. + attachWebsocketHttpLogger(request); + + request.log.info("HTTP Upgrade Requested"); + /** @param {Error} err */ const onSocketError = (err) => { - log.error(`Error with websocket upgrade: ${err}`); + request.log.error({ error: err }, err.message); }; socket.on('error', onSocketError); - // Authenticate: - try { - await accountFromRequest(request); - } catch (err) { - log.error(`Error authenticating request: ${err}`); + /** @type {ResolvedAccount} */ + let resolvedAccount; + try { + resolvedAccount = await accountFromRequest(request); + } catch (err) { // Unfortunately for using the on('upgrade') setup, we need to manually // write a HTTP Response to the Socket to close the connection upgrade // attempt, so the following code is to handle all of that. const statusCode = err.status ?? 401; - /** @type {Record} */ + /** @type {Record} */ const headers = { 'Connection': 'close', 'Content-Type': 'text/plain', 'Content-Length': 0, 'X-Request-Id': request.id, - // TODO: Send the error message via header so it can be debugged in - // developer tools + 'X-Error-Message': err.status ? err.toString() : 'An unexpected error occurred' }; // Ensure the socket is closed once we've finished writing to it: @@ -238,15 +260,28 @@ const startServer = async () => { // Write the HTTP response manually: socket.end(`HTTP/1.1 ${statusCode} ${http.STATUS_CODES[statusCode]}\r\n${Object.keys(headers).map((key) => `${key}: ${headers[key]}`).join('\r\n')}\r\n\r\n`); + // Finally, log the error: + request.log.error({ + err, + res: { + statusCode, + headers + } + }, err.toString()); + return; } + // Remove the error handler, wss.handleUpgrade has its own: + socket.removeListener('error', onSocketError); + wss.handleUpgrade(request, socket, head, function done(ws) { - // Remove the error handler: - socket.removeListener('error', onSocketError); + request.log.info("Authenticated request & upgraded to WebSocket connection"); + + const wsLogger = createWebsocketLogger(request, resolvedAccount); // Start the connection: - wss.emit('connection', ws, request); + wss.emit('connection', ws, request, wsLogger); }); }); @@ -273,9 +308,9 @@ const startServer = async () => { // When checking metrics in the browser, the favicon is requested this // prevents the request from falling through to the API Router, which would // error for this endpoint: - app.get('/favicon.ico', (req, res) => res.status(404).end()); + app.get('/favicon.ico', (_req, res) => res.status(404).end()); - app.get('/api/v1/streaming/health', (req, res) => { + app.get('/api/v1/streaming/health', (_req, res) => { res.writeHead(200, { 'Content-Type': 'text/plain' }); res.end('OK'); }); @@ -285,7 +320,7 @@ const startServer = async () => { res.set('Content-Type', metrics.register.contentType); res.end(await metrics.register.metrics()); } catch (ex) { - log.error(ex); + req.log.error(ex); res.status(500).end(); } }); @@ -319,7 +354,7 @@ const startServer = async () => { const callbacks = subs[channel]; - log.silly(`New message on channel ${redisPrefix}${channel}`); + logger.debug(`New message on channel ${redisPrefix}${channel}`); if (!callbacks) { return; @@ -343,17 +378,16 @@ const startServer = async () => { * @param {SubscriptionListener} callback */ const subscribe = (channel, callback) => { - log.silly(`Adding listener for ${channel}`); + logger.debug(`Adding listener for ${channel}`); subs[channel] = subs[channel] || []; if (subs[channel].length === 0) { - log.verbose(`Subscribe ${channel}`); + logger.debug(`Subscribe ${channel}`); redisSubscribeClient.subscribe(channel, (err, count) => { if (err) { - log.error(`Error subscribing to ${channel}`); - } - else { + logger.error(`Error subscribing to ${channel}`); + } else if (typeof count === 'number') { redisSubscriptions.set(count); } }); @@ -367,7 +401,7 @@ const startServer = async () => { * @param {SubscriptionListener} callback */ const unsubscribe = (channel, callback) => { - log.silly(`Removing listener for ${channel}`); + logger.debug(`Removing listener for ${channel}`); if (!subs[channel]) { return; @@ -376,12 +410,11 @@ const startServer = async () => { subs[channel] = subs[channel].filter(item => item !== callback); if (subs[channel].length === 0) { - log.verbose(`Unsubscribe ${channel}`); + logger.debug(`Unsubscribe ${channel}`); redisSubscribeClient.unsubscribe(channel, (err, count) => { if (err) { - log.error(`Error unsubscribing to ${channel}`); - } - else { + logger.error(`Error unsubscribing to ${channel}`); + } else if (typeof count === 'number') { redisSubscriptions.set(count); } }); @@ -390,45 +423,13 @@ const startServer = async () => { }; /** - * @param {any} req - * @param {any} res - * @param {function(Error=): void} next - */ - const setRequestId = (req, res, next) => { - req.requestId = uuid.v4(); - res.header('X-Request-Id', req.requestId); - - next(); - }; - - /** - * @param {any} req - * @param {any} res - * @param {function(Error=): void} next - */ - const setRemoteAddress = (req, res, next) => { - req.remoteAddress = req.connection.remoteAddress; - - next(); - }; - - /** - * @param {any} req + * @param {http.IncomingMessage & ResolvedAccount} req * @param {string[]} necessaryScopes * @returns {boolean} */ const isInScope = (req, necessaryScopes) => req.scopes.some(scope => necessaryScopes.includes(scope)); - /** - * @typedef ResolvedAccount - * @property {string} accessTokenId - * @property {string[]} scopes - * @property {string} accountId - * @property {string[]} chosenLanguages - * @property {string} deviceId - */ - /** * @param {string} token * @param {any} req @@ -441,6 +442,7 @@ const startServer = async () => { return; } + // @ts-ignore client.query('SELECT oauth_access_tokens.id, oauth_access_tokens.resource_owner_id, users.account_id, users.chosen_languages, oauth_access_tokens.scopes, devices.device_id FROM oauth_access_tokens INNER JOIN users ON oauth_access_tokens.resource_owner_id = users.id LEFT OUTER JOIN devices ON oauth_access_tokens.id = devices.access_token_id WHERE oauth_access_tokens.token = $1 AND oauth_access_tokens.revoked_at IS NULL LIMIT 1', [token], (err, result) => { done(); @@ -451,6 +453,7 @@ const startServer = async () => { if (result.rows.length === 0) { err = new Error('Invalid access token'); + // @ts-ignore err.status = 401; reject(err); @@ -485,6 +488,7 @@ const startServer = async () => { if (!authorization && !accessToken) { const err = new Error('Missing access token'); + // @ts-ignore err.status = 401; reject(err); @@ -529,15 +533,16 @@ const startServer = async () => { }; /** - * @param {any} req + * @param {http.IncomingMessage & ResolvedAccount} req + * @param {import('pino').Logger} logger * @param {string|undefined} channelName * @returns {Promise.} */ - const checkScopes = (req, channelName) => new Promise((resolve, reject) => { - log.silly(req.requestId, `Checking OAuth scopes for ${channelName}`); + const checkScopes = (req, logger, channelName) => new Promise((resolve, reject) => { + logger.debug(`Checking OAuth scopes for ${channelName}`); // When accessing public channels, no scopes are needed - if (PUBLIC_CHANNELS.includes(channelName)) { + if (channelName && PUBLIC_CHANNELS.includes(channelName)) { resolve(); return; } @@ -564,6 +569,7 @@ const startServer = async () => { } const err = new Error('Access token does not cover required scopes'); + // @ts-ignore err.status = 401; reject(err); @@ -577,38 +583,40 @@ const startServer = async () => { /** * @param {any} req * @param {SystemMessageHandlers} eventHandlers - * @returns {function(object): void} + * @returns {SubscriptionListener} */ const createSystemMessageListener = (req, eventHandlers) => { return message => { + if (!message?.event) { + return; + } + const { event } = message; - log.silly(req.requestId, `System message for ${req.accountId}: ${event}`); + req.log.debug(`System message for ${req.accountId}: ${event}`); if (event === 'kill') { - log.verbose(req.requestId, `Closing connection for ${req.accountId} due to expired access token`); + req.log.debug(`Closing connection for ${req.accountId} due to expired access token`); eventHandlers.onKill(); } else if (event === 'filters_changed') { - log.verbose(req.requestId, `Invalidating filters cache for ${req.accountId}`); + req.log.debug(`Invalidating filters cache for ${req.accountId}`); req.cachedFilters = null; } }; }; /** - * @param {any} req - * @param {any} res + * @param {http.IncomingMessage & ResolvedAccount} req + * @param {http.OutgoingMessage} res */ const subscribeHttpToSystemChannel = (req, res) => { const accessTokenChannelId = `timeline:access_token:${req.accessTokenId}`; const systemChannelId = `timeline:system:${req.accountId}`; const listener = createSystemMessageListener(req, { - onKill() { res.end(); }, - }); res.on('close', () => { @@ -641,13 +649,14 @@ const startServer = async () => { // the connection, as there's nothing to stream back if (!channelName) { const err = new Error('Unknown channel requested'); + // @ts-ignore err.status = 400; next(err); return; } - accountFromRequest(req).then(() => checkScopes(req, channelName)).then(() => { + accountFromRequest(req).then(() => checkScopes(req, req.log, channelName)).then(() => { subscribeHttpToSystemChannel(req, res); }).then(() => { next(); @@ -663,22 +672,28 @@ const startServer = async () => { * @param {function(Error=): void} next */ const errorMiddleware = (err, req, res, next) => { - log.error(req.requestId, err.toString()); + req.log.error({ err }, err.toString()); if (res.headersSent) { next(err); return; } - res.writeHead(err.status || 500, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ error: err.status ? err.toString() : 'An unexpected error occurred' })); + const hasStatusCode = Object.hasOwnProperty.call(err, 'status'); + // @ts-ignore + const statusCode = hasStatusCode ? err.status : 500; + const errorMessage = hasStatusCode ? err.toString() : 'An unexpected error occurred'; + + res.writeHead(statusCode, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: errorMessage })); }; /** - * @param {array} arr + * @param {any[]} arr * @param {number=} shift * @returns {string} */ + // @ts-ignore const placeholders = (arr, shift = 0) => arr.map((_, i) => `$${i + 1 + shift}`).join(', '); /** @@ -695,6 +710,7 @@ const startServer = async () => { return; } + // @ts-ignore client.query('SELECT id, account_id FROM lists WHERE id = $1 LIMIT 1', [listId], (err, result) => { done(); @@ -709,34 +725,43 @@ const startServer = async () => { }); /** - * @param {string[]} ids - * @param {any} req + * @param {string[]} channelIds + * @param {http.IncomingMessage & ResolvedAccount} req + * @param {import('pino').Logger} log * @param {function(string, string): void} output * @param {undefined | function(string[], SubscriptionListener): void} attachCloseHandler * @param {'websocket' | 'eventsource'} destinationType * @param {boolean=} needsFiltering * @returns {SubscriptionListener} */ - const streamFrom = (ids, req, output, attachCloseHandler, destinationType, needsFiltering = false) => { - const accountId = req.accountId || req.remoteAddress; - - log.verbose(req.requestId, `Starting stream from ${ids.join(', ')} for ${accountId}`); + const streamFrom = (channelIds, req, log, output, attachCloseHandler, destinationType, needsFiltering = false) => { + log.info({ channelIds }, `Starting stream`); + /** + * @param {string} event + * @param {object|string} payload + */ const transmit = (event, payload) => { // TODO: Replace "string"-based delete payloads with object payloads: const encodedPayload = typeof payload === 'object' ? JSON.stringify(payload) : payload; messagesSent.labels({ type: destinationType }).inc(1); - log.silly(req.requestId, `Transmitting for ${accountId}: ${event} ${encodedPayload}`); + log.debug({ event, payload }, `Transmitting ${event} to ${req.accountId}`); + output(event, encodedPayload); }; // The listener used to process each message off the redis subscription, // message here is an object with an `event` and `payload` property. Some // events also include a queued_at value, but this is being removed shortly. + /** @type {SubscriptionListener} */ const listener = message => { + if (!message?.event || !message?.payload) { + return; + } + const { event, payload } = message; // Streaming only needs to apply filtering to some channels and only to @@ -759,7 +784,7 @@ const startServer = async () => { // Filter based on language: if (Array.isArray(req.chosenLanguages) && payload.language !== null && req.chosenLanguages.indexOf(payload.language) === -1) { - log.silly(req.requestId, `Message ${payload.id} filtered by language (${payload.language})`); + log.debug(`Message ${payload.id} filtered by language (${payload.language})`); return; } @@ -770,6 +795,7 @@ const startServer = async () => { } // Filter based on domain blocks, blocks, mutes, or custom filters: + // @ts-ignore const targetAccountIds = [payload.account.id].concat(payload.mentions.map(item => item.id)); const accountDomain = payload.account.acct.split('@')[1]; @@ -781,6 +807,7 @@ const startServer = async () => { } const queries = [ + // @ts-ignore client.query(`SELECT 1 FROM blocks WHERE (account_id = $1 AND target_account_id IN (${placeholders(targetAccountIds, 2)})) @@ -793,10 +820,13 @@ const startServer = async () => { ]; if (accountDomain) { + // @ts-ignore queries.push(client.query('SELECT 1 FROM account_domain_blocks WHERE account_id = $1 AND domain = $2', [req.accountId, accountDomain])); } + // @ts-ignore if (!payload.filtered && !req.cachedFilters) { + // @ts-ignore queries.push(client.query('SELECT filter.id AS id, filter.phrase AS title, filter.context AS context, filter.expires_at AS expires_at, filter.action AS filter_action, keyword.keyword AS keyword, keyword.whole_word AS whole_word FROM custom_filter_keywords keyword JOIN custom_filters filter ON keyword.custom_filter_id = filter.id WHERE filter.account_id = $1 AND (filter.expires_at IS NULL OR filter.expires_at > NOW())', [req.accountId])); } @@ -819,9 +849,11 @@ const startServer = async () => { // Handling for constructing the custom filters and caching them on the request // TODO: Move this logic out of the message handling lifecycle + // @ts-ignore if (!req.cachedFilters) { const filterRows = values[accountDomain ? 2 : 1].rows; + // @ts-ignore req.cachedFilters = filterRows.reduce((cache, filter) => { if (cache[filter.id]) { cache[filter.id].keywords.push([filter.keyword, filter.whole_word]); @@ -851,7 +883,9 @@ const startServer = async () => { // needs to be done in a separate loop as the database returns one // filterRow per keyword, so we need all the keywords before // constructing the regular expression + // @ts-ignore Object.keys(req.cachedFilters).forEach((key) => { + // @ts-ignore req.cachedFilters[key].regexp = new RegExp(req.cachedFilters[key].keywords.map(([keyword, whole_word]) => { let expr = keyword.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); @@ -872,13 +906,16 @@ const startServer = async () => { // Apply cachedFilters against the payload, constructing a // `filter_results` array of FilterResult entities + // @ts-ignore if (req.cachedFilters) { const status = payload; // TODO: Calculate searchableContent in Ruby on Rails: + // @ts-ignore const searchableContent = ([status.spoiler_text || '', status.content].concat((status.poll && status.poll.options) ? status.poll.options.map(option => option.title) : [])).concat(status.media_attachments.map(att => att.description)).join('\n\n').replace(//g, '\n').replace(/<\/p>

/g, '\n\n'); const searchableTextContent = JSDOM.fragment(searchableContent).textContent; const now = new Date(); + // @ts-ignore const filter_results = Object.values(req.cachedFilters).reduce((results, cachedFilter) => { // Check the filter hasn't expired before applying: if (cachedFilter.expires_at !== null && cachedFilter.expires_at < now) { @@ -926,12 +963,12 @@ const startServer = async () => { }); }; - ids.forEach(id => { + channelIds.forEach(id => { subscribe(`${redisPrefix}${id}`, listener); }); if (typeof attachCloseHandler === 'function') { - attachCloseHandler(ids.map(id => `${redisPrefix}${id}`), listener); + attachCloseHandler(channelIds.map(id => `${redisPrefix}${id}`), listener); } return listener; @@ -943,8 +980,6 @@ const startServer = async () => { * @returns {function(string, string): void} */ const streamToHttp = (req, res) => { - const accountId = req.accountId || req.remoteAddress; - const channelName = channelNameFromPath(req); connectedClients.labels({ type: 'eventsource' }).inc(); @@ -963,7 +998,8 @@ const startServer = async () => { const heartbeat = setInterval(() => res.write(':thump\n'), 15000); req.on('close', () => { - log.verbose(req.requestId, `Ending stream for ${accountId}`); + req.log.info({ accountId: req.accountId }, `Ending stream`); + // We decrement these counters here instead of in streamHttpEnd as in that // method we don't have knowledge of the channel names connectedClients.labels({ type: 'eventsource' }).dec(); @@ -1007,15 +1043,15 @@ const startServer = async () => { */ const streamToWs = (req, ws, streamName) => (event, payload) => { if (ws.readyState !== ws.OPEN) { - log.error(req.requestId, 'Tried writing to closed socket'); + req.log.error('Tried writing to closed socket'); return; } const message = JSON.stringify({ stream: streamName, event, payload }); - ws.send(message, (/** @type {Error} */ err) => { + ws.send(message, (/** @type {Error|undefined} */ err) => { if (err) { - log.error(req.requestId, `Failed to send to websocket: ${err}`); + req.log.error({err}, `Failed to send to websocket`); } }); }; @@ -1032,20 +1068,19 @@ const startServer = async () => { app.use(api); - api.use(setRequestId); - api.use(setRemoteAddress); - api.use(authenticationMiddleware); api.use(errorMiddleware); api.get('/api/v1/streaming/*', (req, res) => { + // @ts-ignore channelNameToIds(req, channelNameFromPath(req), req.query).then(({ channelIds, options }) => { const onSend = streamToHttp(req, res); const onEnd = streamHttpEnd(req, subscriptionHeartbeat(channelIds)); - streamFrom(channelIds, req, onSend, onEnd, 'eventsource', options.needsFiltering); + // @ts-ignore + streamFrom(channelIds, req, req.log, onSend, onEnd, 'eventsource', options.needsFiltering); }).catch(err => { - log.verbose(req.requestId, 'Subscription error:', err.toString()); + res.log.info({ err }, 'Subscription error:', err.toString()); httpNotFound(res); }); }); @@ -1197,6 +1232,7 @@ const startServer = async () => { break; case 'list': + // @ts-ignore authorizeListAccess(params.list, req).then(() => { resolve({ channelIds: [`timeline:list:${params.list}`], @@ -1218,9 +1254,9 @@ const startServer = async () => { * @returns {string[]} */ const streamNameFromChannelName = (channelName, params) => { - if (channelName === 'list') { + if (channelName === 'list' && params.list) { return [channelName, params.list]; - } else if (['hashtag', 'hashtag:local'].includes(channelName)) { + } else if (['hashtag', 'hashtag:local'].includes(channelName) && params.tag) { return [channelName, params.tag]; } else { return [channelName]; @@ -1229,8 +1265,9 @@ const startServer = async () => { /** * @typedef WebSocketSession - * @property {WebSocket} websocket - * @property {http.IncomingMessage} request + * @property {WebSocket & { isAlive: boolean}} websocket + * @property {http.IncomingMessage & ResolvedAccount} request + * @property {import('pino').Logger} logger * @property {Object.} subscriptions */ @@ -1240,8 +1277,8 @@ const startServer = async () => { * @param {StreamParams} params * @returns {void} */ - const subscribeWebsocketToChannel = ({ socket, request, subscriptions }, channelName, params) => { - checkScopes(request, channelName).then(() => channelNameToIds(request, channelName, params)).then(({ + const subscribeWebsocketToChannel = ({ websocket, request, logger, subscriptions }, channelName, params) => { + checkScopes(request, logger, channelName).then(() => channelNameToIds(request, channelName, params)).then(({ channelIds, options, }) => { @@ -1249,9 +1286,9 @@ const startServer = async () => { return; } - const onSend = streamToWs(request, socket, streamNameFromChannelName(channelName, params)); + const onSend = streamToWs(request, websocket, streamNameFromChannelName(channelName, params)); const stopHeartbeat = subscriptionHeartbeat(channelIds); - const listener = streamFrom(channelIds, request, onSend, undefined, 'websocket', options.needsFiltering); + const listener = streamFrom(channelIds, request, logger, onSend, undefined, 'websocket', options.needsFiltering); connectedChannels.labels({ type: 'websocket', channel: channelName }).inc(); @@ -1261,14 +1298,17 @@ const startServer = async () => { stopHeartbeat, }; }).catch(err => { - log.verbose(request.requestId, 'Subscription error:', err.toString()); - socket.send(JSON.stringify({ error: err.toString() })); + logger.error({ err }, 'Subscription error'); + websocket.send(JSON.stringify({ error: err.toString() })); }); }; - - const removeSubscription = (subscriptions, channelIds, request) => { - log.verbose(request.requestId, `Ending stream from ${channelIds.join(', ')} for ${request.accountId}`); + /** + * @param {WebSocketSession} session + * @param {string[]} channelIds + */ + const removeSubscription = ({ request, logger, subscriptions }, channelIds) => { + logger.info({ channelIds, accountId: request.accountId }, `Ending stream`); const subscription = subscriptions[channelIds.join(';')]; @@ -1292,16 +1332,17 @@ const startServer = async () => { * @param {StreamParams} params * @returns {void} */ - const unsubscribeWebsocketFromChannel = ({ socket, request, subscriptions }, channelName, params) => { + const unsubscribeWebsocketFromChannel = (session, channelName, params) => { + const { websocket, request, logger } = session; + channelNameToIds(request, channelName, params).then(({ channelIds }) => { - removeSubscription(subscriptions, channelIds, request); + removeSubscription(session, channelIds); }).catch(err => { - log.verbose(request.requestId, 'Unsubscribe error:', err); + logger.error({err}, 'Unsubscribe error'); // If we have a socket that is alive and open still, send the error back to the client: - // FIXME: In other parts of the code ws === socket - if (socket.isAlive && socket.readyState === socket.OPEN) { - socket.send(JSON.stringify({ error: "Error unsubscribing from channel" })); + if (websocket.isAlive && websocket.readyState === websocket.OPEN) { + websocket.send(JSON.stringify({ error: "Error unsubscribing from channel" })); } }); }; @@ -1309,16 +1350,14 @@ const startServer = async () => { /** * @param {WebSocketSession} session */ - const subscribeWebsocketToSystemChannel = ({ socket, request, subscriptions }) => { + const subscribeWebsocketToSystemChannel = ({ websocket, request, subscriptions }) => { const accessTokenChannelId = `timeline:access_token:${request.accessTokenId}`; const systemChannelId = `timeline:system:${request.accountId}`; const listener = createSystemMessageListener(request, { - onKill() { - socket.close(); + websocket.close(); }, - }); subscribe(`${redisPrefix}${accessTokenChannelId}`, listener); @@ -1355,18 +1394,15 @@ const startServer = async () => { /** * @param {WebSocket & { isAlive: boolean }} ws - * @param {http.IncomingMessage} req + * @param {http.IncomingMessage & ResolvedAccount} req + * @param {import('pino').Logger} log */ - function onConnection(ws, req) { + function onConnection(ws, req, log) { // Note: url.parse could throw, which would terminate the connection, so we // increment the connected clients metric straight away when we establish // the connection, without waiting: connectedClients.labels({ type: 'websocket' }).inc(); - // Setup request properties: - req.requestId = uuid.v4(); - req.remoteAddress = ws._socket.remoteAddress; - // Setup connection keep-alive state: ws.isAlive = true; ws.on('pong', () => { @@ -1377,8 +1413,9 @@ const startServer = async () => { * @type {WebSocketSession} */ const session = { - socket: ws, + websocket: ws, request: req, + logger: log, subscriptions: {}, }; @@ -1386,27 +1423,30 @@ const startServer = async () => { const subscriptions = Object.keys(session.subscriptions); subscriptions.forEach(channelIds => { - removeSubscription(session.subscriptions, channelIds.split(';'), req); + removeSubscription(session, channelIds.split(';')); }); // Decrement the metrics for connected clients: connectedClients.labels({ type: 'websocket' }).dec(); - // ensure garbage collection: - session.socket = null; - session.request = null; - session.subscriptions = {}; + // We need to delete the session object as to ensure it correctly gets + // garbage collected, without doing this we could accidentally hold on to + // references to the websocket, the request, and the logger, causing + // memory leaks. + // + // @ts-ignore + delete session; }); // Note: immediately after the `error` event is emitted, the `close` event // is emitted. As such, all we need to do is log the error here. - ws.on('error', (err) => { - log.error('websocket', err.toString()); + ws.on('error', (/** @type {Error} */ err) => { + log.error(err); }); ws.on('message', (data, isBinary) => { if (isBinary) { - log.warn('websocket', 'Received binary data, closing connection'); + log.warn('Received binary data, closing connection'); ws.close(1003, 'The mastodon streaming server does not support binary messages'); return; } @@ -1441,18 +1481,20 @@ const startServer = async () => { setInterval(() => { wss.clients.forEach(ws => { + // @ts-ignore if (ws.isAlive === false) { ws.terminate(); return; } + // @ts-ignore ws.isAlive = false; ws.ping('', false); }); }, 30000); attachServerWithConfig(server, address => { - log.warn(`Streaming API now listening on ${address}`); + logger.info(`Streaming API now listening on ${address}`); }); const onExit = () => { @@ -1460,8 +1502,10 @@ const startServer = async () => { process.exit(0); }; + /** @param {Error} err */ const onError = (err) => { - log.error(err); + logger.error(err); + server.close(); process.exit(0); }; @@ -1485,7 +1529,7 @@ const attachServerWithConfig = (server, onSuccess) => { } }); } else { - server.listen(+process.env.PORT || 4000, process.env.BIND || '127.0.0.1', () => { + server.listen(+(process.env.PORT || 4000), process.env.BIND || '127.0.0.1', () => { if (onSuccess) { onSuccess(`${server.address().address}:${server.address().port}`); } diff --git a/streaming/logging.js b/streaming/logging.js new file mode 100644 index 000000000..64ee47487 --- /dev/null +++ b/streaming/logging.js @@ -0,0 +1,119 @@ +const { pino } = require('pino'); +const { pinoHttp, stdSerializers: pinoHttpSerializers } = require('pino-http'); +const uuid = require('uuid'); + +/** + * Generates the Request ID for logging and setting on responses + * @param {http.IncomingMessage} req + * @param {http.ServerResponse} [res] + * @returns {import("pino-http").ReqId} + */ +function generateRequestId(req, res) { + if (req.id) { + return req.id; + } + + req.id = uuid.v4(); + + // Allow for usage with WebSockets: + if (res) { + res.setHeader('X-Request-Id', req.id); + } + + return req.id; +} + +/** + * Request log sanitizer to prevent logging access tokens in URLs + * @param {http.IncomingMessage} req + */ +function sanitizeRequestLog(req) { + const log = pinoHttpSerializers.req(req); + if (typeof log.url === 'string' && log.url.includes('access_token')) { + // Doorkeeper uses SecureRandom.urlsafe_base64 per RFC 6749 / RFC 6750 + log.url = log.url.replace(/(access_token)=([a-zA-Z0-9\-_]+)/gi, '$1=[Redacted]'); + } + return log; +} + +const logger = pino({ + name: "streaming", + // Reformat the log level to a string: + formatters: { + level: (label) => { + return { + level: label + }; + }, + }, + redact: { + paths: [ + 'req.headers["sec-websocket-key"]', + // Note: we currently pass the AccessToken via the websocket subprotocol + // field, an anti-pattern, but this ensures it doesn't end up in logs. + 'req.headers["sec-websocket-protocol"]', + 'req.headers.authorization', + 'req.headers.cookie', + 'req.query.access_token' + ] + } +}); + +const httpLogger = pinoHttp({ + logger, + genReqId: generateRequestId, + serializers: { + req: sanitizeRequestLog + } +}); + +/** + * Attaches a logger to the request object received by http upgrade handlers + * @param {http.IncomingMessage} request + */ +function attachWebsocketHttpLogger(request) { + generateRequestId(request); + + request.log = logger.child({ + req: sanitizeRequestLog(request), + }); +} + +/** + * Creates a logger instance for the Websocket connection to use. + * @param {http.IncomingMessage} request + * @param {import('./index.js').ResolvedAccount} resolvedAccount + */ +function createWebsocketLogger(request, resolvedAccount) { + // ensure the request.id is always present. + generateRequestId(request); + + return logger.child({ + req: { + id: request.id + }, + account: { + id: resolvedAccount.accountId ?? null + } + }); +} + +exports.logger = logger; +exports.httpLogger = httpLogger; +exports.attachWebsocketHttpLogger = attachWebsocketHttpLogger; +exports.createWebsocketLogger = createWebsocketLogger; + +/** + * Initializes the log level based on the environment + * @param {Object} env + * @param {string} environment + */ +exports.initializeLogLevel = function initializeLogLevel(env, environment) { + if (env.LOG_LEVEL && Object.keys(logger.levels.values).includes(env.LOG_LEVEL)) { + logger.level = env.LOG_LEVEL; + } else if (environment === 'development') { + logger.level = 'debug'; + } else { + logger.level = 'info'; + } +}; diff --git a/streaming/package.json b/streaming/package.json index 149055ca1..52a997970 100644 --- a/streaming/package.json +++ b/streaming/package.json @@ -21,9 +21,10 @@ "express": "^4.18.2", "ioredis": "^5.3.2", "jsdom": "^23.0.0", - "npmlog": "^7.0.1", "pg": "^8.5.0", "pg-connection-string": "^2.6.0", + "pino": "^8.17.2", + "pino-http": "^9.0.0", "prom-client": "^15.0.0", "uuid": "^9.0.0", "ws": "^8.12.1" @@ -31,11 +32,11 @@ "devDependencies": { "@types/cors": "^2.8.16", "@types/express": "^4.17.17", - "@types/npmlog": "^7.0.0", "@types/pg": "^8.6.6", "@types/uuid": "^9.0.0", "@types/ws": "^8.5.9", "eslint-define-config": "^2.0.0", + "pino-pretty": "^10.3.1", "typescript": "^5.0.4" }, "optionalDependencies": { diff --git a/yarn.lock b/yarn.lock index 6f8381db0..75586ac49 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2536,7 +2536,6 @@ __metadata: dependencies: "@types/cors": "npm:^2.8.16" "@types/express": "npm:^4.17.17" - "@types/npmlog": "npm:^7.0.0" "@types/pg": "npm:^8.6.6" "@types/uuid": "npm:^9.0.0" "@types/ws": "npm:^8.5.9" @@ -2547,9 +2546,11 @@ __metadata: express: "npm:^4.18.2" ioredis: "npm:^5.3.2" jsdom: "npm:^23.0.0" - npmlog: "npm:^7.0.1" pg: "npm:^8.5.0" pg-connection-string: "npm:^2.6.0" + pino: "npm:^8.17.2" + pino-http: "npm:^9.0.0" + pino-pretty: "npm:^10.3.1" prom-client: "npm:^15.0.0" typescript: "npm:^5.0.4" utf-8-validate: "npm:^6.0.3" @@ -3338,15 +3339,6 @@ __metadata: languageName: node linkType: hard -"@types/npmlog@npm:^7.0.0": - version: 7.0.0 - resolution: "@types/npmlog@npm:7.0.0" - dependencies: - "@types/node": "npm:*" - checksum: e94cb1d7dc6b1251d58d0a3cbf0c5b9e9b7c7649774cf816b9277fc10e1a09e65f2854357c4972d04d477f8beca3c8accb5e8546d594776e59e35ddfee79aff2 - languageName: node - linkType: hard - "@types/object-assign@npm:^4.0.30": version: 4.0.33 resolution: "@types/object-assign@npm:4.0.33" @@ -3791,6 +3783,16 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/scope-manager@npm:6.9.1": + version: 6.9.1 + resolution: "@typescript-eslint/scope-manager@npm:6.9.1" + dependencies: + "@typescript-eslint/types": "npm:6.9.1" + "@typescript-eslint/visitor-keys": "npm:6.9.1" + checksum: 53fa7c3813d22b119e464f9b6d7d23407dfe103ee8ad2dcacf9ad6d656fda20e2bb3346df39e62b0e6b6ce71572ce5838071c5d2cca6daa4e0ce117ff22eafe5 + languageName: node + linkType: hard + "@typescript-eslint/type-utils@npm:6.19.0": version: 6.19.0 resolution: "@typescript-eslint/type-utils@npm:6.19.0" @@ -3815,6 +3817,13 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/types@npm:6.9.1": + version: 6.9.1 + resolution: "@typescript-eslint/types@npm:6.9.1" + checksum: 4ba21ba18e256da210a4caedfbc5d4927cf8cb4f2c4d74f8ccc865576f3659b974e79119d3c94db2b68a4cec9cd687e43971d355450b7082d6d1736a5dd6db85 + languageName: node + linkType: hard + "@typescript-eslint/typescript-estree@npm:6.19.0": version: 6.19.0 resolution: "@typescript-eslint/typescript-estree@npm:6.19.0" @@ -3834,7 +3843,25 @@ __metadata: languageName: node linkType: hard -"@typescript-eslint/utils@npm:6.19.0, @typescript-eslint/utils@npm:^6.5.0": +"@typescript-eslint/typescript-estree@npm:6.9.1": + version: 6.9.1 + resolution: "@typescript-eslint/typescript-estree@npm:6.9.1" + dependencies: + "@typescript-eslint/types": "npm:6.9.1" + "@typescript-eslint/visitor-keys": "npm:6.9.1" + debug: "npm:^4.3.4" + globby: "npm:^11.1.0" + is-glob: "npm:^4.0.3" + semver: "npm:^7.5.4" + ts-api-utils: "npm:^1.0.1" + peerDependenciesMeta: + typescript: + optional: true + checksum: 850b1865a90107879186c3f2969968a2c08fc6fcc56d146483c297cf5be376e33d505ac81533ba8e8103ca4d2edfea7d21b178de9e52217f7ee2922f51a445fa + languageName: node + linkType: hard + +"@typescript-eslint/utils@npm:6.19.0": version: 6.19.0 resolution: "@typescript-eslint/utils@npm:6.19.0" dependencies: @@ -3851,6 +3878,23 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/utils@npm:^6.5.0": + version: 6.9.1 + resolution: "@typescript-eslint/utils@npm:6.9.1" + dependencies: + "@eslint-community/eslint-utils": "npm:^4.4.0" + "@types/json-schema": "npm:^7.0.12" + "@types/semver": "npm:^7.5.0" + "@typescript-eslint/scope-manager": "npm:6.9.1" + "@typescript-eslint/types": "npm:6.9.1" + "@typescript-eslint/typescript-estree": "npm:6.9.1" + semver: "npm:^7.5.4" + peerDependencies: + eslint: ^7.0.0 || ^8.0.0 + checksum: 3d329d54c3d155ed29e2b456a602aef76bda1b88dfcf847145849362e4ddefabe5c95de236de750d08d5da9bedcfb2131bdfd784ce4eb87cf82728f0b6662033 + languageName: node + linkType: hard + "@typescript-eslint/visitor-keys@npm:6.19.0": version: 6.19.0 resolution: "@typescript-eslint/visitor-keys@npm:6.19.0" @@ -3861,6 +3905,16 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/visitor-keys@npm:6.9.1": + version: 6.9.1 + resolution: "@typescript-eslint/visitor-keys@npm:6.9.1" + dependencies: + "@typescript-eslint/types": "npm:6.9.1" + eslint-visitor-keys: "npm:^3.4.1" + checksum: ac5f375a177add30489e5b63cafa8d82a196b33624bb36418422ebe0d7973b3ba550dc7e0dda36ea75a94cf9b200b4fb5f5fb4d77c027fd801201c1a269d343b + languageName: node + linkType: hard + "@ungap/structured-clone@npm:^1.2.0": version: 1.2.0 resolution: "@ungap/structured-clone@npm:1.2.0" @@ -4324,13 +4378,6 @@ __metadata: languageName: node linkType: hard -"aproba@npm:^1.0.3 || ^2.0.0": - version: 2.0.0 - resolution: "aproba@npm:2.0.0" - checksum: d06e26384a8f6245d8c8896e138c0388824e259a329e0c9f196b4fa533c82502a6fd449586e3604950a0c42921832a458bb3aa0aa9f0ba449cfd4f50fd0d09b5 - languageName: node - linkType: hard - "are-docs-informative@npm:^0.0.2": version: 0.0.2 resolution: "are-docs-informative@npm:0.0.2" @@ -4338,16 +4385,6 @@ __metadata: languageName: node linkType: hard -"are-we-there-yet@npm:^4.0.0": - version: 4.0.0 - resolution: "are-we-there-yet@npm:4.0.0" - dependencies: - delegates: "npm:^1.0.0" - readable-stream: "npm:^4.1.0" - checksum: 760008e32948e9f738c5a288792d187e235fee0f170e042850bc7ff242f2a499f3f2874d6dd43ac06f5d9f5306137bc51bbdd4ae0bb11379c58b01678e0f684d - languageName: node - linkType: hard - "argparse@npm:^1.0.7": version: 1.0.10 resolution: "argparse@npm:1.0.10" @@ -4669,6 +4706,13 @@ __metadata: languageName: node linkType: hard +"atomic-sleep@npm:^1.0.0": + version: 1.0.0 + resolution: "atomic-sleep@npm:1.0.0" + checksum: e329a6665512736a9bbb073e1761b4ec102f7926cce35037753146a9db9c8104f5044c1662e4a863576ce544fb8be27cd2be6bc8c1a40147d03f31eb1cfb6e8a + languageName: node + linkType: hard + "autoprefixer@npm:^10.4.14": version: 10.4.17 resolution: "autoprefixer@npm:10.4.17" @@ -5763,15 +5807,6 @@ __metadata: languageName: node linkType: hard -"color-support@npm:^1.1.3": - version: 1.1.3 - resolution: "color-support@npm:1.1.3" - bin: - color-support: bin.js - checksum: 8ffeaa270a784dc382f62d9be0a98581db43e11eee301af14734a6d089bd456478b1a8b3e7db7ca7dc5b18a75f828f775c44074020b51c05fc00e6d0992b1cc6 - languageName: node - linkType: hard - "colord@npm:^2.9.1, colord@npm:^2.9.3": version: 2.9.3 resolution: "colord@npm:2.9.3" @@ -5779,7 +5814,7 @@ __metadata: languageName: node linkType: hard -"colorette@npm:^2.0.20": +"colorette@npm:^2.0.20, colorette@npm:^2.0.7": version: 2.0.20 resolution: "colorette@npm:2.0.20" checksum: e94116ff33b0ff56f3b83b9ace895e5bf87c2a7a47b3401b8c3f3226e050d5ef76cf4072fb3325f9dc24d1698f9b730baf4e05eeaf861d74a1883073f4c98a40 @@ -5911,13 +5946,6 @@ __metadata: languageName: node linkType: hard -"console-control-strings@npm:^1.1.0": - version: 1.1.0 - resolution: "console-control-strings@npm:1.1.0" - checksum: 7ab51d30b52d461412cd467721bb82afe695da78fff8f29fe6f6b9cbaac9a2328e27a22a966014df9532100f6dd85370460be8130b9c677891ba36d96a343f50 - languageName: node - linkType: hard - "constants-browserify@npm:^1.0.0": version: 1.0.0 resolution: "constants-browserify@npm:1.0.0" @@ -6445,6 +6473,13 @@ __metadata: languageName: node linkType: hard +"dateformat@npm:^4.6.3": + version: 4.6.3 + resolution: "dateformat@npm:4.6.3" + checksum: e2023b905e8cfe2eb8444fb558562b524807a51cdfe712570f360f873271600b5c94aebffaf11efb285e2c072264a7cf243eadb68f3eba0f8cc85fb86cd25df6 + languageName: node + linkType: hard + "debounce@npm:^1.2.1": version: 1.2.1 resolution: "debounce@npm:1.2.1" @@ -6680,13 +6715,6 @@ __metadata: languageName: node linkType: hard -"delegates@npm:^1.0.0": - version: 1.0.0 - resolution: "delegates@npm:1.0.0" - checksum: ba05874b91148e1db4bf254750c042bf2215febd23a6d3cda2e64896aef79745fbd4b9996488bd3cafb39ce19dbce0fd6e3b6665275638befffe1c9b312b91b5 - languageName: node - linkType: hard - "denque@npm:^2.1.0": version: 2.1.0 resolution: "denque@npm:2.1.0" @@ -7952,6 +7980,13 @@ __metadata: languageName: node linkType: hard +"fast-copy@npm:^3.0.0": + version: 3.0.1 + resolution: "fast-copy@npm:3.0.1" + checksum: a8310dbcc4c94ed001dc3e0bbc3c3f0491bb04e6c17163abe441a54997ba06cdf1eb532c2f05e54777c6f072c84548c23ef0ecd54665cd611be1d42f37eca258 + languageName: node + linkType: hard + "fast-deep-equal@npm:^3.1.1, fast-deep-equal@npm:^3.1.3": version: 3.1.3 resolution: "fast-deep-equal@npm:3.1.3" @@ -7993,6 +8028,20 @@ __metadata: languageName: node linkType: hard +"fast-redact@npm:^3.1.1": + version: 3.3.0 + resolution: "fast-redact@npm:3.3.0" + checksum: d81562510681e9ba6404ee5d3838ff5257a44d2f80937f5024c099049ff805437d0fae0124458a7e87535cc9dcf4de305bb075cab8f08d6c720bbc3447861b4e + languageName: node + linkType: hard + +"fast-safe-stringify@npm:^2.1.1": + version: 2.1.1 + resolution: "fast-safe-stringify@npm:2.1.1" + checksum: d90ec1c963394919828872f21edaa3ad6f1dddd288d2bd4e977027afff09f5db40f94e39536d4646f7e01761d704d72d51dce5af1b93717f3489ef808f5f4e4d + languageName: node + linkType: hard + "fastest-levenshtein@npm:^1.0.16": version: 1.0.16 resolution: "fastest-levenshtein@npm:1.0.16" @@ -8407,22 +8456,6 @@ __metadata: languageName: node linkType: hard -"gauge@npm:^5.0.0": - version: 5.0.1 - resolution: "gauge@npm:5.0.1" - dependencies: - aproba: "npm:^1.0.3 || ^2.0.0" - color-support: "npm:^1.1.3" - console-control-strings: "npm:^1.1.0" - has-unicode: "npm:^2.0.1" - signal-exit: "npm:^4.0.1" - string-width: "npm:^4.2.3" - strip-ansi: "npm:^6.0.1" - wide-align: "npm:^1.1.5" - checksum: 845f9a2534356cd0e9c1ae590ed471bbe8d74c318915b92a34e8813b8d3441ca8e0eb0fa87a48081e70b63b84d398c5e66a13b8e8040181c10b9d77e9fe3287f - languageName: node - linkType: hard - "gensync@npm:^1.0.0-beta.2": version: 1.0.0-beta.2 resolution: "gensync@npm:1.0.0-beta.2" @@ -8771,13 +8804,6 @@ __metadata: languageName: node linkType: hard -"has-unicode@npm:^2.0.1": - version: 2.0.1 - resolution: "has-unicode@npm:2.0.1" - checksum: ebdb2f4895c26bb08a8a100b62d362e49b2190bcfd84b76bc4be1a3bd4d254ec52d0dd9f2fbcc093fc5eb878b20c52146f9dfd33e2686ed28982187be593b47c - languageName: node - linkType: hard - "has-value@npm:^0.3.1": version: 0.3.1 resolution: "has-value@npm:0.3.1" @@ -8854,6 +8880,13 @@ __metadata: languageName: node linkType: hard +"help-me@npm:^5.0.0": + version: 5.0.0 + resolution: "help-me@npm:5.0.0" + checksum: 054c0e2e9ae2231c85ab5e04f75109b9d068ffcc54e58fb22079822a5ace8ff3d02c66fd45379c902ad5ab825e5d2e1451fcc2f7eab1eb49e7d488133ba4cacb + languageName: node + linkType: hard + "history@npm:^4.10.1, history@npm:^4.9.0": version: 4.10.1 resolution: "history@npm:4.10.1" @@ -9320,7 +9353,7 @@ __metadata: languageName: node linkType: hard -"intl-messageformat@npm:10.5.10, intl-messageformat@npm:^10.3.5": +"intl-messageformat@npm:10.5.10": version: 10.5.10 resolution: "intl-messageformat@npm:10.5.10" dependencies: @@ -9332,6 +9365,18 @@ __metadata: languageName: node linkType: hard +"intl-messageformat@npm:^10.3.5": + version: 10.5.8 + resolution: "intl-messageformat@npm:10.5.8" + dependencies: + "@formatjs/ecma402-abstract": "npm:1.18.0" + "@formatjs/fast-memoize": "npm:2.2.0" + "@formatjs/icu-messageformat-parser": "npm:2.7.3" + tslib: "npm:^2.4.0" + checksum: 1d2854aae8471ec48165ca265760d6c5b1814eca831c88db698eb29b5ed20bee21ca8533090c9d28d9c6f1d844dda210b0bc58a2e036446158fae0845e5eed4f + languageName: node + linkType: hard + "invariant@npm:^2.2.2, invariant@npm:^2.2.4": version: 2.2.4 resolution: "invariant@npm:2.2.4" @@ -10570,6 +10615,13 @@ __metadata: languageName: node linkType: hard +"joycon@npm:^3.1.1": + version: 3.1.1 + resolution: "joycon@npm:3.1.1" + checksum: 131fb1e98c9065d067fd49b6e685487ac4ad4d254191d7aa2c9e3b90f4e9ca70430c43cad001602bdbdabcf58717d3b5c5b7461c1bd8e39478c8de706b3fe6ae + languageName: node + linkType: hard + "jpeg-autorotate@npm:^7.1.1": version: 7.1.1 resolution: "jpeg-autorotate@npm:7.1.1" @@ -11966,18 +12018,6 @@ __metadata: languageName: node linkType: hard -"npmlog@npm:^7.0.1": - version: 7.0.1 - resolution: "npmlog@npm:7.0.1" - dependencies: - are-we-there-yet: "npm:^4.0.0" - console-control-strings: "npm:^1.1.0" - gauge: "npm:^5.0.0" - set-blocking: "npm:^2.0.0" - checksum: d4e6a2aaa7b5b5d2e2ed8f8ac3770789ca0691a49f3576b6a8c97d560a4c3305d2c233a9173d62be737e6e4506bf9e89debd6120a3843c1d37315c34f90fef71 - languageName: node - linkType: hard - "nth-check@npm:^1.0.2": version: 1.0.2 resolution: "nth-check@npm:1.0.2" @@ -12150,6 +12190,13 @@ __metadata: languageName: node linkType: hard +"on-exit-leak-free@npm:^2.1.0": + version: 2.1.2 + resolution: "on-exit-leak-free@npm:2.1.2" + checksum: faea2e1c9d696ecee919026c32be8d6a633a7ac1240b3b87e944a380e8a11dc9c95c4a1f8fb0568de7ab8db3823e790f12bda45296b1d111e341aad3922a0570 + languageName: node + linkType: hard + "on-finished@npm:2.4.1": version: 2.4.1 resolution: "on-finished@npm:2.4.1" @@ -12717,6 +12764,80 @@ __metadata: languageName: node linkType: hard +"pino-abstract-transport@npm:^1.0.0, pino-abstract-transport@npm:v1.1.0": + version: 1.1.0 + resolution: "pino-abstract-transport@npm:1.1.0" + dependencies: + readable-stream: "npm:^4.0.0" + split2: "npm:^4.0.0" + checksum: 6e9b9d5a2c0a37f91ecaf224d335daae1ae682b1c79a05b06ef9e0f0a5d289f8e597992217efc857796dae6f1067e9b4882f95c6228ff433ddc153532cae8aca + languageName: node + linkType: hard + +"pino-http@npm:^9.0.0": + version: 9.0.0 + resolution: "pino-http@npm:9.0.0" + dependencies: + get-caller-file: "npm:^2.0.5" + pino: "npm:^8.17.1" + pino-std-serializers: "npm:^6.2.2" + process-warning: "npm:^3.0.0" + checksum: 05496cb76cc9908658e50c4620fbdf7b0b5d99fb529493d601c3e4635b0bf7ce12b8a8eed7b5b520089f643b099233d61dd71f7cdfad8b66e59b9b81d79b6512 + languageName: node + linkType: hard + +"pino-pretty@npm:^10.3.1": + version: 10.3.1 + resolution: "pino-pretty@npm:10.3.1" + dependencies: + colorette: "npm:^2.0.7" + dateformat: "npm:^4.6.3" + fast-copy: "npm:^3.0.0" + fast-safe-stringify: "npm:^2.1.1" + help-me: "npm:^5.0.0" + joycon: "npm:^3.1.1" + minimist: "npm:^1.2.6" + on-exit-leak-free: "npm:^2.1.0" + pino-abstract-transport: "npm:^1.0.0" + pump: "npm:^3.0.0" + readable-stream: "npm:^4.0.0" + secure-json-parse: "npm:^2.4.0" + sonic-boom: "npm:^3.0.0" + strip-json-comments: "npm:^3.1.1" + bin: + pino-pretty: bin.js + checksum: 6964fba5acc7a9f112e4c6738d602e123daf16cb5f6ddc56ab4b6bb05059f28876d51da8f72358cf1172e95fa12496b70465431a0836df693c462986d050686b + languageName: node + linkType: hard + +"pino-std-serializers@npm:^6.0.0, pino-std-serializers@npm:^6.2.2": + version: 6.2.2 + resolution: "pino-std-serializers@npm:6.2.2" + checksum: 8f1c7f0f0d8f91e6c6b5b2a6bfb48f06441abeb85f1c2288319f736f9c6d814fbeebe928d2314efc2ba6018fa7db9357a105eca9fc99fc1f28945a8a8b28d3d5 + languageName: node + linkType: hard + +"pino@npm:^8.17.1, pino@npm:^8.17.2": + version: 8.17.2 + resolution: "pino@npm:8.17.2" + dependencies: + atomic-sleep: "npm:^1.0.0" + fast-redact: "npm:^3.1.1" + on-exit-leak-free: "npm:^2.1.0" + pino-abstract-transport: "npm:v1.1.0" + pino-std-serializers: "npm:^6.0.0" + process-warning: "npm:^3.0.0" + quick-format-unescaped: "npm:^4.0.3" + real-require: "npm:^0.2.0" + safe-stable-stringify: "npm:^2.3.1" + sonic-boom: "npm:^3.7.0" + thread-stream: "npm:^2.0.0" + bin: + pino: bin.js + checksum: 9e55af6cd9d1833a4dbe64924fc73163295acd3c988a9c7db88926669f2574ab7ec607e8487b6dd71dbdad2d7c1c1aac439f37e59233f37220b1a9d88fa2ce01 + languageName: node + linkType: hard + "pirates@npm:^4.0.4": version: 4.0.6 resolution: "pirates@npm:4.0.6" @@ -13319,6 +13440,13 @@ __metadata: languageName: node linkType: hard +"process-warning@npm:^3.0.0": + version: 3.0.0 + resolution: "process-warning@npm:3.0.0" + checksum: 60f3c8ddee586f0706c1e6cb5aa9c86df05774b9330d792d7c8851cf0031afd759d665404d07037e0b4901b55c44a423f07bdc465c63de07d8d23196bb403622 + languageName: node + linkType: hard + "process@npm:^0.11.10": version: 0.11.10 resolution: "process@npm:0.11.10" @@ -13496,6 +13624,13 @@ __metadata: languageName: node linkType: hard +"quick-format-unescaped@npm:^4.0.3": + version: 4.0.4 + resolution: "quick-format-unescaped@npm:4.0.4" + checksum: fe5acc6f775b172ca5b4373df26f7e4fd347975578199e7d74b2ae4077f0af05baa27d231de1e80e8f72d88275ccc6028568a7a8c9ee5e7368ace0e18eff93a4 + languageName: node + linkType: hard + "raf@npm:^3.1.0": version: 3.4.1 resolution: "raf@npm:3.4.1" @@ -13991,15 +14126,16 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:^4.1.0": - version: 4.4.0 - resolution: "readable-stream@npm:4.4.0" +"readable-stream@npm:^4.0.0": + version: 4.4.2 + resolution: "readable-stream@npm:4.4.2" dependencies: abort-controller: "npm:^3.0.0" buffer: "npm:^6.0.3" events: "npm:^3.3.0" process: "npm:^0.11.10" - checksum: 83f5a11285e5ebefb7b22a43ea77a2275075639325b4932a328a1fb0ee2475b83b9cc94326724d71c6aa3b60fa87e2b16623530b1cac34f3825dcea0996fdbe4 + string_decoder: "npm:^1.3.0" + checksum: cf7cc8daa2b57872d120945a20a1458c13dcb6c6f352505421115827b18ac4df0e483ac1fe195cb1f5cd226e1073fc55b92b569269d8299e8530840bcdbba40c languageName: node linkType: hard @@ -14023,6 +14159,13 @@ __metadata: languageName: node linkType: hard +"real-require@npm:^0.2.0": + version: 0.2.0 + resolution: "real-require@npm:0.2.0" + checksum: 23eea5623642f0477412ef8b91acd3969015a1501ed34992ada0e3af521d3c865bb2fe4cdbfec5fe4b505f6d1ef6a03e5c3652520837a8c3b53decff7e74b6a0 + languageName: node + linkType: hard + "redent@npm:^3.0.0": version: 3.0.0 resolution: "redent@npm:3.0.0" @@ -14568,6 +14711,13 @@ __metadata: languageName: node linkType: hard +"safe-stable-stringify@npm:^2.3.1": + version: 2.4.3 + resolution: "safe-stable-stringify@npm:2.4.3" + checksum: 81dede06b8f2ae794efd868b1e281e3c9000e57b39801c6c162267eb9efda17bd7a9eafa7379e1f1cacd528d4ced7c80d7460ad26f62ada7c9e01dec61b2e768 + languageName: node + linkType: hard + "safer-buffer@npm:>= 2.1.2 < 3, safer-buffer@npm:>= 2.1.2 < 3.0.0, safer-buffer@npm:^2.1.0": version: 2.1.2 resolution: "safer-buffer@npm:2.1.2" @@ -14681,6 +14831,13 @@ __metadata: languageName: node linkType: hard +"secure-json-parse@npm:^2.4.0": + version: 2.7.0 + resolution: "secure-json-parse@npm:2.7.0" + checksum: f57eb6a44a38a3eeaf3548228585d769d788f59007454214fab9ed7f01fbf2e0f1929111da6db28cf0bcc1a2e89db5219a59e83eeaec3a54e413a0197ce879e4 + languageName: node + linkType: hard + "select-hose@npm:^2.0.0": version: 2.0.0 resolution: "select-hose@npm:2.0.0" @@ -15084,6 +15241,15 @@ __metadata: languageName: node linkType: hard +"sonic-boom@npm:^3.0.0, sonic-boom@npm:^3.7.0": + version: 3.7.0 + resolution: "sonic-boom@npm:3.7.0" + dependencies: + atomic-sleep: "npm:^1.0.0" + checksum: 57a3d560efb77f4576db111168ee2649c99e7869fda6ce0ec2a4e5458832d290ba58d74b073ddb5827d9a30f96d23cff79157993d919e1a6d5f28d8b6391c7f0 + languageName: node + linkType: hard + "source-list-map@npm:^2.0.0": version: 2.0.1 resolution: "source-list-map@npm:2.0.1" @@ -15242,7 +15408,7 @@ __metadata: languageName: node linkType: hard -"split2@npm:^4.1.0": +"split2@npm:^4.0.0, split2@npm:^4.1.0": version: 4.2.0 resolution: "split2@npm:4.2.0" checksum: b292beb8ce9215f8c642bb68be6249c5a4c7f332fc8ecadae7be5cbdf1ea95addc95f0459ef2e7ad9d45fd1064698a097e4eb211c83e772b49bc0ee423e91534 @@ -15407,7 +15573,7 @@ __metadata: languageName: node linkType: hard -"string-width-cjs@npm:string-width@^4.2.0, string-width@npm:^1.0.2 || 2 || 3 || 4, string-width@npm:^4.1.0, string-width@npm:^4.2.0, string-width@npm:^4.2.3": +"string-width-cjs@npm:string-width@^4.2.0, string-width@npm:^4.1.0, string-width@npm:^4.2.0, string-width@npm:^4.2.3": version: 4.2.3 resolution: "string-width@npm:4.2.3" dependencies: @@ -15500,7 +15666,7 @@ __metadata: languageName: node linkType: hard -"string_decoder@npm:^1.0.0, string_decoder@npm:^1.1.1": +"string_decoder@npm:^1.0.0, string_decoder@npm:^1.1.1, string_decoder@npm:^1.3.0": version: 1.3.0 resolution: "string_decoder@npm:1.3.0" dependencies: @@ -16046,6 +16212,15 @@ __metadata: languageName: node linkType: hard +"thread-stream@npm:^2.0.0": + version: 2.4.1 + resolution: "thread-stream@npm:2.4.1" + dependencies: + real-require: "npm:^0.2.0" + checksum: ce29265810b9550ce896726301ff006ebfe96b90292728f07cfa4c379740585583046e2a8018afc53aca66b18fed12b33a84f3883e7ebc317185f6682898b8f8 + languageName: node + linkType: hard + "thunky@npm:^1.0.2": version: 1.1.0 resolution: "thunky@npm:1.1.0" @@ -17283,15 +17458,6 @@ __metadata: languageName: node linkType: hard -"wide-align@npm:^1.1.5": - version: 1.1.5 - resolution: "wide-align@npm:1.1.5" - dependencies: - string-width: "npm:^1.0.2 || 2 || 3 || 4" - checksum: 1d9c2a3e36dfb09832f38e2e699c367ef190f96b82c71f809bc0822c306f5379df87bab47bed27ea99106d86447e50eb972d3c516c2f95782807a9d082fbea95 - languageName: node - linkType: hard - "wildcard@npm:^2.0.0": version: 2.0.1 resolution: "wildcard@npm:2.0.1"