WIP

apitally · Jul 6, 2024 · 5172eea · 5172eea
1 parent 6d4a557
commit 5172eea
Show file tree

Hide file tree

Showing 6 changed files with 114 additions and 19 deletions.
diff --git a/src/common/client.ts b/src/common/client.ts
@@ -1,5 +1,6 @@
 import { randomUUID } from "crypto";
 import fetchRetry from "fetch-retry";
+import ConsumerRegistry from "./consumerRegistry.js";
 import { Logger, getLogger } from "./logging.js";
 import { isValidClientId, isValidEnv } from "./paramValidation.js";
 import RequestCounter from "./requestCounter.js";
@@ -43,6 +44,7 @@ export class ApitallyClient {
   public requestCounter: RequestCounter;
   public validationErrorCounter: ValidationErrorCounter;
   public serverErrorCounter: ServerErrorCounter;
+  public consumerRegistry: ConsumerRegistry;
   public logger: Logger;
 
   constructor({ clientId, env = "dev", logger }: ApitallyConfig) {
@@ -68,6 +70,7 @@ export class ApitallyClient {
     this.requestCounter = new RequestCounter();
     this.validationErrorCounter = new ValidationErrorCounter();
     this.serverErrorCounter = new ServerErrorCounter();
+    this.consumerRegistry = new ConsumerRegistry();
     this.logger = logger || getLogger();
 
     this.startSync();
@@ -190,6 +193,7 @@ export class ApitallyClient {
       validation_errors:
         this.validationErrorCounter.getAndResetValidationErrors(),
       server_errors: this.serverErrorCounter.getAndResetServerErrors(),
+      consumers: this.consumerRegistry.getAndResetUpdatedConsumers(),
     };
     this.syncDataQueue.push([Date.now(), newPayload]);
 

diff --git a/src/common/consumerRegistry.ts b/src/common/consumerRegistry.ts
@@ -0,0 +1,57 @@
+import { ApitallyConsumer } from "./types.js";
+
+export const consumerFromStringOrObject = (
+  consumer: ApitallyConsumer | string,
+) => {
+  if (typeof consumer === "string") {
+    consumer = String(consumer).trim().substring(0, 128);
+    return consumer ? { identifier: consumer } : null;
+  } else {
+    consumer.identifier = String(consumer.identifier).trim().substring(0, 128);
+    consumer.name = consumer.name?.trim().substring(0, 64);
+    consumer.group = consumer.group?.trim().substring(0, 64);
+    return consumer;
+  }
+};
+
+export default class ConsumerRegistry {
+  private consumers: Map<string, ApitallyConsumer>;
+  private updated: Set<string>;
+
+  constructor() {
+    this.consumers = new Map();
+    this.updated = new Set();
+  }
+
+  public addOrUpdateConsumer(consumer?: ApitallyConsumer | null) {
+    if (!consumer) {
+      return;
+    }
+    const existing = this.consumers.get(consumer.identifier);
+    if (!existing) {
+      this.consumers.set(consumer.identifier, consumer);
+      this.updated.add(consumer.identifier);
+    } else {
+      if (consumer.name && consumer.name !== existing.name) {
+        existing.name = consumer.name;
+        this.updated.add(consumer.identifier);
+      }
+      if (consumer.group && consumer.group !== existing.group) {
+        existing.group = consumer.group;
+        this.updated.add(consumer.identifier);
+      }
+    }
+  }
+
+  public getAndResetUpdatedConsumers() {
+    const data: Array<ApitallyConsumer> = [];
+    this.updated.forEach((identifier) => {
+      const consumer = this.consumers.get(identifier);
+      if (consumer) {
+        data.push(consumer);
+      }
+    });
+    this.updated.clear();
+    return data;
+  }
+}
diff --git a/src/common/types.ts b/src/common/types.ts
@@ -8,6 +8,12 @@ export type ApitallyConfig = {
   logger?: Logger;
 };
 
+export type ApitallyConsumer = {
+  identifier: string;
+  name?: string | null;
+  group?: string | null;
+};
+
 export type PathInfo = {
   method: string;
   path: string;
@@ -74,11 +80,14 @@ export type ServerErrorsItem = ConsumerMethodPath & {
   error_count: number;
 };
 
+export type ConsumerItem = ApitallyConsumer;
+
 export type SyncPayload = {
   time_offset: number;
   instance_uuid: string;
   message_uuid: string;
   requests: Array<RequestsItem>;
   validation_errors: Array<ValidationErrorsItem>;
   server_errors: Array<ServerErrorsItem>;
+  consumers: Array<ConsumerItem>;
 };
diff --git a/src/express/middleware.ts b/src/express/middleware.ts
@@ -2,18 +2,20 @@ import type { Express, NextFunction, Request, Response } from "express";
 import { performance } from "perf_hooks";
 
 import { ApitallyClient } from "../common/client.js";
+import { consumerFromStringOrObject } from "../common/consumerRegistry.js";
 import { getPackageVersion } from "../common/packageVersions.js";
 import {
   ApitallyConfig,
+  ApitallyConsumer,
   StartupData,
   ValidationError,
 } from "../common/types.js";
 import listEndpoints from "./listEndpoints.js";
 
 declare module "express" {
   interface Request {
-    apitallyConsumer?: string;
-    consumerIdentifier?: string; // For backwards compatibility
+    apitallyConsumer?: ApitallyConsumer | string | null;
+    consumerIdentifier?: ApitallyConsumer | string | null; // For backwards compatibility
   }
 }
 
@@ -56,8 +58,9 @@ const getMiddleware = (app: Express, client: ApitallyClient) => {
           if (req.route) {
             const responseTime = performance.now() - startTime;
             const consumer = getConsumer(req);
+            client.consumerRegistry.addOrUpdateConsumer(consumer);
             client.requestCounter.addRequest({
-              consumer: consumer,
+              consumer: consumer?.identifier,
               method: req.method,
               path: req.route.path,
               statusCode: res.statusCode,
@@ -87,7 +90,7 @@ const getMiddleware = (app: Express, client: ApitallyClient) => {
               }
               validationErrors.forEach((error) => {
                 client.validationErrorCounter.addValidationError({
-                  consumer: consumer,
+                  consumer: consumer?.identifier,
                   method: req.method,
                   path: req.route.path,
                   ...error,
@@ -97,7 +100,7 @@ const getMiddleware = (app: Express, client: ApitallyClient) => {
             if (res.statusCode === 500 && res.locals.serverError) {
               const serverError = res.locals.serverError as Error;
               client.serverErrorCounter.addServerError({
-                consumer: consumer,
+                consumer: consumer?.identifier,
                 method: req.method,
                 path: req.route.path,
                 type: serverError.name,
@@ -127,10 +130,14 @@ const getMiddleware = (app: Express, client: ApitallyClient) => {
 
 const getConsumer = (req: Request) => {
   if (req.apitallyConsumer) {
-    return String(req.apitallyConsumer);
+    return consumerFromStringOrObject(req.apitallyConsumer);
   } else if (req.consumerIdentifier) {
     // For backwards compatibility
-    return String(req.consumerIdentifier);
+    process.emitWarning(
+      "The consumerIdentifier property on the request object is deprecated. Use apitallyConsumer instead.",
+      "DeprecationWarning",
+    );
+    return consumerFromStringOrObject(req.consumerIdentifier);
   }
   return null;
 };

diff --git a/src/fastify/plugin.ts b/src/fastify/plugin.ts
@@ -7,8 +7,14 @@ import type {
 import fp from "fastify-plugin";
 
 import { ApitallyClient } from "../common/client.js";
+import { consumerFromStringOrObject } from "../common/consumerRegistry.js";
 import { getPackageVersion } from "../common/packageVersions.js";
-import { ApitallyConfig, PathInfo, ValidationError } from "../common/types.js";
+import {
+  ApitallyConfig,
+  ApitallyConsumer,
+  PathInfo,
+  ValidationError,
+} from "../common/types.js";
 
 declare module "fastify" {
   interface FastifyReply {
@@ -17,8 +23,8 @@ declare module "fastify" {
   }
 
   interface FastifyRequest {
-    apitallyConsumer?: string;
-    consumerIdentifier?: string; // For backwards compatibility
+    apitallyConsumer?: ApitallyConsumer | string | null;
+    consumerIdentifier?: ApitallyConsumer | string | null; // For backwards compatibility
   }
 }
 
@@ -82,8 +88,9 @@ const apitallyPlugin: FastifyPluginAsync<ApitallyConfig> = async (
       if (Array.isArray(responseSize)) {
         responseSize = responseSize[0];
       }
+      client.consumerRegistry.addOrUpdateConsumer(consumer);
       client.requestCounter.addRequest({
-        consumer: consumer,
+        consumer: consumer?.identifier,
         method: request.method,
         path: path,
         statusCode: reply.statusCode,
@@ -100,7 +107,7 @@ const apitallyPlugin: FastifyPluginAsync<ApitallyConfig> = async (
         const validationErrors = extractAjvErrors(reply.payload.message);
         validationErrors.forEach((error) => {
           client.validationErrorCounter.addValidationError({
-            consumer: consumer,
+            consumer: consumer?.identifier,
             method: request.method,
             path: path,
             ...error,
@@ -109,7 +116,7 @@ const apitallyPlugin: FastifyPluginAsync<ApitallyConfig> = async (
       }
       if (reply.statusCode === 500 && reply.serverError) {
         client.serverErrorCounter.addServerError({
-          consumer: consumer,
+          consumer: consumer?.identifier,
           method: request.method,
           path: path,
           type: reply.serverError.name,
@@ -146,10 +153,14 @@ const getAppInfo = (routes: PathInfo[], appVersion?: string) => {
 
 const getConsumer = (request: FastifyRequest) => {
   if (request.apitallyConsumer) {
-    return String(request.apitallyConsumer);
+    return consumerFromStringOrObject(request.apitallyConsumer);
   } else if (request.consumerIdentifier) {
     // For backwards compatibility
-    return String(request.consumerIdentifier);
+    process.emitWarning(
+      "The consumerIdentifier property on the request object is deprecated. Use apitallyConsumer instead.",
+      "DeprecationWarning",
+    );
+    return consumerFromStringOrObject(request.consumerIdentifier);
   }
   return null;
 };

diff --git a/src/koa/middleware.ts b/src/koa/middleware.ts
@@ -1,6 +1,7 @@
 import Koa from "koa";
 
 import { ApitallyClient } from "../common/client.js";
+import { consumerFromStringOrObject } from "../common/consumerRegistry.js";
 import { getPackageVersion } from "../common/packageVersions.js";
 import { ApitallyConfig, PathInfo, StartupData } from "../common/types.js";
 
@@ -25,7 +26,7 @@ const getMiddleware = (client: ApitallyClient) => {
       statusCode = error.statusCode || error.status || 500;
       if (path && statusCode === 500 && error instanceof Error) {
         client.serverErrorCounter.addServerError({
-          consumer: getConsumer(ctx),
+          consumer: getConsumer(ctx)?.identifier,
           method: ctx.request.method,
           path,
           type: error.name,
@@ -40,8 +41,10 @@ const getMiddleware = (client: ApitallyClient) => {
       }
       if (path) {
         try {
+          const consumer = getConsumer(ctx);
+          client.consumerRegistry.addOrUpdateConsumer(consumer);
           client.requestCounter.addRequest({
-            consumer: getConsumer(ctx),
+            consumer: consumer?.identifier,
             method: ctx.request.method,
             path,
             statusCode: statusCode || ctx.response.status,
@@ -66,10 +69,14 @@ const getPath = (ctx: Koa.Context) => {
 
 const getConsumer = (ctx: Koa.Context) => {
   if (ctx.state.apitallyConsumer) {
-    return String(ctx.state.apitallyConsumer);
+    return consumerFromStringOrObject(ctx.state.apitallyConsumer);
   } else if (ctx.state.consumerIdentifier) {
     // For backwards compatibility
-    return String(ctx.state.consumerIdentifier);
+    process.emitWarning(
+      "The consumerIdentifier property on the ctx.state object is deprecated. Use apitallyConsumer instead.",
+      "DeprecationWarning",
+    );
+    return consumerFromStringOrObject(ctx.state.consumerIdentifier);
   }
   return null;
 };