feat: Allow short function names in queue consumer config (#1206)

Lambda Wrapper's current documentation for "direct"-mode offline SQS
is incorrect: queue consumer names must be the full deployed function
name generated by Serverless, typically `service-stage-FunctionName`,
instead of just `FunctionName`.

However, this approach is more difficult for the developer. In some
of our services, this had been set up with hardcoded stage values,
leading to unexpected issues when trying to simulate stages other than
`dev` locally.

This PR allows `SQSService` to add the necessary prefix to queue
consumer function names, making the previously incorrect example in the
documentation work.

In order to maintain backwards compatibility, the prefix will not be
added if it is already present. Correctly configured projects can
continue to function, while projects that are broken (or broken in some
stages where stage was hardcoded) will continue to be broken.

Some design decisions:

- I've placed the helper for getting the Lambda function name prefix
into a public method of `DependencyInjection`, alongside the `isOffline`
helper. It's not SQS-specific, and will be helpful in other scenarios
where we want to invoke one Lambda function from another.

- All changes in `SQSService` are within the `publishOffline` method. I
initially considered mutating the SQS config object within the
constructor so that all `queueConsumer` values were converted to full
function names (requiring no changes to `publishOffline`). However, in a
deployed environment, this creates unnecessary additional work within
every Lambda invocation.

Jira: [ENG-3396]

[ENG-3396]: https://comicrelief.atlassian.net/browse/ENG-3396

Author: seb-cr

docs/services/SQSService.md

@@ -109,22 +109,18 @@ To take advantage of SQS emulation, you will need to do the following in your pr
 
 - Include the `queueConsumers` key in your `SQSService` config.
 
-  This maps the queue name to the fully qualified `FunctionName` that we want to trigger when messages are sent to that queue.
+  This maps the queue name to the name of the Serverless function that we want to trigger when messages are sent to that queue.
 
   Extending the example from above, your config might look like this:
 
   ```ts
   const lambdaWrapper = lw.configure({
     sqs: {
       queues: {
-        // Add an entry for each queue with its AWS name.
-        // Usually we define queue names in our serverless.yml and provide them
-        // to the application via environment variables. If you haven't defined
-        // types for your env vars, you'll need to coerce them to `string`.
         submissions: process.env.SQS_QUEUE_SUBMISSIONS as string,
       },
       queueConsumers: {
-        // See section below about offline SQS emulation.
+        // add an entry mapping each queue to its consumer function name
         submissions: 'SubmissionConsumer',
       },
     }
@@ -151,6 +147,8 @@ To take advantage of SQS emulation, you will need to do the following in your pr
 
 3. If the triggered lambda incurs an exception, this will be propagated upstream, effectively killing the execution of the calling lambda.
 
+4. Queue producer and consumer functions must not have custom deployed Lambda names.
+
 ### Local SQS mode
 
 Use this mode by setting `LAMBDA_WRAPPER_OFFLINE_SQS_MODE=local`. Messages will still be sent to an SQS queue, but using a locally simulated version instead of AWS. This allows you to test your service using a tool like Localstack.

src/core/DependencyInjection.ts

@@ -137,4 +137,39 @@ export default class DependencyInjection<TConfig extends LambdaWrapperConfig = a
       || this.context.invokedFunctionArn.includes('offline')
       || !!process.env.USE_SERVERLESS_OFFLINE;
   }
+
+  /**
+   * Get the `service-stage-` prefix added by Serverless to deployed Lambda
+   * function names. This is handy when you want to invoke other functions,
+   * without having to hardcode the service name and stage.
+   *
+   * The returned prefix includes a trailing dash. To get the deployed name of
+   * another Lambda function, concatenate its serverless function name (its key
+   * in `serverless.yml`) onto the prefix:
+   *
+   * ```js
+   * const serverlessFunctionName = 'MyFunction';
+   * const deployedName = `${di.getLambdaPrefix()}${serverlessFunctionName}`;
+   * ```
+   *
+   * This function relies on looking at the currently running Lambda function's
+   * resource name. It will not work correctly if the Lambda function has been
+   * given a custom resource name.
+   */
+  getLambdaPrefix(): string {
+    const stage = process.env.STAGE;
+    if (!stage) {
+      /* eslint-disable no-template-curly-in-string */
+      throw new Error(
+        'STAGE is not set\n\n'
+        + 'Please add to your Lambda environment:\n\n'
+        + '    STAGE: ${sls:stage}\n',
+      );
+    }
+    if (!this.context.functionName) {
+      throw new Error('Lambda function name is unavailable in context');
+    }
+    const [service] = this.context.functionName.split(`-${stage}-`);
+    return `${service}-${stage}-`;
+  }
 }

src/services/SQSService.ts

@@ -478,15 +478,19 @@ export default class SQSService<
       throw new Error('Can only publishOffline while running serverless offline.');
     }
 
-    const FunctionName = this.queueConsumers[queue];
-
-    if (!FunctionName) {
+    const shortOrLongFunctionName = this.queueConsumers[queue];
+    if (!shortOrLongFunctionName) {
       throw new Error(
         `Queue consumer for queue ${queue} was not found. Please add it to `
         + 'the sqs.queueConsumers key in your Lambda Wrapper config.',
       );
     }
 
+    const prefix = this.di.getLambdaPrefix();
+    const FunctionName = shortOrLongFunctionName.startsWith(prefix)
+      ? shortOrLongFunctionName
+      : `${prefix}${shortOrLongFunctionName}`;
+
     const InvocationType = 'RequestResponse';
 
     const Payload = JSON.stringify({

tests/mocks/aws/context.json

@@ -1,3 +1,4 @@
 {
+  "functionName": "service-stage-FunctionName",
   "invokedFunctionArn": "offline"
 }

tests/unit/core/DependencyInjection.spec.ts

@@ -113,4 +113,39 @@ describe('unit.core.DependencyInjection', () => {
       expect(di.getConfiguration()).toBe(mockConfig);
     });
   });
+
+  describe('getLambdaPrefix', () => {
+    describe('when STAGE and functionName are set', () => {
+      beforeAll(() => {
+        process.env.STAGE = 'stage';
+        mockContext.functionName = 'service-stage-FunctionName';
+      });
+
+      it('should return `service-stage-` prefix', () => {
+        expect(di.getLambdaPrefix()).toEqual('service-stage-');
+      });
+    });
+
+    describe('when STAGE is not set', () => {
+      beforeAll(() => {
+        delete process.env.STAGE;
+        mockContext.functionName = 'service-stage-FunctionName';
+      });
+
+      it('should throw', () => {
+        expect(() => di.getLambdaPrefix()).toThrow('STAGE is not set');
+      });
+    });
+
+    describe('when functionName is not set', () => {
+      beforeAll(() => {
+        process.env.STAGE = 'stage';
+        mockContext.functionName = '';
+      });
+
+      it('should throw', () => {
+        expect(() => di.getLambdaPrefix()).toThrow('function name is unavailable');
+      });
+    });
+  });
 });

tests/unit/services/SQSService.spec.ts

@@ -16,8 +16,10 @@ import {
   SQS_PUBLISH_FAILURE_MODES,
   TimerService,
 } from '@/src';
+import { mockContext } from '@/tests/mocks/aws';
 
 const TEST_QUEUE = 'TEST_QUEUE';
+const TEST_QUEUE_2 = 'TEST_QUEUE_2';
 
 const config = {
   dependencies: {
@@ -28,9 +30,11 @@ const config = {
   sqs: {
     queues: {
       [TEST_QUEUE]: 'QueueName',
+      [TEST_QUEUE_2]: 'QueueName',
     },
     queueConsumers: {
-      [TEST_QUEUE]: 'ConsumerFunctionName',
+      [TEST_QUEUE]: 'ShortFunctionName',
+      [TEST_QUEUE_2]: 'service-stage-FullFunctionName',
     },
   },
 };
@@ -58,9 +62,14 @@ const getService = (
   }: any = {},
   isOffline = false,
 ): MockSQSService => {
-  const di = new DependencyInjection(config, {}, {
-    invokedFunctionArn: isOffline ? 'offline' : 'arn:aws:lambda:eu-west-1:0123456789:test',
-  } as Context);
+  const context = {
+    ...mockContext,
+    invokedFunctionArn: isOffline
+      ? 'offline'
+      : `arn:aws:lambda:eu-west-1:0123456789:${mockContext.functionName}`,
+  };
+
+  const di = new DependencyInjection(config, {}, context);
 
   const logger = di.get(LoggerService);
   jest.spyOn(logger, 'error').mockImplementation();
@@ -116,6 +125,8 @@ describe('unit.services.SQSService', () => {
     envOfflineSqsHost = process.env.LAMBDA_WRAPPER_OFFLINE_SQS_HOST;
     envOfflineSqsPort = process.env.LAMBDA_WRAPPER_OFFLINE_SQS_PORT;
     envRegion = process.env.REGION;
+
+    process.env.STAGE = 'stage';
   });
 
   afterAll(() => {
@@ -228,10 +239,20 @@ describe('unit.services.SQSService', () => {
         const service = getService({}, true);
 
         await service.publish(TEST_QUEUE, { test: 1 });
+        await service.publish(TEST_QUEUE_2, { test: 2 });
 
         expect(service.sqs.send).not.toHaveBeenCalled();
-        expect(service.lambda.send).toHaveBeenCalledTimes(1);
-        expect(service.lambda.send).toHaveBeenCalledWith(expect.any(InvokeCommand));
+        expect(service.lambda.send).toHaveBeenCalledTimes(2);
+
+        // when a short consumer name is given, we should add the prefix
+        const command1: InvokeCommand = service.lambda.send.mock.calls[0][0];
+        expect(command1).toBeInstanceOf(InvokeCommand);
+        expect(command1.input.FunctionName).toEqual('service-stage-ShortFunctionName');
+
+        // when a full consumer name is given, we should _not_ add the prefix
+        const command2: InvokeCommand = service.lambda.send.mock.calls[1][0];
+        expect(command2).toBeInstanceOf(InvokeCommand);
+        expect(command2.input.FunctionName).toEqual('service-stage-FullFunctionName');
       });
 
       it('sends a local SQS request in "local" mode', async () => {