feat(serverless-offline-sqs): opt-in autoStart ElasticMQ for self-contained operation (Fixes #146)

packages/serverless-offline-sqs/README.md

@@ -101,6 +101,50 @@ Start it with `docker compose up elasticmq`, then run `serverless offline` on th
 `http://localhost:9324` (see the [SQS](#sqs) config below). If you'd rather not write a config file,
 the bare `docker run -p 9324:9324 -p 9325:9325 softwaremill/elasticmq-native` works too.
 
+### Zero-setup local SQS with `autoStart` (#146)
+
+If you don't want to manage the container yourself, set `autoStart: true` and the plugin starts an
+ElasticMQ container for you during `serverless offline`, points the SQS client at it, and tears it
+down when the session ends (or on `Ctrl-C`). Combined with `autoCreate: true`, every declared/event
+queue is created inside that container before the listeners attach — so `npm install` plus the
+config below is all you need, **nothing else to run**:
+
+```yml
+custom:
+  serverless-offline-sqs:
+    autoStart: true # spawn + manage an ElasticMQ container for this session
+    autoCreate: true # create the declared/event queues inside it
+    region: eu-west-1
+    accountId: '000000000000'
+```
+
+> **Self-contained _given Docker_, not zero-runtime.** `autoStart` runs the real
+> `softwaremill/elasticmq-native` engine in a container (the same engine the manual setup above uses,
+> so FIFO/DLQ behaviour is identical), driven through the Docker **CLI** — so **Docker must be
+> installed and running**. If it isn't, the plugin fails fast with a clear error and you can fall back
+> to the manual docker-compose setup above. autoStart does not bundle a JVM or a pure-JS emulator.
+
+**Precedence & defaults.** `autoStart` is **opt-in** — unset, the plugin behaves exactly as before.
+If you also set an explicit `endpoint`, `autoStart` is skipped in favour of your endpoint (with a
+warning). When `autoStart` starts a container and you configured no credentials, harmless local
+placeholders are injected (ElasticMQ ignores credential values). Tune the container with the object
+form:
+
+```yml
+custom:
+  serverless-offline-sqs:
+    autoStart:
+      image: softwaremill/elasticmq-native:1.6.11 # default (pinned, not :latest)
+      port: 9324 # host port -> container 9324 (default 9324)
+      pullPolicy: missing # 'missing' (default) pulls only if absent; 'always' always pulls
+      readinessTimeout: 30000 # ms to wait for the port to answer before giving up (default 30000)
+    autoCreate: true
+```
+
+A leaked container from a hard crash (`SIGKILL`) is reclaimed automatically on the next start: the
+container runs `--rm` under a fixed `--name`, and the next `autoStart` force-removes any pre-existing
+container of that name before re-creating it.
+
 ### Dead-letter queues & redrive (`#167`, `#133`, `#65`, `#87`)
 
 When `autoCreate: true`, the plugin now creates **every** `AWS::SQS::Queue` declared in

packages/serverless-offline-sqs/package.json

@@ -15,6 +15,7 @@
     "src/index.js",
     "src/log.js",
     "src/client-config.js",
+    "src/elasticmq.js",
     "src/sqs.js",
     "src/sqs-event.js",
     "src/sqs-event-definition.js"

packages/serverless-offline-sqs/src/elasticmq.js

@@ -0,0 +1,200 @@
+const {execFile: execFileCb} = require('child_process');
+const {promisify} = require('util');
+const {get, isNil, isPlainObject} = require('lodash/fp');
+
+const {normalizeLog} = require('./log');
+
+// #146 (tstackhouse, tristan-mastrodicasa): opt-in `autoStart` — spawn an ElasticMQ container from
+// the plugin's own lifecycle so `autoStart: true` + `autoCreate: true` is a zero-setup local SQS.
+// ALL Docker side effects live here; the plugin class only orchestrates. The Docker CLI is driven
+// through child_process.execFile (argv array, no shell) so we add NO new runtime dependency, and
+// readiness is probed with the built-in fetch (Node 18+). Both are injectable (see `deps`) so the
+// whole lifecycle is unit-testable with mocks and never touches real Docker.
+
+// Pin a concrete image tag (NOT :latest) so a local/CI autoStart is reproducible; `pullPolicy:
+// missing` caches the pull. The exact engine the repo's test suite + docker-compose already use.
+const DEFAULT_AUTOSTART_IMAGE = 'softwaremill/elasticmq-native:1.6.11';
+// ElasticMQ's SQS REST API port, matching docker-compose.yml.
+const DEFAULT_AUTOSTART_PORT = 9324;
+// The container's fixed internal SQS port; only the host port is configurable.
+const CONTAINER_PORT = 9324;
+const DEFAULT_PULL_POLICY = 'missing';
+// Generous default: a cold ElasticMQ container is ready well under 30s.
+const DEFAULT_READINESS_TIMEOUT = 30000;
+// A fixed name so a leaked container (hard crash, SIGKILL) is reclaimed on the next start — the real
+// safety net behind `--rm`, not the SIGINT handler.
+const DEFAULT_CONTAINER_NAME = 'serverless-offline-sqs-autostart';
+const READINESS_POLL_INTERVAL = 250;
+
+// #146/#222: mirror isPluginEnabled's string coercion. autoStart is opt-in, so an absent value is
+// OFF (the inverse default of `enabled`). The string 'false' (YAML/CLI) is off; any other truthy
+// value — including the object config form `{port: ...}` — is on. Pure + non-throwing.
+const isAutoStartEnabled = options => {
+  const autoStart = get('autoStart', options);
+  if (isNil(autoStart)) return false;
+  if (autoStart === 'false') return false;
+  return Boolean(autoStart);
+};
+
+const delay = timeout =>
+  new Promise(resolve => {
+    setTimeout(resolve, timeout);
+  });
+
+// resolveAutoStartOptions(options) -> {image, port, pullPolicy, readinessTimeout, name}
+// Pure + non-mutating. `autoStart` may be `true`/`'true'` (all defaults) or an object of overrides
+// (#146 AC9). A non-object autoStart contributes no overrides, so the pinned defaults stand.
+const resolveAutoStartOptions = options => {
+  const autoStart = get('autoStart', options);
+  const config = isPlainObject(autoStart) ? autoStart : {};
+  return {
+    image: isNil(config.image) ? DEFAULT_AUTOSTART_IMAGE : config.image,
+    port: isNil(config.port) ? DEFAULT_AUTOSTART_PORT : config.port,
+    pullPolicy: isNil(config.pullPolicy) ? DEFAULT_PULL_POLICY : config.pullPolicy,
+    readinessTimeout: isNil(config.readinessTimeout)
+      ? DEFAULT_READINESS_TIMEOUT
+      : config.readinessTimeout,
+    name: isNil(config.name) ? DEFAULT_CONTAINER_NAME : config.name
+  };
+};
+
+// buildContainerArgs(opts) -> the `docker run` argv. Detached, self-removing (`--rm`), fixed
+// `--name` (idempotent reclaim) and the `-Dnode-address.host="*"` flag the docker-compose service
+// uses so the container answers on the host endpoint. Pure (#146 AC2/AC9).
+const buildContainerArgs = ({name, port, image}) => [
+  'run',
+  '-d',
+  '--rm',
+  '--name',
+  name,
+  '-p',
+  `${port}:${CONTAINER_PORT}`,
+  image,
+  '-Dnode-address.host=*'
+];
+
+// The host endpoint the SQS client is pointed at; also the readiness probe target (#146 AC2).
+const buildReadinessUrl = ({port}) => `http://localhost:${port}`;
+
+// #146: to be truly zero-setup, autoStart must not require the user to also configure credentials.
+// ElasticMQ ignores credential VALUES, but the @aws-sdk v3 client still needs SOME credentials to
+// sign — with none, buildCredentials returns undefined and the default provider chain hangs/throws
+// ("Could not load credentials from any providers"). Inject harmless local placeholders, but ONLY
+// when the user supplied neither key (an explicit pair still wins). Pure + non-mutating: returns a
+// new options object.
+const LOCAL_CREDENTIAL = 'localAutoStart';
+const ensureLocalCredentials = options => {
+  const accessKeyId = get('accessKeyId', options);
+  const secretAccessKey = get('secretAccessKey', options);
+  if (!isNil(accessKeyId) || !isNil(secretAccessKey)) return options;
+  return {...options, accessKeyId: LOCAL_CREDENTIAL, secretAccessKey: LOCAL_CREDENTIAL};
+};
+
+// Build the error thrown when `docker version` fails — names Docker and points at the documented
+// manual docker-compose fallback so the message is actionable (#146 AC6).
+const dockerUnavailableError = cause =>
+  new Error(
+    'serverless-offline-sqs autoStart requires Docker, but `docker version` failed ' +
+      `(${cause && cause.message ? cause.message : cause}). Start Docker, or run an ElasticMQ ` +
+      'container yourself and set custom.serverless-offline-sqs.endpoint (see the README).'
+  );
+
+// startElasticMq(opts, log, deps?) -> Promise<{endpoint, stop()}>
+// `opts` is the output of resolveAutoStartOptions. `deps` injects execFile/fetch/delay for tests.
+// Sequence: preflight `docker version` (fail fast, AC6) -> ensure the image per pullPolicy ->
+// reclaim a leaked `<name>` (`rm -f`, idempotent) -> `docker run` -> poll readiness until the port
+// answers or readinessTimeout (then tear down + throw, AC7). The returned `stop()` is
+// guarded-idempotent so SIGINT-then-offline:start:end does not double-remove (AC4).
+const startElasticMq = async (opts, log, deps = {}) => {
+  const logger = normalizeLog(log);
+  const execFile = deps.execFile || promisify(execFileCb);
+  // Node 18+ exposes a global `fetch` (the package `engines` requires >=18). Read it off `global`
+  // (defined in the node ESLint env) so no polyfill/dependency is pulled in.
+  const fetchFn = deps.fetch || global.fetch;
+  const sleep = deps.delay || delay;
+
+  const docker = (...args) => execFile('docker', args);
+  // Never let a probe/teardown throw — these are best-effort checks, not the happy path.
+  const dockerQuiet = async (...args) => {
+    try {
+      await docker(...args);
+      return true;
+    } catch (err) {
+      return false;
+    }
+  };
+
+  // AC6: preflight — fail fast BEFORE any `docker run`, so nothing is left dangling.
+  try {
+    await docker('version');
+  } catch (err) {
+    throw dockerUnavailableError(err);
+  }
+
+  // AC9: honor the pull policy. `missing` => inspect, pull only on a miss; `always` => always pull.
+  if (opts.pullPolicy === 'always') {
+    await docker('pull', opts.image);
+  } else if (opts.pullPolicy === 'missing') {
+    const present = await dockerQuiet('image', 'inspect', opts.image);
+    if (!present) await docker('pull', opts.image);
+  }
+
+  // Idempotent reclaim: force-remove a leaked container of our fixed name before (re)creating it.
+  await dockerQuiet('rm', '-f', opts.name);
+
+  await docker(...buildContainerArgs(opts));
+
+  const endpoint = buildReadinessUrl(opts);
+
+  // Guarded-idempotent teardown (AC4): `stop` removes the container at most once.
+  const stopState = {stopped: false};
+  const stop = async () => {
+    if (stopState.stopped) return;
+    stopState.stopped = true;
+    logger.notice('Stopping autostarted ElasticMQ container');
+    await dockerQuiet('stop', opts.name);
+    await dockerQuiet('rm', '-f', opts.name);
+  };
+
+  // AC7: poll the SQS port until ANY HTTP answer (even a 4xx means the port is up). A
+  // connection-refused while booting is the expected not-ready signal — swallow + retry until the
+  // readinessTimeout, then tear the container down and throw a descriptive error.
+  const deadline = Date.now() + opts.readinessTimeout;
+  const waitUntilReady = async () => {
+    try {
+      await fetchFn(endpoint);
+      return;
+    } catch (err) {
+      if (Date.now() >= deadline) {
+        await stop();
+        throw new Error(
+          `serverless-offline-sqs autoStart: ElasticMQ did not become ready at ${endpoint} ` +
+            `within ${opts.readinessTimeout}ms. Is the port free? Override with ` +
+            'custom.serverless-offline-sqs.autoStart.port.'
+        );
+      }
+      await sleep(READINESS_POLL_INTERVAL);
+      return waitUntilReady();
+    }
+  };
+  await waitUntilReady();
+
+  logger.notice(`Started autostarted ElasticMQ container at ${endpoint}`);
+
+  return {endpoint, stop};
+};
+
+module.exports = {
+  startElasticMq,
+  isAutoStartEnabled,
+  resolveAutoStartOptions,
+  buildContainerArgs,
+  buildReadinessUrl,
+  ensureLocalCredentials,
+  DEFAULT_AUTOSTART_IMAGE,
+  DEFAULT_AUTOSTART_PORT,
+  DEFAULT_PULL_POLICY,
+  DEFAULT_READINESS_TIMEOUT,
+  DEFAULT_CONTAINER_NAME,
+  LOCAL_CREDENTIAL
+};

packages/serverless-offline-sqs/src/index.js

@@ -15,6 +15,12 @@ const {
 
 const {normalizeLog} = require('./log');
 const SQS = require('./sqs');
+const {
+  startElasticMq,
+  isAutoStartEnabled,
+  resolveAutoStartOptions,
+  ensureLocalCredentials
+} = require('./elasticmq');
 
 const OFFLINE_OPTION = 'serverless-offline';
 const CUSTOM_OPTION = 'serverless-offline-sqs';
@@ -90,21 +96,57 @@ class ServerlessOfflineSQS {
 
     this._mergeOptions();
 
-    const {sqsEvents, lambdas} = this._getEvents();
+    // #146 (tstackhouse, tristan-mastrodicasa): opt-in autoStart — spawn an ElasticMQ container and
+    // point the SQS client at it BEFORE any queue is created or polled. Setting this.options.endpoint
+    // is enough: buildClientConfig passes it through and sqs._rewriteQueueUrl rewrites every queue URL
+    // to it (AC2/AC3/AC10). If the user ALSO set an explicit endpoint, autoStart is skipped in its
+    // favour with a warning (AC5). Default-off path is untouched (AC1).
+    await this._startAutoStartElasticMq();
 
-    await this._createLambda(lambdas);
+    try {
+      const {sqsEvents, lambdas} = this._getEvents();
 
-    const eventModules = [];
+      await this._createLambda(lambdas);
+
+      const eventModules = [];
 
-    // #222 (gndelia): skip SQS setup entirely when disabled, but still create the lambdas so plain
-    // HTTP functions keep working under serverless-offline.
-    if (isPluginEnabled(this.options) && sqsEvents.length > 0) {
-      eventModules.push(this._createSqs(sqsEvents));
+      // #222 (gndelia): skip SQS setup entirely when disabled, but still create the lambdas so plain
+      // HTTP functions keep working under serverless-offline.
+      if (isPluginEnabled(this.options) && sqsEvents.length > 0) {
+        eventModules.push(this._createSqs(sqsEvents));
+      }
+
+      await Promise.all(eventModules);
+
+      this.log.notice(
+        `Starting Offline SQS at stage ${this.options.stage} (${this.options.region})`
+      );
+    } catch (err) {
+      // #146: if a later start step fails after autoStart spun up a container, tear it down so a
+      // failed boot never leaks the container (belt-and-suspenders to the --rm + reclaim-on-next-start net).
+      if (this.elasticmq) await this.elasticmq.stop();
+      throw err;
     }
+  }
 
-    await Promise.all(eventModules);
+  // #146: thin orchestration only — all Docker logic lives in the pure/side-effect elasticmq module.
+  async _startAutoStartElasticMq() {
+    if (!isAutoStartEnabled(this.options)) return;
+
+    // AC5: an explicit user endpoint wins; do NOT start a container, just warn.
+    if (this.options.endpoint) {
+      this.log.warning(
+        `serverless-offline-sqs: autoStart is enabled but an explicit endpoint (${this.options.endpoint}) ` +
+          'is set — skipping autoStart and using the configured endpoint.'
+      );
+      return;
+    }
 
-    this.log.notice(`Starting Offline SQS at stage ${this.options.stage} (${this.options.region})`);
+    this.elasticmq = await startElasticMq(resolveAutoStartOptions(this.options), this.log);
+    this.options.endpoint = this.elasticmq.endpoint;
+    // Zero-setup: ElasticMQ ignores credential values, but the v3 SQS client needs SOME credentials
+    // to sign. Inject local placeholders unless the user already configured a pair (#146).
+    this.options = ensureLocalCredentials(this.options);
   }
 
   ready() {
@@ -147,6 +189,13 @@ class ServerlessOfflineSQS {
       eventModules.push(this.sqs.stop(SERVER_SHUTDOWN_TIMEOUT));
     }
 
+    // #146 (AC4): tear down the autostarted ElasticMQ container before the process exits. SIGINT/
+    // SIGTERM already route through end() via _listenForTermination, and elasticmq.stop() is
+    // guarded-idempotent, so a SIGINT followed by offline:start:end is a harmless no-op the 2nd time.
+    if (this.elasticmq) {
+      eventModules.push(this.elasticmq.stop());
+    }
+
     await Promise.all(eventModules);
 
     if (!skipExit) {