diff --git a/packages/mongodb-runner/src/cli.spec.ts b/packages/mongodb-runner/src/cli.spec.ts new file mode 100644 index 00000000..2777c0d4 --- /dev/null +++ b/packages/mongodb-runner/src/cli.spec.ts @@ -0,0 +1,129 @@ +import { expect } from 'chai'; +import { promises as fs } from 'fs'; +import path from 'path'; +import os from 'os'; +import { promisify } from 'util'; +import { execFile } from 'child_process'; +import createDebug from 'debug'; +import sinon from 'sinon'; +import { MongoClient } from 'mongodb'; + +if (process.env.CI) { + createDebug.enable('mongodb-runner,mongodb-downloader'); +} + +const execFileAsync = promisify(execFile); +const tmpDir = path.join(os.tmpdir(), `runner-cli-tests-${Date.now()}`); + +async function runCli(args: string[]): Promise { + const isWin = process.platform === 'win32'; + const runner = isWin ? 'mongodb-runner.cmd' : 'mongodb-runner'; + const { stdout } = await execFileAsync(runner, args); + return stdout; +} + +describe('cli', function () { + this.timeout(1_000_000); // Downloading Windows binaries can take a very long time... + + before(async function () { + await fs.mkdir(tmpDir, { recursive: true }); + }); + + after(async function () { + await fs.rm(tmpDir, { + recursive: true, + maxRetries: 100, + }); + }); + + afterEach(function () { + sinon.restore(); + }); + + it('can manage a standalone cluster with command line args', async function () { + // Start the CLI with arguments and capture stdout. + const stdout = await runCli(['start', '--topology', 'standalone']); + + // stdout is JUST the connection string. + const connectionString = stdout.trim(); + expect(connectionString).to.match(/^mongodb(\+srv)?:\/\//); + + // Connect to the cluster. + const client = new MongoClient(connectionString); + const result = await client.db('admin').command({ ping: 1 }); + await client.close(); + expect(result.ok).to.eq(1); + + // Call `stop` on the CLI + await runCli(['stop', '--all']); + }); + it('can manage a replset cluster with command line args', async function () { + const stdout = await runCli([ + 'start', + '--topology', + 'replset', + '--secondaries', + '2', + '--arbiters', + '1', + '--version', + '8.0.x', + '--', + '--replSet', + 'repl0', + ]); + const connectionString = stdout.trim(); + expect(/repl0/.test(connectionString)).to.be.true; + + // Connect to the cluster. + const client = new MongoClient(connectionString); + const result = await client.db('admin').command({ ping: 1 }); + await client.close(); + expect(result.ok).to.eq(1); + + // Call `stop` on the CLI + await runCli(['stop', '--all']); + }); + it('can manage a sharded cluster with command line args', async function () { + const stdout = await runCli([ + 'start', + '--topology', + 'sharded', + '--shards', + '2', + '--version', + '7.0.x', + ]); + const connectionString = stdout.trim(); + + // Connect to the cluster. + const client = new MongoClient(connectionString); + const result = await client.db('admin').command({ ping: 1 }); + await client.close(); + expect(result.ok).to.eq(1); + + // Call `stop` on the CLI + await runCli(['stop', '--all']); + }); + it('can manage a cluster with a config file', async function () { + const configFile = path.resolve( + __dirname, + '..', + 'test', + 'fixtures', + 'config.json', + ); + const stdout = await runCli(['start', '--config', configFile]); + const connectionString = stdout.trim(); + expect(/repl0/.test(connectionString)).to.be.true; + + // Connect to the cluster. + const client = new MongoClient(connectionString); + const result = await client.db('admin').command({ ping: 1 }); + await client.close(); + expect(result.ok).to.eq(1); + + // Call `stop` on the CLI + await runCli(['stop', '--all']); + }); +}); diff --git a/packages/mongodb-runner/src/cli.ts b/packages/mongodb-runner/src/cli.ts index 6e00db37..633eb19a 100644 --- a/packages/mongodb-runner/src/cli.ts +++ b/packages/mongodb-runner/src/cli.ts @@ -90,6 +90,10 @@ import type { MongoClientOptions } from 'mongodb'; .demandCommand(1, 'A command needs to be provided') .help().argv; const [command, ...args] = argv._.map(String); + // Allow args to be provided by the config file. + if (Array.isArray(argv.args)) { + args.push(...argv.args.map(String)); + } if (argv.debug || argv.verbose) { createDebug.enable('mongodb-runner'); } @@ -111,22 +115,29 @@ import type { MongoClientOptions } from 'mongodb'; async function start() { const { cluster, id } = await utilities.start(argv, args); const cs = new ConnectionString(cluster.connectionString); - console.log(`Server started and running at ${cs.toString()}`); + // Only the connection string should print to stdout so it can be captured + // by a calling process. + console.error(`Server started and running at ${cs.toString()}`); if (cluster.oidcIssuer) { cs.typedSearchParams().set( 'authMechanism', 'MONGODB-OIDC', ); - console.log(`OIDC provider started and running at ${cluster.oidcIssuer}`); - console.log(`Server connection string with OIDC auth: ${cs.toString()}`); + console.error( + `OIDC provider started and running at ${cluster.oidcIssuer}`, + ); + console.error( + `Server connection string with OIDC auth: ${cs.toString()}`, + ); } - console.log('Run the following command to stop the instance:'); - console.log( + console.error('Run the following command to stop the instance:'); + console.error( `${argv.$0} stop --id=${id}` + (argv.runnerDir !== defaultRunnerDir ? `--runnerDir=${argv.runnerDir}` : ''), ); + console.log(cs.toString()); cluster.unref(); } diff --git a/packages/mongodb-runner/src/mongocluster.spec.ts b/packages/mongodb-runner/src/mongocluster.spec.ts index edcb20b0..8ff02ed4 100644 --- a/packages/mongodb-runner/src/mongocluster.spec.ts +++ b/packages/mongodb-runner/src/mongocluster.spec.ts @@ -630,4 +630,51 @@ describe('MongoCluster', function () { { user: 'testuser', db: 'admin' }, ]); }); + it('can use a keyFile', async function () { + const keyFile = path.join(tmpDir, 'keyFile'); + await fs.writeFile(keyFile, 'secret', { mode: 0o400 }); + cluster = await MongoCluster.start({ + version: '8.x', + topology: 'replset', + tmpDir, + secondaries: 1, + arbiters: 1, + args: ['--keyFile', keyFile], + users: [ + { + username: 'testuser', + password: 'testpass', + roles: [ + { role: 'userAdminAnyDatabase', db: 'admin' }, + { role: 'clusterAdmin', db: 'admin' }, + ], + }, + ], + }); + expect(cluster.connectionString).to.be.a('string'); + expect(cluster.serverVersion).to.match(/^8\./); + expect(cluster.connectionString).to.include('testuser:testpass@'); + cluster = await MongoCluster.deserialize(cluster.serialize()); + expect(cluster.connectionString).to.include('testuser:testpass@'); + }); + it('can support requireApiVersion', async function () { + cluster = await MongoCluster.start({ + version: '8.x', + topology: 'sharded', + tmpDir, + secondaries: 1, + shards: 1, + requireApiVersion: 1, + args: ['--setParameter', 'enableTestCommands=1'], + }); + expect(cluster.connectionString).to.be.a('string'); + expect(cluster.serverVersion).to.match(/^8\./); + await cluster.withClient((client) => { + expect(client.serverApi?.version).to.eq('1'); + }); + cluster = await MongoCluster.deserialize(cluster.serialize()); + await cluster.withClient((client) => { + expect(client.serverApi?.version).to.eq('1'); + }); + }); }); diff --git a/packages/mongodb-runner/src/mongocluster.ts b/packages/mongodb-runner/src/mongocluster.ts index 20b0a4eb..b5b93959 100644 --- a/packages/mongodb-runner/src/mongocluster.ts +++ b/packages/mongodb-runner/src/mongocluster.ts @@ -110,6 +110,11 @@ export interface CommonOptions { */ tlsAddClientKey?: boolean; + /** + * Whether to require an API version for commands. + */ + requireApiVersion?: number; + /** * Topology of the cluster. */ @@ -488,6 +493,7 @@ export class MongoCluster extends EventEmitter { ...options, ...s, topology: 'replset', + requireApiVersion: undefined, users: isConfig ? undefined : options.users, // users go on the mongos/config server only for the config set }); return [cluster, isConfig] as const; @@ -528,6 +534,7 @@ export class MongoCluster extends EventEmitter { } await cluster.addAuthIfNeeded(); + await cluster.addRequireApiVersionIfNeeded(options); return cluster; } @@ -536,6 +543,32 @@ export class MongoCluster extends EventEmitter { yield* this.shards; } + async addRequireApiVersionIfNeeded({ + ...options + }: MongoClusterOptions): Promise { + // Set up requireApiVersion if requested. + if (options.requireApiVersion === undefined) { + return; + } + if (options.topology === 'replset') { + throw new Error( + 'requireApiVersion is not supported for replica sets, see SERVER-97010', + ); + } + await Promise.all( + [...this.servers].map( + async (child) => + await child.withClient(async (client) => { + const admin = client.db('admin'); + await admin.command({ setParameter: 1, requireApiVersion: true }); + }), + ), + ); + await this.updateDefaultConnectionOptions({ + serverApi: String(options.requireApiVersion) as '1', + }); + } + async addAuthIfNeeded(): Promise { if (!this.users?.length) return; // Sleep to give time for a possible replset election to settle. diff --git a/packages/mongodb-runner/src/mongoserver.ts b/packages/mongodb-runner/src/mongoserver.ts index 7e7fab1c..eab2f9e0 100644 --- a/packages/mongodb-runner/src/mongoserver.ts +++ b/packages/mongodb-runner/src/mongoserver.ts @@ -20,6 +20,7 @@ import { debugVerbose, jsonClone, makeConnectionString, + sleep, } from './util'; /** @@ -286,9 +287,13 @@ export class MongoServer extends EventEmitter { logEntryStream.resume(); srv.port = port; - const buildInfoError = await srv._populateBuildInfo('insert-new'); - if (buildInfoError) { - debug('failed to get buildInfo', buildInfoError); + // If a keyFile is present, we cannot read or write on the server until + // a user is added to the primary. + if (!options.args?.includes('--keyFile')) { + const buildInfoError = await srv._populateBuildInfo('insert-new'); + if (buildInfoError) { + debug('failed to get buildInfo', buildInfoError); + } } } catch (err) { await srv.close(); @@ -301,24 +306,78 @@ export class MongoServer extends EventEmitter { async updateDefaultConnectionOptions( options: Partial, ): Promise { + // Assume we need these new options to connect. + this.defaultConnectionOptions = { + ...this.defaultConnectionOptions, + ...options, + }; + + // If there is no auth in the connection options, do an immediate metadata refresh and return. let buildInfoError: Error | null = null; + if (!options.auth) { + buildInfoError = await this._populateBuildInfo('restore-check'); + if (buildInfoError) { + debug( + 'failed to refresh buildInfo when updating connection options', + buildInfoError, + options, + ); + throw buildInfoError; + } + return; + } + + debug('Waiting for authorization on', this.port); + + // Wait until we can get connectionStatus. + let supportsAuth = false; + let error: unknown = null; for (let attempts = 0; attempts < 10; attempts++) { - buildInfoError = await this._populateBuildInfo('restore-check', { - ...options, - }); - if (!buildInfoError) break; + error = null; + try { + supportsAuth = await this.withClient(async (client) => { + const status = await client + .db('admin') + .command({ connectionStatus: 1 }); + if (status.authInfo.authenticatedUsers.length > 0) { + debug('Server supports authorization', this.port); + return true; + } + // The server is most likely an arbiter, which does not support + // authenticated users but does support getting the buildInfo. + debug('Server does not support authorization', this.port); + this.buildInfo = await client.db('admin').command({ buildInfo: 1 }); + return false; + }); + } catch (e) { + error = e; + await sleep(2 ** attempts * 10); + } + if (error === null) { + break; + } + } + + if (error !== null) { + throw error; + } + + if (!supportsAuth) { + return; + } + + const mode = this.hasInsertedMetadataCollEntry + ? 'restore-check' + : 'insert-new'; + buildInfoError = await this._populateBuildInfo(mode); + if (buildInfoError) { debug( - 'failed to get buildInfo when setting new options', + 'failed to refresh buildInfo when updating connection options', buildInfoError, options, - this.connectionString, ); + throw buildInfoError; } - if (buildInfoError) throw buildInfoError; - this.defaultConnectionOptions = { - ...this.defaultConnectionOptions, - ...options, - }; } async close(): Promise { diff --git a/packages/mongodb-runner/test/fixtures/config.json b/packages/mongodb-runner/test/fixtures/config.json new file mode 100644 index 00000000..5235578a --- /dev/null +++ b/packages/mongodb-runner/test/fixtures/config.json @@ -0,0 +1,38 @@ +{ + "topology": "replset", + "args": ["--replSet", "repl0"], + "rsMembers": [ + { + "args": [ + "--oplogSize", + "500", + "--setParameter", + "enableTestCommands=true" + ], + "tags": { + "ordinal": "one", + "dc": "ny" + }, + "priority": 1 + }, + { + "args": [ + "--oplogSize", + "500", + "--setParameter", + "enableTestCommands=true" + ], + "tags": { + "ordinal": "two", + "dc": "pa" + }, + "priority": 1 + }, + { + "args": ["--setParameter", "enableTestCommands=true"], + "tags": {}, + "priority": 0, + "arbiterOnly": true + } + ] +}