} */
+ this.suites = new Map();
+ this.xunitWritten = false;
+ runner.on(EVENT_RUN_BEGIN, () => this.start());
+ runner.on(EVENT_RUN_END, () => this.end());
+ runner.on(EVENT_SUITE_BEGIN, suite => this.onSuite(suite));
+ runner.on(EVENT_TEST_BEGIN, test => this.onTest(test));
+ runner.on(EVENT_TEST_PASS, test => this.pass(test));
+ runner.on(EVENT_TEST_FAIL, (test, error) => this.fail(test, error));
+ runner.on(EVENT_TEST_PENDING, test => this.pending(test));
+ runner.on(EVENT_SUITE_END, suite => this.suiteEnd(suite));
+ runner.on(EVENT_TEST_END, test => this.testEnd(test));
+
+ process.on('SIGINT', () => this.end(true));
+ }
+ start() {}
+
+ end(ctrlC) {
+ try {
+ if (ctrlC) console.log('emergency exit!');
+ const output = { testSuites: [] };
+
+ for (const [id, [className, { suite }]] of [...this.suites.entries()].entries()) {
+ let totalSuiteTime = 0;
+ let testCases = [];
+ let failureCount = 0;
+
+ const tests = /** @type {MongoMochaTest[]}*/ (suite.tests);
+ for (const test of tests) {
+ let time = test.elapsedTime / 1000;
+ time = Number.isNaN(time) ? 0 : time;
+
+ totalSuiteTime += time;
+ failureCount += test.state === 'failed' ? 1 : 0;
+
+ /** @type {string | Date | number} */
+ let startTime = test.startTime;
+ startTime = startTime ? startTime.toISOString() : 0;
+
+ /** @type {string | Date | number} */
+ let endTime = test.endTime;
+ endTime = endTime ? endTime.toISOString() : 0;
+
+ let error = test.error;
+ let failure = error
+ ? {
+ type: error.constructor.name,
+ message: error.message,
+ stack: error.stack
+ }
+ : undefined;
+
+ let skipped = !!test.skipped;
+
+ testCases.push({
+ name: test.title,
+ className,
+ time,
+ startTime,
+ endTime,
+ skipped,
+ failure
+ });
+ }
+
+ /** @type {string | Date | number} */
+ let timestamp = suite.timestamp;
+ timestamp = timestamp ? timestamp.toISOString().split('.')[0] : '';
+
+ output.testSuites.push({
+ package: suite.file.includes('integration') ? 'Integration' : 'Unit',
+ id,
+ name: className,
+ timestamp,
+ hostname: os.hostname(),
+ tests: suite.tests.length,
+ failures: failureCount,
+ errors: '0',
+ time: totalSuiteTime,
+ testCases,
+ stdout: suite.stdout,
+ stderr: suite.stderr
+ });
+ }
+
+ if (!this.xunitWritten) {
+ fs.writeFileSync('xunit.xml', outputToXML(output), { encoding: 'utf8' });
+ }
+ this.xunitWritten = true;
+ console.log(chalk.bold('wrote xunit.xml'));
+ } catch (error) {
+ console.error(chalk.red(`Failed to output xunit report! ${error}`));
+ } finally {
+ if (ctrlC) process.exit(1);
+ }
+ }
+
+ /**
+ * @param {MongoMochaSuite} suite
+ */
+ onSuite(suite) {
+ if (suite.root) return;
+ if (!this.suites.has(suite.fullTitle())) {
+ suite.timestamp = new Date();
+ this.suites.set(suite.fullTitle(), {
+ suite,
+ stdout: captureStream(process.stdout),
+ stderr: captureStream(process.stderr)
+ });
+ } else {
+ console.warn(`${chalk.yellow('WARNING:')} ${suite.fullTitle()} started twice`);
+ }
+ }
+
+ /**
+ * @param {MongoMochaSuite} suite
+ */
+ suiteEnd(suite) {
+ if (suite.root) return;
+ const currentSuite = this.suites.get(suite.fullTitle());
+ if (!currentSuite) {
+ console.error('Suite never started >:(');
+ process.exit(1);
+ }
+ if (currentSuite.stdout || currentSuite.stderr) {
+ suite.stdout = currentSuite.stdout.unhook();
+ suite.stderr = currentSuite.stderr.unhook();
+ delete currentSuite.stdout;
+ delete currentSuite.stderr;
+ }
+ }
+
+ /**
+ * @param {MongoMochaTest} test
+ */
+ onTest(test) {
+ test.startTime = new Date();
+ }
+
+ /**
+ * @param {MongoMochaTest} test
+ */
+ testEnd(test) {
+ test.endTime = new Date();
+ test.elapsedTime = Number(test.endTime) - Number(test.startTime);
+ }
+
+ /**
+ * @param {MongoMochaTest} test
+ */
+ pass(test) {
+ if (REPORT_TO_STDIO) console.log(chalk.green(`✔ ${test.fullTitle()}`));
+ }
+
+ /**
+ * @param {MongoMochaTest} test
+ * @param {Error} error
+ */
+ fail(test, error) {
+ if (REPORT_TO_STDIO) console.log(chalk.red(`⨯ ${test.fullTitle()} -- ${error.message}`));
+ test.error = error;
+ }
+
+ /**
+ * @param {MongoMochaTest & {skipReason?: string}} test
+ */
+ pending(test) {
+ if (REPORT_TO_STDIO) console.log(chalk.cyan(`↬ ${test.fullTitle()}`));
+ if (typeof test.skipReason === 'string') {
+ console.log(chalk.cyan(`${' '.repeat(test.titlePath().length + 1)}↬ ${test.skipReason}`));
+ }
+ test.skipped = true;
+ }
+}
+
+module.exports = MongoDBMochaReporter;
+
+function replaceIllegalXMLCharacters(string) {
+ // prettier-ignore
+ return String(string)
+ .split('"').join('＂')
+ .split('<').join('﹤')
+ .split('>').join('﹥')
+ .split('&').join('﹠');
+}
+
+const ANSI_ESCAPE_REGEX =
+ // eslint-disable-next-line no-control-regex
+ /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g;
+function outputToXML(output) {
+ function cdata(str) {
+ return `')
+ .join('\\]\\]\\>')}]]>`;
+ }
+
+ function makeTag(name, attributes, selfClose, content) {
+ const attributesString = Object.entries(attributes || {})
+ .map(([k, v]) => `${k}="${replaceIllegalXMLCharacters(v)}"`)
+ .join(' ');
+ let tag = `<${name}${attributesString ? ' ' + attributesString : ''}`;
+ if (selfClose) return tag + '/>\n';
+ else tag += '>';
+ if (content) return tag + content + ``;
+ return tag;
+ }
+
+ let s =
+ '\n\n\n';
+
+ for (const suite of output.testSuites) {
+ s += makeTag('testsuite', {
+ package: suite.package,
+ id: suite.id,
+ name: suite.name,
+ timestamp: suite.timestamp,
+ hostname: suite.hostname,
+ tests: suite.tests,
+ failures: suite.failures,
+ errors: suite.errors,
+ time: suite.time
+ });
+ s += '\n\t' + makeTag('properties') + '\n'; // can put metadata here?
+ for (const test of suite.testCases) {
+ s +=
+ '\t' +
+ makeTag(
+ 'testcase',
+ {
+ name: test.name,
+ classname: test.className,
+ time: test.time,
+ start: test.startTime,
+ end: test.endTime
+ },
+ !test.failure && !test.skipped
+ );
+ if (test.failure) {
+ s +=
+ '\n\t\t' +
+ makeTag('failure', { type: test.failure.type }, false, cdata(test.failure.stack)) +
+ '\n';
+ s += `\t\n`;
+ }
+ if (test.skipped) {
+ s += makeTag('skipped', {}, true);
+ s += `\t\n`;
+ }
+ }
+ s += '\t' + makeTag('system-out', {}, false, cdata(suite.stdout)) + '\n';
+ s += '\t' + makeTag('system-err', {}, false, cdata(suite.stderr)) + '\n';
+ s += `\n`;
+ }
+
+ return s + '\n';
+}
diff -pruN 1.3.0-1/bindings/python/CHANGELOG.rst 1.5.2-1/bindings/python/CHANGELOG.rst
--- 1.3.0-1/bindings/python/CHANGELOG.rst 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/bindings/python/CHANGELOG.rst 2022-07-30 15:14:43.000000000 +0000
@@ -1,6 +1,32 @@
Changelog
=========
+Changes in Version 1.3.0
+------------------------
+
+- Add support for Queryable Encryption with MongoDB 6.0.
+- Add support for the crypt_shared library which can be used instead
+ of mongocryptd.
+- **Remove support for libmongocrypt 1.3, libmongocrypt >=1.5 is now
+ required.** Note this is only relevant for users that install from
+ source or use the ``PYMONGOCRYPT_LIB`` environment variable.
+
+Changes in Version 1.2.0
+------------------------
+
+- Add support for the "kmip" KMS provider.
+- Add MongoCryptKmsContext.kms_provider property.
+- Bundle libmongocrypt 1.3.0 in release wheels.
+- **Remove support for libmongocrypt 1.2, libmongocrypt >=1.3 is now
+ required.** Note this is only relevant for users that install from
+ source or use the ``PYMONGOCRYPT_LIB`` environment variable.
+
+Changes in Version 1.1.2
+------------------------
+
+- Fix a bug where decrypting from a memoryview was not supported.
+- Bundle libmongocrypt 1.2.2 in release wheels.
+
Changes in Version 1.1.1
------------------------
@@ -14,7 +40,7 @@ Changes in Version 1.1.0
- Bundle libmongocrypt 1.2.0 in release wheels.
- **Remove support for libmongocrypt 1.0 and 1.1, libmongocrypt >=1.2
is now required.** Note this is only relevant for users that install from
- source or use the ``PYMONGOCRYPT_LIB`` envirnoment variable.
+ source or use the ``PYMONGOCRYPT_LIB`` environment variable.
Changes in Version 1.0.1
------------------------
diff -pruN 1.3.0-1/bindings/python/.evergreen/test.sh 1.5.2-1/bindings/python/.evergreen/test.sh
--- 1.3.0-1/bindings/python/.evergreen/test.sh 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/bindings/python/.evergreen/test.sh 2022-07-30 15:14:43.000000000 +0000
@@ -10,6 +10,7 @@ set -o errexit # Exit the script with e
# MONGOCRYPT_DIR is set by libmongocrypt/.evergreen/config.yml
MONGOCRYPT_DIR="$MONGOCRYPT_DIR"
+git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git
if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
PYMONGOCRYPT_LIB=${MONGOCRYPT_DIR}/nocrypto/bin/mongocrypt.dll
@@ -20,7 +21,11 @@ if [ "Windows_NT" = "$OS" ]; then # Magi
"C:/python/Python36/python.exe"
"C:/python/Python37/python.exe"
"C:/python/Python38/python.exe"
- "C:/python/Python39/python.exe")
+ "C:/python/Python39/python.exe"
+ "C:/python/Python310/python.exe")
+ export CRYPT_SHARED_PATH=../crypt_shared/bin/mongo_crypt_v1.dll
+ C:/python/Python310/python.exe drivers-evergreen-tools/.evergreen/mongodl.py --component crypt_shared \
+ --version latest --out ../crypt_shared/
elif [ "Darwin" = "$(uname -s)" ]; then
export PYMONGOCRYPT_LIB=${MONGOCRYPT_DIR}/nocrypto/lib/libmongocrypt.dylib
PYTHONS=("python" # Python 2.7 from brew
@@ -31,7 +36,11 @@ elif [ "Darwin" = "$(uname -s)" ]; then
"/Library/Frameworks/Python.framework/Versions/3.6/bin/python3"
"/Library/Frameworks/Python.framework/Versions/3.7/bin/python3"
"/Library/Frameworks/Python.framework/Versions/3.8/bin/python3"
- "/Library/Frameworks/Python.framework/Versions/3.9/bin/python3")
+ "/Library/Frameworks/Python.framework/Versions/3.9/bin/python3"
+ "/Library/Frameworks/Python.framework/Versions/3.10/bin/python3")
+ export CRYPT_SHARED_PATH="../crypt_shared/lib/mongo_crypt_v1.dylib"
+ python3 drivers-evergreen-tools/.evergreen/mongodl.py --component crypt_shared \
+ --version latest --out ../crypt_shared/
else
export PYMONGOCRYPT_LIB=${MONGOCRYPT_DIR}/nocrypto/lib64/libmongocrypt.so
PYTHONS=("/opt/python/2.7/bin/python"
@@ -40,14 +49,23 @@ else
"/opt/python/3.6/bin/python3"
"/opt/python/pypy/bin/pypy"
"/opt/python/pypy3.6/bin/pypy3")
+ export CRYPT_SHARED_PATH="../crypt_shared/lib/mongo_crypt_v1.so"
+ /opt/mongodbtoolchain/v3/bin/python3 drivers-evergreen-tools/.evergreen/mongodl.py --component \
+ crypt_shared --version latest --out ../crypt_shared/ --target rhel70
fi
+
for PYTHON_BINARY in "${PYTHONS[@]}"; do
echo "Running test with python: $PYTHON_BINARY"
$PYTHON_BINARY -c 'import sys; print(sys.version)'
createvirtualenv $PYTHON_BINARY .venv
python -m pip install --prefer-binary -r test-requirements.txt
python setup.py test
+ echo "Running tests with CSFLE on dynamic library path..."
+ TEST_CRYPT_SHARED=1 DYLD_FALLBACK_LIBRARY_PATH=../crypt_shared/lib/:$DYLD_FALLBACK_LIBRARY_PATH \
+ LD_LIBRARY_PATH=../crypt_shared/lib:$LD_LIBRARY_PATH \
+ PATH=../crypt_shared/bin:$PATH \
+ python setup.py test
deactivate
rm -rf .venv
done
diff -pruN 1.3.0-1/bindings/python/pymongocrypt/binding.py 1.5.2-1/bindings/python/pymongocrypt/binding.py
--- 1.3.0-1/bindings/python/pymongocrypt/binding.py 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/bindings/python/pymongocrypt/binding.py 2022-07-30 15:14:43.000000000 +0000
@@ -33,8 +33,7 @@ except ImportError:
ffi = cffi.FFI()
# Generated with strip_header.py
-ffi.cdef("""
-/*
+ffi.cdef("""/*
* Copyright 2019-present MongoDB, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -57,11 +56,8 @@ ffi.cdef("""
* See all public API documentation in: @ref mongocrypt.h
*/
-/**
- * @def MONGOCRYPT_VERSION
- * The version string describing libmongocrypt.
- * Has the form x.y.z-++git.
- */
+/* clang-format off */
+/* clang-format on */
/**
* Returns the version string for libmongocrypt.
@@ -164,7 +160,8 @@ typedef struct _mongocrypt_status_t mong
typedef enum {
MONGOCRYPT_STATUS_OK = 0,
MONGOCRYPT_STATUS_ERROR_CLIENT = 1,
- MONGOCRYPT_STATUS_ERROR_KMS = 2
+ MONGOCRYPT_STATUS_ERROR_KMS = 2,
+ MONGOCRYPT_STATUS_ERROR_CRYPT_SHARED = 3,
} mongocrypt_status_type_t;
/**
@@ -311,7 +308,7 @@ mongocrypt_new (void);
* @param[in] crypt The @ref mongocrypt_t object.
* @param[in] log_fn The log callback.
* @param[in] log_ctx A context passed as an argument to the log callback every
- * invokation.
+ * invocation.
* @pre @ref mongocrypt_init has not been called on @p crypt.
* @returns A boolean indicating success. If false, an error status is set.
* Retrieve it with @ref mongocrypt_ctx_status
@@ -324,6 +321,9 @@ mongocrypt_setopt_log_handler (mongocryp
/**
* Configure an AWS KMS provider on the @ref mongocrypt_t object.
*
+ * This has been superseded by the more flexible:
+ * @ref mongocrypt_setopt_kms_providers
+ *
* @param[in] crypt The @ref mongocrypt_t object.
* @param[in] aws_access_key_id The AWS access key ID used to generate KMS
* messages.
@@ -349,6 +349,9 @@ mongocrypt_setopt_kms_provider_aws (mong
/**
* Configure a local KMS provider on the @ref mongocrypt_t object.
*
+ * This has been superseded by the more flexible:
+ * @ref mongocrypt_setopt_kms_providers
+ *
* @param[in] crypt The @ref mongocrypt_t object.
* @param[in] key A 96 byte master key used to encrypt and decrypt key vault
* keys. The viewed data is copied. It is valid to destroy @p key with @ref
@@ -363,11 +366,11 @@ mongocrypt_setopt_kms_provider_local (mo
/**
* Configure KMS providers with a BSON document.
- * Currently only applies to Azure.
*
* @param[in] crypt The @ref mongocrypt_t object.
* @param[in] kms_providers A BSON document mapping the KMS provider names
- * to credentials.
+ * to credentials. Set a KMS provider value to an empty document to supply
+ * credentials on-demand with @ref mongocrypt_ctx_provide_kms_providers.
* @pre @ref mongocrypt_init has not been called on @p crypt.
* @returns A boolean indicating success. If false, an error status is set.
* Retrieve it with @ref mongocrypt_ctx_status
@@ -393,6 +396,98 @@ mongocrypt_setopt_schema_map (mongocrypt
mongocrypt_binary_t *schema_map);
/**
+ * Set a local EncryptedFieldConfigMap for encryption.
+ *
+ * @param[in] crypt The @ref mongocrypt_t object.
+ * @param[in] efc_map A BSON document representing the EncryptedFieldConfigMap
+ * supplied by the user. The keys are collection namespaces and values are
+ * EncryptedFieldConfigMap documents. The viewed data copied. It is valid to
+ * destroy @p efc_map with @ref mongocrypt_binary_destroy immediately after.
+ * @pre @p crypt has not been initialized.
+ * @returns A boolean indicating success. If false, an error status is set.
+ * Retrieve it with @ref mongocrypt_status
+ */
+bool
+mongocrypt_setopt_encrypted_field_config_map (mongocrypt_t *crypt,
+ mongocrypt_binary_t *efc_map);
+
+/**
+ * @brief Append an additional search directory to the search path for loading
+ * the crypt_shared dynamic library.
+ *
+ * @param[in] crypt The @ref mongocrypt_t object to update
+ * @param[in] path A null-terminated sequence of bytes for the search path. On
+ * some filesystems, this may be arbitrary bytes. On other filesystems, this may
+ * be required to be a valid UTF-8 code unit sequence. If the leading element of
+ * the path is the literal string "$ORIGIN", that substring will be replaced
+ * with the directory path containing the executable libmongocrypt module. If
+ * the path string is literal "$SYSTEM", then libmongocrypt will defer to the
+ * system's library resolution mechanism to find the crypt_shared library.
+ *
+ * @note If no crypt_shared dynamic library is found in any of the directories
+ * specified by the search paths loaded here, @ref mongocrypt_init() will still
+ * succeed and continue to operate without crypt_shared.
+ *
+ * @note The search paths are searched in the order that they are appended. This
+ * allows one to provide a precedence in how the library will be discovered. For
+ * example, appending known directories before appending "$SYSTEM" will allow
+ * one to supersede the system's installed library, but still fall-back to it if
+ * the library wasn't found otherwise. If one does not ever append "$SYSTEM",
+ * then the system's library-search mechanism will never be consulted.
+ *
+ * @note If an absolute path to the library is specified using
+ * @ref mongocrypt_setopt_set_crypt_shared_lib_path_override, then paths
+ * appended here will have no effect.
+ */
+void
+mongocrypt_setopt_append_crypt_shared_lib_search_path (mongocrypt_t *crypt,
+ const char *path);
+
+/**
+ * @brief Set a single override path for loading the crypt_shared dynamic
+ * library.
+ *
+ * @param[in] crypt The @ref mongocrypt_t object to update
+ * @param[in] path A null-terminated sequence of bytes for a path to the
+ * crypt_shared dynamic library. On some filesystems, this may be arbitrary
+ * bytes. On other filesystems, this may be required to be a valid UTF-8 code
+ * unit sequence. If the leading element of the path is the literal string
+ * `$ORIGIN`, that substring will be replaced with the directory path containing
+ * the executable libmongocrypt module.
+ *
+ * @note This function will do no IO nor path validation. All validation will
+ * occur during the call to @ref mongocrypt_init.
+ *
+ * @note If a crypt_shared library path override is specified here, then no
+ * paths given to @ref mongocrypt_setopt_append_crypt_shared_lib_search_path
+ * will be consulted when opening the crypt_shared library.
+ *
+ * @note If a path is provided via this API and @ref mongocrypt_init fails to
+ * initialize a valid crypt_shared library instance for the path specified, then
+ * the initialization of mongocrypt_t will fail with an error.
+ */
+void
+mongocrypt_setopt_set_crypt_shared_lib_path_override (mongocrypt_t *crypt,
+ const char *path);
+
+/**
+ * @brief Opt-into handling the MONGOCRYPT_CTX_NEED_KMS_CREDENTIALS state.
+ *
+ * If set, before entering the MONGOCRYPT_CTX_NEED_KMS state,
+ * contexts may enter the MONGOCRYPT_CTX_NEED_KMS_CREDENTIALS state
+ * and then wait for credentials to be supplied through
+ * @ref mongocrypt_ctx_provide_kms_providers.
+ *
+ * A context will only enter MONGOCRYPT_CTX_NEED_KMS_CREDENTIALS
+ * if an empty document was set for a KMS provider in @ref
+ * mongocrypt_setopt_kms_providers.
+ *
+ * @param[in] crypt The @ref mongocrypt_t object to update
+ */
+void
+mongocrypt_setopt_use_need_kms_credentials_state (mongocrypt_t *crypt);
+
+/**
* Initialize new @ref mongocrypt_t object.
*
* Set options before using @ref mongocrypt_setopt_kms_provider_local, @ref
@@ -429,6 +524,50 @@ void
mongocrypt_destroy (mongocrypt_t *crypt);
/**
+ * Obtain a nul-terminated version string of the loaded crypt_shared dynamic
+ * library, if available.
+ *
+ * If no crypt_shared was successfully loaded, this function returns NULL.
+ *
+ * @param[in] crypt The mongocrypt_t object after a successful call to
+ * mongocrypt_init.
+ * @param[out] len An optional output parameter to which the length of the
+ * returned string is written. If provided and no crypt_shared library was
+ * loaded, zero is written to *len.
+ *
+ * @return A nul-terminated string of the dynamically loaded crypt_shared
+ * library.
+ *
+ * @note For a numeric value that can be compared against, use
+ * @ref mongocrypt_crypt_shared_lib_version.
+ */
+const char *
+mongocrypt_crypt_shared_lib_version_string (const mongocrypt_t *crypt,
+ uint32_t *len);
+
+/**
+ * @brief Obtain a 64-bit constant encoding the version of the loaded
+ * crypt_shared library, if available.
+ *
+ * @param[in] crypt The mongocrypt_t object after a successul call to
+ * mongocrypt_init.
+ *
+ * @return A 64-bit encoded version number, with the version encoded as four
+ * sixteen-bit integers, or zero if no crypt_shared library was loaded.
+ *
+ * The version is encoded as four 16-bit numbers, from high to low:
+ *
+ * - Major version
+ * - Minor version
+ * - Revision
+ * - Reserved
+ *
+ * For example, version 6.2.1 would be encoded as: 0x0006'0002'0001'0000
+ */
+uint64_t
+mongocrypt_crypt_shared_lib_version (const mongocrypt_t *crypt);
+
+/**
* Manages the state machine for encryption or decryption.
*/
typedef struct _mongocrypt_ctx_t mongocrypt_ctx_t;
@@ -504,6 +643,25 @@ mongocrypt_ctx_setopt_key_alt_name (mong
mongocrypt_binary_t *key_alt_name);
/**
+ * Set the keyMaterial to use for encrypting data.
+ *
+ * Pass the binary encoding of a BSON document like the following:
+ *
+ * { "keyMaterial" : (BSON BINARY value) }
+ *
+ * @param[in] ctx The @ref mongocrypt_ctx_t object.
+ * @param[in] key_material The data encryption key to use. The viewed data is
+ * copied. It is valid to destroy @p key_material with @ref
+ * mongocrypt_binary_destroy immediately after.
+ * @pre @p ctx has not been initialized.
+ * @returns A boolean indicating success. If false, an error status is set.
+ * Retrieve it with @ref mongocrypt_ctx_status
+ */
+bool
+mongocrypt_ctx_setopt_key_material (mongocrypt_ctx_t *ctx,
+ mongocrypt_binary_t *key_material);
+
+/**
* Set the algorithm used for encryption to either
* deterministic or random encryption. This value
* should only be set when using explicit encryption.
@@ -528,9 +686,17 @@ mongocrypt_ctx_setopt_algorithm (mongocr
const char *algorithm,
int len);
+/// String constant for setopt_algorithm "Deterministic" encryption
+/// String constant for setopt_algorithm "Random" encryption
+/// String constant for setopt_algorithm "Indexed" explicit encryption
+/// String constant for setopt_algorithm "Unindexed" explicit encryption
+
/**
* Identify the AWS KMS master key to use for creating a data key.
*
+ * This has been superseded by the more flexible:
+ * @ref mongocrypt_ctx_setopt_key_encryption_key
+ *
* @param[in] ctx The @ref mongocrypt_ctx_t object.
* @param[in] region The AWS region.
* @param[in] region_len The string length of @p region. Pass -1 to determine
@@ -557,6 +723,9 @@ mongocrypt_ctx_setopt_masterkey_aws (mon
* is persisted in the new data key, and will be returned via
* @ref mongocrypt_kms_ctx_endpoint.
*
+ * This has been superseded by the more flexible:
+ * @ref mongocrypt_ctx_setopt_key_encryption_key
+ *
* @param[in] ctx The @ref mongocrypt_ctx_t object.
* @param[in] endpoint The endpoint.
* @param[in] endpoint_len The string length of @p endpoint. Pass -1 to
@@ -571,6 +740,8 @@ mongocrypt_ctx_setopt_masterkey_aws_endp
/**
* Set the master key to "local" for creating a data key.
+ * This has been superseded by the more flexible:
+ * @ref mongocrypt_ctx_setopt_key_encryption_key
*
* @param[in] ctx The @ref mongocrypt_ctx_t object.
* @pre @p ctx has not been initialized.
@@ -581,11 +752,52 @@ bool
mongocrypt_ctx_setopt_masterkey_local (mongocrypt_ctx_t *ctx);
/**
- * Set key encryption key document for creating a data key.
- * Currently only applies to Azure.
+ * Set key encryption key document for creating a data key or for rewrapping
+ * datakeys.
*
* @param[in] ctx The @ref mongocrypt_ctx_t object.
- * @param[in] bin BSON representing the key encryption key document.
+ * @param[in] bin BSON representing the key encryption key document with
+ * an additional "provider" field. The following forms are accepted:
+ *
+ * AWS
+ * {
+ * provider: "aws",
+ * region: ,
+ * key: ,
+ * endpoint:
+ * }
+ *
+ * Azure
+ * {
+ * provider: "azure",
+ * keyVaultEndpoint: ,
+ * keyName: ,
+ * keyVersion:
+ * }
+ *
+ * GCP
+ * {
+ * provider: "gcp",
+ * projectId: ,
+ * location: ,
+ * keyRing: ,
+ * keyName: ,
+ * keyVersion: ,
+ * endpoint:
+ * }
+ *
+ * Local
+ * {
+ * provider: "local"
+ * }
+ *
+ * KMIP
+ * {
+ * provider: "kmip",
+ * keyId:
+ * endpoint:
+ * }
+ *
* @pre @p ctx has not been initialized.
* @returns A boolean indicating success. If false, and error status is set.
* Retrieve it with @ref mongocrypt_ctx_status.
@@ -641,11 +853,20 @@ mongocrypt_ctx_encrypt_init (mongocrypt_
* This method expects the passed-in BSON to be of the form:
* { "v" : BSON value to encrypt }
*
- * Associated options:
+ * Associated options for FLE 1:
* - @ref mongocrypt_ctx_setopt_key_id
* - @ref mongocrypt_ctx_setopt_key_alt_name
* - @ref mongocrypt_ctx_setopt_algorithm
*
+ * Associated options for Queryable Encryption:
+ * - @ref mongocrypt_ctx_setopt_key_id
+ * - @ref mongocrypt_ctx_setopt_index_key_id
+ * - @ref mongocrypt_ctx_setopt_contention_factor
+ * - @ref mongocrypt_ctx_setopt_query_type
+ *
+ * An error is returned if FLE 1 and Queryable Encryption incompatible options
+ * are set.
+ *
* @param[in] ctx A @ref mongocrypt_ctx_t.
* @param[in] msg A @ref mongocrypt_binary_t the plaintext BSON value. The
* viewed data is copied. It is valid to destroy @p msg with @ref
@@ -676,6 +897,10 @@ mongocrypt_ctx_decrypt_init (mongocrypt_
/**
* Explicit helper method to decrypt a single BSON object.
*
+ * Pass the binary encoding of a BSON document containing the BSON value to
+ * encrypt like the following:
+ *
+ * { "v" : (BSON BINARY value of subtype 6) }
*
* @param[in] ctx A @ref mongocrypt_ctx_t.
* @param[in] msg A @ref mongocrypt_binary_t the encrypted BSON. The viewed data
@@ -687,6 +912,22 @@ mongocrypt_ctx_explicit_decrypt_init (mo
mongocrypt_binary_t *msg);
/**
+ * @brief Initialize a context to rewrap datakeys.
+ *
+ * Associated options:
+ * - @ref mongocrypt_ctx_setopt_key_encryption_key
+ *
+ * @param[in] ctx The @ref mongocrypt_ctx_t object.
+ * @param[in] filter The filter to use for the find command on the key vault
+ * collection to retrieve datakeys to rewrap.
+ * @return A boolean indicating success. If false, an error status is set.
+ * Retrieve it with @ref mongocrypt_ctx_status.
+ */
+bool
+mongocrypt_ctx_rewrap_many_datakey_init (mongocrypt_ctx_t *ctx,
+ mongocrypt_binary_t *filter);
+
+/**
* Indicates the state of the @ref mongocrypt_ctx_t. Each state requires
* different handling. See [the integration
* guide](https://github.com/mongodb/libmongocrypt/blob/master/integrating.md#state-machine)
@@ -698,8 +939,9 @@ typedef enum {
MONGOCRYPT_CTX_NEED_MONGO_MARKINGS = 2, /* run on mongocryptd. */
MONGOCRYPT_CTX_NEED_MONGO_KEYS = 3, /* run on key vault */
MONGOCRYPT_CTX_NEED_KMS = 4,
+ MONGOCRYPT_CTX_NEED_KMS_CREDENTIALS = 7, /* fetch/renew KMS credentials */
MONGOCRYPT_CTX_READY = 5, /* ready for encryption/decryption */
- MONGOCRYPT_CTX_DONE = 6
+ MONGOCRYPT_CTX_DONE = 6,
} mongocrypt_ctx_state_t;
/**
@@ -814,9 +1056,10 @@ mongocrypt_kms_ctx_message (mongocrypt_k
* is valid until calling @ref mongocrypt_ctx_kms_done.
*
* @param[in] kms A @ref mongocrypt_kms_ctx_t.
- * @param[out] endpoint The output hostname as a NULL terminated string. This
- * may include a port (e.g. "example.com:123"). If it does not, default to port
- * 443.
+ * @param[out] endpoint The output endpoint as a NULL terminated string.
+ * The endpoint consists of a hostname and port separated by a colon.
+ * E.g. "example.com:123". A port is always present.
+ *
* @returns A boolean indicating success. If false, an error status is set.
* Retrieve it with @ref mongocrypt_kms_ctx_status
*/
@@ -860,6 +1103,24 @@ mongocrypt_kms_ctx_status (mongocrypt_km
mongocrypt_status_t *status);
/**
+ * Get the KMS provider identifier associated with this KMS request.
+ *
+ * This is used to conditionally configure TLS connections based on the KMS
+ * request. It is useful for KMIP, which authenticates with a client
+ * certificate.
+ *
+ * @param[in] kms The @ref mongocrypt_kms_ctx_t object.
+ * @param[out] len Receives the length of the returned string. It may be NULL.
+ * If it is not NULL, it is set to the length of the returned string without
+ * the NULL terminator.
+ *
+ * @returns One of the NULL terminated static strings: "aws", "azure", "gcp", or
+ * "kmip".
+ */
+const char *
+mongocrypt_kms_ctx_get_kms_provider (mongocrypt_kms_ctx_t *kms, uint32_t *len);
+
+/**
* Call when done handling all KMS contexts.
*
* @param[in] ctx The @ref mongocrypt_ctx_t object.
@@ -871,6 +1132,24 @@ bool
mongocrypt_ctx_kms_done (mongocrypt_ctx_t *ctx);
/**
+ * Call in response to the MONGOCRYPT_CTX_NEED_KMS_CREDENTIALS state
+ * to set per-context KMS provider settings. These follow the same format
+ * as @ref mongocrypt_setopt_kms_providers. If no keys are present in the
+ * BSON input, the KMS provider settings configured for the @ref mongocrypt_t
+ * at initialization are used.
+ *
+ * @param[in] ctx The @ref mongocrypt_ctx_t object.
+ * @param[in] kms_providers A BSON document mapping the KMS provider names
+ * to credentials.
+ *
+ * @returns A boolean indicating success. If false, an error status is set.
+ * Retrieve it with @ref mongocrypt_ctx_status.
+ */
+bool
+mongocrypt_ctx_provide_kms_providers (
+ mongocrypt_ctx_t *ctx, mongocrypt_binary_t *kms_providers_definition);
+
+/**
* Perform the final encryption or decryption.
*
* @param[in] ctx A @ref mongocrypt_ctx_t.
@@ -896,6 +1175,11 @@ mongocrypt_ctx_kms_done (mongocrypt_ctx_
* this BSON is the document containing the new data key to be inserted into
* the key vault collection.
*
+ * If @p ctx was initialized with @ref mongocrypt_ctx_rewrap_many_datakey_init,
+ * then this BSON has the form { "v": [(BSON document), ...] } where each BSON
+ * document in the array is a document containing a rewrapped datakey to be
+ * bulk-updated into the key vault collection.
+ *
* @returns a bool indicating success. If false, an error status is set.
* Retrieve it with @ref mongocrypt_ctx_status
*/
@@ -925,7 +1209,7 @@ mongocrypt_ctx_destroy (mongocrypt_ctx_t
* @param[out] status An optional status to pass error messages. See @ref
* mongocrypt_status_set.
* @returns A boolean indicating success. If returning false, set @p status
- * with a message indiciating the error using @ref mongocrypt_status_ste.
+ * with a message indiciating the error using @ref mongocrypt_status_set.
*/
typedef bool (*mongocrypt_crypto_fn) (void *ctx,
mongocrypt_binary_t *key,
@@ -950,7 +1234,7 @@ typedef bool (*mongocrypt_crypto_fn) (vo
* @param[out] status An optional status to pass error messages. See @ref
* mongocrypt_status_set.
* @returns A boolean indicating success. If returning false, set @p status
- * with a message indiciating the error using @ref mongocrypt_status_ste.
+ * with a message indiciating the error using @ref mongocrypt_status_set.
*/
typedef bool (*mongocrypt_hmac_fn) (void *ctx,
mongocrypt_binary_t *key,
@@ -969,7 +1253,7 @@ typedef bool (*mongocrypt_hmac_fn) (void
* @param[out] status An optional status to pass error messages. See @ref
* mongocrypt_status_set.
* @returns A boolean indicating success. If returning false, set @p status
- * with a message indiciating the error using @ref mongocrypt_status_ste.
+ * with a message indiciating the error using @ref mongocrypt_status_set.
*/
typedef bool (*mongocrypt_hash_fn) (void *ctx,
mongocrypt_binary_t *in,
@@ -987,7 +1271,7 @@ typedef bool (*mongocrypt_hash_fn) (void
* @param[out] status An optional status to pass error messages. See @ref
* mongocrypt_status_set.
* @returns A boolean indicating success. If returning false, set @p status
- * with a message indiciating the error using @ref mongocrypt_status_ste.
+ * with a message indiciating the error using @ref mongocrypt_status_set.
*/
typedef bool (*mongocrypt_random_fn) (void *ctx,
mongocrypt_binary_t *out,
@@ -1004,13 +1288,132 @@ mongocrypt_setopt_crypto_hooks (mongocry
mongocrypt_hash_fn sha_256,
void *ctx);
+/**
+ * Set a crypto hook for the AES256-CTR operations.
+ *
+ * @param[in] crypt The @ref mongocrypt_t object.
+ * @param[in] aes_256_ctr_encrypt The crypto callback function for encrypt
+ * operation.
+ * @param[in] aes_256_ctr_decrypt The crypto callback function for decrypt
+ * operation.
+ * @param[in] ctx A context passed as an argument to the crypto callback
+ * every invocation.
+ * @pre @ref mongocrypt_init has not been called on @p crypt.
+ * @returns A boolean indicating success. If false, an error status is set.
+ * Retrieve it with @ref mongocrypt_status
+ *
+ */
+bool
+mongocrypt_setopt_aes_256_ctr (mongocrypt_t *crypt,
+ mongocrypt_crypto_fn aes_256_ctr_encrypt,
+ mongocrypt_crypto_fn aes_256_ctr_decrypt,
+ void *ctx);
+
+/**
+ * Set an AES256-ECB crypto hook for the AES256-CTR operations. If CTR hook was
+ * configured using @ref mongocrypt_setopt_aes_256_ctr, ECB hook will be
+ * ignored.
+ *
+ * @param[in] crypt The @ref mongocrypt_t object.
+ * @param[in] aes_256_ecb_encrypt The crypto callback function for encrypt
+ * operation.
+ * @param[in] ctx A context passed as an argument to the crypto callback
+ * every invocation.
+ * @pre @ref mongocrypt_init has not been called on @p crypt.
+ * @returns A boolean indicating success. If false, an error status is set.
+ * Retrieve it with @ref mongocrypt_status
+ *
+ */
+bool
+mongocrypt_setopt_aes_256_ecb (mongocrypt_t *crypt,
+ mongocrypt_crypto_fn aes_256_ecb_encrypt,
+ void *ctx);
+
+/**
+ * Set a crypto hook for the RSASSA-PKCS1-v1_5 algorithm with a SHA-256 hash.
+ *
+ * See: https://tools.ietf.org/html/rfc3447#section-8.2
+ *
+ * Note: this function has the wrong name. It should be:
+ * mongocrypt_setopt_crypto_hook_sign_rsassa_pkcs1_v1_5
+ *
+ * @param[in] crypt The @ref mongocrypt_t object.
+ * @param[in] sign_rsaes_pkcs1_v1_5 The crypto callback function.
+ * @param[in] sign_ctx A context passed as an argument to the crypto callback
+ * every invocation.
+ * @pre @ref mongocrypt_init has not been called on @p crypt.
+ * @returns A boolean indicating success. If false, an error status is set.
+ * Retrieve it with @ref mongocrypt_status
+ *
+ */
bool
mongocrypt_setopt_crypto_hook_sign_rsaes_pkcs1_v1_5 (
mongocrypt_t *crypt,
mongocrypt_hmac_fn sign_rsaes_pkcs1_v1_5,
void *sign_ctx);
-""")
+/**
+ * @brief Opt-into skipping query analysis.
+ *
+ * If opted in:
+ * - The crypt_shared library will not attempt to be loaded.
+ * - A mongocrypt_ctx_t will never enter the MONGOCRYPT_CTX_NEED_MARKINGS state.
+ *
+ * @param[in] crypt The @ref mongocrypt_t object to update
+ */
+void
+mongocrypt_setopt_bypass_query_analysis (mongocrypt_t *crypt);
+
+/**
+ * Set the contention factor used for explicit encryption.
+ * The contention factor is only used for indexed Queryable Encryption.
+ *
+ * @param[in] ctx The @ref mongocrypt_ctx_t object.
+ * @param[in] contention_factor
+ * @pre @p ctx has not been initialized.
+ * @returns A boolean indicating success. If false, an error status is set.
+ * Retrieve it with @ref mongocrypt_ctx_status.
+ */
+bool
+mongocrypt_ctx_setopt_contention_factor (mongocrypt_ctx_t *ctx,
+ int64_t contention_factor);
+
+/**
+ * Set the index key id to use for explicit Queryable Encryption.
+ *
+ * If the index key id not set, the key id from @ref
+ * mongocrypt_ctx_setopt_key_id is used.
+ *
+ * @param[in] ctx The @ref mongocrypt_ctx_t object.
+ * @param[in] key_id The binary corresponding to the _id (a UUID) of the data
+ * key to use from the key vault collection. Note, the UUID must be encoded with
+ * RFC-4122 byte order. The viewed data is copied. It is valid to destroy
+ * @p key_id with @ref mongocrypt_binary_destroy immediately after.
+ * @pre @p ctx has not been initialized.
+ * @returns A boolean indicating success. If false, an error status is set.
+ * Retrieve it with @ref mongocrypt_ctx_status
+ */
+bool
+mongocrypt_ctx_setopt_index_key_id (mongocrypt_ctx_t *ctx,
+ mongocrypt_binary_t *key_id);
+
+/**
+ * Set the query type to use for explicit Queryable Encryption.
+ *
+ * @param[in] ctx The @ref mongocrypt_ctx_t object.
+ * @param[in] query_type The query type string
+ * @param[in] len The length of query_type, or -1 for automatic
+ * @pre @p ctx has not been initialized.
+ * @returns A boolean indicating success. If false, an error status is set.
+ * Retrieve it with @ref mongocrypt_ctx_status
+ */
+bool
+mongocrypt_ctx_setopt_query_type (mongocrypt_ctx_t *ctx,
+ const char *query_type,
+ int len);
+
+/// String constant for setopt_query_type_v2, "equality" query type
+""")
if PY3:
def _to_string(cdata):
diff -pruN 1.3.0-1/bindings/python/pymongocrypt/crypto.py 1.5.2-1/bindings/python/pymongocrypt/crypto.py
--- 1.3.0-1/bindings/python/pymongocrypt/crypto.py 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/bindings/python/pymongocrypt/crypto.py 2022-07-30 15:14:43.000000000 +0000
@@ -44,6 +44,24 @@ def _callback_error_handler(exception, e
return False
+def _aes_256_encrypt(key, mode, input, output, bytes_written):
+ cipher = Cipher(algorithms.AES(_to_bytes(key)), mode,
+ backend=default_backend())
+ encryptor = cipher.encryptor()
+ data = encryptor.update(_to_bytes(input)) + encryptor.finalize()
+ _write_bytes(output, data)
+ bytes_written[0] = len(data)
+
+
+def _aes_256_decrypt(key, mode, input, output, bytes_written):
+ cipher = Cipher(algorithms.AES(_to_bytes(key)), mode,
+ backend=default_backend())
+ decryptor = cipher.decryptor()
+ data = decryptor.update(_to_bytes(input)) + decryptor.finalize()
+ _write_bytes(output, data)
+ bytes_written[0] = len(data)
+
+
@ffi.callback(
"bool(void *, mongocrypt_binary_t *, mongocrypt_binary_t *,"
" mongocrypt_binary_t *, mongocrypt_binary_t *, uint32_t *,"
@@ -51,12 +69,7 @@ def _callback_error_handler(exception, e
onerror=_callback_error_handler)
def aes_256_cbc_encrypt(ctx, key, iv, input, output, bytes_written, status):
# Note that libmongocrypt pads the input before calling this method.
- cipher = Cipher(algorithms.AES(_to_bytes(key)), modes.CBC(_to_bytes(iv)),
- backend=default_backend())
- encryptor = cipher.encryptor()
- data = encryptor.update(_to_bytes(input)) + encryptor.finalize()
- _write_bytes(output, data)
- bytes_written[0] = len(data)
+ _aes_256_encrypt(key, modes.CBC(_to_bytes(iv)), input, output, bytes_written)
return True
@@ -67,12 +80,27 @@ def aes_256_cbc_encrypt(ctx, key, iv, in
onerror=_callback_error_handler)
def aes_256_cbc_decrypt(ctx, key, iv, input, output, bytes_written, status):
# Note that libmongocrypt pads the input before calling this method.
- cipher = Cipher(algorithms.AES(_to_bytes(key)), modes.CBC(_to_bytes(iv)),
- backend=default_backend())
- decryptor = cipher.decryptor()
- data = decryptor.update(_to_bytes(input)) + decryptor.finalize()
- _write_bytes(output, data)
- bytes_written[0] = len(data)
+ _aes_256_decrypt(key, modes.CBC(_to_bytes(iv)), input, output, bytes_written)
+ return True
+
+
+@ffi.callback(
+ "bool(void *, mongocrypt_binary_t *, mongocrypt_binary_t *,"
+ " mongocrypt_binary_t *, mongocrypt_binary_t *, uint32_t *,"
+ " mongocrypt_status_t *)",
+ onerror=_callback_error_handler)
+def aes_256_ctr_encrypt(ctx, key, iv, input, output, bytes_written, status):
+ _aes_256_encrypt(key, modes.CTR(_to_bytes(iv)), input, output, bytes_written)
+ return True
+
+
+@ffi.callback(
+ "bool(void *, mongocrypt_binary_t *, mongocrypt_binary_t *,"
+ " mongocrypt_binary_t *, mongocrypt_binary_t *, uint32_t *,"
+ " mongocrypt_status_t *)",
+ onerror=_callback_error_handler)
+def aes_256_ctr_decrypt(ctx, key, iv, input, output, bytes_written, status):
+ _aes_256_decrypt(key, modes.CTR(_to_bytes(iv)), input, output, bytes_written)
return True
diff -pruN 1.3.0-1/bindings/python/pymongocrypt/explicit_encrypter.py 1.5.2-1/bindings/python/pymongocrypt/explicit_encrypter.py
--- 1.3.0-1/bindings/python/pymongocrypt/explicit_encrypter.py 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/bindings/python/pymongocrypt/explicit_encrypter.py 2022-07-30 15:14:43.000000000 +0000
@@ -17,19 +17,37 @@ from pymongocrypt.state_machine import r
class ExplicitEncryptOpts(object):
- def __init__(self, algorithm, key_id=None, key_alt_name=None):
+ def __init__(self, algorithm, key_id=None, key_alt_name=None, index_key_id=None,
+ query_type=None, contention_factor=None):
"""Options for explicit encryption.
:Parameters:
- - `algorithm`: The algorithm to use.
+ - `algorithm` (str): The algorithm to use.
- `key_id`: The data key _id.
- `key_alt_name` (bytes): Identifies a key vault document by
'keyAltName'. Must be BSON encoded document in the form:
{ "keyAltName" : (BSON UTF8 value) }
+ - `index_key_id` (bytes): the index key id to use for Queryable Encryption.
+ - `query_type` (str): The query type to execute.
+ - `contention_factor` (int): The contention factor to use
+ when the algorithm is "Indexed".
+
+ .. versionchanged:: 1.3
+ Added the `index_key_id`, `query_type`, and `contention_factor` parameters.
"""
self.algorithm = algorithm
self.key_id = key_id
self.key_alt_name = key_alt_name
+ self.index_key_id = index_key_id
+ if query_type is not None:
+ if not isinstance(query_type, str):
+ raise TypeError(
+ 'query_type must be str or None, not: %r' % (type(query_type),))
+ self.query_type = query_type
+ if contention_factor is not None and not isinstance(contention_factor, int):
+ raise TypeError(
+ 'contention_factor must be an int or None, not: %r' % (type(contention_factor),))
+ self.contention_factor = contention_factor
class DataKeyOpts(object):
@@ -71,6 +89,16 @@ class DataKeyOpts(object):
- `endpoint` (string): Optional. Host with optional port.
Defaults to "cloudkms.googleapis.com".
+ If the `kms_provider` is "kmip" it is optional and has the
+ following fields::
+
+ - `keyId` (string): Optional. `keyId` is the KMIP Unique
+ Identifier to a 96 byte KMIP Secret Data managed object. If
+ keyId is omitted, the driver creates a random 96 byte KMIP
+ Secret Data managed object.
+ - `endpoint` (string): Optional. Host with optional
+ port, e.g. "example.vault.azure.net:".
+
- `key_alt_names`: An optional list of bytes suitable to be passed to
mongocrypt_ctx_setopt_key_alt_name. Each element must be BSON
encoded document in the form: { "keyAltName" : (BSON UTF8 value) }
@@ -101,19 +129,8 @@ class ExplicitEncrypter(object):
:Parameters:
- `kms_provider`: The KMS provider to use. Supported values are
- "aws" and "local".
- - `master_key`: Identifies a KMS-specific key used to encrypt the
- new data key. If the kmsProvider is "local" the `master_key` is
- not applicable and may be omitted. If the `kms_provider` is "aws"
- it is required and has the following fields::
-
- - `region` (string): Required. The AWS region, e.g. "us-east-1".
- - `key` (string): Required. The Amazon Resource Name (ARN) to
- the AWS customer.
- - `endpoint` (string): Optional. An alternate host to send KMS
- requests to. May include port number, e.g.
- "kms.us-east-1.amazonaws.com:443".
-
+ "aws", "azure", "gcp", "kmip", and "local".
+ - `master_key`: See DataKeyOpts.
- `key_alt_names` (optional): An optional list of string alternate
names used to reference a key. If a key is created with alternate
names, then encryption may refer to the key by the unique
@@ -135,7 +152,8 @@ class ExplicitEncrypter(object):
key = run_state_machine(ctx, self.callback)
return self.callback.insert_data_key(key)
- def encrypt(self, value, algorithm, key_id=None, key_alt_name=None):
+ def encrypt(self, value, algorithm, key_id=None, key_alt_name=None, index_key_id=None,
+ query_type=None, contention_factor=None):
"""Encrypts a BSON value.
Note that exactly one of ``key_id`` or ``key_alt_name`` must be
@@ -149,15 +167,23 @@ class ExplicitEncrypter(object):
key. For example, ``uuid.bytes`` or ``bytes(bson_binary)``.
- `key_alt_name` (string): Identifies a key vault document by
'keyAltName'.
+ - `index_key_id` (bytes): the index key id to use for Queryable Encryption.
+ - `query_type` (str): The query type to execute.
+ - `contention_factor` (int): The contention factor to use
+ when the algorithm is "Indexed".
:Returns:
The encrypted BSON value.
+
+ .. versionchanged:: 1.3
+ Added the `index_key_id`, `query_type`, and `contention_factor` parameters.
"""
# CDRIVER-3275 key_alt_name needs to be wrapped in a bson document.
if key_alt_name is not None:
key_alt_name = self.callback.bson_encode(
{'keyAltName': key_alt_name})
- opts = ExplicitEncryptOpts(algorithm, key_id, key_alt_name)
+ opts = ExplicitEncryptOpts(
+ algorithm, key_id, key_alt_name, index_key_id, query_type, contention_factor)
with self.mongocrypt.explicit_encryption_context(value, opts) as ctx:
return run_state_machine(ctx, self.callback)
diff -pruN 1.3.0-1/bindings/python/pymongocrypt/mongocrypt.py 1.5.2-1/bindings/python/pymongocrypt/mongocrypt.py
--- 1.3.0-1/bindings/python/pymongocrypt/mongocrypt.py 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/bindings/python/pymongocrypt/mongocrypt.py 2022-07-30 15:14:43.000000000 +0000
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import base64
import copy
from pymongocrypt.binary import (MongoCryptBinaryIn,
@@ -25,6 +24,8 @@ from pymongocrypt.state_machine import M
from pymongocrypt.crypto import (aes_256_cbc_encrypt,
aes_256_cbc_decrypt,
+ aes_256_ctr_decrypt,
+ aes_256_ctr_encrypt,
hmac_sha_256,
hmac_sha_512,
sha_256,
@@ -33,19 +34,21 @@ from pymongocrypt.crypto import (aes_256
class MongoCryptOptions(object):
- def __init__(self, kms_providers, schema_map=None):
+ def __init__(self, kms_providers, schema_map=None, encrypted_fields_map=None,
+ bypass_query_analysis=False, crypt_shared_lib_path=None,
+ crypt_shared_lib_required=False, bypass_encryption=False):
"""Options for :class:`MongoCrypt`.
:Parameters:
- - `kms_providers`: Map of KMS provider options. Two KMS providers
- are supported: "aws" and "local". The kms_providers map values
- differ by provider:
+ - `kms_providers`: Map of KMS provider options. The kms_providers
+ map values differ by provider:
- `aws`: Map with "accessKeyId" and "secretAccessKey" as strings,
and optionally a "sessionToken" for temporary credentials.
- `azure`: Map with "clientId" and "clientSecret" as strings.
- `gcp`: Map with "email" as a string and "privateKey" as
a byte array or a base64-encoded string. On Python 2,
base64-encoded strings must be passed as unicode literals.
+ - `kmip`: Map with "endpoint" as a string.
- `local`: Map with "key" as a 96-byte array or the equivalent
base64-encoded string. On Python 2, base64-encoded strings
must be passed as unicode literals.
@@ -63,6 +66,11 @@ class MongoCryptOptions(object):
automatic encryption for client side encryption. Other validation
rules in the JSON schema will not be enforced by the driver and
will result in an error.
+ - `encrypted_fields_map`: Optional map encoded to BSON `bytes`.
+ - `bypass_query_analysis`: If ``True``, disable automatic analysis of
+ outgoing commands. Set `bypass_query_analysis` to use explicit
+ encryption on indexed fields without the MongoDB Enterprise Advanced
+ licensed crypt_shared library.
.. versionadded:: 1.1
Support for "azure" and "gcp" kms_providers.
@@ -106,6 +114,18 @@ class MongoCryptOptions(object):
"be an instance of bytes or str "
"(unicode in Python 2)")
+ if 'kmip' in kms_providers:
+ kmip = kms_providers['kmip']
+ if not isinstance(kmip, dict):
+ raise ValueError("kms_providers['kmip'] must be a dict")
+ if 'endpoint' not in kmip:
+ raise ValueError("kms_providers['kmip'] must contain "
+ "'endpoint'")
+ if not isinstance(kms_providers['kmip']['endpoint'],
+ (str, unicode_type)):
+ raise TypeError("kms_providers['kmip']['endpoint'] must "
+ "be an instance of str")
+
if 'local' in kms_providers:
local = kms_providers['local']
if not isinstance(local, dict):
@@ -121,11 +141,20 @@ class MongoCryptOptions(object):
if schema_map is not None and not isinstance(schema_map, bytes):
raise TypeError("schema_map must be bytes or None")
+ if encrypted_fields_map is not None and not isinstance(encrypted_fields_map, bytes):
+ raise TypeError("encrypted_fields_map must be bytes or None")
+
self.kms_providers = kms_providers
self.schema_map = schema_map
+ self.encrypted_fields_map = encrypted_fields_map
+ self.bypass_query_analysis = bypass_query_analysis
+ self.crypt_shared_lib_path = crypt_shared_lib_path
+ self.crypt_shared_lib_required = crypt_shared_lib_required
+ self.bypass_encryption = bypass_encryption
class MongoCrypt(object):
+
def __init__(self, options, callback):
"""Abstracts libmongocrypt's mongocrypt_t type.
@@ -133,7 +162,7 @@ class MongoCrypt(object):
- `options`: A :class:`MongoCryptOptions`.
- `callback`: A :class:`MongoCryptCallback`.
"""
- self.__opts = options
+ self.__opts = options # type: MongoCryptOptions
self.__callback = callback
self.__crypt = None
@@ -181,6 +210,16 @@ class MongoCrypt(object):
self.__crypt, binary_schema_map.bin):
self.__raise_from_status()
+ encrypted_fields_map = self.__opts.encrypted_fields_map
+ if encrypted_fields_map is not None:
+ with MongoCryptBinaryIn(encrypted_fields_map) as binary_encrypted_fields_map:
+ if not lib.mongocrypt_setopt_encrypted_field_config_map(
+ self.__crypt, binary_encrypted_fields_map.bin):
+ self.__raise_from_status()
+
+ if self.__opts.bypass_query_analysis:
+ lib.mongocrypt_setopt_bypass_query_analysis(self.__crypt)
+
if not lib.mongocrypt_setopt_crypto_hooks(
self.__crypt, aes_256_cbc_encrypt, aes_256_cbc_decrypt,
secure_random, hmac_sha_512, hmac_sha_256, sha_256, ffi.NULL):
@@ -190,9 +229,27 @@ class MongoCrypt(object):
self.__crypt, sign_rsaes_pkcs1_v1_5, ffi.NULL):
self.__raise_from_status()
+ if not lib.mongocrypt_setopt_aes_256_ctr(
+ self.__crypt, aes_256_ctr_encrypt, aes_256_ctr_decrypt, ffi.NULL):
+ self.__raise_from_status()
+
+ if self.__opts.crypt_shared_lib_path is not None:
+ lib.mongocrypt_setopt_set_crypt_shared_lib_path_override(
+ self.__crypt, self.__opts.crypt_shared_lib_path.encode("utf-8"))
+
+ if not self.__opts.bypass_encryption:
+ lib.mongocrypt_setopt_append_crypt_shared_lib_search_path(self.__crypt, b"$SYSTEM")
+
if not lib.mongocrypt_init(self.__crypt):
self.__raise_from_status()
+ if self.__opts.crypt_shared_lib_required and self.crypt_shared_lib_version is None:
+ raise MongoCryptError(
+ "crypt_shared_lib_required=True but the crypt_shared library could not be loaded "
+ "from crypt_shared_lib_path={}".format(
+ self.__opts.crypt_shared_lib_path) +
+ " or the operating system's dynamic library search path")
+
def __raise_from_status(self):
status = lib.mongocrypt_status_new()
try:
@@ -202,6 +259,13 @@ class MongoCrypt(object):
lib.mongocrypt_status_destroy(status)
raise exc
+ @property
+ def crypt_shared_lib_version(self):
+ ver = lib.mongocrypt_crypt_shared_lib_version_string(self.__crypt, ffi.NULL)
+ if ver == ffi.NULL:
+ return None
+ return ver
+
def close(self):
"""Cleanup resources."""
if self.__crypt is None:
@@ -437,13 +501,25 @@ class ExplicitEncryptionContext(MongoCry
if opts.key_alt_name is not None:
with MongoCryptBinaryIn(opts.key_alt_name) as binary:
- if not lib.mongocrypt_ctx_setopt_key_alt_name(ctx,
- binary.bin):
+ if not lib.mongocrypt_ctx_setopt_key_alt_name(ctx, binary.bin):
self._raise_from_status()
+ if opts.index_key_id is not None:
+ with MongoCryptBinaryIn(opts.index_key_id) as binary:
+ if not lib.mongocrypt_ctx_setopt_index_key_id(ctx, binary.bin):
+ self._raise_from_status()
+
+ if opts.query_type is not None:
+ qt = str_to_bytes(opts.query_type)
+ if not lib.mongocrypt_ctx_setopt_query_type(ctx, qt, -1):
+ self._raise_from_status()
+
+ if opts.contention_factor is not None:
+ if not lib.mongocrypt_ctx_setopt_contention_factor(ctx, opts.contention_factor):
+ self._raise_from_status()
+
with MongoCryptBinaryIn(value) as binary:
- if not lib.mongocrypt_ctx_explicit_encrypt_init(ctx,
- binary.bin):
+ if not lib.mongocrypt_ctx_explicit_encrypt_init(ctx, binary.bin):
self._raise_from_status()
except Exception:
# Destroy the context on error.
@@ -490,11 +566,11 @@ class DataKeyContext(MongoCryptContext):
"""
super(DataKeyContext, self).__init__(ctx)
try:
- if kms_provider not in ['aws', 'gcp', 'azure', 'local']:
+ if kms_provider not in ['aws', 'gcp', 'azure', 'kmip', 'local']:
raise ValueError('unknown kms_provider: %s' % (kms_provider,))
if opts is None or opts.master_key is None:
- if kms_provider == 'local':
+ if kms_provider in ['kmip', 'local']:
master_key = {}
else:
raise ValueError(
@@ -504,22 +580,22 @@ class DataKeyContext(MongoCryptContext):
master_key = opts.master_key.copy()
if kms_provider == 'aws':
- if ('region' not in opts.master_key or
- 'key' not in opts.master_key):
+ if ('region' not in master_key or
+ 'key' not in master_key):
raise ValueError(
'master_key must include "region" and "key" for '
'kms_provider: "aws"')
elif kms_provider == 'azure':
- if ('keyName' not in opts.master_key or
- 'keyVaultEndpoint' not in opts.master_key):
+ if ('keyName' not in master_key or
+ 'keyVaultEndpoint' not in master_key):
raise ValueError(
'master key must include "keyName" and '
'"keyVaultEndpoint" for kms_provider: "azure"')
elif kms_provider == 'gcp':
- if ('projectId' not in opts.master_key or
- 'location' not in opts.master_key or
- 'keyRing' not in opts.master_key or
- 'keyName' not in opts.master_key):
+ if ('projectId' not in master_key or
+ 'location' not in master_key or
+ 'keyRing' not in master_key or
+ 'keyName' not in master_key):
raise ValueError(
'master key must include "projectId", "location",'
'"keyRing", and "keyName" for kms_provider: "gcp"')
@@ -591,6 +667,18 @@ class MongoCryptKmsContext(object):
"""Indicates how many bytes to send to :meth:`feed`."""
return lib.mongocrypt_kms_ctx_bytes_needed(self.__ctx)
+ @property
+ def kms_provider(self):
+ """The KMS provider identifier associated with this KMS request.
+
+ :Returns:
+ The KMS provider as a string, eg "aws", "azure", "gcp", or "kmip".
+
+ .. versionadded:: 1.2
+ """
+ return _to_string(
+ lib.mongocrypt_kms_ctx_get_kms_provider(self.__ctx, ffi.NULL))
+
def feed(self, data):
"""Feed bytes from the HTTP response.
diff -pruN 1.3.0-1/bindings/python/pymongocrypt/version.py 1.5.2-1/bindings/python/pymongocrypt/version.py
--- 1.3.0-1/bindings/python/pymongocrypt/version.py 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/bindings/python/pymongocrypt/version.py 2022-07-30 15:14:43.000000000 +0000
@@ -12,6 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-__version__ = '1.1.2.dev0'
+__version__ = '1.3.0b1.dev0'
-_MIN_LIBMONGOCRYPT_VERSION = '1.2.0'
+_MIN_LIBMONGOCRYPT_VERSION = '1.5.0rc1'
diff -pruN 1.3.0-1/bindings/python/release.sh 1.5.2-1/bindings/python/release.sh
--- 1.3.0-1/bindings/python/release.sh 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/bindings/python/release.sh 2022-07-30 15:14:43.000000000 +0000
@@ -14,7 +14,7 @@ set -o xtrace # Write all commands fir
set -o errexit # Exit the script with error if any of the commands fail
# The libmongocrypt git revision release to embed in our wheels.
-REVISION=$(git rev-list -n 1 1.2.1)
+REVISION=$(git rev-list -n 1 1.5.0-rc1)
if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
rm -rf venv37
diff -pruN 1.3.0-1/bindings/python/setup.py 1.5.2-1/bindings/python/setup.py
--- 1.3.0-1/bindings/python/setup.py 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/bindings/python/setup.py 2022-07-30 15:14:43.000000000 +0000
@@ -57,7 +57,7 @@ setup(
keywords=["mongo", "mongodb", "pymongocrypt", "pymongo", "mongocrypt",
"bson"],
test_suite="test",
- tests_require=["pymongo"],
+ tests_require=["pymongo>=3.11"],
license="Apache License, Version 2.0",
python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*",
classifiers=[
@@ -76,6 +76,7 @@ setup(
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
"Programming Language :: Python :: Implementation :: CPython",
"Programming Language :: Python :: Implementation :: PyPy",
"Topic :: Database"],
diff -pruN 1.3.0-1/bindings/python/strip_header.py 1.5.2-1/bindings/python/strip_header.py
--- 1.3.0-1/bindings/python/strip_header.py 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/bindings/python/strip_header.py 2022-07-30 15:14:43.000000000 +0000
@@ -14,41 +14,35 @@
"""Generate a CFFI.cdef() string from a C header file
-Usage (on macOS):: python strip_header.py ../../src/mongocrypt.h.in | pbcopy
+Usage (on macOS):: python strip_header.py ../../src/mongocrypt.h | pbcopy
"""
+import itertools
import re
import sys
-HEADER_RE = re.compile(r'^(#|MONGOCRYPT_EXPORT)')
-BLANK_RE = re.compile(r'^\s*$')
+DROP_RE = re.compile(r'^\s*(#|MONGOCRYPT_EXPORT)')
-def strip_file(fp, out):
- for line in fp:
- if not HEADER_RE.match(line):
- out.append(line)
+def strip_file(content):
+ fold = content.replace('\\\n', ' ')
+ all_lines = fold.split('\n') + ['']
+ keep_lines = (line for line in all_lines if not DROP_RE.match(line))
+ fin = ''
+ for line, peek in itertools.pairwise(keep_lines):
+ if peek == '' and line == '':
+ # Drop adjacent empty lines
+ continue
+ yield line
+ fin = peek
+ yield fin
+
def strip(hdr):
- out = []
with open(hdr) as fp:
- strip_file(fp, out)
-
- # Strip consecutive blank lines
- last_blank = True
- new = []
- for line in out:
- if BLANK_RE.match(line):
- if last_blank:
- continue
- last_blank = True
- else:
- last_blank = False
- new.append(line)
-
- if new:
- print(''.join(new))
+ out = strip_file(fp.read())
+ print('\n'.join(out))
if __name__ == "__main__":
diff -pruN 1.3.0-1/bindings/python/test/data/collection-info.json 1.5.2-1/bindings/python/test/data/collection-info.json
--- 1.3.0-1/bindings/python/test/data/collection-info.json 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/bindings/python/test/data/collection-info.json 2022-07-30 15:14:43.000000000 +0000
@@ -19,14 +19,16 @@
"properties": {
"ssn": {
"encrypt": {
- "keyId": {
- "$binary": {
- "base64": "YWFhYWFhYWFhYWFhYWFhYQ==",
- "subType": "04"
+ "keyId": [
+ {
+ "$binary": {
+ "base64": "YWFhYWFhYWFhYWFhYWFhYQ==",
+ "subType": "04"
+ }
}
- },
- "type": "string",
- "algorithm": "AEAD_AES_CBC_HMAC_SHA512-Deterministic"
+ ],
+ "bsonType": "string",
+ "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
}
},
diff -pruN 1.3.0-1/bindings/python/test/data/compact/success/cmd.json 1.5.2-1/bindings/python/test/data/compact/success/cmd.json
--- 1.3.0-1/bindings/python/test/data/compact/success/cmd.json 1970-01-01 00:00:00.000000000 +0000
+++ 1.5.2-1/bindings/python/test/data/compact/success/cmd.json 2022-07-30 15:14:43.000000000 +0000
@@ -0,0 +1 @@
+{ "compactStructuredEncryptionData": "test" }
diff -pruN 1.3.0-1/bindings/python/test/data/compact/success/encrypted-field-config-map.json 1.5.2-1/bindings/python/test/data/compact/success/encrypted-field-config-map.json
--- 1.3.0-1/bindings/python/test/data/compact/success/encrypted-field-config-map.json 1970-01-01 00:00:00.000000000 +0000
+++ 1.5.2-1/bindings/python/test/data/compact/success/encrypted-field-config-map.json 2022-07-30 15:14:43.000000000 +0000
@@ -0,0 +1,47 @@
+{
+ "db.test": {
+ "escCollection": "esc",
+ "eccCollection": "ecc",
+ "ecocCollection": "ecoc",
+ "fields": [
+ {
+ "keyId": {
+ "$binary": {
+ "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+ "subType": "04"
+ }
+ },
+ "path": "encrypted",
+ "bsonType": "string",
+ "queries": {
+ "queryType": "equality",
+ "contention": 0
+ }
+ },
+ {
+ "keyId": {
+ "$binary": {
+ "base64": "q83vqxI0mHYSNBI0VniQEg==",
+ "subType": "04"
+ }
+ },
+ "path": "nested.encrypted",
+ "bsonType": "string",
+ "queries": {
+ "queryType": "equality",
+ "contention": 0
+ }
+ },
+ {
+ "keyId": {
+ "$binary": {
+ "base64": "EjRWeBI0mHYSNBI0VniQEw==",
+ "subType": "04"
+ }
+ },
+ "path": "nested.notindexed",
+ "bsonType": "string"
+ }
+ ]
+ }
+}
diff -pruN 1.3.0-1/bindings/python/test/data/compact/success/encrypted-payload.json 1.5.2-1/bindings/python/test/data/compact/success/encrypted-payload.json
--- 1.3.0-1/bindings/python/test/data/compact/success/encrypted-payload.json 1970-01-01 00:00:00.000000000 +0000
+++ 1.5.2-1/bindings/python/test/data/compact/success/encrypted-payload.json 2022-07-30 15:14:43.000000000 +0000
@@ -0,0 +1,23 @@
+{
+ "compactStructuredEncryptionData": "test",
+ "compactionTokens": {
+ "nested.notindexed": {
+ "$binary": {
+ "base64": "27J6DZqcjkRzZ3lWEsxH7CsQHr4CZirrGmuPS8ZkRO0=",
+ "subType": "00"
+ }
+ },
+ "nested.encrypted": {
+ "$binary": {
+ "base64": "SWO8WEoZ2r2Kx/muQKb7+COizy85nIIUFiHh4K9kcvA=",
+ "subType": "00"
+ }
+ },
+ "encrypted": {
+ "$binary": {
+ "base64": "noN+05JsuO1oDg59yypIGj45i+eFH6HOTXOPpeZ//Mk=",
+ "subType": "00"
+ }
+ }
+ }
+}
\ No newline at end of file
diff -pruN 1.3.0-1/bindings/python/test/data/encrypted-command.json 1.5.2-1/bindings/python/test/data/encrypted-command.json
--- 1.3.0-1/bindings/python/test/data/encrypted-command.json 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/bindings/python/test/data/encrypted-command.json 2022-07-30 15:14:43.000000000 +0000
@@ -1,11 +1,13 @@
{
+ "find": "test",
"filter": {
"ssn": {
- "$binary": {
- "base64": "AWFhYWFhYWFhYWFhYWFhYWECRTOW9yZzNDn5dGwuqsrJQNLtgMEKaujhs9aRWRp+7Yo3JK8N8jC8P0Xjll6C1CwLsE/iP5wjOMhVv1KMMyOCSCrHorXRsb2IKPtzl2lKTqQ=",
- "subType": "06"
+ "$eq": {
+ "$binary": {
+ "base64": "AWFhYWFhYWFhYWFhYWFhYWECRTOW9yZzNDn5dGwuqsrJQNLtgMEKaujhs9aRWRp+7Yo3JK8N8jC8P0Xjll6C1CwLsE/iP5wjOMhVv1KMMyOCSCrHorXRsb2IKPtzl2lKTqQ=",
+ "subType": "06"
+ }
}
}
- },
- "find": "test"
+ }
}
diff -pruN 1.3.0-1/bindings/python/test/data/encrypted-field-config-map.json 1.5.2-1/bindings/python/test/data/encrypted-field-config-map.json
--- 1.3.0-1/bindings/python/test/data/encrypted-field-config-map.json 1970-01-01 00:00:00.000000000 +0000
+++ 1.5.2-1/bindings/python/test/data/encrypted-field-config-map.json 2022-07-30 15:14:43.000000000 +0000
@@ -0,0 +1,48 @@
+{
+ "test.test": {
+ "escCollection": "fle2.basic.esc",
+ "eccCollection": "fle2.basic.ecc",
+ "ecocCollection": "fle2.basic.ecoc",
+ "fields": [
+ {
+ "keyId": {
+ "$binary": {
+ "base64": "KEY1+AAAAAAAAAAAAAAAAA==",
+ "subType": "04"
+ }
+ },
+ "path": "firstName",
+ "bsonType": "string",
+ "queries": {
+ "queryType": "equality",
+ "contention": {
+ "$numberLong": "0"
+ }
+ }
+ }
+ ]
+ },
+ "test.test2": {
+ "escCollection": "fle2.basic.esc",
+ "eccCollection": "fle2.basic.ecc",
+ "ecocCollection": "fle2.basic.ecoc",
+ "fields": [
+ {
+ "keyId": {
+ "$binary": {
+ "base64": "KEY2+AAAAAAAAAAAAAAAAA==",
+ "subType": "04"
+ }
+ },
+ "path": "firstName",
+ "bsonType": "string",
+ "queries": {
+ "queryType": "equality",
+ "contention": {
+ "$numberLong": "0"
+ }
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
diff -pruN 1.3.0-1/bindings/python/test/data/keys/12345678123498761234123456789012-local-document.json 1.5.2-1/bindings/python/test/data/keys/12345678123498761234123456789012-local-document.json
--- 1.3.0-1/bindings/python/test/data/keys/12345678123498761234123456789012-local-document.json 1970-01-01 00:00:00.000000000 +0000
+++ 1.5.2-1/bindings/python/test/data/keys/12345678123498761234123456789012-local-document.json 2022-07-30 15:14:43.000000000 +0000
@@ -0,0 +1,30 @@
+{
+ "_id": {
+ "$binary": {
+ "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+ "subType": "04"
+ }
+ },
+ "keyMaterial": {
+ "$binary": {
+ "base64": "1ZbBTB1i/z4LcmBKi9+nnWqkVB4Wl6P4G7/TFQvXATRF2fX0lhBLIM6rT1U547FX2YgMtaP7sid+jpd4Vhz5kS+UlgtmCFfjeO4qOnJ78KEXRzeIebzKWKQz1pMhYZ3OURDL4wCtNqt3tbSr11kfTADmCMuzgp8U8P8T21RWWBU0f2XDcxiIShYncOS3poKu7GJaPCTav4r3h5h2xRklDA==",
+ "subType": "00"
+ }
+ },
+ "creationDate": {
+ "$date": {
+ "$numberLong": "1648914851981"
+ }
+ },
+ "updateDate": {
+ "$date": {
+ "$numberLong": "1648914851981"
+ }
+ },
+ "status": {
+ "$numberInt": "0"
+ },
+ "masterKey": {
+ "provider": "local"
+ }
+}
diff -pruN 1.3.0-1/bindings/python/test/data/keys/12345678123498761234123456789013-local-document.json 1.5.2-1/bindings/python/test/data/keys/12345678123498761234123456789013-local-document.json
--- 1.3.0-1/bindings/python/test/data/keys/12345678123498761234123456789013-local-document.json 1970-01-01 00:00:00.000000000 +0000
+++ 1.5.2-1/bindings/python/test/data/keys/12345678123498761234123456789013-local-document.json 2022-07-30 15:14:43.000000000 +0000
@@ -0,0 +1,30 @@
+{
+ "_id": {
+ "$binary": {
+ "base64": "EjRWeBI0mHYSNBI0VniQEw==",
+ "subType": "04"
+ }
+ },
+ "keyMaterial": {
+ "$binary": {
+ "base64": "YQXu48YyDbXvVQ1OhPsodQQNA1gLVWZSV0udYVmCTpVrgyAZePHQmsWWnQzNZj+ZsTxRm02soje/FJCqWGLeth3gKdvIsRg15CDEUOqLdDEpHl46hadosXyJIfo0umZ/LVTkvxRhmDCDxAkd0+Dg4/vWSiG0FgNzGrlvOUsTLGbqWtNMuOdZ8pKAdnFRrqce5cwBGQmd2VVBA2OQ0/IMxQ==",
+ "subType": "00"
+ }
+ },
+ "creationDate": {
+ "$date": {
+ "$numberLong": "1650631142512"
+ }
+ },
+ "updateDate": {
+ "$date": {
+ "$numberLong": "1650631142512"
+ }
+ },
+ "status": {
+ "$numberInt": "0"
+ },
+ "masterKey": {
+ "provider": "local"
+ }
+}
diff -pruN 1.3.0-1/bindings/python/test/data/keys/ABCDEFAB123498761234123456789012-local-document.json 1.5.2-1/bindings/python/test/data/keys/ABCDEFAB123498761234123456789012-local-document.json
--- 1.3.0-1/bindings/python/test/data/keys/ABCDEFAB123498761234123456789012-local-document.json 1970-01-01 00:00:00.000000000 +0000
+++ 1.5.2-1/bindings/python/test/data/keys/ABCDEFAB123498761234123456789012-local-document.json 2022-07-30 15:14:43.000000000 +0000
@@ -0,0 +1,30 @@
+{
+ "_id": {
+ "$binary": {
+ "base64": "q83vqxI0mHYSNBI0VniQEg==",
+ "subType": "04"
+ }
+ },
+ "keyMaterial": {
+ "$binary": {
+ "base64": "27OBvUqHAuYFy60nwCdvq2xmZ4kFzVySphXzBGq+HEot13comCoydEfnltBzLTuXLbV9cnREFJIO5f0jMqrlkxIuvAV8yO84p5VJTEa8j/xSNe7iA594rx7UeKT0fOt4VqM47fht8h+8PZYc5JVezvEMvwk115IBCwENxDjLtT0g+y8Hf+aTUEGtxrYToH8zf1/Y7S16mHiIc4jK3/vxHw==",
+ "subType": "00"
+ }
+ },
+ "creationDate": {
+ "$date": {
+ "$numberLong": "1648915408923"
+ }
+ },
+ "updateDate": {
+ "$date": {
+ "$numberLong": "1648915408923"
+ }
+ },
+ "status": {
+ "$numberInt": "0"
+ },
+ "masterKey": {
+ "provider": "local"
+ }
+}
diff -pruN 1.3.0-1/bindings/python/test/data/mongocryptd-command.json 1.5.2-1/bindings/python/test/data/mongocryptd-command.json
--- 1.3.0-1/bindings/python/test/data/mongocryptd-command.json 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/bindings/python/test/data/mongocryptd-command.json 2022-07-30 15:14:43.000000000 +0000
@@ -1,22 +1,26 @@
{
- "find": "test",
- "filter": {
- "ssn": "457-55-5462"
- },
- "jsonSchema": {
- "properties": {
- "ssn": {
- "encrypt": {
- "keyId": {
- "$binary": "YWFhYWFhYWFhYWFhYWFhYQ==",
- "$type": "04"
- },
- "type": "string",
- "algorithm": "AEAD_AES_CBC_HMAC_SHA512-Deterministic"
- }
- }
+ "find": "test",
+ "filter": {
+ "ssn": "457-55-5462"
},
- "bsonType": "object"
- },
- "isRemoteSchema": true
+ "jsonSchema": {
+ "properties": {
+ "ssn": {
+ "encrypt": {
+ "keyId": [
+ {
+ "$binary": {
+ "base64": "YWFhYWFhYWFhYWFhYWFhYQ==",
+ "subType": "04"
+ }
+ }
+ ],
+ "bsonType": "string",
+ "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+ }
+ }
+ },
+ "bsonType": "object"
+ },
+ "isRemoteSchema": true
}
\ No newline at end of file
diff -pruN 1.3.0-1/bindings/python/test/data/mongocryptd-reply.json 1.5.2-1/bindings/python/test/data/mongocryptd-reply.json
--- 1.3.0-1/bindings/python/test/data/mongocryptd-reply.json 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/bindings/python/test/data/mongocryptd-reply.json 2022-07-30 15:14:43.000000000 +0000
@@ -4,15 +4,17 @@
"$numberInt": "1"
},
"result": {
+ "find": "test",
"filter": {
"ssn": {
- "$binary": {
- "base64": "ADgAAAAQYQABAAAABWtpABAAAAAEYWFhYWFhYWFhYWFhYWFhYQJ2AAwAAAA0NTctNTUtNTQ2MgAA",
- "subType": "06"
+ "$eq": {
+ "$binary": {
+ "base64": "ADgAAAAQYQABAAAABWtpABAAAAAEYWFhYWFhYWFhYWFhYWFhYQJ2AAwAAAA0NTctNTUtNTQ2MgAA",
+ "subType": "06"
+ }
}
}
- },
- "find": "test"
+ }
},
"hasEncryptedPlaceholders": true
}
\ No newline at end of file
diff -pruN 1.3.0-1/bindings/python/test/data/schema-map.json 1.5.2-1/bindings/python/test/data/schema-map.json
--- 1.3.0-1/bindings/python/test/data/schema-map.json 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/bindings/python/test/data/schema-map.json 2022-07-30 15:14:43.000000000 +0000
@@ -3,14 +3,16 @@
"properties": {
"ssn": {
"encrypt": {
- "keyId": {
- "$binary": {
- "base64": "AAAAAAAAAAAAAAAAAAAAAA==",
- "subType": "04"
+ "keyId": [
+ {
+ "$binary": {
+ "base64": "AAAAAAAAAAAAAAAAAAAAAA==",
+ "subType": "04"
+ }
}
- },
- "type": "string",
- "algorithm": "AEAD_AES_CBC_HMAC_SHA512-Deterministic"
+ ],
+ "bsonType": "string",
+ "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
}
}
},
@@ -20,14 +22,16 @@
"properties": {
"ssn": {
"encrypt": {
- "keyId": {
- "$binary": {
- "base64": "AAAAAAAAAAAAAAAAAAAAAA==",
- "subType": "04"
+ "keyId": [
+ {
+ "$binary": {
+ "base64": "AAAAAAAAAAAAAAAAAAAAAA==",
+ "subType": "04"
+ }
}
- },
- "type": "string",
- "algorithm": "AEAD_AES_CBC_HMAC_SHA512-Random"
+ ],
+ "bsonType": "string",
+ "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random"
}
}
},
diff -pruN 1.3.0-1/bindings/python/test/test_mongocrypt.py 1.5.2-1/bindings/python/test/test_mongocrypt.py
--- 1.3.0-1/bindings/python/test/test_mongocrypt.py 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/bindings/python/test/test_mongocrypt.py 2022-07-30 15:14:43.000000000 +0000
@@ -20,8 +20,9 @@ import os
import sys
import uuid
-from bson import json_util, BSON
-from bson.binary import Binary, STANDARD
+import bson
+from bson import json_util
+from bson.binary import Binary, UuidRepresentation
from bson.codec_options import CodecOptions
from bson.json_util import JSONOptions
from bson.son import SON
@@ -103,6 +104,14 @@ class TestMongoCryptOptions(unittest.Tes
opts = MongoCryptOptions(kms_providers, schema_map)
self.assertEqual(opts.kms_providers, kms_providers, msg=kms_providers)
self.assertEqual(opts.schema_map, schema_map)
+ self.assertIsNone(opts.encrypted_fields_map)
+ self.assertFalse(opts.bypass_query_analysis)
+
+ encrypted_fields_map = bson_data('encrypted-field-config-map.json')
+ opts = MongoCryptOptions(valid[0][0], schema_map, encrypted_fields_map=encrypted_fields_map,
+ bypass_query_analysis=True)
+ self.assertEqual(opts.encrypted_fields_map, encrypted_fields_map)
+ self.assertTrue(opts.bypass_query_analysis)
def test_mongocrypt_options_validation(self):
with self.assertRaisesRegex(
@@ -131,8 +140,13 @@ class TestMongoCryptOptions(unittest.Tes
TypeError, "schema_map must be bytes or None"):
MongoCryptOptions(valid_kms, schema_map={})
+ with self.assertRaisesRegex(
+ TypeError, "encrypted_fields_map must be bytes or None"):
+ MongoCryptOptions(valid_kms, encrypted_fields_map={})
+
class TestMongoCrypt(unittest.TestCase):
+ maxDiff = None
def test_mongocrypt(self):
kms_providers = {
@@ -214,10 +228,10 @@ class TestMongoCrypt(unittest.TestCase):
MongoCrypt(options, callback)
@staticmethod
- def create_mongocrypt():
+ def create_mongocrypt(**kwargs):
return MongoCrypt(MongoCryptOptions({
'aws': {'accessKeyId': 'example', 'secretAccessKey': 'example'},
- 'local': {'key': b'\x00'*96}}), MockCallback())
+ 'local': {'key': b'\x00'*96}}, **kwargs), MockCallback())
def _test_kms_context(self, ctx):
key_filter = ctx.mongo_operation()
@@ -229,18 +243,22 @@ class TestMongoCrypt(unittest.TestCase):
km_contexts = list(ctx.kms_contexts())
self.assertEqual(len(km_contexts), 1)
with km_contexts[0] as kms_ctx:
+ self.assertEqual(kms_ctx.kms_provider, "aws")
self.assertEqual(kms_ctx.endpoint, "kms.us-east-1.amazonaws.com:443")
self.assertEqual(len(kms_ctx.message), 781)
self.assertEqual(kms_ctx.bytes_needed, 1024)
kms_ctx.feed(http_data('kms-reply.txt'))
self.assertEqual(kms_ctx.bytes_needed, 0)
+ self.assertEqual(kms_ctx.kms_provider, "aws")
ctx.complete_kms()
def test_encrypt(self):
mc = self.create_mongocrypt()
self.addCleanup(mc.close)
+ if mc.crypt_shared_lib_version is not None:
+ self.skipTest("this test must be skipped when crypt_shared is loaded")
with mc.encryption_context('text', bson_data('command.json')) as ctx:
self.assertEqual(ctx.state, lib.MONGOCRYPT_CTX_NEED_MONGO_COLLINFO)
@@ -253,6 +271,8 @@ class TestMongoCrypt(unittest.TestCase):
self.assertEqual(ctx.state, lib.MONGOCRYPT_CTX_NEED_MONGO_MARKINGS)
mongocryptd_cmd = ctx.mongo_operation()
+ self.assertEqual(bson.decode(mongocryptd_cmd, OPTS),
+ json_data('mongocryptd-command.json'))
self.assertEqual(mongocryptd_cmd,
bson_data('mongocryptd-command.json'))
@@ -265,8 +285,8 @@ class TestMongoCrypt(unittest.TestCase):
self.assertEqual(ctx.state, lib.MONGOCRYPT_CTX_READY)
encrypted = ctx.finish()
- self.assertEqual(
- BSON(encrypted).decode(), json_data('encrypted-command.json'))
+ self.assertEqual(bson.decode(encrypted, OPTS),
+ json_data('encrypted-command.json'))
self.assertEqual(encrypted, bson_data('encrypted-command.json'))
self.assertEqual(ctx.state, lib.MONGOCRYPT_CTX_DONE)
@@ -282,11 +302,39 @@ class TestMongoCrypt(unittest.TestCase):
self.assertEqual(ctx.state, lib.MONGOCRYPT_CTX_READY)
encrypted = ctx.finish()
- self.assertEqual(
- BSON(encrypted).decode(), json_data('command-reply.json'))
+ self.assertEqual(bson.decode(encrypted, OPTS),
+ json_data('command-reply.json'))
self.assertEqual(encrypted, bson_data('command-reply.json'))
self.assertEqual(ctx.state, lib.MONGOCRYPT_CTX_DONE)
+ def test_encrypt_encrypted_fields_map(self):
+ encrypted_fields_map = bson_data('compact/success/encrypted-field-config-map.json')
+ mc = self.create_mongocrypt(encrypted_fields_map=encrypted_fields_map)
+ self.addCleanup(mc.close)
+ with mc.encryption_context('db', bson_data('compact/success/cmd.json')) as ctx:
+ self.assertEqual(ctx.state, lib.MONGOCRYPT_CTX_NEED_MONGO_KEYS)
+
+ ctx.mongo_operation()
+ ctx.add_mongo_operation_result(bson_data(
+ 'keys/12345678123498761234123456789012-local-document.json'))
+ self.assertEqual(ctx.state, lib.MONGOCRYPT_CTX_NEED_MONGO_KEYS)
+ ctx.mongo_operation()
+ ctx.add_mongo_operation_result(bson_data(
+ 'keys/ABCDEFAB123498761234123456789012-local-document.json'))
+ self.assertEqual(ctx.state, lib.MONGOCRYPT_CTX_NEED_MONGO_KEYS)
+ ctx.mongo_operation()
+ ctx.add_mongo_operation_result(bson_data(
+ 'keys/12345678123498761234123456789013-local-document.json'))
+ ctx.complete_mongo_operation()
+
+ self.assertEqual(ctx.state, lib.MONGOCRYPT_CTX_READY)
+
+ encrypted = ctx.finish()
+ self.assertEqual(bson.decode(encrypted, OPTS),
+ json_data('compact/success/encrypted-payload.json'))
+ self.assertEqual(encrypted, bson_data('compact/success/encrypted-payload.json'))
+ self.assertEqual(ctx.state, lib.MONGOCRYPT_CTX_DONE)
+
class MockCallback(MongoCryptCallback):
def __init__(self,
@@ -317,13 +365,14 @@ class MockCallback(MongoCryptCallback):
raise NotImplementedError
def bson_encode(self, doc):
- return BSON.encode(doc)
+ return bson.encode(doc)
def close(self):
pass
class TestMongoCryptCallback(unittest.TestCase):
+ maxDiff = None
@staticmethod
def mongo_crypt_opts():
@@ -331,6 +380,31 @@ class TestMongoCryptCallback(unittest.Te
'aws': {'accessKeyId': 'example', 'secretAccessKey': 'example'},
'local': {'key': b'\x00'*96}})
+ @unittest.skipUnless(os.getenv("TEST_CRYPT_SHARED"), "this test requires TEST_CRYPT_SHARED=1")
+ def test_crypt_shared(self):
+ kms_providers = {
+ 'aws': {'accessKeyId': 'example', 'secretAccessKey': 'example'},
+ 'local': {'key': b'\x00'*96}}
+ mc = MongoCrypt(MongoCryptOptions(kms_providers), MockCallback())
+ self.addCleanup(mc.close)
+ self.assertIsNotNone(mc.crypt_shared_lib_version)
+ # Test that we can pick up crypt_shared automatically
+ encrypter = AutoEncrypter(MockCallback(), MongoCryptOptions(
+ kms_providers,
+ bypass_encryption=False,
+ crypt_shared_lib_required=True))
+ self.addCleanup(encrypter.close)
+ encrypter = AutoEncrypter(MockCallback(), MongoCryptOptions(
+ kms_providers,
+ crypt_shared_lib_path=os.environ["CRYPT_SHARED_PATH"],
+ crypt_shared_lib_required=True))
+ self.addCleanup(encrypter.close)
+ with self.assertRaisesRegex(MongoCryptError, "/doesnotexist"):
+ AutoEncrypter(MockCallback(), MongoCryptOptions(
+ kms_providers,
+ crypt_shared_lib_path="/doesnotexist",
+ crypt_shared_lib_required=True))
+
def test_encrypt(self):
encrypter = AutoEncrypter(MockCallback(
list_colls_result=bson_data('collection-info.json'),
@@ -338,9 +412,9 @@ class TestMongoCryptCallback(unittest.Te
key_docs=[bson_data('key-document.json')],
kms_reply=http_data('kms-reply.txt')), self.mongo_crypt_opts())
self.addCleanup(encrypter.close)
- encrypted = encrypter.encrypt('text', bson_data('command.json'))
- self.assertEqual(
- BSON(encrypted).decode(), json_data('encrypted-command.json'))
+ encrypted = encrypter.encrypt('test', bson_data('command.json'))
+ self.assertEqual(bson.decode(encrypted, OPTS),
+ json_data('encrypted-command.json'))
self.assertEqual(encrypted, bson_data('encrypted-command.json'))
def test_decrypt(self):
@@ -352,8 +426,8 @@ class TestMongoCryptCallback(unittest.Te
self.addCleanup(encrypter.close)
decrypted = encrypter.decrypt(
bson_data('encrypted-command-reply.json'))
- self.assertEqual(
- BSON(decrypted).decode(), json_data('command-reply.json'))
+ self.assertEqual(bson.decode(decrypted, OPTS),
+ json_data('command-reply.json'))
self.assertEqual(decrypted, bson_data('command-reply.json'))
@@ -367,10 +441,11 @@ class KeyVaultCallback(MockCallback):
def insert_data_key(self, data_key):
self.data_key = data_key
- return BSON(data_key).decode()['_id']
+ return bson.decode(data_key, OPTS)['_id']
class TestExplicitEncryption(unittest.TestCase):
+ maxDiff = None
@staticmethod
def mongo_crypt_opts():
@@ -385,26 +460,90 @@ class TestExplicitEncryption(unittest.Te
self.addCleanup(encrypter.close)
val = {'v': 'hello'}
- encoded_val = BSON.encode(val)
+ encoded_val = bson.encode(val)
algo = "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
encrypted = encrypter.encrypt(
encoded_val, algo, key_id=key_id, key_alt_name=key_alt_name)
- self.assertEqual(
- BSON(encrypted).decode(), json_data('encrypted-value.json'))
+ self.assertEqual(bson.decode(encrypted, OPTS),
+ json_data('encrypted-value.json'))
self.assertEqual(encrypted, bson_data('encrypted-value.json'))
decrypted = encrypter.decrypt(encrypted)
- self.assertEqual(BSON(decrypted).decode(), val)
+ self.assertEqual(bson.decode(decrypted, OPTS), val)
self.assertEqual(encoded_val, decrypted)
def test_encrypt_decrypt(self):
key_id = json_data('key-document.json')['_id']
- self._test_encrypt_decrypt(key_id=key_id.bytes)
+ self._test_encrypt_decrypt(key_id=key_id)
def test_encrypt_decrypt_key_alt_name(self):
key_alt_name = json_data('key-document.json')['keyAltNames'][0]
self._test_encrypt_decrypt(key_alt_name=key_alt_name)
+ def test_encrypt_errors(self):
+ key_id = json_data('key-document.json')['_id']
+ encrypter = ExplicitEncrypter(MockCallback(key_docs=[]), self.mongo_crypt_opts())
+ self.addCleanup(encrypter.close)
+
+ val = {'v': 'value123'}
+ encoded_val = bson.encode(val)
+ # Invalid algorithm.
+ with self.assertRaisesRegex(MongoCryptError, "algorithm"):
+ encrypter.encrypt(encoded_val, "Invalid", key_id)
+ # Invalid index_key_id type.
+ with self.assertRaises(TypeError):
+ encrypter.encrypt(encoded_val, "Indexed", key_id, index_key_id=1)
+ with self.assertRaises(MongoCryptError):
+ encrypter.encrypt(encoded_val, "Indexed", key_id, index_key_id=b'1234')
+ # Invalid query_type type.
+ with self.assertRaisesRegex(TypeError, "query_type"):
+ encrypter.encrypt(encoded_val, "Indexed", key_id, query_type=42)
+ # Invalid query_type string.
+ with self.assertRaisesRegex(MongoCryptError, "query_type"):
+ encrypter.encrypt(encoded_val, "Indexed", key_id, query_type='invalid query type string')
+ # Invalid contention_factor type.
+ with self.assertRaisesRegex(TypeError, "contention_factor"):
+ encrypter.encrypt(encoded_val, "Indexed", key_id, contention_factor='not an int')
+ with self.assertRaisesRegex(MongoCryptError, "contention"):
+ encrypter.encrypt(encoded_val, "Indexed", key_id, contention_factor=-1)
+ # Invalid: Unindexed + query_type is an error.
+ with self.assertRaisesRegex(MongoCryptError, "query"):
+ encrypter.encrypt(encoded_val, "Unindexed", key_id, query_type='equality')
+ # Invalid: Unindexed + contention_factor is an error.
+ with self.assertRaisesRegex(MongoCryptError, "contention"):
+ encrypter.encrypt(encoded_val, "Unindexed", key_id, contention_factor=1)
+
+ def test_encrypt_indexed(self):
+ key_path = 'keys/ABCDEFAB123498761234123456789012-local-document.json'
+ index_key_path = 'keys/12345678123498761234123456789012-local-document.json'
+ key_id = json_data(key_path)['_id']
+ index_key_id = json_data(index_key_path)['_id']
+ encrypter = ExplicitEncrypter(MockCallback(
+ key_docs=[bson_data(key_path), bson_data(index_key_path)],
+ kms_reply=http_data('kms-reply.txt')), self.mongo_crypt_opts())
+ self.addCleanup(encrypter.close)
+
+ val = {'v': 'value123'}
+ encoded_val = bson.encode(val)
+ for kwargs in [
+ dict(algorithm='Indexed', contention_factor=0),
+ dict(algorithm='Indexed', query_type='equality', contention_factor=0),
+ dict(algorithm='Indexed', contention_factor=100),
+ dict(algorithm='Unindexed'),
+ ]:
+ kwargs['key_id'] = key_id
+ kwargs['index_key_id'] = index_key_id
+ encrypted = encrypter.encrypt(encoded_val, **kwargs)
+ encrypted_val = bson.decode(encrypted, OPTS)['v']
+ self.assertIsInstance(encrypted_val, Binary)
+ self.assertEqual(encrypted_val.subtype, 6)
+
+ # Queryable Encryption find payloads cannot be round-tripped.
+ if 'query_type' not in kwargs:
+ decrypted = encrypter.decrypt(encrypted)
+ self.assertEqual(bson.decode(decrypted, OPTS), val)
+ self.assertEqual(encoded_val, decrypted)
+
def test_data_key_creation(self):
mock_key_vault = KeyVaultCallback(
kms_reply=http_data('kms-encrypt-reply.txt'))
@@ -424,8 +563,9 @@ class TestExplicitEncryption(unittest.Te
key_id = encrypter.create_data_key(
kms_provider, master_key=master_key,
key_alt_names=key_alt_names)
- self.assertIsInstance(key_id, uuid.UUID)
- data_key = BSON(mock_key_vault.data_key).decode()
+ self.assertIsInstance(key_id, Binary)
+ self.assertEqual(key_id.subtype, 4)
+ data_key = bson.decode(mock_key_vault.data_key, OPTS)
# CDRIVER-3277 The order of key_alt_names is not maintained.
for name in key_alt_names:
self.assertIn(name, data_key['keyAltNames'])
@@ -445,10 +585,15 @@ def read(filename, **kwargs):
return fp.read()
-OPTS = CodecOptions(uuid_representation=STANDARD)
+OPTS = CodecOptions(uuid_representation=UuidRepresentation.UNSPECIFIED)
# Use SON to preserve the order of fields while parsing json.
-JSON_OPTS = JSONOptions(document_class=SON, uuid_representation=STANDARD)
+if sys.version_info[:2] < (3, 6):
+ document_class = SON
+else:
+ document_class = dict
+JSON_OPTS = JSONOptions(document_class=document_class,
+ uuid_representation=UuidRepresentation.UNSPECIFIED)
def json_data(filename):
@@ -456,7 +601,7 @@ def json_data(filename):
def bson_data(filename):
- return BSON.encode(json_data(filename), codec_options=OPTS)
+ return bson.encode(json_data(filename), codec_options=OPTS)
def http_data(filename):
diff -pruN 1.3.0-1/CHANGELOG.md 1.5.2-1/CHANGELOG.md
--- 1.3.0-1/CHANGELOG.md 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/CHANGELOG.md 2022-07-30 15:14:43.000000000 +0000
@@ -1,4 +1,70 @@
# ChangeLog
+## 1.5.2
+### Fixed
+- Fix datakey decryption requiring multiple rounds of KMS requests.
+## 1.5.1
+### New Features
+- Update Java bindings to support remaining 1.5.0 API.
+
+## 1.5.0
+## Fixed
+- Update to use new payload for FLE 2.0 find.
+- Require contention factor.
+## 1.5.0-rc2
+### Fixed
+- Fix handling of create command with $jsonSchema.
+- Fix leak on encrypt or decrypt error.
+## Improved
+- Accept string values for QueryType and IndexType.
+
+## 1.4.1
+### Fixed
+- Add missing MONGOCRYPT_EXPORT to mongocrypt_ctx_provide_kms_providers
+## 1.5.0-rc1
+## Fixed
+- Revert new payload for FLE 2.0 find.
+- Do not send "create" and "createIndexes" to mongocryptd when bypassing query analysis.
+
+## 1.5.0-rc0
+## Fixed
+- Account for shared library rename.
+- Update to use new payload for FLE 2.0 find.
+
+## 1.5.0-alpha2
+## New Features
+- Fix explain when using csfle shared library.
+- Do not bypass "create" or "createIndexes" commands. Support "collMod".
+- Bypass "hello", "buildInfo", "getCmdLineOpts", and "getLog" commands.
+## Fixed
+- Preserve $db in output command.
+- Add missing MONGOCRYPT_EXPORT to mongocrypt_ctx_provide_kms_providers
+## 1.5.0-alpha1
+### Fixed
+- Pick a random contention factor on FLE 2.0 insert.
+
+## 1.5.0-alpha0
+### New Features
+- Support FLE 2.0.
+- Support FLE 1.0 Shared Library.
+- Support Key Management API.
+
+## 1.4.0
+### New Features
+- Support on-demand credentials with `MONGOCRYPT_CTX_NEED_KMS_CREDENTIALS` state and `mongocrypt_ctx_provide_kms_providers`.
+
+## 1.4.0-alpha0
+### New Features
+- Support on-demand AWS credentials with `MONGOCRYPT_CTX_NEED_KMS_CREDENTIALS` state and `mongocrypt_ctx_provide_kms_providers`.
+### Fixed
+- Resolve 32 bit Windows compile errors.
+
+## 1.3.1
+
+### New Features
+- Support custom key material through `mongocrypt_ctx_setopt_key_material`.
+### Fixed
+- Fix deprecation warnings with OpenSSL 3.0.
+- Resolve possible symbol conflicts with OpenSSL.
## 1.3.0
- Support "kmip" KMS provider.
diff -pruN 1.3.0-1/cmake/FetchMongoC.cmake 1.5.2-1/cmake/FetchMongoC.cmake
--- 1.3.0-1/cmake/FetchMongoC.cmake 1970-01-01 00:00:00.000000000 +0000
+++ 1.5.2-1/cmake/FetchMongoC.cmake 2022-07-30 15:14:43.000000000 +0000
@@ -0,0 +1,20 @@
+include (FetchContent)
+
+# Set the tag that we will fetch.
+set (MONGOC_FETCH_TAG_FOR_LIBBSON "1.17.7" CACHE STRING "The Git tag of mongo-c-driver that will be fetched to obtain libbson")
+
+# Fetch the source archive for the requested tag from GitHub
+FetchContent_Declare (
+ embedded_mcd
+ URL "https://github.com/mongodb/mongo-c-driver/archive/refs/tags/${MONGOC_FETCH_TAG_FOR_LIBBSON}.tar.gz"
+ )
+# Populate it:
+FetchContent_GetProperties (embedded_mcd)
+if (NOT embedded_mcd_POPULATED)
+ message (STATUS "Downloading mongo-c-driver ${MONGOC_FETCH_TAG_FOR_LIBBSON} for libbson")
+ FetchContent_Populate (embedded_mcd)
+endif ()
+# Store the directory path to the external mongoc project:
+get_filename_component (MONGOCRYPT_MONGOC_DIR "${embedded_mcd_SOURCE_DIR}" ABSOLUTE)
+# The project wants a VERSION_CURRENT file. We know that based on the tag.
+file (WRITE "${embedded_mcd_SOURCE_DIR}/VERSION_CURRENT" "${MONGOC_FETCH_TAG_FOR_LIBBSON}")
diff -pruN 1.3.0-1/cmake/GetVersion.cmake 1.5.2-1/cmake/GetVersion.cmake
--- 1.3.0-1/cmake/GetVersion.cmake 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/cmake/GetVersion.cmake 2022-07-30 15:14:43.000000000 +0000
@@ -2,10 +2,18 @@
# from git history.
#
# This may be run as an independent script, like so:
-# cmake -P ./cmake/GetVersion.cmake
+# cmake -DRELEASE_BRANCH_REF=origin/r1.5 -P ./cmake/GetVersion.cmake
# And the computed version is printed to stderr (since printing to stdout with
# cmake's "message" function would prefix with "-- ")
#
+# Note that specifying -DRELEASE_BRANCH_REF=origin/r1.5 tells the script to base
+# its version calculation on that particular branch. It should point to the
+# latest available release branch. This is necessary to ensure that calling the
+# script while checked out to a branch other than the latest release branch
+# returns a sensible version number. Put more concretely, given the latest
+# release branch as origin/r1.5, running this script while checked out on the
+# master branch will return a version number like 1.6.0-dev+...
+#
# The general approach of this function is to produce a sequence of versions
# which distinguish between development versions, release candidates, prerelease
# builds which fall between release candidates, and actual releases. This
@@ -25,7 +33,8 @@
#
function (GetVersion OUTVAR)
execute_process (
- COMMAND git describe --tags --match "1.*"
+ COMMAND git describe --tags --match "1.*" ${RELEASE_BRANCH_REF}
+ WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
OUTPUT_VARIABLE VERSION_WITH_SUFFIX
RESULT_VARIABLE GIT_STATUS
ERROR_VARIABLE GIT_ERROR
@@ -37,7 +46,8 @@ function (GetVersion OUTVAR)
endif ()
execute_process (
- COMMAND git describe --tags --abbrev=0 --match "1.*"
+ COMMAND git describe --tags --abbrev=0 --match "1.*" ${RELEASE_BRANCH_REF}
+ WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
OUTPUT_VARIABLE VERSION
RESULT_VARIABLE GIT_STATUS
ERROR_VARIABLE GIT_ERROR
@@ -56,6 +66,19 @@ function (GetVersion OUTVAR)
endif ()
# Otherwise, construct a version based on the next release version
+ execute_process (
+ COMMAND git rev-parse --abbrev-ref HEAD
+ WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+ OUTPUT_VARIABLE CURRENT_BRANCH
+ RESULT_VARIABLE GIT_STATUS
+ ERROR_VARIABLE GIT_ERROR
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ )
+
+ if (NOT GIT_STATUS STREQUAL 0)
+ message (FATAL_ERROR "Unable to determine branch: 'git rev-parse' failed: '${GIT_ERROR}'")
+ endif ()
+
# A list of version components separated by dots and dashes: "1.0.0-[prerelease-marker]"
string (REGEX MATCHALL "[^.-]+" VERSION_PARTS ${VERSION})
@@ -65,10 +88,15 @@ function (GetVersion OUTVAR)
list (GET VERSION_PARTS 1 MINOR_VERSION)
list (GET VERSION_PARTS 2 PATCH_VERSION)
set (PRERELEASE_VERSION "")
- if (VERSION_LENGTH GREATER 3)
+ if (CURRENT_BRANCH STREQUAL "master")
+ # We need a version in the next minor sequence
+ math (EXPR MINOR_VERSION "${MINOR_VERSION} + 1")
+ set (PATCH_VERSION "0")
+ set (PRERELEASE_VERSION "-dev")
+ elseif (VERSION_LENGTH GREATER 3)
# The version we are starting with is already a pre-release of the next
list (GET VERSION_PARTS 3 PRERELEASE_VERSION)
- string (REGEX MATCHALL "(beta|rc|[0-9]+)" PRERELEASE_PARTS ${PRERELEASE_VERSION})
+ string (REGEX MATCHALL "(alpha|beta|rc|[0-9]+)" PRERELEASE_PARTS ${PRERELEASE_VERSION})
list (LENGTH PRERELEASE_PARTS PRERELEASE_LENGTH)
if (PRERELEASE_LENGTH EQUAL 2)
list (GET PRERELEASE_PARTS 0 PRE_PT_ONE)
@@ -88,6 +116,7 @@ function (GetVersion OUTVAR)
# Append our custom suffix +git
execute_process (
COMMAND git rev-parse --revs-only --short=10 HEAD
+ WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
OUTPUT_VARIABLE SUFFIX_SHA
RESULT_VARIABLE GIT_STATUS
ERROR_VARIABLE GIT_ERROR
diff -pruN 1.3.0-1/cmake/ImportBSON.cmake 1.5.2-1/cmake/ImportBSON.cmake
--- 1.3.0-1/cmake/ImportBSON.cmake 1970-01-01 00:00:00.000000000 +0000
+++ 1.5.2-1/cmake/ImportBSON.cmake 2022-07-30 15:14:43.000000000 +0000
@@ -0,0 +1,202 @@
+#[[
+ This file defines, exports, and installs two INTERFACE targets: '_mongocrypt::libbson_for_static'
+ and '_mongocrypt::libbson_for_shared', that are used to link libbson correctly for the build
+ configuration of libmongocrypt. At find_package() time, we can resolve these interface targets
+ to link to the appropriate libbson based on the build configurations of libmongocrypt.
+
+ mongo::mongocrypt must link to _mongocrypt::libbson_for_shared, and mongo::mongocrypt_static must
+ link to _mongocrypt::libbson_for_static.
+
+ At configure+build time, these target will create BUILD_INTERFACE-only usage requirements
+ appropriate for libmongocrypt to build against a libbson. Once these targets are installed,
+ they retain no usage requirements defined here.
+
+ Instead, the installed version of these targets will be manipulated in mongocrypt-config.cmake
+ based on user settings and build configuration options of the installed libmongocrypt in order
+ to ensure that users have satisfied the linking requirements of libmongocrypt.
+ Refer to mongocrypt-config.cmake for more information
+
+ This file calls add_subdirectory(EXCLUDE_FROM_ALL) on a mongo-c-driver project directory. This
+ will expose libbson targets that we can link and use for the libmongocrypt build.
+
+ The boolean option USE_SHARED_LIBBSON controls the behavior of libbson_for_shared:
+
+ If USE_SHARED_LIBBSON=FALSE:
+
+ - libbson_for_shared will transitively link the static libbson from the MONGOCRYPT_MONGOC_DIR.
+ - The result is that mongo::mongocrypt (which is a SHARED library) will have the translation
+ units of libbson directly embedded into the resulting binary.
+ - The symbols from libbson that are merged into mongo::mongocrypt will be suppressed using
+ linker scripts such that consumers of mongo::mongocrypt will not see the libbson symbols that
+ were statically linked into the shared library. This allows consumers to link against a
+ completely independent libbson without interfering with the libbson symbols that were merged
+ into mongo::mongocrypt
+ - The installed libbson_for_shared will have no usage requirements.
+
+ If USE_SHARED_LIBBSON=TRUE:
+
+ - libbson_for_shared will transitively use the shared libbson library from
+ the MONGOCRYPT_MONGOC_DIR.
+ - mongo::mongocrypt will be built with a dynamic link requirement on a libbson dynamic
+ library, which must be resolved at runtime by consumers. The translation units from the
+ MONGOCRYPT_MONGOC_DIR *will not* be included in the mongo::mongocrypt library.
+ - The installed libbson_for_shared will dynamically link to a libbson on the user's system by
+ using a find_library() call.
+
+ In both of the above cases, libbson_for_static will require that the final consumer
+ provide their own definitions of the libbson symbols, regardless of the value
+ of USE_SHARED_LIBBSON.
+]]
+
+include (CheckCSourceCompiles)
+include (CMakePushCheckState)
+
+cmake_push_check_state ()
+ # Even though we aren't going to use the system's libbson, try to detect whether it has
+ # extra-alignment enabled. We want to match that setting as our default, for convenience
+ # purposes only.
+ find_path (SYSTEM_BSON_INCLUDE_DIR bson/bson.h PATH_SUFFIXES libbson-1.0)
+ if (SYSTEM_BSON_INCLUDE_DIR AND NOT DEFINED ENABLE_EXTRA_ALIGNMENT)
+ set (CMAKE_REQUIRED_INCLUDES "${SYSTEM_BSON_INCLUDE_DIR}")
+ set (_extra_alignment_default OFF)
+ check_c_source_compiles ([[
+ #include
+
+ int main() { }
+ ]] HAVE_SYSTEM_LIBBSON)
+
+ if (HAVE_SYSTEM_LIBBSON)
+ # We have a libbson, check for extra alignment
+ check_c_source_compiles ([[
+ #include
+
+ #ifndef BSON_EXTRA_ALIGN
+ #error "Not extra-aligned"
+ #endif
+
+ int main() {}
+ ]] SYSTEM_LIBBSON_IS_EXTRA_ALIGNED)
+ if (SYSTEM_LIBBSON_IS_EXTRA_ALIGNED)
+ # Extra aligned! We'll use extra alignment by default.
+ set (_extra_alignment_default ON)
+ endif ()
+ endif ()
+ endif ()
+cmake_pop_check_state ()
+
+set (init OFF)
+if (DEFINED ENABLED_SHARED_BSON)
+ message (STATUS "ENABLE_SHARED_BSON is now named USE_SHARED_LIBBSON")
+ set (init "${ENABLE_SHARED_BSON}")
+endif ()
+option (USE_SHARED_LIBBSON "Dynamically link libbson for the libmongocrypt dynamic library (default is static)" ${init})
+
+if (NOT DEFINED MONGOCRYPT_MONGOC_DIR)
+ # The user did not provide a MONGOCRYPT_MONGOC_DIR, so we'll get one
+ include (FetchContent OPTIONAL)
+ if (NOT COMMAND FetchContent_Declare)
+ # We need FetchContent in order to download the project.
+ message (FATAL_ERROR
+ "No MONGOCRYPT_MONGOC_DIR setting was defined, and the FetchContent.cmake "
+ "module is not available. Upgrade your CMake version, or provide a "
+ "MONGOCRYPT_MONGOC_DIR path to a mongo-c-driver directory (This is required "
+ "for libmongocrypt to find a libbson to use and link against).")
+ endif ()
+ include (FetchMongoC)
+ # The FetchMongoC module defines a MONGOCRYPT_MONGOC_DIR for us to use
+endif ()
+
+function (_import_bson)
+ if (MONGOCRYPT_MONGOC_DIR STREQUAL "USE-SYSTEM" AND USE_SHARED_LIBBSON AND NOT ENABLE_ONLINE_TESTS)
+ message (STATUS "NOTE: Using system-wide libbson library. This is intended only for package maintainers.")
+ find_library (_MONGOCRYPT_SYSTEM_LIBBSON_SHARED "${CMAKE_SHARED_LIBRARY_PREFIX}bson-1.0${CMAKE_SHARED_LIBRARY_SUFFIX}")
+ find_library (_MONGOCRYPT_SYSTEM_LIBBSON_STATIC "${CMAKE_STATIC_LIBRARY_PREFIX}bson-static-1.0${CMAKE_STATIC_LIBRARY_SUFFIX}")
+ find_path (_MONGOCRYPT_SYSTEM_LIBBSON_INCLUDE_DIR bson/bson.h PATH_SUFFIXES libbson-1.0)
+ add_library (bson_shared SHARED IMPORTED)
+ add_library (bson_static STATIC IMPORTED)
+ set_target_properties (bson_shared bson_static PROPERTIES
+ IMPORTED_CONFIGURATIONS "Release"
+ INTERFACE_INCLUDE_DIRECTORIES "${_MONGOCRYPT_SYSTEM_LIBBSON_INCLUDE_DIR}"
+ )
+ set_property (TARGET bson_shared PROPERTY IMPORTED_LOCATION "${_MONGOCRYPT_SYSTEM_LIBBSON_SHARED}")
+ set_property (TARGET bson_static PROPERTY IMPORTED_LOCATION "${_MONGOCRYPT_SYSTEM_LIBBSON_STATIC}")
+ set_property (
+ CACHE _MONGOCRYPT_SYSTEM_LIBBSON_SHARED
+ _MONGOCRYPT_SYSTEM_LIBBSON_INCLUDE_DIR
+ PROPERTY ADVANCED
+ TRUE
+ )
+ else ()
+ message (STATUS "Using [${MONGOCRYPT_MONGOC_DIR}] as a sub-project for libbson")
+ # Disable AWS_AUTH, to prevent it from building the kms-message symbols, which we build ourselves
+ set (ENABLE_MONGODB_AWS_AUTH OFF CACHE BOOL "Disable kms-message content in mongoc for libmongocrypt" FORCE)
+ # Disable install() for the libbson static library. We'll do it ourselves
+ set (ENABLE_STATIC BUILD_ONLY)
+ # Disable libzstd, which isn't necessary for libmongocrypt and isn't necessarily available.
+ set (ENABLE_ZSTD OFF CACHE BOOL "Toggle libzstd for the mongoc subproject (not required by libmongocrypt)")
+ # Disable deprecated automatic init and cleanup. (May be overridden by the user)
+ set (ENABLE_AUTOMATIC_INIT_AND_CLEANUP OFF CACHE BOOL "Enable automatic init and cleanup (GCC only)")
+ # Disable over-alignment of bson types. (May be overridden by the user)
+ set (ENABLE_EXTRA_ALIGNMENT ${_extra_alignment_default} CACHE BOOL "Toggle extra alignment of bson_t")
+ # We don't want the subproject to find libmongocrypt
+ set (ENABLE_CLIENT_SIDE_ENCRYPTION OFF CACHE BOOL "Disable client-side encryption for the libmongoc subproject")
+ # Add the subdirectory as a project. EXCLUDE_FROM_ALL to inhibit building and installing of components unless requested
+ add_subdirectory ("${MONGOCRYPT_MONGOC_DIR}" _mongo-c-driver EXCLUDE_FROM_ALL)
+ # Workaround: Embedded mongoc_static does not set its INCLUDE_DIRECTORIES for user targets
+ target_include_directories (mongoc_static
+ PUBLIC
+ "$"
+ "$"
+ )
+ endif ()
+endfunction ()
+
+# Do the import in a function to isolate variable scope
+_import_bson ()
+
+# Define interface targets to be used to control the libbson used at both build and import time.
+# Refer to mongocrypt-config.cmake to see how these targets are used by consumers
+add_library (_mongocrypt-libbson_for_static INTERFACE)
+add_library (_mongocrypt-libbson_for_shared INTERFACE)
+add_library (_mongocrypt::libbson_for_static ALIAS _mongocrypt-libbson_for_static)
+add_library (_mongocrypt::libbson_for_shared ALIAS _mongocrypt-libbson_for_shared)
+install (
+ TARGETS _mongocrypt-libbson_for_static _mongocrypt-libbson_for_shared
+ EXPORT mongocrypt_targets
+)
+
+# Link to the requested libbson, only exporting that usage for the local build tree.
+# The mongocrypt-config file will later add the appropriate link library for downstream
+# users during find_package()
+if (USE_SHARED_LIBBSON)
+ target_link_libraries (_mongocrypt-libbson_for_shared INTERFACE $)
+else ()
+ target_link_libraries (_mongocrypt-libbson_for_shared INTERFACE $)
+endif ()
+# libbson_for_static always links to the static libbson:
+target_link_libraries (_mongocrypt-libbson_for_static INTERFACE $)
+
+if (TARGET mongoc_static)
+ # And an alias to the mongoc target for use in some test cases
+ add_library (_mongocrypt::mongoc ALIAS mongoc_static)
+endif ()
+
+# Put the libbson dynamic library into the current binary directory (plus possible config suffix).
+# This ensures that libbson DLL will resolve on Windows when it searches during tests
+set_property (TARGET bson_shared PROPERTY RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+
+if (ENABLE_STATIC)
+ # We are going to build a static libmongocrypt.
+ # We want the static libbson target from the embedded mongoc. Enable the static library as
+ # part of "all", and install the archive alongside the rest of our static libraries.
+ # (Useful for some users for convenience of static-linking libmongocrypt: CDRIVER-3187)
+ set_target_properties (bson_static PROPERTIES
+ EXCLUDE_FROM_ALL FALSE
+ OUTPUT_NAME bson-static-for-libmongocrypt
+ )
+ install (
+ FILES $
+ DESTINATION "${CMAKE_INSTALL_LIBDIR}"
+ RENAME ${CMAKE_STATIC_LIBRARY_PREFIX}bson-static-for-libmongocrypt${CMAKE_STATIC_LIBRARY_SUFFIX}
+ )
+endif ()
diff -pruN 1.3.0-1/cmake/mongocrypt-config.cmake 1.5.2-1/cmake/mongocrypt-config.cmake
--- 1.3.0-1/cmake/mongocrypt-config.cmake 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/cmake/mongocrypt-config.cmake 2022-07-30 15:14:43.000000000 +0000
@@ -1,3 +1,32 @@
include(CMakeFindDependencyMacro)
find_dependency(kms_message 0.0.1)
include("${CMAKE_CURRENT_LIST_DIR}/mongocrypt_targets.cmake")
+
+if (DEFINED MONGOCRYPT_LIBBSON_STATIC_USE)
+ # The user has named a library that should be linked as the static libbson library
+ set_property (
+ TARGET mongo::_mongocrypt-libbson_for_static
+ APPEND PROPERTY INTERFACE_LINK_LIBRARIES "$"
+ )
+endif ()
+
+
+# BOOL: Whether the libmongocrypt dynamic library in this package needs to link to an external libbson.
+# In the default configuration, the shared lib will include the TUs for a pinned version of libbson
+# and will use linker scripts to "hide" these symbols from the outside world.
+#
+# If the libmongocrypt package was built to link against a shared libbson library, then the
+# libmongocrypt dynamic library will contain pending references to libbson symbols that will
+# need to be resolved before the library can be used.
+#
+# (Note: static libmongocrypt *always* needs to link against an external libbson, as it does not
+# embed the libbson symbols.)
+set (_using_shared_libbson "@USE_SHARED_LIBBSON@")
+
+if (_using_shared_libbson AND DEFINED MONGOCRYPT_LIBBSON_SHARED_USE)
+ # The user has named a library that should be linked as the shared libbson library
+ set_property (
+ TARGET mongo::_mongocrypt-libbson_for_shared
+ APPEND PROPERTY INTERFACE_LINK_LIBRARIES "$"
+ )
+endif ()
diff -pruN 1.3.0-1/cmake/MongoC-Warnings.cmake 1.5.2-1/cmake/MongoC-Warnings.cmake
--- 1.3.0-1/cmake/MongoC-Warnings.cmake 1970-01-01 00:00:00.000000000 +0000
+++ 1.5.2-1/cmake/MongoC-Warnings.cmake 2022-07-30 15:14:43.000000000 +0000
@@ -0,0 +1,74 @@
+#[[
+ This file sets warning options for the directories in which it is include()'d
+
+ These warnings are intended to be ported to each supported platform, and
+ especially for high-value warnings that are very likely to catch latent bugs
+ early in the process before the code is even run.
+]]
+
+set (__is_gnu "$")
+set (__is_clang "$,$>")
+set (__is_gnu_like "$")
+set (__is_msvc "$")
+
+# "Old" GNU is GCC < 5, which is missing several warning options
+set (__is_old_gnu "$,5>>")
+set (__not_old_gnu "$")
+
+#[[
+ Define additional compile options, conditional on the compiler being used.
+ Each option should be prefixed by `gnu:`, `clang:`, `msvc:`, or `gnu-like:`.
+ Those options will be conditionally enabled for GCC, Clang, or MSVC.
+
+ These options are attached to the source directory and its children.
+]]
+function (mongoc_add_platform_compile_options)
+ foreach (opt IN LISTS ARGV)
+ if (NOT opt MATCHES "^(gnu-like|gnu|clang|msvc):(.*)")
+ message (SEND_ERROR "Invalid option '${opt}' (Should be prefixed by 'msvc:', 'gnu:', 'clang:', or 'gnu-like:'")
+ continue ()
+ endif ()
+ if (CMAKE_MATCH_1 STREQUAL "gnu-like")
+ add_compile_options ("$<${__is_gnu_like}:${CMAKE_MATCH_2}>")
+ elseif (CMAKE_MATCH_1 STREQUAL gnu)
+ add_compile_options ("$<${__is_gnu}:${CMAKE_MATCH_2}>")
+ elseif (CMAKE_MATCH_1 STREQUAL clang)
+ add_compile_options ("$<${__is_clang}:${CMAKE_MATCH_2}>")
+ elseif (CMAKE_MATCH_1 STREQUAL "msvc")
+ add_compile_options ("$<${__is_msvc}:${CMAKE_MATCH_2}>")
+ else ()
+ message (SEND_ERROR "Invalid option to mongoc_add_platform_compile_options(): '${opt}'")
+ endif ()
+ endforeach ()
+endfunction ()
+
+if (CMAKE_VERSION VERSION_LESS 3.3)
+ # On older CMake versions, we'll just always pass the warning options, even
+ # if the generate warnings for the C++ check file
+ set (is_c_lang "1")
+else ()
+ # $ is only valid in CMake 3.3+
+ set (is_c_lang "$")
+endif ()
+
+# These below warnings should always be unconditional hard errors, as the code is
+# almost definitely broken
+mongoc_add_platform_compile_options (
+ # Implicit function or variable declarations
+ gnu-like:$<${is_c_lang}:-Werror=implicit> msvc:/we4013 msvc:/we4431
+ # Missing return types/statements
+ gnu-like:-Werror=return-type msvc:/we4716
+ # Incompatible pointer types
+ gnu-like:$<$:-Werror=incompatible-pointer-types> msvc:/we4113
+ # Integral/pointer conversions
+ gnu-like:$<$:-Werror=int-conversion> msvc:/we4047
+ # Discarding qualifiers
+ gnu:$<$:-Werror=discarded-qualifiers>
+ clang:$<${is_c_lang}:-Werror=ignored-qualifiers>
+ msvc:/we4090
+ # Definite use of uninitialized value
+ gnu-like:-Werror=uninitialized msvc:/we4700
+
+ # Aside: Disable CRT insecurity warnings
+ msvc:/D_CRT_SECURE_NO_WARNINGS
+ )
diff -pruN 1.3.0-1/CMakeLists.txt 1.5.2-1/CMakeLists.txt
--- 1.3.0-1/CMakeLists.txt 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/CMakeLists.txt 2022-07-30 15:14:43.000000000 +0000
@@ -2,6 +2,9 @@ cmake_minimum_required (VERSION 3.5)
if (CMAKE_VERSION VERSION_EQUAL 3.12 OR CMAKE_VERSION VERSION_GREATER 3.12)
project (mongocrypt C)
+
+ # Used for the csfle-markup util:
+ enable_language (CXX OPTIONAL)
else ()
# GenerateExportHeader only works with C with 3.12 - https://gitlab.kitware.com/cmake/cmake/commit/de348a9638bd51af4523f36c68884b901d4aff18
project (mongocrypt C CXX)
@@ -9,58 +12,46 @@ endif ()
set (CMAKE_C_STANDARD 99)
-option (ENABLE_SHARED_BSON "Dynamically link libbson (default is static)" OFF)
option (ENABLE_STATIC "Install static libraries" ON)
option (ENABLE_PIC
"Enables building of position independent code for static library components."
ON
)
option (ENABLE_BUILD_FOR_PPA "Maintainer-only option for preparing PPA build" OFF)
-if (ENABLE_SHARED_BSON AND ENABLE_BUILD_FOR_PPA)
+option (ENABLE_ONLINE_TESTS "Enable online tests and the csfle utility" ON)
+
+list (APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
+
+include (GNUInstallDirs)
+
+include (ImportBSON)
+
+if (USE_SHARED_LIBBSON AND ENABLE_BUILD_FOR_PPA)
message (FATAL_ERROR "PPA build requires static linking to libbson")
endif ()
-set (CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/cmake)
-
-# Attempt to find libbson by new package name.
-find_package (bson-1.0 1.11 QUIET)
-if (bson-1.0_FOUND)
- message ("-- libbson found version \"${bson-1.0_VERSION}\"")
- set (BSON_TARGET mongo::bson_static)
- if (ENABLE_SHARED_BSON)
- set (BSON_TARGET mongo::bson_shared)
- endif ()
-elseif (ENABLE_SHARED_BSON)
- # Try old package name for libbson.
- find_package (libbson-1.0 1.11 REQUIRED)
- message ("-- libbson found version \"${BSON_VERSION}\"")
- message ("-- libbson include path \"${BSON_INCLUDE_DIRS}\"")
- message ("-- libbson libraries \"${BSON_LIBRARIES}\"")
- set (BSON_TARGET ${BSON_LIBRARIES})
- set (BSON_INCLUDES ${BSON_INCLUDE_DIRS})
- set (BSON_DEFINITIONS ${BSON_DEFINITIONS})
-else ()
- # Try old package name for libbson.
- find_package (libbson-static-1.0 1.11 REQUIRED)
- message ("-- libbson-static found version \"${BSON_STATIC_VERSION}\"")
- message ("-- libbson-static include path \"${BSON_STATIC_INCLUDE_DIRS}\"")
- message ("-- libbson-static libraries \"${BSON_STATIC_LIBRARIES}\"")
- set (BSON_TARGET ${BSON_STATIC_LIBRARIES})
- set (BSON_INCLUDES ${BSON_STATIC_INCLUDE_DIRS})
- set (BSON_DEFINITIONS ${BSON_STATIC_DEFINITIONS})
+# Enable a higher warning level and warnings-as-errors
+include (MongoC-Warnings)
+option (ENABLE_MORE_WARNINGS_AS_ERRORS "Enable extra warnings-as-errors in the build." OFF)
+if (ENABLE_MORE_WARNINGS_AS_ERRORS)
+ mongoc_add_platform_compile_options (
+ msvc:/W3 msvc:/WX msvc:/wd4996 msvc:/wd4359
+ gnu-like:-Wall gnu-like:-Werror
+ gnu-like:-Wswitch-enum gnu-like:-Wswitch-default
+ )
endif ()
-find_package ( Threads REQUIRED )
+find_package (Threads REQUIRED)
add_subdirectory (bindings/cs)
include (GenerateExportHeader)
-include (GNUInstallDirs)
-enable_testing ()
+include (CTest)
set (MONGOCRYPT_PUBLIC_HEADERS
src/mongocrypt-compat.h
+ src/mongocrypt.h
)
set (MONGOCRYPT_SOURCES
@@ -68,6 +59,13 @@ set (MONGOCRYPT_SOURCES
src/crypto/commoncrypto.c
src/crypto/libcrypto.c
src/crypto/none.c
+ src/mc-efc.c
+ src/mc-fle2-insert-update-payload.c
+ src/mc-fle2-encryption-placeholder.c
+ src/mc-fle2-find-equality-payload.c
+ src/mc-fle2-payload-ieev.c
+ src/mc-fle2-payload-uev.c
+ src/mc-tokens.c
src/mongocrypt-binary.c
src/mongocrypt-buffer.c
src/mongocrypt-cache.c
@@ -79,6 +77,7 @@ set (MONGOCRYPT_SOURCES
src/mongocrypt-ctx-datakey.c
src/mongocrypt-ctx-decrypt.c
src/mongocrypt-ctx-encrypt.c
+ src/mongocrypt-ctx-rewrap-many-datakey.c
src/mongocrypt-ctx.c
src/mongocrypt-endpoint.c
src/mongocrypt-kek.c
@@ -93,27 +92,19 @@ set (MONGOCRYPT_SOURCES
src/mongocrypt-util.c
src/mongocrypt.c
src/os_win/os_mutex.c
- src/os_win/os_once.c
src/os_posix/os_mutex.c
- src/os_posix/os_once.c
+ src/os_win/os_dll.c
+ src/os_posix/os_dll.c
)
-if ( MSVC )
- # W4996 - POSIX name for this item is deprecated
- set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W3 /wd4996 /D_CRT_SECURE_NO_WARNINGS /WX")
-
- # TODO: add support for clang-cl which is detected as MSVC
-else ()
- # GNU, Clang, AppleClang
- set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror -Wno-missing-braces")
-endif ()
-
-# Choose a Crypto provider
-set (MONGOCRYPT_CRYPTO OpenSSL)
-if (APPLE)
- set (MONGOCRYPT_CRYPTO CommonCrypto)
-elseif (WIN32)
- set (MONGOCRYPT_CRYPTO CNG)
+# If MONGOCRYPT_CRYPTO is not set, choose a system default.
+if (NOT MONGOCRYPT_CRYPTO)
+ set (MONGOCRYPT_CRYPTO OpenSSL)
+ if (APPLE)
+ set (MONGOCRYPT_CRYPTO CommonCrypto)
+ elseif (WIN32)
+ set (MONGOCRYPT_CRYPTO CNG)
+ endif ()
endif ()
# Otherwise, override with crypto hooks.
@@ -152,26 +143,80 @@ if (ENABLE_TRACE)
set (MONGOCRYPT_ENABLE_TRACE 1)
endif ()
+set (BUILD_VERSION "0.0.0" CACHE STRING "Library version")
+if (BUILD_VERSION STREQUAL "0.0.0")
+ if (EXISTS ${PROJECT_SOURCE_DIR}/VERSION_CURRENT)
+ file (STRINGS ${PROJECT_SOURCE_DIR}/VERSION_CURRENT BUILD_VERSION)
+ message (STATUS "File VERSION_CURRENT contained BUILD_VERSION ${BUILD_VERSION}")
+ else ()
+ set (RELEASE_BRANCH_REF "origin/r1.5")
+ include (GetVersion)
+ GetVersion (BUILD_VERSION)
+ message (STATUS "Storing BUILD_VERSION ${BUILD_VERSION} in file VERSION_CURRENT for later use")
+ file (WRITE ${PROJECT_SOURCE_DIR}/VERSION_CURRENT ${BUILD_VERSION})
+ endif ()
+else ()
+ message (STATUS "Storing BUILD_VERSION ${BUILD_VERSION} in file VERSION_CURRENT for later use")
+ file (WRITE ${PROJECT_SOURCE_DIR}/VERSION_CURRENT ${BUILD_VERSION})
+endif ()
+
+set (MONGOCRYPT_BUILD_VERSION ${BUILD_VERSION})
+
configure_file (
"${PROJECT_SOURCE_DIR}/src/mongocrypt-config.h.in"
"${PROJECT_BINARY_DIR}/src/mongocrypt-config.h"
)
+# Define the mlib target, which is private and header-only. It is not exported
+# nor are its headers installed.
+add_library (_mongo-mlib INTERFACE)
+add_library (mongo::mlib ALIAS _mongo-mlib)
+set_property(
+ TARGET _mongo-mlib
+ APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS
+ MLIB_USER
+ )
+set_property(
+ TARGET _mongo-mlib
+ APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES
+ "$"
+ )
+
# kms-message
add_subdirectory (kms-message)
# Define mongocrypt library
add_library (mongocrypt SHARED ${MONGOCRYPT_SOURCES})
-target_include_directories (mongocrypt PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/kms-message/src")
-target_include_directories (mongocrypt PUBLIC $)
-target_include_directories (mongocrypt PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
-target_include_directories (mongocrypt PRIVATE ${BSON_INCLUDES})
-target_compile_definitions (mongocrypt PRIVATE ${BSON_DEFINITIONS})
-target_link_libraries (mongocrypt PRIVATE ${BSON_TARGET})
-target_link_libraries (mongocrypt PRIVATE ${CMAKE_THREAD_LIBS_INIT})
-target_link_libraries (mongocrypt PRIVATE kms_message_static)
+target_include_directories (
+ mongocrypt
+ PRIVATE
+ "${CMAKE_CURRENT_SOURCE_DIR}/kms-message/src"
+ "${CMAKE_CURRENT_SOURCE_DIR}/src"
+ PUBLIC
+ "$"
+ )
+include (TestBigEndian)
+TEST_BIG_ENDIAN (MONGOCRYPT_BIG_ENDIAN)
+if (MONGOCRYPT_BIG_ENDIAN)
+ message ("Detected byte order: big endian")
+ list (APPEND MONGOCRYPT_DEFINITIONS MONGOCRYPT_BIG_ENDIAN)
+else ()
+ message ("Detected byte order: little endian")
+ list (APPEND MONGOCRYPT_DEFINITIONS MONGOCRYPT_LITTLE_ENDIAN)
+endif ()
+target_compile_definitions (mongocrypt PRIVATE ${MONGOCRYPT_DEFINITIONS})
+target_link_libraries (
+ mongocrypt
+ PRIVATE
+ _mongocrypt::libbson_for_shared
+ Threads::Threads
+ kms_message_static
+ $
+ PUBLIC
+ ${CMAKE_DL_LIBS}
+ )
-if (NOT ENABLE_SHARED_BSON)
+if (NOT USE_SHARED_LIBBSON)
if (APPLE)
message ("compiling with unexported symbols list to hide bson symbols")
set_target_properties (mongocrypt PROPERTIES LINK_FLAGS "-Wl,-unexported_symbols_list,\"${CMAKE_CURRENT_SOURCE_DIR}/cmake/libmongocrypt-hidden-symbols.txt\"")
@@ -193,16 +238,37 @@ if (NOT WIN32 AND ENABLE_PIC AND "${FPIC
target_compile_options (mongocrypt_static PUBLIC -fPIC)
message ("Adding -fPIC to compilation of mongocrypt_static components")
endif ()
-target_include_directories (mongocrypt_static PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/kms-message/src")
-target_include_directories (mongocrypt_static PUBLIC $)
-target_include_directories (mongocrypt_static PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
-target_include_directories (mongocrypt_static PRIVATE ${BSON_INCLUDES})
-target_compile_definitions (mongocrypt_static PRIVATE ${BSON_DEFINITIONS})
+target_include_directories (
+ mongocrypt_static
+ PRIVATE
+ "${CMAKE_CURRENT_SOURCE_DIR}/kms-message/src"
+ "${CMAKE_CURRENT_SOURCE_DIR}/src"
+ PUBLIC
+ "$"
+ )
+target_compile_definitions (
+ mongocrypt_static
+ PRIVATE
+ ${MONGOCRYPT_DEFINITIONS}
+ PUBLIC
+ MONGOCRYPT_STATIC_DEFINE
+ KMS_MSG_STATIC
+ )
+target_link_libraries (
+ mongocrypt_static
+ PRIVATE
+ _mongocrypt::libbson_for_static
+ kms_message_static
+ $
+ PUBLIC
+ ${CMAKE_THREAD_LIBS_INIT}
+ ${CMAKE_DL_LIBS}
+ )
set (PKG_CONFIG_STATIC_LIBS "\${prefix}/${CMAKE_INSTALL_LIBDIR}/libmongocrypt-static.a")
-target_link_libraries (mongocrypt_static PRIVATE ${BSON_TARGET})
-target_link_libraries (mongocrypt_static PRIVATE ${CMAKE_THREAD_LIBS_INIT})
set (PKG_CONFIG_STATIC_LIBS "${PKG_CONFIG_STATIC_LIBS} ${CMAKE_THREAD_LIBS_INIT}")
-target_link_libraries (mongocrypt_static PRIVATE kms_message_static)
+if (CMAKE_DL_LIBS)
+ string (APPEND PKG_CONFIG_STATIC_LIBS " -l${CMAKE_DL_LIBS}")
+endif ()
set (PKG_CONFIG_STATIC_LIBS "${PKG_CONFIG_STATIC_LIBS} \${prefix}/${CMAKE_INSTALL_LIBDIR}/libkms_message-static.a")
if (ENABLE_BUILD_FOR_PPA)
set (PKG_CONFIG_STATIC_LIBS "${PKG_CONFIG_STATIC_LIBS} \${prefix}/${CMAKE_INSTALL_LIBDIR}/libbson-static-1.0.a")
@@ -213,7 +279,6 @@ if (ENABLE_BUILD_FOR_PPA)
endif ()
set (PKG_CONFIG_STATIC_LIBS "${PKG_CONFIG_STATIC_LIBS} -pthread")
endif ()
-target_compile_definitions (mongocrypt_static PUBLIC MONGOCRYPT_STATIC_DEFINE KMS_MSG_STATIC)
if (ENABLE_WINDOWS_STATIC_RUNTIME)
target_compile_options (mongocrypt_static PUBLIC /MT)
target_compile_options (kms_message_static PUBLIC /MT)
@@ -234,7 +299,6 @@ elseif (MONGOCRYPT_CRYPTO STREQUAL OpenS
set (PKG_CONFIG_STATIC_LIBS "${PKG_CONFIG_STATIC_LIBS} -lssl -lcrypto")
endif ()
-
set_target_properties (mongocrypt PROPERTIES
SOVERSION 0
VERSION "0.0.0"
@@ -247,19 +311,74 @@ set_target_properties (mongocrypt_static
OUTPUT_NAME "mongocrypt-static"
)
+if (BUILD_TESTING)
+ # Use C++ in the testing DLL to ensure we can load a library with the C++ runtime
+ enable_language (CXX)
+ add_library (test-dll MODULE test/test-dll.cpp)
+ set_target_properties (test-dll PROPERTIES
+ SUFFIX ".dll"
+ PREFIX ""
+ )
+
+ # Create two stubbed crypt_shared libraries
+ add_library (stubbed-crypt_shared SHARED test/crypt_shared-stub.cpp)
+ add_library (stubbed-crypt_shared-2 SHARED test/crypt_shared-stub.cpp)
+
+ set_target_properties(stubbed-crypt_shared stubbed-crypt_shared-2 PROPERTIES
+ INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/src"
+ LINK_LIBRARIES _mongocrypt::libbson_for_static
+ COMPILE_FEATURES cxx_std_11
+ PREFIX ""
+ )
+
+ set_target_properties(stubbed-crypt_shared-2 PROPERTIES SUFFIX ".dll")
+
+ if (MONGOCRYPT_TESTING_CRYPT_SHARED_FILE)
+ # Generate a target that copies the CSFLE library into the binary directory of test-mongocrypt
+ set (stamp "${CMAKE_CURRENT_BINARY_DIR}/mongo_crypt_v1.copied.$.stamp")
+ add_custom_command (
+ OUTPUT "${stamp}"
+ COMMAND "${CMAKE_COMMAND}" -E copy
+ "${MONGOCRYPT_TESTING_CRYPT_SHARED_FILE}"
+ "$/mongo_crypt_v1${CMAKE_SHARED_LIBRARY_SUFFIX}"
+ COMMAND "${CMAKE_COMMAND}" -E touch "${stamp}"
+ DEPENDS "${MONGOCRYPT_TESTING_CRYPT_SHARED_FILE}"
+ COMMENT "Getting mongo_crypt library"
+ )
+ add_custom_target (copy-crypt_shared ALL DEPENDS "${stamp}")
+ else ()
+ # The first stubbed crypt_shared library will take the place of the actual crypt_shared for testing
+ message (STATUS "Generating a stubbed crypt_shared dynamic library for use in testing.")
+ message (STATUS "Provide a MONGOCRYPT_TESTING_CRYPT_SHARED_FILE= to provide a crypt_shared for use in testing")
+ set_target_properties (stubbed-crypt_shared PROPERTIES
+ # Normalize the output name expected by libmongocrypt
+ OUTPUT_NAME "mongo_crypt_v1"
+ )
+ endif ()
+endif ()
set (TEST_MONGOCRYPT_SOURCES
+ test/test-mc-efc.c
+ test/test-mc-fle2-payload-ieev.c
+ test/test-mc-fle2-payload-iup.c
+ test/test-mc-fle2-payload-uev.c
+ test/test-mc-tokens.c
test/test-mongocrypt-assert-match-bson.c
test/test-mongocrypt-buffer.c
test/test-mongocrypt-cache.c
test/test-mongocrypt-cache-oauth.c
test/test-mongocrypt-ciphertext.c
+ test/test-mongocrypt-compact.c
test/test-mongocrypt-crypto.c
test/test-mongocrypt-crypto-hooks.c
+ test/test-mongocrypt-crypto-std-hooks.c
+ test/test-mongocrypt-csfle-lib.c
test/test-mongocrypt-ctx-decrypt.c
test/test-mongocrypt-ctx-encrypt.c
+ test/test-mongocrypt-ctx-rewrap-many-datakey.c
test/test-mongocrypt-ctx-setopt.c
test/test-mongocrypt-datakey.c
+ test/test-mongocrypt-dll.c
test/test-mongocrypt-endpoint.c
test/test-mongocrypt-kek.c
test/test-mongocrypt-key.c
@@ -274,44 +393,61 @@ set (TEST_MONGOCRYPT_SOURCES
test/test-mongocrypt-traverse-util.c
test/test-mongocrypt-util.c
test/test-mongocrypt.c
-)
+ )
# Define test-mongocrypt
add_executable (test-mongocrypt ${TEST_MONGOCRYPT_SOURCES})
# Use the static version since it allows the test binary to use private symbols
-target_link_libraries (test-mongocrypt PRIVATE mongocrypt_static)
target_include_directories (test-mongocrypt PRIVATE ./src "${CMAKE_CURRENT_SOURCE_DIR}/kms-message/src")
-target_link_libraries (test-mongocrypt PRIVATE ${BSON_TARGET})
-target_include_directories (test-mongocrypt PRIVATE ${BSON_INCLUDES})
-target_compile_definitions (test-mongocrypt PRIVATE ${BSON_DEFINITIONS})
+target_link_libraries (test-mongocrypt PRIVATE _mongocrypt::libbson_for_static mongocrypt_static mongo::mlib)
+target_include_directories (test-mongocrypt PRIVATE "${CMAKE_CURRENT_LIST_DIR}/test")
+target_compile_definitions (test-mongocrypt PRIVATE
+ ${BSON_DEFINITIONS}
+ ${MONGOCRYPT_DEFINITIONS}
+ # Set a definition so that testcases can know where test-mongocrypt.exe was written to
+ "TEST_MONGOCRYPT_OUTPUT_PATH=\"$\""
+ # Tell test-mongocrypt whether we have a real csfle library for testing
+ TEST_MONGOCRYPT_HAVE_REAL_CRYPT_SHARED_LIB=$
+ )
-add_test (mongocrypt test-mongocrypt WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+add_test (
+ NAME mongocrypt
+ COMMAND test-mongocrypt
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+ )
+
+foreach (test IN ITEMS path str)
+ add_executable (mlib.${test}.test src/mlib/${test}.test.c)
+ add_test (mlib.${test} mlib.${test}.test)
+ target_link_libraries (mlib.${test}.test PRIVATE mongo::mlib)
+endforeach ()
+
+if ("cxx_std_20" IN_LIST CMAKE_CXX_COMPILE_FEATURES)
+ add_executable (csfle-markup src/csfle-markup.cpp)
+ target_link_libraries (csfle-markup PRIVATE mongocrypt_static _mongocrypt::libbson_for_static mongo::mlib)
+ target_compile_features (csfle-markup PRIVATE cxx_std_20)
+endif ()
# Exclude example-state-machine since it requires native crypto.
if (NOT MONGOCRYPT_CRYPTO STREQUAL none)
# Define example-state-machine
add_executable (example-state-machine test/example-state-machine.c)
- target_link_libraries (example-state-machine PRIVATE mongocrypt ${BSON_TARGET})
- target_include_directories (example-state-machine PRIVATE ${BSON_INCLUDES})
- target_compile_definitions (example-state-machine PRIVATE ${BSON_DEFINITIONS})
+ target_link_libraries (example-state-machine PRIVATE mongocrypt _mongocrypt::libbson_for_shared)
target_include_directories (example-state-machine PRIVATE ./src "${CMAKE_CURRENT_SOURCE_DIR}/kms-message/src")
# Define example-state-machine-static
add_executable (example-state-machine-static test/example-state-machine.c)
- target_link_libraries (example-state-machine-static PRIVATE mongocrypt_static ${BSON_TARGET})
- target_include_directories (example-state-machine-static PRIVATE ${BSON_INCLUDES})
- target_compile_definitions (example-state-machine-static PRIVATE ${BSON_DEFINITIONS})
+ target_link_libraries (example-state-machine-static PRIVATE mongocrypt_static _mongocrypt::libbson_for_static)
target_include_directories (example-state-machine-static PRIVATE ./src)
- find_package (mongoc-1.0)
- if (ENABLE_ONLINE_TESTS AND mongoc-1.0_FOUND)
+ if (ENABLE_ONLINE_TESTS)
message ("compiling utilities")
add_executable (csfle test/util/csfle.c test/util/util.c)
target_link_libraries (csfle PRIVATE mongocrypt_static)
target_include_directories (csfle PRIVATE ${CMAKE_BINARY_DIR}/src)
target_include_directories (csfle PRIVATE ./src)
target_include_directories (csfle PRIVATE ./kms-message/src)
- target_link_libraries (csfle PRIVATE mongo::mongoc_shared)
+ target_link_libraries (csfle PRIVATE _mongocrypt::mongoc)
endif ()
endif ()
@@ -329,43 +465,25 @@ install (
INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
)
+# This export set is not installed, and is only to allow export() of the mlib-using targets
+install (TARGETS _mongo-mlib EXPORT _exports_for_export)
+export (EXPORT _exports_for_export)
+
install (
FILES
${MONGOCRYPT_PUBLIC_HEADERS}
- ${CMAKE_CURRENT_BINARY_DIR}/src/mongocrypt.h
${CMAKE_CURRENT_BINARY_DIR}/src/mongocrypt-config.h
${CMAKE_CURRENT_BINARY_DIR}/src/mongocrypt-export.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/mongocrypt
COMPONENT Devel
)
-set (BUILD_VERSION "0.0.0" CACHE STRING "Library version")
-
-if (BUILD_VERSION STREQUAL "0.0.0")
- if (EXISTS ${PROJECT_SOURCE_DIR}/VERSION_CURRENT)
- file (STRINGS ${PROJECT_SOURCE_DIR}/VERSION_CURRENT BUILD_VERSION)
- message ("file VERSION_CURRENT contained BUILD_VERSION ${BUILD_VERSION}")
- else ()
- include (GetVersion)
- GetVersion (BUILD_VERSION)
- message ("storing BUILD_VERSION ${BUILD_VERSION} in file VERSION_CURRENT for later use")
- file (WRITE ${PROJECT_SOURCE_DIR}/VERSION_CURRENT ${BUILD_VERSION})
- endif ()
-else ()
- message ("storing BUILD_VERSION ${BUILD_VERSION} in file VERSION_CURRENT for later use")
- file (WRITE ${PROJECT_SOURCE_DIR}/VERSION_CURRENT ${BUILD_VERSION})
-endif ()
-
-message ("Configuring libmongocrypt version ${BUILD_VERSION}")
-set (MONGOCRYPT_BUILD_VERSION ${BUILD_VERSION})
-configure_file (src/mongocrypt.h.in src/mongocrypt.h)
-
set (PROJECT_VERSION "${BUILD_VERSION}")
set (PROJECT_DESCRIPTION "The libmongocrypt client-side field level encryption library.")
if (NOT ENABLE_BUILD_FOR_PPA)
set (PKG_CONFIG_STATIC_REQUIRES "libbson-static-1.0")
endif ()
-if (ENABLE_SHARED_BSON)
+if (USE_SHARED_LIBBSON)
set (PKG_CONFIG_REQUIRES "libbson-1.0")
set (PKG_CONFIG_STATIC_REQUIRES "libbson-1.0")
endif ()
@@ -412,14 +530,9 @@ write_basic_package_version_file (
COMPATIBILITY AnyNewerVersion
)
-export (EXPORT mongocrypt_targets
- NAMESPACE mongo::
- FILE "${CMAKE_CURRENT_BINARY_DIR}/mongocrypt/mongocrypt_targets.cmake"
-)
-
configure_file (cmake/mongocrypt-config.cmake
"${CMAKE_CURRENT_BINARY_DIR}/mongocrypt/mongocrypt-config.cmake"
- COPYONLY
+ @ONLY
)
install (EXPORT mongocrypt_targets
@@ -430,7 +543,7 @@ install (EXPORT mongocrypt_targets
install (
FILES
- cmake/mongocrypt-config.cmake
+ "${CMAKE_CURRENT_BINARY_DIR}/mongocrypt/mongocrypt-config.cmake"
"${CMAKE_CURRENT_BINARY_DIR}/mongocrypt/mongocrypt-config-version.cmake"
DESTINATION
${CMAKE_INSTALL_LIBDIR}/cmake/mongocrypt
diff -pruN 1.3.0-1/CODEOWNERS 1.5.2-1/CODEOWNERS
--- 1.3.0-1/CODEOWNERS 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/CODEOWNERS 2022-07-30 15:14:43.000000000 +0000
@@ -1,10 +1,10 @@
# Code Owners will automatically be added as reviewers on PRs
# NodeJS Bindings
-bindings/node @nbbeeken @emadum @durran
+bindings/node @nbbeeken @durran @baileympearson @dariakp @addaleax
# Python Bindings
-bindings/python @ShaneHarvey @prashantmital
+bindings/python @ShaneHarvey @blink1073 @juliusgeo
# Java Bindings
bindings/java @jyemin @rozza
diff -pruN 1.3.0-1/debian/changelog 1.5.2-1/debian/changelog
--- 1.3.0-1/debian/changelog 2021-11-04 21:38:47.000000000 +0000
+++ 1.5.2-1/debian/changelog 2022-07-30 15:07:08.000000000 +0000
@@ -1,3 +1,28 @@
+libmongocrypt (1.5.2-1) unstable; urgency=medium
+
+ * New upstream release.
+
+ -- Roberto C. Sanchez Sat, 30 Jul 2022 11:07:08 -0400
+
+libmongocrypt (1.5.0-1) unstable; urgency=medium
+
+ * New upstream release.
+ * Update to Standards-Version 4.6.1 (no changes)
+
+ -- Roberto C. Sanchez Tue, 28 Jun 2022 16:05:33 -0400
+
+libmongocrypt (1.4.0-1) unstable; urgency=medium
+
+ * New upstream release.
+
+ -- Roberto C. Sanchez Mon, 18 Apr 2022 18:46:10 -0400
+
+libmongocrypt (1.3.1-1) unstable; urgency=medium
+
+ * New upstream release.
+
+ -- Roberto C. Sanchez Wed, 02 Mar 2022 14:05:42 -0500
+
libmongocrypt (1.3.0-1) unstable; urgency=medium
* New upstream release.
diff -pruN 1.3.0-1/debian/control 1.5.2-1/debian/control
--- 1.3.0-1/debian/control 2021-11-04 21:38:47.000000000 +0000
+++ 1.5.2-1/debian/control 2022-07-30 15:07:08.000000000 +0000
@@ -8,7 +8,7 @@ Build-Depends: debhelper (>= 10),
libssl-dev,
pkg-config,
libbson-dev
-Standards-Version: 4.6.0
+Standards-Version: 4.6.1
Section: libs
Homepage: https://github.com/mongodb/libmongocrypt
diff -pruN 1.3.0-1/debian/gbp.conf 1.5.2-1/debian/gbp.conf
--- 1.3.0-1/debian/gbp.conf 2021-11-04 21:38:47.000000000 +0000
+++ 1.5.2-1/debian/gbp.conf 2022-07-30 15:07:08.000000000 +0000
@@ -8,18 +8,12 @@ prebuild = bash -c "
set -x &&
# Use debian/changelog (not VERSION_CURRENT) to get upstream version to build
upstream_version=$(dpkg-parsechangelog | sed -E -n 's/^Version: +(.*)-.*/\1/p') &&
- # Determine if we are on a release branch and set the reference to use later
- # to create the upstream tarball (when on a release branch use the most
- # recent release tag, and when on any other branch use HEAD)
- release_branch=$(cd $GBP_GIT_DIR/..; git symbolic-ref --short HEAD | sed -n '/^r[0-9]\./p') &&
- if [ -n \"\${release_branch}\" ]; then archive_ref=$(cd $GBP_GIT_DIR/..; git describe --tags --abbrev=0 --match '1.*'); else archive_ref=HEAD; fi &&
# Create upstream tarball from reference, exclude items that do not belong
pushd $GBP_GIT_DIR/.. &&
- ls -1 &&
- git archive --format=tar --prefix=libmongocrypt-\${upstream_version}/ \${archive_ref} | tar -f - --delete libmongocrypt-\${upstream_version}/debian | gzip > $GBP_BUILD_DIR/../libmongocrypt_\${upstream_version}.orig.tar.gz &&
+ git archive --format=tar --prefix=libmongocrypt-\${upstream_version}/ HEAD | tar -f - --delete libmongocrypt-\${upstream_version}/debian | gzip > $GBP_BUILD_DIR/../libmongocrypt_\${upstream_version}.orig.tar.gz &&
popd"
upstream-tree = BRANCH
-upstream-branch = debian/1.3.0
+upstream-branch = r1.5
upstream-tag = %(version)s
-debian-branch = debian/1.3.0
+debian-branch = r1.5
diff -pruN 1.3.0-1/debian/rules 1.5.2-1/debian/rules
--- 1.3.0-1/debian/rules 2021-11-04 21:38:47.000000000 +0000
+++ 1.5.2-1/debian/rules 2022-07-30 15:07:08.000000000 +0000
@@ -25,7 +25,9 @@ export DEB_CFLAGS_MAINT_APPEND = -fPIC
override_dh_auto_configure:
dh_auto_configure -B$(CURDIR)/cmake-build -- \
-DBUILD_VERSION=$(DEB_VERSION_UPSTREAM) \
- -DENABLE_SHARED_BSON=ON \
+ -DUSE_SHARED_LIBBSON=ON \
+ -DMONGOCRYPT_MONGOC_DIR=USE-SYSTEM \
+ -DENABLE_ONLINE_TESTS=OFF \
-DCMAKE_BUILD_TYPE=RelWithDebInfo
override_dh_auto_build:
diff -pruN 1.3.0-1/debian/source/lintian-overrides 1.5.2-1/debian/source/lintian-overrides
--- 1.3.0-1/debian/source/lintian-overrides 2021-11-04 21:38:47.000000000 +0000
+++ 1.5.2-1/debian/source/lintian-overrides 2022-07-30 15:07:08.000000000 +0000
@@ -1,3 +1,3 @@
# This file is a source file. The presence of base64-encoded cryptographic
# keys results in very long lines.
-libmongocrypt source: source-is-missing bindings/node/test/cryptoCallbacks.test.js
+libmongocrypt source: source-is-missing [bindings/node/test/cryptoCallbacks.test.js]
diff -pruN 1.3.0-1/etc/fle2_aead_generate_tests.py 1.5.2-1/etc/fle2_aead_generate_tests.py
--- 1.3.0-1/etc/fle2_aead_generate_tests.py 1970-01-01 00:00:00.000000000 +0000
+++ 1.5.2-1/etc/fle2_aead_generate_tests.py 2022-07-30 15:14:43.000000000 +0000
@@ -0,0 +1,41 @@
+# fle2_aead_generate_tests.py is used to generate the test file: ./test/data/fle2-aead.cstructs
+import os
+import itertools
+import fle2_crypto
+
+# Generate test cases by taking the cross-product of AD (associated data) and plaintext (M) lengths.
+AD_lens = [5, 10, 20]
+M_lens = [1, 16, 64, 100]
+
+tmpl = """{{
+ .testname = "{testname}",
+ .iv = "{iv}",
+ .associated_data = "{associated_data}",
+ .key = "{key}",
+ .plaintext = "{plaintext}",
+ .ciphertext = "{ciphertext}",
+ .bytes_written_expected = {bytes_written_expected}
+}},"""
+
+print("/* This file is generated by ./etc/fle2_aead_generate_tests.py. */")
+for (AD_len, M_len) in itertools.product(AD_lens, M_lens):
+ M = os.urandom(M_len)
+ AD = os.urandom(AD_len)
+ IV = os.urandom(16)
+ Ke = os.urandom(32)
+ Km = os.urandom(32)
+ C = fle2_crypto.fle2aead_encrypt(M=M, Ke=Ke, IV=IV, Km=Km, AD=AD)
+
+ # Create the 96 byte data encryption key. The last 32 are unused.
+ key = Ke + Km + (b"\x00" * 32)
+ args = {
+ "testname": "generated test. AD length={}, M length={}".format(AD_len, M_len),
+ "iv": IV.hex(),
+ "associated_data": AD.hex(),
+ "key": key.hex(),
+ "plaintext": M.hex(),
+ "ciphertext": C.hex(),
+ "bytes_written_expected": len(C)
+ }
+ testcase = tmpl.format(**args)
+ print(testcase)
diff -pruN 1.3.0-1/etc/fle2_crypto.py 1.5.2-1/etc/fle2_crypto.py
--- 1.3.0-1/etc/fle2_crypto.py 1970-01-01 00:00:00.000000000 +0000
+++ 1.5.2-1/etc/fle2_crypto.py 2022-07-30 15:14:43.000000000 +0000
@@ -0,0 +1,105 @@
+from cryptography.hazmat.primitives import hashes, hmac
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+
+import struct
+
+ENCRYPTION_KEY_LENGTH = 32
+MAC_KEY_LENGTH = 32
+HMAC_SHA256_TAG_LENGTH = 32
+IV_LENGTH = 16
+DEK_LENGTH = 96
+
+def _hmacsha256 (Km, input):
+ assert (len(Km) == MAC_KEY_LENGTH)
+ hm = hmac.HMAC(Km, hashes.SHA256())
+ hm.update (input)
+ return hm.finalize()
+
+class DEK ():
+ """
+ Class representing a Data Encryption Key (DEK)
+ """
+ def __init__ (self, bytesIn):
+ assert (len(bytesIn) == DEK_LENGTH)
+ self.Ke = bytesIn[0:32]
+ self.Km = bytesIn[32:64]
+ self.TokenKey = bytesIn[64:96]
+
+def fle2_encrypt (M, Ke, IV):
+ """
+ Compute
+ S = AES-CTR.Enc(Ke, IV, M)
+
+ Output
+ C = IV || S
+ """
+ assert (len(Ke) == ENCRYPTION_KEY_LENGTH)
+ assert (len(IV) == IV_LENGTH)
+
+ # S = AES-CTR.Enc(Ke, IV, M)
+ cipher = Cipher(algorithms.AES(Ke), modes.CTR(IV))
+ encryptor = cipher.encryptor()
+ S = encryptor.update(M) + encryptor.finalize()
+
+ C = IV + S
+ return C
+
+def fle2_decrypt (C, Ke):
+ assert (len(Ke) == ENCRYPTION_KEY_LENGTH)
+ assert (len(C) > IV_LENGTH)
+
+ IV = C[0:IV_LENGTH]
+ # S = AES-CTR.Enc(Ke, IV, M)
+ cipher = Cipher(algorithms.AES(Ke), modes.CTR(IV))
+ encryptor = cipher.decryptor()
+ M = encryptor.update(C[IV_LENGTH:]) + encryptor.finalize()
+ return M
+
+def fle2aead_encrypt(M, Ke, IV, Km, AD):
+ """
+ Do FLE 2 AEAD encryption.
+ See [AEAD with CTR](https://docs.google.com/document/d/1eCU7R8Kjr-mdyz6eKvhNIDVmhyYQcAaLtTfHeK7a_vE/edit#heading=h.35kjadvlcbty)
+ See [aead_encryption_fle2_test_vectors.sh](https://github.com/mongodb/mongo/blob/ecc66915ac757cbeaa7c40eb443d7ec7bffcb80a/src/mongo/crypto/scripts/aead_encryption_fle2_test_vectors.sh#L15) for how server team is generating this.
+ """
+ assert (len(Ke) == ENCRYPTION_KEY_LENGTH)
+ assert (len(IV) == IV_LENGTH)
+ assert (len(Km) == MAC_KEY_LENGTH)
+
+ # S = AES-CTR.Enc(Ke, IV, M)
+ cipher = Cipher(algorithms.AES(Ke), modes.CTR(IV))
+ encryptor = cipher.encryptor()
+ S = encryptor.update(M) + encryptor.finalize()
+
+ # T = HMAC-SHA256(Km, AD || IV || S)
+ T = _hmacsha256 (Km, AD + IV + S)
+
+ # C = IV || S || T
+ C = IV + S + T
+ return C
+
+
+def fle2aead_decrypt(C, Km, AD, Ke):
+ assert (len(Ke) == ENCRYPTION_KEY_LENGTH)
+ assert (len(C) > HMAC_SHA256_TAG_LENGTH + IV_LENGTH)
+ assert (len(Km) == MAC_KEY_LENGTH)
+
+ # Parse C as IV || S || T
+ IV = C[0:IV_LENGTH]
+ S = C[IV_LENGTH:-HMAC_SHA256_TAG_LENGTH]
+ T = C[-HMAC_SHA256_TAG_LENGTH:]
+
+ # Compute T' = HMAC-SHA256(Km, AD || IV || S)
+ Tp = _hmacsha256 (Km, AD + IV + S)
+ if Tp != T:
+ raise Exception("decryption error")
+
+ # Else compute and output M = AES-CTR.Dec(Ke, S)
+ cipher = Cipher(algorithms.AES(Ke), modes.CTR(IV))
+ decryptor = cipher.decryptor()
+ M = decryptor.update(S) + decryptor.finalize()
+
+ return M
+
+
+def ServerDataEncryptionLevel1Token (rootKey):
+ return _hmacsha256 (rootKey, struct.pack("
"""
+from fnmatch import fnmatch
import sys
import json
+import os
if len(sys.argv) != 2:
print(description)
@@ -11,5 +13,8 @@ if len(sys.argv) != 2:
compile_db = json.load(open(sys.argv[1] + "compile_commands.json", "r"))
for entry in compile_db:
- print(entry["file"])
-print("")
\ No newline at end of file
+ fpath = entry['file']
+ fname = os.path.basename(fpath)
+ if fnmatch(fname, 'mc-*') or fnmatch(fname, 'mongocrypt-*'):
+ print(fpath)
+print("")
diff -pruN 1.3.0-1/etc/packager.py 1.5.2-1/etc/packager.py
--- 1.3.0-1/etc/packager.py 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/etc/packager.py 2022-07-30 15:14:43.000000000 +0000
@@ -208,6 +208,8 @@ class Distro(object):
return "ppc64le"
elif arch == "s390x":
return "s390x"
+ elif arch == "arm64":
+ return "aarch64"
elif arch.endswith("86"):
return "i686"
return "x86_64"
diff -pruN 1.3.0-1/.evergreen/build_all.sh 1.5.2-1/.evergreen/build_all.sh
--- 1.3.0-1/.evergreen/build_all.sh 2021-11-04 21:48:25.000000000 +0000
+++ 1.5.2-1/.evergreen/build_all.sh 2022-07-30 15:14:43.000000000 +0000
@@ -18,6 +18,28 @@ evergreen_root="$(pwd)"
. ${evergreen_root}/libmongocrypt/.evergreen/setup-env.sh
+# We may need some more C++ flags
+_cxxflags=""
+
+# Use C driver helper script to find cmake binary, stored in $CMAKE.
+if [ "$OS" == "Windows_NT" ]; then
+ : "${CMAKE:=/cygdrive/c/cmake/bin/cmake}"
+ # Enable exception handling for MSVC
+ _cxxflags="-EHsc"
+ if [ "$WINDOWS_32BIT" != "ON" ]; then
+ ADDITIONAL_CMAKE_FLAGS="-Thost=x64 -A x64"
+ fi
+else
+ # Amazon Linux 2 (arm64) has a very old system CMake we want to ignore
+ IGNORE_SYSTEM_CMAKE=1 . ${evergreen_root}/libmongocrypt/.evergreen/find-cmake.sh
+ # Check if on macOS with arm64. Use system cmake. See BUILD-14565.
+ OS_NAME=$(uname -s | tr '[:upper:]' '[:lower:]')
+ MARCH=$(uname -m | tr '[:upper:]' '[:lower:]')
+ if [ "darwin" = "$OS_NAME" -a "arm64" = "$MARCH" ]; then
+ CMAKE=cmake
+ fi
+fi
+
if [ "$PPA_BUILD_ONLY" ]; then
# Clean-up from previous build iteration
cd $evergreen_root
@@ -25,18 +47,26 @@ if [ "$PPA_BUILD_ONLY" ]; then
ADDITIONAL_CMAKE_FLAGS="${ADDITIONAL_CMAKE_FLAGS} -DENABLE_BUILD_FOR_PPA=ON"
fi
-. ${evergreen_root}/libmongocrypt/.evergreen/build_install_bson.sh
+if [ "$MACOS_UNIVERSAL" = "ON" ]; then
+ ADDITIONAL_CMAKE_FLAGS="$ADDITIONAL_CMAKE_FLAGS -DCMAKE_OSX_ARCHITECTURES='arm64;x86_64'"
+fi
cd