mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-101469 Introduce Bazel targets for certificate generation (#45530)

Browse Source 
GitOrigin-RevId: 74bb3718d95f7b34f171ec68a328bfa5886ae290
This commit is contained in:
Gabriel Marks 2025-12-29 13:56:06 -05:00 committed by MongoDB Bot
parent d4ffabf953
commit fbf7499622
16 changed files with 2457 additions and 1328 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/CODEOWNERS vendored
View File

@ -1303,6 +1303,9 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
# The following patterns are parsed from ./jstests/noPassthrough/libs/query/OWNERS.yml
/jstests/noPassthrough/libs/query/**/* @10gen/query @svc-auto-approve-bot
# The following patterns are parsed from ./jstests/noPassthrough/libs/x509/OWNERS.yml
/jstests/noPassthrough/libs/x509/**/* @10gen/server-security @svc-auto-approve-bot
# The following patterns are parsed from ./jstests/noPassthrough/local_catalog/OWNERS.yml
/jstests/noPassthrough/local_catalog/**/* @10gen/server-catalog-and-routing-shard-catalog @svc-auto-approve-bot
@ -1475,6 +1478,9 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
# The following patterns are parsed from ./jstests/noPassthrough/wt_integration/OWNERS.yml
/jstests/noPassthrough/wt_integration/**/* @10gen/server-storage-engine-integration @svc-auto-approve-bot
# The following patterns are parsed from ./jstests/noPassthrough/x509/OWNERS.yml
/jstests/noPassthrough/x509/**/* @10gen/server-security @svc-auto-approve-bot
# The following patterns are parsed from ./jstests/noPassthrough/zones/OWNERS.yml
/jstests/noPassthrough/zones/**/* @10gen/server-cluster-scalability @svc-auto-approve-bot

12
jstests/noPassthrough/libs/x509/BUILD.bazel Normal file
View File

@ -0,0 +1,12 @@
load("//bazel:mongo_js_rules.bzl", "all_subpackage_javascript_files", "mongo_js_library")
package(default_visibility = ["//visibility:public"])
mongo_js_library(
name = "all_javascript_files",
srcs = glob([
"*.js",
]),
)
all_subpackage_javascript_files()

5
jstests/noPassthrough/libs/x509/OWNERS.yml Normal file
View File

@ -0,0 +1,5 @@
version: 1.0.0
filters:
- "*":
approvers:
- 10gen/server-security

1
jstests/noPassthrough/libs/x509/apple_certs.json Normal file
View File

@ -0,0 +1 @@
{"certs":[{"Issuer":"self","Subject":{"CN":"Trusted MacOS Kernel Test CA"},"description":"CA for trusted MacOS client/server certificate chain.","extensions":{"basicConstraints":{"CA":true},"subjectAltName":{"DNS":"localhost","IP":"127.0.0.1"}},"keyfile":"macos_ca_key.pem","name":"macos-trusted-ca.pem"},{"Issuer":"macos-trusted-ca.pem","Subject":{"CN":"Trusted MacOS Kernel Test Client"},"description":"Client certificate for trusted MacOS chain.","extensions":{"extendedKeyUsage":["clientAuth"],"subjectAltName":{"DNS":"localhost","IP":"127.0.0.1"}},"name":"macos-trusted-client.pem","pkcs12":{"name":"macos-trusted-client.pfx","passphrase":"qwerty"}},{"Issuer":"macos-trusted-ca.pem","Subject":{"CN":"Trusted MacOS Kernel Test Server"},"description":"Server certificate for trusted MacOS chain.","extensions":{"extendedKeyUsage":["serverAuth"],"subjectAltName":{"DNS":"localhost","IP":"127.0.0.1"}},"name":"macos-trusted-server.pem","pkcs12":{"name":"macos-trusted-server.pfx","passphrase":"qwerty"}}],"global":{"Subject":{"C":"US","L":"New York City","O":"MongoDB","OU":"Kernel","ST":"New York"},"keyfile":"macos_key.pem"}}

1
jstests/noPassthrough/libs/x509/main_certs.json Normal file
View File

File diff suppressed because one or more lines are too long

5
jstests/noPassthrough/x509/OWNERS.yml Normal file
View File

@ -0,0 +1,5 @@
version: 1.0.0
filters:
- "*":
approvers:
- 10gen/server-security

33
jstests/noPassthrough/x509/mkcert_is_deterministic.js
View File

@ -1,11 +1,16 @@
// Test that mkcert.py generates certificates deterministically, and that pkcs12 certificates, while
// not deterministic, can be generated.
/**
* Test that mkcert.py generates certificates deterministically, and that pkcs12 certificates, while
* not deterministic, can be generated. Uses the libs/x509/apple_certs.json and main_certs.json
* files, which are static copies of the JSON files generated by the x509:generate_main_certificates
* and x509:generate_apple_certificates bazel targets.
*/
import {getPython3Binary} from "jstests/libs/python.js";
const now = new Date();
// Note - month is 0-indexed, so 11 is December.
if (now.getMonth() == 11 && now.getDate() == 31 && now.getHours() == 23) {
jsTestLog(
jsTest.log.warning(
"Deterministic certificate generation relies on the current year being constant; skipping test as there is less than an hour until the year changes.",
);
quit();
@ -14,17 +19,19 @@ if (now.getMonth() == 11 && now.getDate() == 31 && now.getHours() == 23) {
// Print the openssl version to help with debugging.
clearRawMongoProgramOutput();
assert.eq(runNonMongoProgram("openssl", "version"), 0);
jsTest.log(rawMongoProgramOutput(".*"));
jsTest.log.info(rawMongoProgramOutput(".*"));
const main_cert_json_file = "jstests/noPassthrough/libs/x509/main_certs.json";
const apple_cert_json_file = "jstests/noPassthrough/libs/x509/apple_certs.json";
const basedir = MongoRunner.dataPath + "certs/";
const genpath = basedir + "generated/";
mkdir(genpath);
// Run mkcert, and ensure it succeeds and that expected results are generated successfully.
jsTest.log("Running mkcert");
jsTest.log.info("Running mkcert");
clearRawMongoProgramOutput();
let res = runNonMongoProgram(getPython3Binary(), "-m", "x509.mkcert", "--mkcrl", "-o", genpath);
jsTest.log(rawMongoProgramOutput(".*"));
let res = runNonMongoProgram(getPython3Binary(), "x509/mkcert.py", main_cert_json_file, "--mkcrl", "-o", genpath);
jsTest.log.info(rawMongoProgramOutput(".*"));
assert.eq(res, 0);
assert(fileExists(genpath + "ca.pem"));
assert(fileExists(genpath + "crl.pem"));
@ -32,21 +39,21 @@ assert(fileExists(genpath + "crl.pem"));
// Run mkcert again, to a different path.
const genpath2 = basedir + "generated2/";
mkdir(genpath2);
jsTest.log("Running mkcert again");
res = runNonMongoProgram(getPython3Binary(), "-m", "x509.mkcert", "--mkcrl", "-o", genpath2);
jsTest.log.info("Running mkcert again");
res = runNonMongoProgram(getPython3Binary(), "x509/mkcert.py", main_cert_json_file, "--mkcrl", "-o", genpath2);
assert.eq(res, 0);
// Diff the two generation paths to make sure the contents of the paths are identical.
jsTest.log("Running diff");
jsTest.log.info("Running diff");
clearRawMongoProgramOutput();
res = runNonMongoProgram("diff", "-r", genpath, genpath2);
assert.eq(res, 0);
const diffout = rawMongoProgramOutput(".*").trim();
assert.eq("", diffout, diffout);
// Run mkcert on the apple-certs.yml definitions file, which contains pkcs12 certificates, and
// Run mkcert on the apple cert definitions file, which contains pkcs12 certificates, and
// ensure a .pfx file was generated.
jsTest.log("Running apple certs");
res = runNonMongoProgram(getPython3Binary(), "-m", "x509.mkcert", "-o", genpath, "--config", "x509/apple-certs.yml");
jsTest.log.info("Running apple certs");
res = runNonMongoProgram(getPython3Binary(), "x509/mkcert.py", apple_cert_json_file, "-o", genpath);
assert.eq(res, 0);
assert(fileExists(genpath + "macos-trusted-server.pfx"));

17
x509/BUILD.bazel Normal file
View File

@ -0,0 +1,17 @@
load(":generate_certificates.bzl", "generate_certificates")
load(":main_certs_def.bzl", main_certs_def = "certs_def")
load(":apple_certs_def.bzl", apple_certs_def = "certs_def")
package(default_visibility = ["//visibility:public"])
generate_certificates(
name = "generate_main_certificates",
certs_def = main_certs_def,
static_inputs = glob(["static/**"]),
)
generate_certificates(
name = "generate_apple_certificates",
certs_def = apple_certs_def,
static_inputs = glob(["static/**"]),
)

140
x509/README
View File

@ -2,20 +2,23 @@ This directory contains:
- mkcert.py
Python script; uses the cryptography package to deterministically generate X509 certificates,
CRLs, and digests based on the contents of certs.yml.
- certs.yml
Main certificate definition file.
- apple-certs.yml
CRLs, and digests based on the contents of the specified certificate definition file.
- main_certs_def.bzl
Main certificate definitions, embedded in a Bazel definition file.
- apple_certs_def.bzl
Certificate definitions for certs to be installed on provision of OSX machines.
To run:
python -m x509.mkcert [--config CONFIG] [--mkcrl | --no-mkcrl] [-o OUTPUT] [--static-dir STATIC_DIR] [certs ...]
- CONFIG is the path to the certs.yml file specifying a list of certificates, default x509/certs.yml
python x509/mkcert.py CONFIG [--mkcrl | --no-mkcrl] [-o OUTPUT] [--static-dir STATIC_DIR]
[--dry-run] [certs ...]
- CONFIG is the path to the JSON file specifying a list of certificates, required
- OUTPUT is the path to a directory where the generated items will be stored, default .
- STATIC_DIR is the path where signing keys needed by certificates are stored, default x509/static
- If --mkcrl is specified, CRLs will be generated after certificate generation ends. Default false.
These are hardcoded and require certain certificates to be generated to work.
- If --dry-run is specified, no files will be written out. This can be used to test what files will
be written and where.
- certs is an optional list of certificate names to generate. If it is not specified, all
certificates specified in the config are generated.
@ -33,61 +36,72 @@ Future work:
- Define keys in the definition file, and make a script to generate all necessary keys which would
be run whenever a new key was defined.
certs.yml format:
Certificate definition format:
global: # Optional, default value to use for Key1 for all certs, overridden by values in cert entries.
Key1: Value1
...
certs:
# Required, this will be used as the name of the file, and for referencing issuers.
- name: 'name-of-cert.pem'
# Required, this will be included in the header of the generated certificate.
description: Tell us about yourself.
# Required, The X509 subject name.
Subject: { C: US, ST: New York, etc... }
# Required, Who is the (intermediate) CA for this certificate. May be 'self'.
Issuer: 'ca.pem'
# Required, relative (within static directory) path to the keyfile to sign this certificate with.
keyfile: 'key.pem'
# Optional, set to true to ignore global.Subject values.
explicit_subject: false
# Optional, serial number to assign this certificate (default: sequential numbers starting from 1000)
serial: 42
# Optional, validity start date, expressed in seconds relative to midnight on the first day of the current year.
not_before: -86400 # 1 day before
# Optional, validity end date, currently expressed in seconds relative to midnight on the first day of the current year.
# Note that not_after - not_before, the validity period, should be less than or equal to 825 days, see:
# https://support.apple.com/en-us/HT210176
not_after: 71107200 # 823 days after
# Optional, IDs of other public keys to append to the file
append_certs: ['ca.pem', 'intermediate-ca.pem', ...]
# Optional, passphrase to encript private key with
passphrase: 'secret'
# Optional, make a pkcs12 copy of the certificate
pkcs12: true | map with keys below
# Optional, all PKCS#12 keys must be encrypted. Will use cert.passphase if not provided.
passphrase: 'secret'
# Optional, name of PKCS#12 version of certificate. If not provided, the original cert will be overwritten with the PKCS#12 version
name: 'name-of-cert.pfx'
# Optional, in addition to the .pem file, write just the certificate to a .crt file and just the signing key to a .key file
split_cert_and_key: true
# Optional, don't write a header comment to this cert
include_header: false
# Optional, X.509 extensions to include in the certificate
extensions: # All extensions are optional.
- basicConstraints: {}
- keyUsage: {}
- extendedKeyUsage: {}
- subjectAltName: {DNS: [...], IP: [...]}
- subjectKeyIdentifier: hash
- authorityKeyIdentifier: keyid | issuer
- authorityInfoAccess:
- method: OCSP
- location: uri-to-OCSP-server
- mustStaple: true
- nsComment: "Comment"
- mongoRoles:
- {role: readWrite, db: test1}
- {role: read, db: test2}
- mongoClusterMembership: clusterName
{
"global": {
# Optional, default value to use for Key1 for all certs, overridden by values in cert entries.
"Key1": "Value1",
...
},
"certs": [
{
# Required, this will be used as the name of the file, and for referencing issuers.
"name": "name-of-cert.pem",
# Required, this will be included in the header of the generated certificate.
"description": "Tell us about yourself.",
# Required, The X509 subject name.
"Subject": { "C": "US", "ST": "New York", ... },
# Required, Who is the (intermediate) CA for this certificate. May be 'self'.
"Issuer": "ca.pem",
# Required, relative (within static directory) path to the keyfile to sign this certificate with.
"keyfile": "key.pem",
# Optional, set to true to ignore global.Subject values.
"explicit_subject": False,
# Optional, serial number to assign this certificate (default: sequential numbers starting from 1000)
"serial": 42,
# Optional, validity start date, expressed in seconds relative to midnight on the first day of the current year.
"not_before": -86400, # 1 day before
# Optional, validity end date, currently expressed in seconds relative to midnight on the first day of the current year.
# Note that not_after - not_before, the validity period, should be less than or equal to 825 days, see:
# https://support.apple.com/en-us/HT210176
"not_after": 71107200, # 823 days after
# Optional, IDs of other public keys to append to the file
"append_certs": ["ca.pem", "intermediate-ca.pem", ...],
# Optional, passphrase to encript private key with
"passphrase": "secret",
# Optional, make a pkcs12 copy of the certificate
"pkcs12": True | {
# Optional, all PKCS#12 keys must be encrypted. Will use cert.passphase if not provided.
"passphrase": "secret",
# Optional, name of PKCS#12 version of certificate. If not provided, the original cert will be overwritten with the PKCS#12 version
"name": "name-of-cert.pfx",
},
# Optional, in addition to the .pem file, write just the certificate to a .crt file and just the signing key to a .key file
"split_cert_and_key": True,
# Optional, don't write a header comment to this cert
"include_header": False,
# Optional, X.509 extensions to include in the certificate
"extensions": { # All extensions are optional.
"basicConstraints": {},
"keyUsage": {},
"extendedKeyUsage": {},
"subjectAltName": {"DNS": [...], "IP": [...]},
"subjectKeyIdentifier": "hash",
"authorityKeyIdentifier": "keyid" | "issuer",
"authorityInfoAccess": {
"method": "OCSP",
"location": "uri-to-OCSP-server",
},
"mustStaple": True,
"nsComment": "Comment",
"mongoRoles": [
{"role": "readWrite", "db": "test1"},
{"role": "read", "db": "test2"}
],
"mongoClusterMembership": "clusterName",
}
},
...
]
}

49
x509/apple-certs.yml
View File

@ -1,49 +0,0 @@
# Definition for testing certificates to be created and installed into the MacOS trusted keychain upon provision.
# On MacOS provision, these certificates will be written to /opt/x509.
global:
Subject:
C: "US"
ST: "New York"
L: "New York City"
O: "MongoDB"
OU: "Kernel"
keyfile: "macos_key.pem"
certs:
- name: "macos-trusted-ca.pem"
description: CA for trusted MacOS client/server certificate chain.
Subject: {CN: "Trusted MacOS Kernel Test CA"}
Issuer: self
keyfile: "macos_ca_key.pem"
extensions:
basicConstraints: {CA: true}
subjectAltName:
DNS: localhost
IP: 127.0.0.1
- name: "macos-trusted-client.pem"
description: Client certificate for trusted MacOS chain.
Subject: {CN: "Trusted MacOS Kernel Test Client"}
Issuer: "macos-trusted-ca.pem"
pkcs12:
passphrase: "qwerty"
name: "macos-trusted-client.pfx"
extensions:
extendedKeyUsage: [clientAuth]
subjectAltName:
DNS: localhost
IP: 127.0.0.1
- name: "macos-trusted-server.pem"
description: Server certificate for trusted MacOS chain.
Subject: {CN: "Trusted MacOS Kernel Test Server"}
Issuer: "macos-trusted-ca.pem"
pkcs12:
passphrase: "qwerty"
name: "macos-trusted-server.pfx"
extensions:
extendedKeyUsage: [serverAuth]
subjectAltName:
DNS: localhost
IP: 127.0.0.1

76
x509/apple_certs_def.bzl Normal file
View File

@ -0,0 +1,76 @@
# Definitions for certificates for MacOS-only certificate generation (the
# generate_apple_certificates target).
certs_def = json.encode({
"global": {
"Subject": {
"C": "US",
"ST": "New York",
"L": "New York City",
"O": "MongoDB",
"OU": "Kernel",
},
"keyfile": "macos_key.pem",
},
"certs": [
{
"name": "macos-trusted-ca.pem",
"description": "CA for trusted MacOS client/server certificate chain.",
"Subject": {
"CN": "Trusted MacOS Kernel Test CA",
},
"Issuer": "self",
"keyfile": "macos_ca_key.pem",
"extensions": {
"basicConstraints": {
"CA": True,
},
"subjectAltName": {
"DNS": "localhost",
"IP": "127.0.0.1",
},
},
},
{
"name": "macos-trusted-client.pem",
"description": "Client certificate for trusted MacOS chain.",
"Subject": {
"CN": "Trusted MacOS Kernel Test Client",
},
"Issuer": "macos-trusted-ca.pem",
"pkcs12": {
"passphrase": "qwerty",
"name": "macos-trusted-client.pfx",
},
"extensions": {
"extendedKeyUsage": [
"clientAuth",
],
"subjectAltName": {
"DNS": "localhost",
"IP": "127.0.0.1",
},
},
},
{
"name": "macos-trusted-server.pem",
"description": "Server certificate for trusted MacOS chain.",
"Subject": {
"CN": "Trusted MacOS Kernel Test Server",
},
"Issuer": "macos-trusted-ca.pem",
"pkcs12": {
"passphrase": "qwerty",
"name": "macos-trusted-server.pfx",
},
"extensions": {
"extendedKeyUsage": [
"serverAuth",
],
"subjectAltName": {
"DNS": "localhost",
"IP": "127.0.0.1",
},
},
},
],
})

1076
x509/certs.yml
View File

File diff suppressed because it is too large Load Diff

111
x509/generate_certificates.bzl Normal file
View File

@ -0,0 +1,111 @@
load("@poetry//:dependencies.bzl", "dependency")
load("//bazel/config:render_template.bzl", "render_template")
def _generate_certificates(ctx):
python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime
python_libs = [py_dep[PyInfo].transitive_sources for py_dep in ctx.attr.py_libs]
python_path = []
for py_dep in ctx.attr.py_libs:
for path in py_dep[PyInfo].imports.to_list():
if path not in python_path:
python_path.append(ctx.expand_make_variables("python_library_imports", "$(BINDIR)/external/" + path, ctx.var))
# Write the cert definitions to a temporary file, which mkcert.py will read from.
certfile = ctx.actions.declare_file("." + ctx.label.name + ".certs.json")
ctx.actions.write(
output = certfile,
content = ctx.attr.certs_def,
)
certs_def = json.decode(ctx.attr.certs_def)
out_set = {}
# Statically compute output files from cert definitions
for cert in certs_def["certs"]:
# Each cert def generates cert.name and a SHA-256 and SHA-1 digest of cert.name.
out_set[cert["name"]] = None
out_set[cert["name"] + ".digest.sha256"] = None
out_set[cert["name"] + ".digest.sha1"] = None
if cert.get("split_cert_and_key", False):
# Split certs additionally generate .crt and .key files.
crt_name = cert["name"][:-len(".pem")] + ".crt"
key_name = cert["name"][:-len(".pem")] + ".key"
out_set[crt_name] = None
out_set[key_name] = None
if cert.get("pkcs12", None) != None:
# PKCS12 certs generate a separate cert bundle at cert.pkcs12.name.
pkcs12_name = cert["pkcs12"].get("name", cert["name"])
out_set[pkcs12_name] = None
should_gen_crls = False
if "crls" in certs_def:
for crl in certs_def["crls"]:
# Each CRL def generates crl and the two digests.
out_set[crl] = None
out_set[crl + ".digest.sha256"] = None
out_set[crl + ".digest.sha1"] = None
should_gen_crls = True
outputs = [ctx.actions.declare_file(out) for out in out_set]
# Run the Python script to generate the certificates, sending stdout to /dev/null to avoid
# cluttering the build log.
args = ctx.actions.args()
args.add(ctx.expand_location(ctx.attr.main))
args.add(certfile.path)
args.add("--mkcrl" if should_gen_crls else "--no-mkcrl")
args.add("--quiet")
args.add("--output", outputs[0].dirname)
ctx.actions.run(
executable = python.interpreter.path,
outputs = outputs,
inputs = depset(
direct = [certfile] + ctx.files.static_inputs,
transitive = [python.files, depset([arg.files.to_list()[0] for arg in ctx.attr.srcs])] + python_libs,
),
arguments = [args],
env = {"PYTHONPATH": ctx.configuration.host_path_separator.join(python_path)},
mnemonic = "CertificateGenerator",
)
return [DefaultInfo(files = depset(outputs))]
generate_certificates = rule(
implementation = _generate_certificates,
attrs = {
"static_inputs": attr.label_list(mandatory = True, allow_files = True, doc = "Static input files required to generate certificates."),
"certs_def": attr.string(mandatory = True, doc = "Definitions for all certificates."),
"srcs": attr.label_list(
doc = "The input files of this rule.",
allow_files = True,
default = [
Label("//x509:mkcert.py"),
],
),
"main": attr.string(
doc = "The main Python file to execute.",
default = "$(location //x509:mkcert.py)",
),
"py_libs": attr.label_list(
default = [
dependency(
"ecdsa",
group = "testing",
),
dependency(
"asn1crypto",
group = "testing",
),
dependency(
"pyyaml",
group = "core",
),
dependency(
"cryptography",
group = "platform",
),
],
),
},
toolchains = ["@bazel_tools//tools/python:toolchain_type"],
)

1955
x509/main_certs_def.bzl Normal file
View File

File diff suppressed because it is too large Load Diff

296
x509/mkcert.py
View File

@ -3,12 +3,12 @@ import argparse
import datetime
import hashlib
import ipaddress
import json
from pathlib import PurePath
from typing import Any, Dict
import asn1crypto.core as asn1
import cryptography.hazmat.primitives.serialization.pkcs12 as pkcs12
import yaml
from cryptography import x509
from cryptography.hazmat._oid import _OID_NAMES
from cryptography.hazmat.primitives import hashes, serialization
@ -16,6 +16,8 @@ from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.x509.extensions import UnrecognizedExtension
from cryptography.x509.oid import ObjectIdentifier
from ecdsa import SigningKey
import os
import sys
# Dictionary from common names to OIDs.
NAME_TO_OID = {v: k for k, v in _OID_NAMES.items()}
@ -38,8 +40,6 @@ CONFIG = Dict[str, Any]
MAX_VALIDITY_PERIOD_DAYS = 824
# Datetime to specify as the start time for all certs.
# TODO SERVER-101469 Make this a command-line argument and add it as a Bazel input so that Bazel
# knows when to rerun certificate generation.
DEFAULT_START_TIME = datetime.datetime(datetime.datetime.now().year, 1, 1)
# Allocate serial numbers sequentially; this is the last-used serial.
LAST_SERIAL_NUMBER = 999
@ -52,6 +52,13 @@ STATIC_PATH = None
LOADED_CERT_AND_KEYS = {}
# True if this is a dry run.
DRY_RUN = False
class CertificateGenerationError(Exception):
pass
def get_next_serial():
"""Get the next sequential serial number to use."""
@ -66,17 +73,20 @@ def get_key(cert):
"""Get the private key object loaded from keyfile."""
keyfile = idx(cert, "keyfile")
if keyfile is None:
raise ValueError("All certificates require a keyfile")
raise CertificateGenerationError("All certificates require a keyfile")
if keyfile not in LOADED_KEYS:
passphrase = cert.get("passphrase")
if passphrase is not None:
passphrase = bytes(passphrase, "ascii")
with open(str(STATIC_PATH / keyfile), "rb") as f:
LOADED_KEYS[keyfile] = serialization.load_pem_private_key(
f.read(),
password=passphrase,
)
if DRY_RUN:
LOADED_KEYS[keyfile] = "dummy"
else:
passphrase = cert.get("passphrase")
if passphrase is not None:
passphrase = bytes(passphrase, "ascii")
with open(str(STATIC_PATH / keyfile), "rb") as f:
LOADED_KEYS[keyfile] = serialization.load_pem_private_key(
f.read(),
password=passphrase,
)
return LOADED_KEYS[keyfile]
@ -109,16 +119,14 @@ def get_header_comment(cert):
if not cert.get("include_header", True):
return ""
"""Header comment for every generated file."""
comment = "# Autogenerated file, do not edit.\n"
comment = comment + "# Generate using python -m x509.mkcert --config " + CONFIGFILE
comment = comment + " " + cert["name"] + "\n#\n"
comment = comment + "# " + cert.get("description", "").replace("\n", "\n# ")
comment = comment + "\n"
comment = "# " + cert.get("description", "").replace("\n", "\n# ") + "\n"
return comment
def get_cert_and_key(cert_name):
"""Locate the cert and key file for a given cert name, load them, and return them."""
if DRY_RUN:
return "dummy", "dummy"
if cert_name in LOADED_CERT_AND_KEYS: # Cache hit, don't need to load again
return LOADED_CERT_AND_KEYS[cert_name]
ca_cert = find_certificate_definition(cert_name)
@ -164,7 +172,7 @@ def get_oid(cn_or_oid):
try:
return ObjectIdentifier(cn_or_oid)
except:
raise ValueError(f"Name attribute {cn_or_oid} not recognized")
raise CertificateGenerationError(f"Name attribute {cn_or_oid} not recognized")
def set_subject(builder, cert, set_issuer=False):
@ -175,7 +183,7 @@ def set_subject(builder, cert, set_issuer=False):
if set_issuer:
builder = builder.issuer_name(x509.Name([]))
return builder.subject_name(x509.Name([]))
raise ValueError(cert["name"] + " requires a Subject")
raise CertificateGenerationError(cert["name"] + " requires a Subject")
attr_dict = {}
if not cert.get("explicit_subject", False):
@ -220,7 +228,7 @@ def set_validity(builder, cert):
def to_der_varint(val):
"""Translate a native int to a variable length ASN.1 encoded integer."""
if val < 0:
raise ValueError("Negative values nor permitted in DER payload")
raise CertificateGenerationError("Negative values nor permitted in DER payload")
if val < 0x80:
return chr(val).encode("ascii")
@ -231,7 +239,7 @@ def to_der_varint(val):
val = val >> 8
if val > 0:
raise ValueError("Length is too large to represent in 64bits")
raise CertificateGenerationError("Length is too large to represent in 64bits")
ret.insert(0, 0x80 + len(ret))
return ret
@ -288,7 +296,7 @@ class ExtensionParser:
"OCSPSigning": 9,
}
if name not in ext_usage_name_map:
raise ValueError(f'Unknown extended key usage identifier: "{name}"')
raise CertificateGenerationError(f'Unknown extended key usage identifier: "{name}"')
return ObjectIdentifier("1.3.6.1.5.5.7.3." + str(ext_usage_name_map[name]))
@staticmethod
@ -312,7 +320,7 @@ class ExtensionParser:
for ip in val:
names.append(x509.IPAddress(ipaddress.ip_address(ip)))
else:
raise ValueError(f'Unknown subject alt name type: "{key}"')
raise CertificateGenerationError(f'Unknown subject alt name type: "{key}"')
return x509.SubjectAlternativeName(names)
@staticmethod
@ -326,7 +334,9 @@ class ExtensionParser:
pair = b""
for role in v:
if (len(role) != 2) or ("role" not in role) or ("db" not in role):
raise ValueError("mongoRoles must consist of a series of role/db pairs")
raise CertificateGenerationError(
"mongoRoles must consist of a series of role/db pairs"
)
pair = pair + to_der_sequence_pair(role["role"], role["db"])
val = b"\x31" + to_der_varint(len(pair)) + pair
@ -336,7 +346,7 @@ class ExtensionParser:
@staticmethod
def authority_key_identifier(v, issuer_public_key, issuer_ski, **_):
if v not in ["keyid", "issuer"]:
raise ValueError(
raise CertificateGenerationError(
"Only the 'keyid' or 'issuer' values are accepted for authorityKeyIdentifier"
)
@ -401,7 +411,7 @@ def set_extensions(builder, cert, **kwargs):
for key, val in extensions.items():
handler = ExtensionParser.parsers.get(key)
if handler is None:
raise ValueError(f'Extension "{key}" is not handled yet')
raise CertificateGenerationError(f'Extension "{key}" is not handled yet')
ext = handler(val, **kwargs)
if isinstance(val, list):
critical = "critical" in val
@ -410,7 +420,7 @@ def set_extensions(builder, cert, **kwargs):
elif isinstance(val, str) or isinstance(val, bool):
critical = False
else:
raise ValueError(f"Could not parse extension: {key} -> {val}")
raise CertificateGenerationError(f"Could not parse extension: {key} -> {val}")
builder = builder.add_extension(ext, critical=critical)
return builder
@ -474,9 +484,11 @@ def write_cert_as_pkcs12(cert, key, cert_obj, issuer_obj):
"""Makes a new copy of the cert/key pair using PKCS#12 encoding."""
pkcs12_opts = cert.get("pkcs12")
if not pkcs12_opts.get("passphrase"):
raise ValueError("PKCS#12 requires a passphrase")
raise CertificateGenerationError("PKCS#12 requires a passphrase")
fname = pkcs12_opts.get("name", cert["name"])
if DRY_RUN:
return
serialized = pkcs12.serialize_key_and_certificates(
fname.encode("ascii"),
key,
@ -493,73 +505,73 @@ def write_cert_as_pkcs12(cert, key, cert_obj, issuer_obj):
def process_normal_cert(cert):
"""Given a certificate definition which has a subject, deterministically generate its corresponding certificate file and store it in the output path."""
key = get_key(cert)
issuer_cert, issuer_key = get_issuer_cert_and_key(cert, key)
# Get SKI of issuer if it exists; we need it for the AuthorityKeyIdentifier extension
if issuer_cert == "self":
my_ski = cert.get("extensions", {}).get("subjectKeyIdentifier")
if my_ski is None:
issuer_ski = None
if not DRY_RUN:
issuer_cert, issuer_key = get_issuer_cert_and_key(cert, key)
# Get SKI of issuer if it exists; we need it for the AuthorityKeyIdentifier extension
if issuer_cert == "self":
my_ski = cert.get("extensions", {}).get("subjectKeyIdentifier")
if my_ski is None:
issuer_ski = None
else:
issuer_ski = ExtensionParser.subject_key_identifier(my_ski, key.public_key())
else:
issuer_ski = ExtensionParser.subject_key_identifier(my_ski, key.public_key())
else:
try:
issuer_ski = issuer_cert.extensions.get_extension_for_class(x509.SubjectKeyIdentifier)
except:
issuer_ski = None
try:
issuer_ski = issuer_cert.extensions.get_extension_for_class(
x509.SubjectKeyIdentifier
)
except:
issuer_ski = None
# Set all fields of the certificate.
builder = x509.CertificateBuilder()
builder = builder.public_key(key.public_key())
serial = cert.get("serial")
if serial is None:
serial = get_next_serial()
else:
serial = int(serial)
builder = builder.serial_number(serial)
builder = set_subject(builder, cert, set_issuer=issuer_cert == "self")
if issuer_cert != "self":
builder = builder.issuer_name(issuer_cert.subject)
builder = set_validity(builder, cert)
builder = set_extensions(
builder,
cert,
public_key=key.public_key(),
issuer_public_key=issuer_key.public_key(),
issuer_ski=issuer_ski,
)
# Set all fields of the certificate.
builder = x509.CertificateBuilder()
builder = builder.public_key(key.public_key())
serial = cert.get("serial")
if serial is None:
serial = get_next_serial()
else:
serial = int(serial)
builder = builder.serial_number(serial)
builder = set_subject(builder, cert, set_issuer=issuer_cert == "self")
if issuer_cert != "self":
builder = builder.issuer_name(issuer_cert.subject)
builder = set_validity(builder, cert)
builder = set_extensions(
builder,
cert,
public_key=key.public_key(),
issuer_public_key=issuer_key.public_key(),
issuer_ski=issuer_ski,
)
if isinstance(key, ec.EllipticCurvePrivateKey):
# For EC, we need to compute a deterministic signature ourselves. While newer versions of OpenSSL support deterministic signing with ECDSA, some of the platforms we run tests on use old versions, so we unfortunately cannot use this feature.
bad_sig_obj = builder.sign(key, hashes.SHA256())
cert_obj = sign_ecdsa_deterministic(key, bad_sig_obj)
else:
cert_obj = builder.sign(key, hashes.SHA256())
if isinstance(key, ec.EllipticCurvePrivateKey):
# For EC, we need to compute a deterministic signature ourselves. While newer versions of OpenSSL support deterministic signing with ECDSA, some of the platforms we run tests on use old versions, so we unfortunately cannot use this feature.
bad_sig_obj = builder.sign(key, hashes.SHA256())
cert_obj = sign_ecdsa_deterministic(key, bad_sig_obj)
else:
cert_obj = builder.sign(key, hashes.SHA256())
header = get_header_comment(cert)
cert_path = make_filename(cert)
# Write header + certificate PEM + key PEM to the output file.
with open(cert_path, "wt") as f:
f.write(header + cert_obj.public_bytes(serialization.Encoding.PEM).decode("ascii"))
with open(str(STATIC_PATH / idx(cert, "keyfile")), "r") as keyf:
f.write(keyf.read())
LOADED_CERT_AND_KEYS[cert["name"]] = (cert_obj, key)
header = get_header_comment(cert)
cert_path = make_filename(cert)
# Write header + certificate PEM + key PEM to the output file.
with open(cert_path, "wt") as f:
f.write(header + cert_obj.public_bytes(serialization.Encoding.PEM).decode("ascii"))
with open(str(STATIC_PATH / idx(cert, "keyfile")), "r") as keyf:
f.write(keyf.read())
LOADED_CERT_AND_KEYS[cert["name"]] = (cert_obj, key)
if cert.get("pkcs12", None) is not None:
write_cert_as_pkcs12(cert, key, cert_obj, issuer_cert)
if cert.get("split_cert_and_key", False):
# Write just the certificate to <path>.crt, and just the key to <path>.key
assert cert_path.endswith(".pem")
crt_path = cert_path[: -len(".pem")] + ".crt"
key_path = cert_path[: -len(".pem")] + ".key"
with open(crt_path, "wt") as f:
f.write(header + cert_obj.public_bytes(serialization.Encoding.PEM).decode("ascii"))
with open(key_path, "wt") as f:
with open(str(STATIC_PATH / idx(cert, "keyfile")), "r") as keyf:
f.write(header + keyf.read())
if cert.get("pkcs12", None) is not None:
write_cert_as_pkcs12(cert, key, cert_obj, issuer_cert)
assert cert["name"].endswith(".pem")
crt_name = cert["name"][: -len(".pem")] + ".crt"
key_name = cert["name"][: -len(".pem")] + ".key"
if not DRY_RUN:
with open(OUTPUT_PATH / crt_name, "wt") as f:
f.write(header + cert_obj.public_bytes(serialization.Encoding.PEM).decode("ascii"))
with open(OUTPUT_PATH / key_name, "wt") as f:
with open(str(STATIC_PATH / idx(cert, "keyfile")), "r") as keyf:
f.write(header + keyf.read())
def process_cert(cert):
@ -576,13 +588,16 @@ def process_cert(cert):
process_normal_cert(cert)
elif append_certs:
# Pure composing certificate. Start with a basic preamble.
with open(make_filename(cert), "wt") as f:
f.write(get_header_comment(cert) + "\n")
if not DRY_RUN:
with open(make_filename(cert), "wt") as f:
f.write(get_header_comment(cert) + "\n")
else:
raise ValueError(
raise CertificateGenerationError(
"Certificate definitions must have at least one of 'Subject' and/or 'append_cert'"
)
if DRY_RUN:
return
for cert_name in append_certs:
append_cert = get_cert_and_key(cert_name)[0]
header = (
@ -599,6 +614,9 @@ def write_digest(filename, item_type, digest_type):
"""Calculate the given digest of the certificate/CRL passed in and write it out to <filename>.digest.<digest_type>"""
assert item_type in {"cert", "crl"}
assert digest_type in DIGEST_NAME_TO_HASH
digest_path = str(filename) + ".digest." + digest_type
if DRY_RUN:
return
with open(filename, "rb") as f:
data = f.read()
@ -609,8 +627,7 @@ def write_digest(filename, item_type, digest_type):
rawdigest = obj.fingerprint(DIGEST_NAME_TO_HASH[digest_type])
towrite = rawdigest.hex().upper()
with open(str(filename) + ".digest." + digest_type, "w") as f:
with open(digest_path, "w") as f:
f.write(towrite)
@ -622,25 +639,26 @@ def generate_crl(issuer_cert, issuer_key, dest, cert_to_revoke=None):
:param cert_to_revoke: x509.Certificate object which this CRL should revoke. Empty for no revocation.
"""
print(f"Writing CRL: {dest}")
builder = (
x509.CertificateRevocationListBuilder()
.issuer_name(issuer_cert.subject)
.last_update(DEFAULT_START_TIME)
.next_update(DEFAULT_START_TIME + datetime.timedelta(days=MAX_VALIDITY_PERIOD_DAYS))
)
if cert_to_revoke is not None:
revoked_builder = (
x509.RevokedCertificateBuilder()
.serial_number(cert_to_revoke.serial_number)
.revocation_date(DEFAULT_START_TIME)
if not DRY_RUN:
builder = (
x509.CertificateRevocationListBuilder()
.issuer_name(issuer_cert.subject)
.last_update(DEFAULT_START_TIME)
.next_update(DEFAULT_START_TIME + datetime.timedelta(days=MAX_VALIDITY_PERIOD_DAYS))
)
builder = builder.add_revoked_certificate(revoked_builder.build())
crl = builder.sign(issuer_key, hashes.SHA256())
if cert_to_revoke is not None:
revoked_builder = (
x509.RevokedCertificateBuilder()
.serial_number(cert_to_revoke.serial_number)
.revocation_date(DEFAULT_START_TIME)
)
builder = builder.add_revoked_certificate(revoked_builder.build())
with open(dest, "wb") as f:
f.write(crl.public_bytes(serialization.Encoding.PEM))
crl = builder.sign(issuer_key, hashes.SHA256())
with open(dest, "wb") as f:
f.write(crl.public_bytes(serialization.Encoding.PEM))
write_digest(dest, "crl", "sha256")
write_digest(dest, "crl", "sha1")
@ -654,7 +672,7 @@ def generate_all_crls():
client_revoked, _ = get_cert_and_key("client_revoked.pem")
intermediate_ca, intermediate_ca_key = get_cert_and_key("ca.pem")
except FileNotFoundError as e:
raise ValueError(
raise CertificateGenerationError(
"ca.pem, trusted-ca.pem, client_revoked.pem, and intermediate-ca-B.pem are required in order to generate CRLs"
) from e
@ -667,14 +685,13 @@ def generate_all_crls():
)
def parse_command_line():
def parse_command_line(argv):
"""Parse and return the command line arguments."""
parser = argparse.ArgumentParser(description="X509 Test Certificate Generator")
parser.add_argument(
"--config",
"config",
help="Certificate definition file",
type=str,
default=str(PurePath("x509/certs.yml")),
)
parser.add_argument(
"--mkcrl",
@ -682,23 +699,38 @@ def parse_command_line():
help="Set to generate the default list of CRLs as well",
default=False,
)
parser.add_argument("-o", "--output", help="Output path", type=str, default=str(PurePath(".")))
parser.add_argument(
"-o", "--output", help="Output path for certs", type=str, default=str(PurePath("."))
)
parser.add_argument(
"--static-dir",
help="Path to directory containing signing keys for certs",
type=str,
default=str(PurePath("x509/static")),
)
parser.add_argument(
"-d",
"--dry-run",
help="If set, just parse the config, but don't generate any certs. If the file/input list paths are set, they will be written.",
action=argparse.BooleanOptionalAction,
default=False,
)
parser.add_argument(
"--quiet",
action=argparse.BooleanOptionalAction,
help="If set, suppresses all output",
default=False,
)
parser.add_argument("cert", nargs="*", help="Certificate to generate (blank for all)")
args = parser.parse_args()
args = parser.parse_args(argv)
return args
def validate_config():
"""Perform basic start up time validation of config file."""
if not CONFIG.get("certs"):
raise ValueError("No certificates defined")
raise CertificateGenerationError("No certificates defined")
permissible = [
"name",
@ -720,12 +752,16 @@ def validate_config():
for cert in CONFIG.get("certs", []):
keys = cert.keys()
if "name" not in keys:
raise ValueError("Name field required for all certificate definitions")
raise CertificateGenerationError("Name field required for all certificate definitions")
if "description" not in keys:
raise ValueError("description field required for all certificate definitions")
raise CertificateGenerationError(
"description field required for all certificate definitions"
)
for key in keys:
if key not in permissible:
raise ValueError("Unknown element '" + key + "' in certificate: " + cert["name"])
raise CertificateGenerationError(
"Unknown element '" + key + "' in certificate: " + cert["name"]
)
def select_items(names):
@ -739,7 +775,7 @@ def select_items(names):
for name in names:
cert = find_certificate_definition(name)
if not cert:
raise ValueError("Unknown certificate: " + name)
raise CertificateGenerationError("Unknown certificate: " + name)
ret[name] = cert
last_count = -1
@ -787,25 +823,33 @@ def sort_items(items):
def setup_global_state(parsed_args):
"""Set up various global state based on the commandline arguments."""
global CONFIG, CONFIGFILE, OUTPUT_PATH, STATIC_PATH
global CONFIG, CONFIGFILE, OUTPUT_PATH, STATIC_PATH, DRY_RUN
CONFIGFILE = parsed_args.config
OUTPUT_PATH = PurePath(parsed_args.output)
STATIC_PATH = PurePath(parsed_args.static_dir)
with open(CONFIGFILE, "r") as f:
CONFIG = yaml.load(f, Loader=yaml.FullLoader)
DRY_RUN = parsed_args.dry_run
with open(CONFIGFILE, "r", encoding="utf-8") as f:
CONFIG = json.load(f)
if parsed_args.quiet:
sys.stdout = open(os.devnull, "w")
validate_config()
def main():
def main(argv=None):
"""Go go go."""
args = parse_command_line()
args = parse_command_line(argv)
setup_global_state(args)
items_to_process = args.cert or []
items = select_items(items_to_process)
items = sort_items(items)
for item in items:
process_cert(item)
try:
process_cert(item)
except Exception as e:
raise CertificateGenerationError(
f"Failed to process certificate {item['name']}: {str(e)}"
) from e
filename = make_filename(item)
write_digest(filename, "cert", "sha256")
write_digest(filename, "cert", "sha1")

2
x509/static/README.md
View File

@ -1 +1 @@
CAs, certificates, digests, keys, etc. which are not generated by mkcert.py are stored here. Contains all of the keys needed by mkcert.py during certificate generation with certs.yml and apple-certs.yml.
CAs, certificates, digests, keys, etc. which are not generated by mkcert.py are stored here. Contains all of the keys needed by mkcert.py during certificate generation with the main and apple certs.