diff --git a/.evergreen/build-mac.sh b/.evergreen/build-mac.sh deleted file mode 100755 index 4e8be8cf5..000000000 --- a/.evergreen/build-mac.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -ex - -# Get access to testinstall. -. .evergreen/utils.sh - -# Create temp directory for validated files. -rm -rf validdist -mkdir -p validdist -mv dist/* validdist || true - -VERSION=${VERSION:-3.10} - -PYTHON=/Library/Frameworks/Python.framework/Versions/$VERSION/bin/python3 -rm -rf build - -createvirtualenv $PYTHON releasevenv -python -m pip install build -python -m build --wheel . -deactivate || true -rm -rf releasevenv - -# Test that each wheel is installable. -for release in dist/*; do - testinstall $PYTHON $release - mv $release validdist/ -done - -mv validdist/* dist -rm -rf validdist -ls dist diff --git a/.evergreen/build-manylinux-internal.sh b/.evergreen/build-manylinux-internal.sh deleted file mode 100755 index 267e647ff..000000000 --- a/.evergreen/build-manylinux-internal.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -ex -cd /src - -# Get access to testinstall. -. .evergreen/utils.sh - -# Create temp directory for validated files. -rm -rf validdist -mkdir -p validdist -mv dist/* validdist || true - -# Compile wheels -for PYTHON in /opt/python/*/bin/python; do - if [[ ! $PYTHON =~ (cp38|cp39|cp310|cp311|cp312) ]]; then - continue - fi - # https://github.com/pypa/manylinux/issues/49 - rm -rf build - $PYTHON -m pip install build - $PYTHON -m build --wheel . - rm -rf build - - # Audit wheels and write manylinux tag - for whl in dist/*.whl; do - # Skip already built manylinux wheels. - if [[ "$whl" != *"manylinux"* ]]; then - auditwheel repair $whl -w dist - rm $whl - fi - done - - # Test that each wheel is installable. - # Test without virtualenv because it's not present on manylinux containers. - for release in dist/*; do - testinstall $PYTHON $release "without-virtualenv" - mv $release validdist/ - done -done - -mv validdist/* dist -rm -rf validdist -ls dist diff --git a/.evergreen/build-manylinux.sh b/.evergreen/build-manylinux.sh deleted file mode 100755 index 19f2b7f4a..000000000 --- a/.evergreen/build-manylinux.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -ex - -docker version - -# Set up qemu support using the method used in docker/setup-qemu-action -# https://github.com/docker/setup-qemu-action/blob/2b82ce82d56a2a04d2637cd93a637ae1b359c0a7/README.md?plain=1#L46 -docker run --rm --privileged tonistiigi/binfmt:latest --install all - -# manylinux1 2021-05-05-b64d921 and manylinux2014 2021-05-05-1ac6ef3 were -# the last releases to generate pip < 20.3 compatible wheels. After that -# auditwheel was upgraded to v4 which produces PEP 600 manylinux_x_y wheels -# which requires pip >= 20.3. We use the older docker image to support older -# pip versions. -BUILD_WITH_TAG="$1" -if [ -n "$BUILD_WITH_TAG" ]; then - images=(quay.io/pypa/manylinux1_x86_64:2021-05-05-b64d921 \ - quay.io/pypa/manylinux1_i686:2021-05-05-b64d921 \ - quay.io/pypa/manylinux2014_x86_64:2021-05-05-1ac6ef3 \ - quay.io/pypa/manylinux2014_i686:2021-05-05-1ac6ef3 \ - quay.io/pypa/manylinux2014_aarch64:2021-05-05-1ac6ef3 \ - quay.io/pypa/manylinux2014_ppc64le:2021-05-05-1ac6ef3 \ - quay.io/pypa/manylinux2014_s390x:2021-05-05-1ac6ef3) -else - images=(quay.io/pypa/manylinux1_x86_64 \ - quay.io/pypa/manylinux1_i686 \ - quay.io/pypa/manylinux2014_x86_64 \ - quay.io/pypa/manylinux2014_i686 \ - quay.io/pypa/manylinux2014_aarch64 \ - quay.io/pypa/manylinux2014_ppc64le \ - quay.io/pypa/manylinux2014_s390x) -fi - -for image in "${images[@]}"; do - docker pull $image - docker run --rm -v "`pwd`:/src" $image /src/.evergreen/build-manylinux-internal.sh -done - -ls dist - -# Check for any unexpected files. -unexpected=$(find dist \! \( -iname dist -or \ - -iname '*cp38*' -or \ - -iname '*cp39*' -or \ - -iname '*cp310*' -or \ - -iname '*cp311*' -or \ - -iname '*cp312*' \)) -if [ -n "$unexpected" ]; then - echo "Unexpected files:" $unexpected - exit 1 -fi diff --git a/.evergreen/build-windows.sh b/.evergreen/build-windows.sh deleted file mode 100755 index d30382fce..000000000 --- a/.evergreen/build-windows.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -ex - -# Get access to testinstall. -. .evergreen/utils.sh - -# Create temp directory for validated files. -rm -rf validdist -mkdir -p validdist -mv dist/* validdist || true - -for VERSION in 38 39 310 311 312; do - _pythons=("C:/Python/Python${VERSION}/python.exe" \ - "C:/Python/32/Python${VERSION}/python.exe") - for PYTHON in "${_pythons[@]}"; do - rm -rf build - $PYTHON -m pip install build - $PYTHON -m build --wheel . - - # Test that each wheel is installable. - for release in dist/*; do - testinstall $PYTHON $release - mv $release validdist/ - done - done -done - -mv validdist/* dist -rm -rf validdist -ls dist diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 8f32b8a1b..1e218e2c7 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -786,92 +786,6 @@ functions: args: - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh - "build release": - - command: shell.exec - type: test - params: - working_dir: "src" - script: | - ${PREPARE_SHELL} - set -o xtrace - VERSION=${VERSION} ENSURE_UNIVERSAL2=${ENSURE_UNIVERSAL2} .evergreen/release.sh - - "upload release": - - command: ec2.assume_role - params: - role_arn: ${assume_role_arn} - - command: archive.targz_pack - params: - target: "release-files.tgz" - source_dir: "src/dist" - include: - - "*" - - command: s3.put - params: - aws_key: ${AWS_ACCESS_KEY_ID} - aws_secret: ${AWS_SECRET_ACCESS_KEY} - aws_session_token: ${AWS_SESSION_TOKEN} - local_file: release-files.tgz - remote_file: release/${revision}/${task_id}-${execution}-release-files.tar.gz - bucket: ${bucket_name} - permissions: public-read - content_type: ${content_type|application/gzip} - display_name: Release files - - "download and merge releases": - - command: ec2.assume_role - params: - role_arn: ${assume_role_arn} - - command: shell.exec - params: - silent: true - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - script: | - # Download all the task coverage files. - aws s3 cp --recursive s3://${bucket_name}/release/${revision}/ release/ - - command: shell.exec - params: - shell: "bash" - script: | - ${PREPARE_SHELL} - set -o xtrace - # Combine releases into one directory. - ls -la release/ - mkdir releases - # Copy old manylinux release first since we want the newer manylinux - # wheels to override them. - mkdir old_manylinux - if mv release/*old_manylinux* old_manylinux; then - for REL in old_manylinux/*; do - tar zxvf $REL -C releases/ - done - fi - for REL in release/*; do - tar zxvf $REL -C releases/ - done - # Build source distribution. - cd src/ - /opt/python/3.8/bin/python3 -m pip install build - /opt/python/3.8/bin/python3 -m build --sdist . - cp dist/* ../releases - - command: archive.targz_pack - params: - target: "release-files-all.tgz" - source_dir: "releases/" - include: - - "*" - - command: s3.put - params: - aws_key: ${AWS_ACCESS_KEY_ID} - aws_secret: ${AWS_SECRET_ACCESS_KEY} - aws_session_token: ${AWS_SESSION_TOKEN} - local_file: release-files-all.tgz - remote_file: release-all/${revision}/${task_id}-${execution}-release-files-all.tar.gz - bucket: ${bucket_name} - permissions: public-read - content_type: ${content_type|application/gzip} - display_name: Release files all - "run perf tests": - command: shell.exec type: test @@ -1116,72 +1030,6 @@ tasks: genhtml --version || true valgrind --version || true - - name: "release-mac" - tags: ["release_tag"] - run_on: macos-11 - commands: - - func: "build release" - vars: - VERSION: "3.12" - ENSURE_UNIVERSAL2: "1" - - func: "build release" - vars: - VERSION: "3.11" - ENSURE_UNIVERSAL2: "1" - - func: "build release" - vars: - VERSION: "3.10" - ENSURE_UNIVERSAL2: "1" - - func: "build release" - vars: - VERSION: "3.9" - ENSURE_UNIVERSAL2: "1" - - func: "upload release" - - func: "build release" - vars: - VERSION: "3.8" - - func: "upload release" - - - name: "release-windows" - tags: ["release_tag"] - run_on: windows-64-vsMulti-small - commands: - - func: "build release" - - func: "upload release" - - - name: "release-manylinux" - tags: ["release_tag"] - run_on: ubuntu2204-large - exec_timeout_secs: 216000 # 60 minutes (manylinux task is slow). - commands: - - func: "build release" - - func: "upload release" - - - name: "release-old-manylinux" - tags: ["release_tag"] - run_on: ubuntu2204-large - exec_timeout_secs: 216000 # 60 minutes (manylinux task is slow). - commands: - - command: shell.exec - type: test - params: - working_dir: "src" - script: | - ${PREPARE_SHELL} - set -o xtrace - .evergreen/build-manylinux.sh BUILD_WITH_TAG - - func: "upload release" - - - name: "release-combine" - tags: ["release_tag"] - run_on: rhel84-small - depends_on: - - name: "*" - variant: ".release_tag" - patch_optional: true - commands: - - func: "download and merge releases" - # Standard test tasks {{{ - name: "mockupdb" @@ -3177,13 +3025,6 @@ buildvariants: tasks: - name: "check-import-time" -- name: Release - display_name: Release - batchtime: 20160 # 14 days - tags: ["release_tag"] - tasks: - - ".release_tag" - - name: "perf-tests" display_name: "Performance Benchmark Tests" batchtime: 10080 # 7 days diff --git a/.evergreen/hatch.sh b/.evergreen/hatch.sh index e24320b9d..843839410 100644 --- a/.evergreen/hatch.sh +++ b/.evergreen/hatch.sh @@ -13,8 +13,11 @@ if $PYTHON_BINARY -m hatch --version; then $PYTHON_BINARY -m hatch run "$@" } else # No toolchain hatch present, set up virtualenv before installing hatch - createvirtualenv "$PYTHON_BINARY" hatchenv - trap "deactivate; rm -rf hatchenv" EXIT HUP + # Use a random venv name because the encryption tasks run this script multiple times in the same run. + ENV_NAME=hatchenv-$RANDOM + createvirtualenv "$PYTHON_BINARY" $ENV_NAME + # shellcheck disable=SC2064 + trap "deactivate; rm -rf $ENV_NAME" EXIT HUP python -m pip install -q hatch run_hatch() { python -m hatch run "$@" diff --git a/.evergreen/release.sh b/.evergreen/release.sh deleted file mode 100755 index 1fdd459ad..000000000 --- a/.evergreen/release.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -ex - -if [ "$(uname -s)" = "Darwin" ]; then - .evergreen/build-mac.sh -elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin - .evergreen/build-windows.sh -else - .evergreen/build-manylinux.sh -fi diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index ff186c6e1..1a5e2a153 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -43,7 +43,7 @@ find_python3() { createvirtualenv () { PYTHON=$1 VENVPATH=$2 - rm -rf $VENVPATH + # Prefer venv VENV="$PYTHON -m venv" if [ "$(uname -s)" = "Darwin" ]; then diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 4d514726c..ba04e8e41 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -78,9 +78,6 @@ jobs: - name: Run tests run: | hatch run test:test - - name: Run async tests - run: | - hatch run test:test-async doctest: runs-on: ubuntu-latest diff --git a/bson/objectid.py b/bson/objectid.py index 57efdc798..a5500872d 100644 --- a/bson/objectid.py +++ b/bson/objectid.py @@ -29,6 +29,9 @@ from bson.errors import InvalidId from bson.tz_util import utc _MAX_COUNTER_VALUE = 0xFFFFFF +_PACK_INT = struct.Struct(">I").pack +_PACK_INT_RANDOM = struct.Struct(">I5s").pack +_UNPACK_INT = struct.Struct(">I").unpack def _raise_invalid_id(oid: str) -> NoReturn: @@ -132,7 +135,7 @@ class ObjectId: if offset is not None: generation_time = generation_time - offset timestamp = calendar.timegm(generation_time.timetuple()) - oid = struct.pack(">I", int(timestamp)) + b"\x00\x00\x00\x00\x00\x00\x00\x00" + oid = _PACK_INT(int(timestamp)) + b"\x00\x00\x00\x00\x00\x00\x00\x00" return cls(oid) @classmethod @@ -163,18 +166,12 @@ class ObjectId: def __generate(self) -> None: """Generate a new value for this ObjectId.""" - # 4 bytes current time - oid = struct.pack(">I", int(time.time())) - - # 5 bytes random - oid += ObjectId._random() - - # 3 bytes inc with ObjectId._inc_lock: - oid += struct.pack(">I", ObjectId._inc)[1:4] - ObjectId._inc = (ObjectId._inc + 1) % (_MAX_COUNTER_VALUE + 1) + inc = ObjectId._inc + ObjectId._inc = (inc + 1) % (_MAX_COUNTER_VALUE + 1) - self.__id = oid + # 4 bytes current time, 5 bytes random, 3 bytes inc. + self.__id = _PACK_INT_RANDOM(int(time.time()), ObjectId._random()) + _PACK_INT(inc)[1:4] def __validate(self, oid: Any) -> None: """Validate and use the given id for this ObjectId. @@ -212,7 +209,7 @@ class ObjectId: represents the generation time in UTC. It is precise to the second. """ - timestamp = struct.unpack(">I", self.__id[0:4])[0] + timestamp = _UNPACK_INT(self.__id[0:4])[0] return datetime.datetime.fromtimestamp(timestamp, utc) def __getstate__(self) -> bytes: diff --git a/test/asynchronous/test_cursor.py b/test/asynchronous/test_cursor.py index 833493ce3..d6d56244f 100644 --- a/test/asynchronous/test_cursor.py +++ b/test/asynchronous/test_cursor.py @@ -1380,7 +1380,6 @@ class TestCursor(AsyncIntegrationTest): self.assertEqual("getMore", started[1].command_name) self.assertNotIn("$readPreference", started[1].command) - @async_client_context.require_version_min(4, 0) @async_client_context.require_replica_set async def test_to_list_tailable(self): oplog = self.client.local.oplog.rs @@ -1391,7 +1390,10 @@ class TestCursor(AsyncIntegrationTest): {"ts": {"$gte": ts}}, cursor_type=pymongo.CursorType.TAILABLE_AWAIT, oplog_replay=True ).max_await_time_ms(1) self.addAsyncCleanup(c.close) - docs = await c.to_list() + # Wait for the change to be read. + docs = [] + while not docs: + docs = await c.to_list() self.assertGreaterEqual(len(docs), 1) async def test_to_list_empty(self): diff --git a/test/test_cursor.py b/test/test_cursor.py index e995bd529..0d6186519 100644 --- a/test/test_cursor.py +++ b/test/test_cursor.py @@ -1371,7 +1371,6 @@ class TestCursor(IntegrationTest): self.assertEqual("getMore", started[1].command_name) self.assertNotIn("$readPreference", started[1].command) - @client_context.require_version_min(4, 0) @client_context.require_replica_set def test_to_list_tailable(self): oplog = self.client.local.oplog.rs @@ -1382,7 +1381,10 @@ class TestCursor(IntegrationTest): {"ts": {"$gte": ts}}, cursor_type=pymongo.CursorType.TAILABLE_AWAIT, oplog_replay=True ).max_await_time_ms(1) self.addCleanup(c.close) - docs = c.to_list() + # Wait for the change to be read. + docs = [] + while not docs: + docs = c.to_list() self.assertGreaterEqual(len(docs), 1) def test_to_list_empty(self):