SERVER-30210 Script to fetch the test lifecycle tags

buildscripts/fetch_test_lifecycle.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python
+
+"""Script to retrieve the etc/test_lifecycle.yml tag file from the metadata repository that
+corresponds to the current repository.
+
+Usage:
+    python buildscsripts/fetch_test_lifecycle.py evergreen-project revision
+"""
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import logging
+import optparse
+import os
+import posixpath
+import shutil
+import sys
+import textwrap
+
+import yaml
+
+# Get relative imports to work when the package is not installed on the PYTHONPATH.
+if __name__ == "__main__" and __package__ is None:
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from buildscripts import git
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+class MetadataRepository(object):
+    """Represent the metadata repository containing the test lifecycle tags file."""
+
+    def __init__(self, repository, references_file, lifecycle_file):
+        """Initlialize the MetadataRepository.
+
+        Args:
+            repository: the git.Repository object for the repository.
+            references_file: the relative path from the root of the repository to the references
+                yaml file.
+            lifecycle_file: the relative path from the root of the repository to the test lifecycle
+                tags yaml file.
+        """
+        self._repository = repository
+        self._references_file = references_file
+        self._lifecycle_file = lifecycle_file
+        # The path to the lifecycle file, absolute or relative to the current working directory.
+        self.lifecycle_path = os.path.join(repository.directory, lifecycle_file)
+
+    def list_revisions(self):
+        """List the revisions from the HEAD of this repository.
+
+        Returns:
+            A list of str containing the git hashes for all the revisions from the newest (HEAD)
+            to the oldest.
+            """
+        return self._repository.git_rev_list(["HEAD", "--", self._lifecycle_file]).splitlines()
+
+    def _get_references_content(self, revision):
+        references_content = self._repository.git_cat_file(
+            ["blob", "%s:%s" % (revision, self._references_file)])
+        return references_content
+
+    def get_reference(self, metadata_revision, project):
+        """Retrieve the reference revision (a revision of the project 'project') associated with
+         the test lifecycle file present in the metadata repository at revision 'metadata_revision'.
+
+         Args:
+             metadata_revision: a revision (git hash) of this repository.
+             project: an Evergreen project name (e.g. mongodb-mongo-master).
+         """
+        references_content = self._get_references_content(metadata_revision)
+        references = yaml.safe_load(references_content)
+        return references.get("test-lifecycle", {}).get(project)
+
+    def get_lifecycle_file_content(self, metadata_revision):
+        """Return the content of the test lifecycle file as it was at the given revision."""
+        return self._repository.git_cat_file(["blob", "%s:%s" % (metadata_revision,
+                                                                 self._lifecycle_file)])
+
+
+def _clone_repository(url, branch):
+    """Clone the repository present at the URL 'url' and use the branch 'branch'."""
+    target_directory = posixpath.splitext(posixpath.basename(url))[0]
+    LOGGER.info("Cloning the repository %s into the directory %s", url, target_directory)
+    return git.Repository.clone(url, target_directory, branch)
+
+
+def _get_metadata_revision(metadata_repo, mongo_repo, project, revision):
+    """Get the metadata revision that corresponds to a given repository, project, revision."""
+    for metadata_revision in metadata_repo.list_revisions():
+        reference = metadata_repo.get_reference(metadata_revision, project)
+        if not reference:
+            # No reference for this revision. This should not happen but we keep trying in
+            # case we can find an older revision with a reference.
+            continue
+        if mongo_repo.is_ancestor(reference, revision):
+            # We found a reference that is a parent of the current revision.
+            return metadata_revision
+    return None
+
+
+def fetch_test_lifecycle(metadata_repo_url, references_file, lifecycle_file, project, revision):
+    """Fetch the test lifecycle file that corresponds to the given revision of the repository this
+    script is called from.
+
+    Args:
+        metadata_repo_url: the git repository URL for the metadata repository containing the test
+            lifecycle file.
+        references_file: the relative path from the root of the metadata repository to the
+            references file.
+        lifecycle_file: the relative path from the root of the metadata repository to the test
+            lifecycle file.
+        project: the Evergreen project name.
+        revision: the current repository revision.
+    """
+    metadata_repo = MetadataRepository(_clone_repository(metadata_repo_url, project),
+                                       references_file, lifecycle_file)
+    mongo_repo = git.Repository(os.getcwd())
+    metadata_revision = _get_metadata_revision(metadata_repo, mongo_repo, project, revision)
+    if metadata_revision:
+        LOGGER.info("Using metadata repository revision '%s'", metadata_revision)
+        result = metadata_repo.get_lifecycle_file_content(metadata_revision)
+    else:
+        result = None
+    return result
+
+
+def main():
+    """
+    Utility to fetch the etc/test_lifecycle.yml file corresponding to a given revision from
+    the mongo-test-metadata repository.
+    """
+    parser = optparse.OptionParser(description=textwrap.dedent(main.__doc__),
+                                   usage="Usage: %prog [options] evergreen-project")
+
+    parser.add_option("--revision", dest="revision",
+                      metavar="<revision>",
+                      default="HEAD",
+                      help=("The project revision for which to retrieve the test lifecycle tags"
+                            " file."))
+
+    parser.add_option("--metadataRepo", dest="metadata_repo_url",
+                      metavar="<metadata-repo-url>",
+                      default="git@github.com:mongodb/mongo-test-metadata.git",
+                      help=("The URL to the metadata repository that contains the test lifecycle"
+                            " tags file."))
+
+    parser.add_option("--lifecycleFile", dest="lifecycle_file",
+                      metavar="<lifecycle-file>",
+                      default="etc/test_lifecycle.yml",
+                      help=("The path to the test lifecycle tags file, relative to the root of the"
+                            " metadata repository. Defaults to '%default'."))
+
+    parser.add_option("--referencesFile", dest="references_file",
+                      metavar="<references-file>",
+                      default="references.yml",
+                      help=("The path to the metadata references file, relative to the root of the"
+                            " metadata repository. Defaults to '%default'."))
+
+    parser.add_option("--destinationFile", dest="destination_file",
+                      metavar="<destination-file>",
+                      default="etc/test_lifecycle.yml",
+                      help=("The path where the lifecycle file should be available when this script"
+                            " completes successfully. This path is absolute or relative to the"
+                            " current working directory. Defaults to '%default'."))
+
+    parser.add_option("--logLevel", dest="log_level",
+                      metavar="<log-level>",
+                      choices=["DEBUG", "INFO", "WARNING", "ERROR"],
+                      default="INFO",
+                      help="The log level: DEBUG, INFO, WARNING or ERROR. Defaults to '%default'.")
+
+    parser.add_option("--logFile", dest="log_file",
+                      metavar="<log-file>",
+                      default=None,
+                      help=("The destination file for the logs. If not set the script will log to"
+                            " the standard output"))
+
+    options, args = parser.parse_args()
+
+    if len(args) != 1:
+        parser.print_help(file=sys.stderr)
+        print(file=sys.stderr)
+        parser.error("Must specify an Evergreen project")
+    evergreen_project = args[0]
+
+    logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s",
+                        level=options.log_level, filename=options.log_file)
+
+    lifecycle_file_content = fetch_test_lifecycle(options.metadata_repo_url,
+                                                  options.references_file,
+                                                  options.lifecycle_file,
+                                                  evergreen_project,
+                                                  options.revision)
+    if not lifecycle_file_content:
+        LOGGER.error("Failed to fetch the test lifecycle tag file.")
+        sys.exit(1)
+    else:
+        LOGGER.info("Writing the test lifecycle file to '%s'.", options.destination_file)
+        with open(options.destination_file, "wb") as destf:
+            destf.write(lifecycle_file_content)
+        LOGGER.info("Done.")
+
+
+if __name__ == "__main__":
+    main()

buildscripts/tests/test_fetch_test_lifecycle.py

@@ -0,0 +1,79 @@
+"""Unit tests for the fetch_test_lifecycle.py script."""
+
+import unittest
+
+import buildscripts.fetch_test_lifecycle as fetch
+
+
+class TestFetchTestLifecycle(unittest.TestCase):
+    def test_get_metadata_revision(self):
+        metadata_repo = MockMetadataRepository([("metadata_revision_05", "mongo_revision_06"),
+                                                ("metadata_revision_04", "mongo_revision_06"),
+                                                ("metadata_revision_03", "mongo_revision_02"),
+                                                ("metadata_revision_02", "mongo_revision_02"),
+                                                ("metadata_revision_01", None)])
+
+        mongo_repo = MockMongoRepository(["mongo_revision_07",
+                                          "mongo_revision_06",
+                                          "mongo_revision_05",
+                                          "mongo_revision_04",
+                                          "mongo_revision_03",
+                                          "mongo_revision_02",
+                                          "mongo_revision_01"])
+
+        self._check_metadata_revision(metadata_repo, mongo_repo,
+                                      "mongo_revision_07",
+                                      "metadata_revision_05")
+
+        self._check_metadata_revision(metadata_repo, mongo_repo,
+                                      "mongo_revision_06",
+                                      "metadata_revision_05")
+
+        self._check_metadata_revision(metadata_repo, mongo_repo,
+                                      "mongo_revision_05",
+                                      "metadata_revision_03")
+
+        self._check_metadata_revision(metadata_repo, mongo_repo,
+                                      "mongo_revision_04",
+                                      "metadata_revision_03")
+
+        self._check_metadata_revision(metadata_repo, mongo_repo,
+                                      "mongo_revision_03",
+                                      "metadata_revision_03")
+
+        self._check_metadata_revision(metadata_repo, mongo_repo,
+                                      "mongo_revision_02",
+                                      "metadata_revision_03")
+
+        self._check_metadata_revision(metadata_repo, mongo_repo,
+                                      "mongo_revision_01",
+                                      None)
+
+    def _check_metadata_revision(self, metadata_repo, mongo_repo, mongo_revision,
+                                 expected_metadata_revision):
+            metadata_revision = fetch._get_metadata_revision(metadata_repo, mongo_repo, "project",
+                                                             mongo_revision)
+            self.assertEqual(expected_metadata_revision, metadata_revision)
+
+
+class MockMongoRepository(object):
+    def __init__(self, revisions):
+        self.revisions = revisions
+
+    def is_ancestor(self, parent, child):
+        return (parent in self.revisions and child in self.revisions and
+                self.revisions.index(parent) >= self.revisions.index(child))
+
+
+class MockMetadataRepository(object):
+    def __init__(self, references_revisions):
+        self.references_revisions = references_revisions
+
+    def list_revisions(self):
+        return [r[0] for r in self.references_revisions]
+
+    def get_reference(self, revision, project):
+        for (metadata_revision, mongo_revision) in self.references_revisions:
+            if metadata_revision == revision:
+                return mongo_revision
+        return None

etc/evergreen.yml

@@ -1024,6 +1024,28 @@ functions:
         kitchen destroy "${packager_distro}" || true
         test "$verified" = "true"
 
+  "fetch test_lifecycle.yml":
+  - command: shell.exec
+    type: test
+    params:
+      working_dir: src
+      script: |
+        set -o verbose
+
+        ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/fetch_test_lifecycle.py \
+            --metadataRepo git@github.com:mongodb/mongo-test-metadata.git \
+            --lifecycleFile etc/test_lifecycle.yml \
+            --referencesFile references.yml \
+            --destinationFile etc/test_lifecycle.yml \
+            --revision ${revision} \
+            ${project}
+        exit_code=$?
+        if [ ${fail_task_on_error|false} = true ]; then
+          exit $exit_code
+        else
+          exit 0
+        fi
+
 pre:
   - func: "kill processes"
   - func: "cleanup environment"
@@ -1514,6 +1536,11 @@ tasks:
           ${python|/opt/mongodbtoolchain/v2/bin/python2} ../buildscripts/make_archive.py -o mongodb-shell.${ext|tgz} $(find mongodb-* -type f)
           cd ..
 
+    - func: "fetch test_lifecycle.yml"
+      vars:
+        # Do not fail the task if we fail to fetch the test_lifecycle.yml file
+        fail_task_on_error: false
+
     - command: archive.targz_pack
       params:
         target: "artifacts.tgz"
@@ -3861,6 +3888,16 @@ tasks:
             - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}-${task_id}.${ext|tgz}.md5', 'bucket': 'build-push-testing'},
                'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.md5', 'bucket': '${push_bucket}'}}
 
+- name: fetch_test_lifecycle
+  depends_on: []
+  commands:
+  - *git_get_project
+  - func: "fetch test_lifecycle.yml"
+    vars:
+      # This task is meant to fail if there is an error while fetching test_lifecycle.yml since
+      # the compile task won't fail.
+      fail_task_on_error: true
+
 - name: update_test_lifecycle
   exec_timeout_secs: 21600 # 6 hour timeout for the task overall
   depends_on: []
@@ -9383,3 +9420,6 @@ buildvariants:
   stepback: false
   tasks:
   - name: update_test_lifecycle
+  - name: fetch_test_lifecycle
+    distros:
+    - rhel62-small

etc/test_lifecycle.yml

@@ -1,7 +1,4 @@
 # This file is used to tag JS tests that run under resmoke.py.
-selector:
-  js_test:
-    # jstests/core/example.js:  # Single file tag
-    # - example_tag
-    # jstests/core/all*.js  # Multiple file tag
-    # - multiple_file_tag
+# The content of this file lives in the https://github.com/mongodb/mongo-test-metadata repository.
+# It is fetched automatically as part of the compile task.
+selector: {}