mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-90907 Gather BF data from Jira and analyze for lockdown criteria (#22737)

Browse Source 
GitOrigin-RevId: b47ef4ec2cf38417d05e2457b6f1354c285e5220
This commit is contained in:
Mikhail Shchatko 2024-05-31 08:24:45 +03:00 committed by MongoDB Bot
parent 30ae610eb7
commit 1554355d50
8 changed files with 609 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
buildscripts/client/jiraclient.py
View File

@ -1,8 +1,10 @@
"""Module to access a JIRA server."""
from enum import Enum
from typing import Any, Dict, Iterable, Optional
import jira
from jira import JIRA, Issue
from jira.client import ResultList
from pydantic import BaseSettings
@ -14,20 +16,34 @@ class SecurityLevel(Enum):
class JiraAuth(BaseSettings):
"""OAuth information to connect to Jira."""
"""Auth information to connect to Jira."""
access_token: str
access_token_secret: str
consumer_key: str
key_cert: str
access_token: Optional[str]
access_token_secret: Optional[str]
consumer_key: Optional[str]
key_cert: Optional[str]
pat: Optional[str]
class Config:
"""Configuration for JiraAuth."""
env_prefix = "JIRA_AUTH_"
def get_token_auth(self) -> Optional[str]:
return self.pat
class JiraClient(object):
def get_oauth(self) -> Optional[Dict[str, Any]]:
if self.access_token and self.access_token_secret and self.consumer_key and self.key_cert:
return {
"access_token": self.access_token,
"access_token_secret": self.access_token_secret,
"consumer_key": self.consumer_key,
"key_cert": self.key_cert,
}
return None
class JiraClient:
"""A client for JIRA."""
def __init__(self, server: str, jira_auth: JiraAuth) -> None:
@ -35,10 +51,14 @@ class JiraClient(object):
Initialize the JiraClient with the server URL and user credentials.
:param server: Jira Server to connect to.
:param jira_auth: OAuth connection information.
:param jira_auth: Auth connection information.
"""
opts = {"server": server, "verify": True}
self._jira = jira.JIRA(options=opts, oauth=jira_auth.dict(), validate=True)
token_auth = jira_auth.get_token_auth()
if token_auth:
self._jira = JIRA(options=opts, validate=True, token_auth=token_auth)
else:
self._jira = JIRA(options=opts, validate=True, oauth=jira_auth.get_oauth())
def get_ticket_security_level(self, key: str) -> SecurityLevel:
"""
@ -52,3 +72,17 @@ class JiraClient(object):
security_level = ticket.fields.security
return SecurityLevel(security_level.name)
return SecurityLevel.NONE
def get_issues(self, query: str) -> Iterable[Issue]:
start_at = 0
max_results = 50
while True:
results: ResultList[Issue] = self._jira.search_issues(
jql_str=query, startAt=start_at, maxResults=max_results
)
for item in results:
yield item
start_at = results.startAt + results.maxResults
if start_at > results.total:
break

0
buildscripts/monitor_build_status/__init__.py Normal file
View File

127
buildscripts/monitor_build_status/bfs_report.py Normal file
View File

@ -0,0 +1,127 @@
from __future__ import annotations
from typing import Any, Dict, List, NamedTuple, Optional, Set
from buildscripts.monitor_build_status.evergreen_service import EvgProjectsInfo
from buildscripts.monitor_build_status.jira_service import BfIssue, BfTemperature, TestType
class BFsTemperatureReport(NamedTuple):
hot: Dict[str, int]
cold: Dict[str, int]
none: Dict[str, int]
@classmethod
def empty(cls) -> BFsTemperatureReport:
return cls(hot={}, cold={}, none={})
def add_bf_data(self, bf_temperature: BfTemperature, assigned_team: str) -> None:
"""
Add BF data to report.
:param bf_temperature: BF temperature.
:param assigned_team: Assigned team.
"""
match bf_temperature:
case BfTemperature.HOT:
self._increment_bf_count(self.hot, assigned_team)
case BfTemperature.COLD:
self._increment_bf_count(self.cold, assigned_team)
case BfTemperature.NONE:
self._increment_bf_count(self.none, assigned_team)
@staticmethod
def _increment_bf_count(bf_count_dict: Dict[str, int], assigned_team: str) -> None:
if assigned_team not in bf_count_dict:
bf_count_dict[assigned_team] = 0
bf_count_dict[assigned_team] += 1
class BFsReport(NamedTuple):
correctness: BFsTemperatureReport
performance: BFsTemperatureReport
unknown: BFsTemperatureReport
all_assigned_teams: Set[str]
@classmethod
def empty(cls) -> BFsReport:
return cls(
correctness=BFsTemperatureReport.empty(),
performance=BFsTemperatureReport.empty(),
unknown=BFsTemperatureReport.empty(),
all_assigned_teams=set(),
)
def add_bf_data(self, bf: BfIssue, evg_projects_info: EvgProjectsInfo) -> None:
"""
Add BF data to report.
:param bf: BF issue.
:param evg_projects_info: Evergreen project information.
"""
for evg_project in bf.evergreen_projects:
if evg_project not in evg_projects_info.active_project_names:
continue
self.all_assigned_teams.add(bf.assigned_team)
test_type = TestType.from_evg_project_name(evg_project)
match test_type:
case TestType.CORRECTNESS:
self.correctness.add_bf_data(bf.temperature, bf.assigned_team)
case TestType.PERFORMANCE:
self.performance.add_bf_data(bf.temperature, bf.assigned_team)
case TestType.UNKNOWN:
self.unknown.add_bf_data(bf.temperature, bf.assigned_team)
def get_bf_count(
self,
test_types: List[TestType],
bf_temperatures: List[BfTemperature],
assigned_team: Optional[str] = None,
) -> int:
"""
Calculate BFs count for a given criteria.
:param test_types: List of test types (correctness, performance or unknown) criteria.
:param bf_temperatures: List of BF temperatures (hot, cold or none) criteria.
:param assigned_team: Assigned team criterion, all teams if None.
:return: BFs count.
"""
total_bf_count = 0
test_type_reports = []
for test_type in test_types:
match test_type:
case TestType.CORRECTNESS:
test_type_reports.append(self.correctness)
case TestType.PERFORMANCE:
test_type_reports.append(self.performance)
case TestType.UNKNOWN:
test_type_reports.append(self.unknown)
bf_temp_reports = []
for test_type_report in test_type_reports:
for bf_temperature in bf_temperatures:
match bf_temperature:
case BfTemperature.HOT:
bf_temp_reports.append(test_type_report.hot)
case BfTemperature.COLD:
bf_temp_reports.append(test_type_report.cold)
case BfTemperature.NONE:
bf_temp_reports.append(test_type_report.none)
for bf_temp_report in bf_temp_reports:
if assigned_team is None:
total_bf_count += sum(bf_temp_report.values())
else:
total_bf_count += bf_temp_report.get(assigned_team, 0)
return total_bf_count
def as_dict(self) -> Dict[str, Any]:
return {
TestType.CORRECTNESS.value: self.correctness._asdict(),
TestType.PERFORMANCE.value: self.performance._asdict(),
TestType.UNKNOWN.value: self.unknown._asdict(),
}

225
buildscripts/monitor_build_status/cli.py Normal file
View File

@ -0,0 +1,225 @@
from __future__ import annotations
from typing import Iterable, List
import structlog
import typer
from typing_extensions import Annotated
from buildscripts.client.jiraclient import JiraAuth, JiraClient
from buildscripts.monitor_build_status.bfs_report import BFsReport
from buildscripts.monitor_build_status.evergreen_service import EvergreenService, EvgProjectsInfo
from buildscripts.monitor_build_status.jira_service import (
BfTemperature,
JiraCustomFieldNames,
JiraService,
TestType,
)
from buildscripts.resmokelib.utils.evergreen_conn import get_evergreen_api
from buildscripts.util.cmdutils import enable_logging
LOGGER = structlog.get_logger(__name__)
JIRA_SERVER = "https://jira.mongodb.org"
DEFAULT_REPO = "10gen/mongo"
DEFAULT_BRANCH = "master"
GLOBAL_CORRECTNESS_HOT_BF_COUNT_LIMIT = 30
GLOBAL_CORRECTNESS_COLD_BF_COUNT_LIMIT = 100
GLOBAL_PERFORMANCE_BF_COUNT_LIMIT = 30
PER_TEAM_CORRECTNESS_HOT_BF_COUNT_LIMIT = 7
PER_TEAM_CORRECTNESS_COLD_BF_COUNT_LIMIT = 20
PER_TEAM_PERFORMANCE_BF_COUNT_LIMIT = 10
def iterable_to_jql(entries: Iterable[str]) -> str:
return ", ".join(f'"{entry}"' for entry in entries)
JIRA_PROJECTS = {"Build Failures"}
END_STATUSES = {"Closed", "Resolved"}
ACTIVE_BFS_QUERY = (
f"project in ({iterable_to_jql(JIRA_PROJECTS)})"
f" AND status not in ({iterable_to_jql(END_STATUSES)})"
)
class MonitorBuildStatusOrchestrator:
def __init__(
self,
jira_service: JiraService,
evg_service: EvergreenService,
) -> None:
self.jira_service = jira_service
self.evg_service = evg_service
def evaluate_build_redness(self, repo: str, branch: str) -> None:
failures = []
LOGGER.info("Getting Evergreen projects data")
evg_projects_info = self.evg_service.get_evg_project_info(repo, branch)
LOGGER.info("Got Evergreen projects data")
bfs_report = self._make_bfs_report(evg_projects_info)
failures.extend(self._check_bf_counts(bfs_report, branch))
# TODO SERVER-90908: fetch evergreen data
# TODO SERVER-90908: check evergreen redness
for failure in failures:
LOGGER.error(failure)
def _make_bfs_report(self, evg_projects_info: EvgProjectsInfo) -> BFsReport:
query = (
f'{ACTIVE_BFS_QUERY} AND "{JiraCustomFieldNames.EVERGREEN_PROJECT}" in'
f" ({iterable_to_jql(evg_projects_info.active_project_names)})"
)
LOGGER.info("Getting active BFs from Jira", query=query)
active_bfs = self.jira_service.fetch_bfs(query)
LOGGER.info("Got active BFs", count=len(active_bfs))
bfs_report = BFsReport.empty()
for bf in active_bfs:
bfs_report.add_bf_data(bf, evg_projects_info)
return bfs_report
@staticmethod
def _check_bf_counts(bfs_report: BFsReport, branch: str) -> List[str]:
failures = []
bf_count_failure_msg = (
"BF count check failed: {scope}: {bf_type} count ({bf_count}) is more than {bf_limit}"
)
global_correctness_hot_bf_count = bfs_report.get_bf_count(
test_types=[TestType.CORRECTNESS],
bf_temperatures=[BfTemperature.HOT],
)
if global_correctness_hot_bf_count > GLOBAL_CORRECTNESS_HOT_BF_COUNT_LIMIT:
failures.append(
bf_count_failure_msg.format(
scope=f"Branch({branch}): Global",
bf_type="Correctness HOT BF",
bf_count=global_correctness_hot_bf_count,
bf_limit=GLOBAL_CORRECTNESS_HOT_BF_COUNT_LIMIT,
)
)
global_correctness_cold_bf_count = bfs_report.get_bf_count(
test_types=[TestType.CORRECTNESS],
bf_temperatures=[BfTemperature.COLD, BfTemperature.NONE],
)
if global_correctness_cold_bf_count > GLOBAL_CORRECTNESS_COLD_BF_COUNT_LIMIT:
failures.append(
bf_count_failure_msg.format(
scope=f"Branch({branch}): Global",
bf_type="Correctness COLD BF",
bf_count=global_correctness_cold_bf_count,
bf_limit=GLOBAL_CORRECTNESS_COLD_BF_COUNT_LIMIT,
)
)
global_performance_bf_count = bfs_report.get_bf_count(
test_types=[TestType.PERFORMANCE],
bf_temperatures=[BfTemperature.HOT, BfTemperature.COLD, BfTemperature.NONE],
)
if global_performance_bf_count > GLOBAL_PERFORMANCE_BF_COUNT_LIMIT:
failures.append(
bf_count_failure_msg.format(
scope=f"Branch({branch}): Global",
bf_type="Performance BF",
bf_count=global_performance_bf_count,
bf_limit=GLOBAL_PERFORMANCE_BF_COUNT_LIMIT,
)
)
for team in bfs_report.all_assigned_teams:
per_team_correctness_hot_bf_count = bfs_report.get_bf_count(
test_types=[TestType.CORRECTNESS],
bf_temperatures=[BfTemperature.HOT],
assigned_team=team,
)
if per_team_correctness_hot_bf_count > PER_TEAM_CORRECTNESS_HOT_BF_COUNT_LIMIT:
failures.append(
bf_count_failure_msg.format(
scope=f"Branch({branch}): Team({team})",
bf_type="Correctness HOT BF",
bf_count=per_team_correctness_hot_bf_count,
bf_limit=PER_TEAM_CORRECTNESS_HOT_BF_COUNT_LIMIT,
)
)
per_team_correctness_cold_bf_count = bfs_report.get_bf_count(
test_types=[TestType.CORRECTNESS],
bf_temperatures=[BfTemperature.COLD, BfTemperature.NONE],
assigned_team=team,
)
if per_team_correctness_cold_bf_count > PER_TEAM_CORRECTNESS_COLD_BF_COUNT_LIMIT:
failures.append(
bf_count_failure_msg.format(
scope=f"Branch({branch}): Team({team})",
bf_type="Correctness COLD BF",
bf_count=per_team_correctness_cold_bf_count,
bf_limit=PER_TEAM_CORRECTNESS_COLD_BF_COUNT_LIMIT,
)
)
per_team_performance_bf_count = bfs_report.get_bf_count(
test_types=[TestType.PERFORMANCE],
bf_temperatures=[BfTemperature.HOT, BfTemperature.COLD, BfTemperature.NONE],
assigned_team=team,
)
if per_team_performance_bf_count > PER_TEAM_PERFORMANCE_BF_COUNT_LIMIT:
failures.append(
bf_count_failure_msg.format(
scope=f"Branch({branch}): Team({team})",
bf_type="Performance BF",
bf_count=per_team_performance_bf_count,
bf_limit=PER_TEAM_PERFORMANCE_BF_COUNT_LIMIT,
)
)
return failures
def main(
github_repo: Annotated[
str, typer.Option(help="Github repository name that Evergreen projects track")
] = DEFAULT_REPO,
branch: Annotated[
str, typer.Option(help="Branch name that Evergreen projects track")
] = DEFAULT_BRANCH,
) -> None:
"""
Analyze Jira BFs count and Evergreen redness data.
For Jira API authentication please use `JIRA_AUTH_PAT` env variable.
More about Jira Personal Access Tokens (PATs) here:
- https://wiki.corp.mongodb.com/pages/viewpage.action?pageId=218995581
For Evergreen API authentication please create `~/.evergreen.yml`.
More about Evergreen auth here:
- https://spruce.mongodb.com/preferences/cli
Example:
JIRA_AUTH_PAT=<auth-token> python buildscripts/monitor_build_status/cli.py --help
"""
enable_logging(verbose=False)
jira_client = JiraClient(JIRA_SERVER, JiraAuth())
evg_api = get_evergreen_api()
jira_service = JiraService(jira_client=jira_client)
evg_service = EvergreenService(evg_api=evg_api)
orchestrator = MonitorBuildStatusOrchestrator(
jira_service=jira_service, evg_service=evg_service
)
orchestrator.evaluate_build_redness(github_repo, branch)
if __name__ == "__main__":
typer.run(main)

59
buildscripts/monitor_build_status/evergreen_service.py Normal file
View File

@ -0,0 +1,59 @@
from __future__ import annotations
from typing import Dict, List, NamedTuple
from evergreen import EvergreenApi
class EvgProjectsInfo(NamedTuple):
project_to_branch_map: Dict[str, str]
branch_to_projects_map: Dict[str, List[str]]
active_project_names: List[str]
tracking_branches: List[str]
@classmethod
def from_project_branch_map(cls, project_to_branch_map: Dict[str, str]) -> EvgProjectsInfo:
"""
Build EvgProjectsInfo object from evergreen project name to its tracking branch map.
:param project_to_branch_map: Evergreen project name to its tracking branch map.
:return: Evergreen projects information.
"""
branch_to_projects_map = {}
for project, branch in project_to_branch_map.items():
if branch not in branch_to_projects_map:
branch_to_projects_map[branch] = []
branch_to_projects_map[branch].append(project)
return cls(
project_to_branch_map=project_to_branch_map,
branch_to_projects_map=branch_to_projects_map,
active_project_names=[name for name in project_to_branch_map.keys()],
tracking_branches=list({branch for branch in project_to_branch_map.values()}),
)
class EvergreenService:
def __init__(self, evg_api: EvergreenApi) -> None:
self.evg_api = evg_api
def get_evg_project_info(self, tracking_repo: str, tracking_branch: str) -> EvgProjectsInfo:
"""
Accumulate information about active evergreen projects that
track provided repo and branch.
:param tracking_repo: Repo name in `{github_org}/{github_repo}` format.
:param tracking_branch: Branch name.
:return: Evergreen projects information.
"""
evg_projects = [
project
for project in self.evg_api.all_projects()
if project.enabled
and f"{project.owner_name}/{project.repo_name}" == tracking_repo
and project.branch_name == tracking_branch
]
project_branch_map = {project.identifier: project.branch_name for project in evg_projects}
return EvgProjectsInfo.from_project_branch_map(project_branch_map)

106
buildscripts/monitor_build_status/jira_service.py Normal file
View File

@ -0,0 +1,106 @@
from __future__ import annotations
import re
from enum import Enum
from typing import List, NamedTuple, Optional
from jira import Issue
from buildscripts.client.jiraclient import JiraClient
UNASSIGNED_LABEL = "~ Unassigned"
CORRECTNESS_EVG_PROJECT_REGEX = re.compile(r"mongodb-mongo.*")
PERFORMANCE_EVG_PROJECT_REGEX = re.compile(r"sys-perf.*")
class JiraCustomFieldIds(str, Enum):
ASSIGNED_TEAMS = "customfield_12751"
EVERGREEN_PROJECT = "customfield_14278"
TEMPERATURE = "customfield_24859"
class JiraCustomFieldNames(str, Enum):
EVERGREEN_PROJECT = "Evergreen Project"
class TestType(str, Enum):
CORRECTNESS = "correctness"
PERFORMANCE = "performance"
UNKNOWN = "unknown"
@classmethod
def from_evg_project_name(cls, name: str) -> TestType:
if CORRECTNESS_EVG_PROJECT_REGEX.match(name):
return cls.CORRECTNESS
if PERFORMANCE_EVG_PROJECT_REGEX.match(name):
return cls.PERFORMANCE
return cls.UNKNOWN
class BfTemperature(str, Enum):
HOT = "hot"
COLD = "cold"
NONE = "none"
@classmethod
def from_str(cls, temperature: Optional[str]) -> BfTemperature:
match temperature:
case "hot":
return cls.HOT
case "cold":
return cls.COLD
case _:
return cls.NONE
class BfIssue(NamedTuple):
key: str
assigned_team: str
evergreen_projects: List[str]
temperature: BfTemperature
@classmethod
def from_jira_issue(cls, issue: Issue) -> BfIssue:
"""
Build BfIssue object from Jira Issue object.
:param issue: Jira issue object.
:return: BF issue object.
"""
assigned_team = UNASSIGNED_LABEL
assigned_teams_field = getattr(issue.fields, JiraCustomFieldIds.ASSIGNED_TEAMS)
if isinstance(assigned_teams_field, list) and len(assigned_teams_field) > 0:
assigned_teams_values = [getattr(item, "value") for item in assigned_teams_field]
assigned_team = assigned_teams_values[0]
evergreen_projects = getattr(issue.fields, JiraCustomFieldIds.EVERGREEN_PROJECT)
if not isinstance(evergreen_projects, list):
evergreen_projects = []
temperature_value = getattr(issue.fields, JiraCustomFieldIds.TEMPERATURE)
if isinstance(temperature_value, str):
temperature = BfTemperature.from_str(temperature_value)
else:
temperature = BfTemperature.NONE
return cls(
key=issue.key,
assigned_team=assigned_team,
evergreen_projects=evergreen_projects,
temperature=temperature,
)
class JiraService:
def __init__(self, jira_client: JiraClient) -> None:
self.jira_client = jira_client
def fetch_bfs(self, query: str) -> List[BfIssue]:
"""
Fetch BFs issues from Jira and transform it into consumable form.
:param query: Jira query string.
:return: BF issues.
"""
jira_issues = self.jira_client.get_issues(query)
return [BfIssue.from_jira_issue(issue) for issue in jira_issues]

0
buildscripts/tests/monitor_build_status/__init__.py Normal file
View File

49
buildscripts/tests/monitor_build_status/test_jira_service.py Normal file
View File

@ -0,0 +1,49 @@
import unittest
from unittest.mock import MagicMock
import buildscripts.monitor_build_status.jira_service as under_test
class TestBfIssue(unittest.TestCase):
def test_parse_assigned_team_from_jira_issue(self):
team_name = "Team Name"
jira_issue_1 = MagicMock(fields=MagicMock(customfield_12751=[MagicMock(value=team_name)]))
bf_issue = under_test.BfIssue.from_jira_issue(jira_issue_1)
self.assertEqual(bf_issue.assigned_team, team_name)
jira_issue_2 = MagicMock(fields=MagicMock(customfield_12751=[]))
bf_issue = under_test.BfIssue.from_jira_issue(jira_issue_2)
self.assertEqual(bf_issue.assigned_team, under_test.UNASSIGNED_LABEL)
jira_issue_3 = MagicMock(fields=MagicMock())
bf_issue = under_test.BfIssue.from_jira_issue(jira_issue_3)
self.assertEqual(bf_issue.assigned_team, under_test.UNASSIGNED_LABEL)
def test_parse_evergreen_projects_from_jira_issue(self):
evergreen_projects = ["evg-project"]
jira_issue_1 = MagicMock(fields=MagicMock(customfield_14278=evergreen_projects))
bf_issue = under_test.BfIssue.from_jira_issue(jira_issue_1)
self.assertEqual(bf_issue.evergreen_projects, evergreen_projects)
jira_issue_2 = MagicMock(fields=MagicMock(customfield_14278=[]))
bf_issue = under_test.BfIssue.from_jira_issue(jira_issue_2)
self.assertEqual(bf_issue.evergreen_projects, [])
jira_issue_3 = MagicMock(fields=MagicMock())
bf_issue = under_test.BfIssue.from_jira_issue(jira_issue_3)
self.assertEqual(bf_issue.evergreen_projects, [])
def test_parse_bf_temperature_from_jira_issue(self):
bf_temperature = "hot"
jira_issue_1 = MagicMock(fields=MagicMock(customfield_24859=bf_temperature))
bf_issue = under_test.BfIssue.from_jira_issue(jira_issue_1)
self.assertEqual(bf_issue.temperature, under_test.BfTemperature.HOT)
bf_temperature = "cold"
jira_issue_2 = MagicMock(fields=MagicMock(customfield_24859=bf_temperature))
bf_issue = under_test.BfIssue.from_jira_issue(jira_issue_2)
self.assertEqual(bf_issue.temperature, under_test.BfTemperature.COLD)
jira_issue_3 = MagicMock(fields=MagicMock())
bf_issue = under_test.BfIssue.from_jira_issue(jira_issue_3)
self.assertEqual(bf_issue.temperature, under_test.BfTemperature.NONE)