Compare commits

...

7 Commits

Author SHA1 Message Date
John Daniels
e21b694884 Make sure that schedule_patch_auto_tasks works.
This needs to depend on schedule_global_auto_tasks.
We also move activate: true to only apply to
generate_all_variant_auto_tasks in the task_generation
variant.
2023-09-19 10:42:45 -04:00
John Daniels
f88e4b1315 Fix accidentally commented _compile_amazon2 anchor 2023-09-18 20:12:12 -04:00
John Daniels
a56b7f94a4 Update bypass compile expansions. 2023-09-18 20:10:50 -04:00
John Daniels
563f5ceff4 Get rid of schedule_global_auto_tasks. 2023-09-18 19:58:49 -04:00
John Daniels
6c8d262e10 Test taskgen. 2023-09-18 17:06:32 -04:00
John Daniels
f12fa6401e Server-80310: Fix duplicate anchors. 2023-09-18 17:00:58 -04:00
John Daniels
f23c684c3f SERVER-80310: sysperf: Always generate all genny workloads on waterfall.
This will make it so that we always do the genny task generation for
all variants on every commit, but those tasks shouldn't run until
the variant is scheduled.
2023-09-18 16:59:51 -04:00

View File

@ -17,30 +17,20 @@ variables:
_real_compile_amazon2: &_compile_amazon2
- name: compile
variant: compile-amazon2
- name: schedule_global_auto_tasks
variant: task_generation
_real_compile_rhel70: &_compile_rhel70
- name: compile
variant: compile-rhel70
- name: schedule_global_auto_tasks
variant: task_generation
_real_compile_amazon_linux2_arm64: &_compile_amazon_linux2_arm64
- name: compile
variant: compile-amazon-linux2-arm64
- name: schedule_global_auto_tasks
variant: task_generation
_real_compile_amazon_linux2_arm64_with_mongocrypt_shlib: &_compile_amazon_linux2_arm64_with_mongocrypt_shlib
- name: compile
variant: compile-amazon-linux2-arm64
- name: compile_mongocrypt_shlib
variant: compile-amazon-linux2-arm64-mongocrypt-shlib
- name: schedule_global_auto_tasks
variant: task_generation
_real_compile_amazon2_streams: &_compile_amazon2_streams
- name: compile
variant: compile-amazon2-streams
- name: schedule_global_auto_tasks
variant: task_generation
_real_expansions: &_expansion_updates
[]
@ -48,27 +38,22 @@ variables:
###
# **Or**: Leave this section uncommented to bypass/skip compile.
# _skip_compile_amazon2: &_compile_amazon2
# - name: schedule_global_auto_tasks
# variant: task_generation
# _skip_compile_rhel70: &_compile_rhel70
# - name: schedule_global_auto_tasks
# variant: task_generation
# _skip_compile_amazon_linux2_arm64: &_compile_amazon_linux2_arm64
# - name: schedule_global_auto_tasks
# variant: task_generation
# _skip_compile_amazon_linux2_arm64_with_mongocrypt_shlib: &_compile_amazon_linux2_arm64_with_mongocrypt_shlib
# - name: schedule_global_auto_tasks
# variant: task_generation
# _skip_compile_amazon2_streams: &_compile_amazon2_streams
# - name: schedule_global_auto_tasks
# variant: task_generation
# _skip_expansions: &_expansion_updates
# # This is the normal (amazon2) "compile" artifact from https://evergreen.mongodb.com/version/sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9
# - key: mdb_binary_for_client
# value: https://mciuploads.s3.amazonaws.com/dsi/sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9/97c6a9e443ff7e171b7310a1fa5c05d0768faff9/linux/mongodb-sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9.tar.gz
# - key: mdb_binary_for_server
# value: https://mciuploads.s3.amazonaws.com/dsi/sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9/97c6a9e443ff7e171b7310a1fa5c05d0768faff9/linux/mongodb-sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9.tar.gz
# _skip_compile_amazon2: &_compile_amazon2
# []
# _skip_compile_rhel70: &_compile_rhel70
# []
# _skip_compile_amazon_linux2_arm64: &_compile_amazon_linux2_arm64
# []
# _skip_compile_amazon_linux2_arm64_with_mongocrypt_shlib: &_compile_amazon_linux2_arm64_with_mongocrypt_shlib
# []
# _skip_compile_amazon2_streams: &_compile_amazon2_streams
# []
# _skip_expansions: &_expansion_updates
# # This is the normal (amazon2) "compile" artifact from https://evergreen.mongodb.com/version/sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9
# - key: mdb_binary_for_client
# value: https://mciuploads.s3.amazonaws.com/dsi/sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9/97c6a9e443ff7e171b7310a1fa5c05d0768faff9/linux/mongodb-sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9.tar.gz
# - key: mdb_binary_for_server
# value: https://mciuploads.s3.amazonaws.com/dsi/sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9/97c6a9e443ff7e171b7310a1fa5c05d0768faff9/linux/mongodb-sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9.tar.gz
###
_src_dir: &src_dir src/mongo
@ -542,24 +527,57 @@ functions:
files:
- build/TaskJSON/Tasks.json
f_generate_all_variant_auto_tasks:
- command: git.get_project
params:
directory: *src_dir
revisions:
dsi: ${dsi_rev}
genny: 28766e31bba4c937f9711c4e18824cb347c32523
linkbench: ${linkbench_rev}
linkbench2: ${linkbench2_rev}
tsbs: ${tsbs_rev}
workloads: ${workloads_rev}
mongo-perf: ${mongo-perf_rev}
YCSB: ${YCSB_rev}
py-tpcc: ${py-tpcc_rev}
PrivateWorkloads: ${PrivateWorkloads_rev}
- command: expansions.write
params:
file: ./expansions.yml
- command: shell.exec
params:
# The script below makes sure that if we generate all tasks in a patch, we will only generate
# them and not run all of them, since activating all tasks is the default behavior for task
# generation in a patch.
script: |
if [ "${is_patch|false}" = "true" ]; then
./src/genny/run-genny auto-tasks-all --project-file ./src/mongo/etc/system_perf.yml --no-activate
else
./src/genny/run-genny auto-tasks-all --project-file ./src/mongo/etc/system_perf.yml
fi
- command: generate.tasks
params:
files:
- build/TaskJSON/Tasks.json
tasks:
###
# Same in every DSI project
- name: generate_all_variant_auto_tasks
priority: 5
commands:
- func: f_generate_all_variant_auto_tasks
- name: schedule_global_auto_tasks
priority: 5
commands:
- func: f_schedule_tasks
vars:
tasks: all_tasks
- name: schedule_variant_auto_tasks
priority: 5
commands:
- func: f_schedule_tasks
vars:
tasks: variant_tasks
- name: schedule_patch_auto_tasks
priority: 5
depends_on:
- name: schedule_global_auto_tasks
variant: task_generation
commands:
- func: f_schedule_tasks
vars:
@ -1748,8 +1766,10 @@ buildvariants:
platform: linux
project_dir: dsi
run_on:
- amazon2-build
- amazon2-small
tasks:
- name: generate_all_variant_auto_tasks
activate: true
- name: schedule_global_auto_tasks
- &compile-amazon2
@ -1854,7 +1874,6 @@ buildvariants:
depends_on: *_compile_amazon2
tasks: &classic_engine_tasks
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: ycsb_60GB
- name: ycsb_60GB.long
@ -1911,7 +1930,6 @@ buildvariants:
depends_on: *_compile_amazon2
tasks: &classic_engine_1nodereplset_tasks
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: linkbench
- name: linkbench2
- name: snapshot_reads
@ -1955,7 +1973,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: bestbuy_4_analytics
- name: bestbuy_4_analytics_columnstore
- name: bestbuy_4_inserts
@ -2006,7 +2023,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: ycsb_60GB
- name: tpcc
@ -2033,7 +2049,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: ycsb_60GB
- name: tpcc
@ -2062,15 +2077,10 @@ buildvariants:
depends_on:
- name: compile
variant: compile-amazon2
- name: schedule_global_auto_tasks
variant: task_generation
- name: compile
variant: compile-amazon-linux2-arm64
- name: schedule_global_auto_tasks
variant: task_generation
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: tpcc
- name: linkbench2
@ -2096,15 +2106,10 @@ buildvariants:
depends_on:
- name: compile
variant: compile-amazon2
- name: schedule_global_auto_tasks
variant: task_generation
- name: compile
variant: compile-amazon-linux2-arm64
- name: schedule_global_auto_tasks
variant: task_generation
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
cron: &high-value-workload-cron "0 */4 * * 1-5" # High-value workloads run every weekday, every 4 hours
- name: ycsb_60GB
@ -2132,12 +2137,8 @@ buildvariants:
run_on:
- "rhel70-perf-atlas-large"
depends_on:
- name: schedule_global_auto_tasks
variant: task_generation
- name: compile
variant: compile-amazon-linux2-arm64
- name: schedule_global_auto_tasks
variant: task_generation
tasks:
- name: locust_bulk_insert
@ -2162,15 +2163,10 @@ buildvariants:
depends_on:
- name: compile
variant: compile-amazon2
- name: schedule_global_auto_tasks
variant: task_generation
- name: compile
variant: compile-rhel70
- name: schedule_global_auto_tasks
variant: task_generation
tasks: # Cannot use *3nodetasks because secondary_performance uses a special mongodb setup
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: ycsb_60GB
- name: tpcc
@ -2209,7 +2205,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks: &standalonetasks
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: dsi_integ_test_run_command_simple
- name: smoke_test
- name: industry_benchmarks
@ -2258,7 +2253,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: ycsb_60GB
- name: ycsb_60GB.long
@ -2303,7 +2297,6 @@ buildvariants:
depends_on: *_compile_amazon2_streams
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: linux-standalone-classic-query-engine.2022-11
display_name: Linux Standalone (Classic Query Engine) 2022-11
@ -2365,7 +2358,11 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks: &bonsai_standalone_tasks
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: ycsb_60GB
- name: ycsb_60GB.long
- name: bestbuy_agg
- name: bestbuy_query
- name: linux-intel-standalone-sampling-bonsai
display_name: Linux Intel Standalone (Bonsai with Sampling CE)
@ -2444,7 +2441,6 @@ buildvariants:
depends_on: *_compile_amazon2
tasks: &bonsai_1nodereplset_tasks
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: linkbench2
- name: snapshot_reads
@ -2568,7 +2564,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: ycsb_60GB
- name: ycsb_60GB.long
@ -2596,7 +2591,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: linux-1-node-replSet-all-feature-flags.2022-11
display_name: Linux 1-Node ReplSet (all feature flags) 2022-11
@ -2618,7 +2612,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: linkbench
- name: linkbench2
@ -2642,7 +2635,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks: &1nodetasks
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: ycsb_60GB
- name: ycsb_60GB.long
@ -2716,7 +2708,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks: &audit-tasks
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: crud_workloads_majority
- name: smoke_test
@ -2781,7 +2772,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: ycsb_60GB
- name: ycsb_60GB.long
@ -2811,7 +2801,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64_with_mongocrypt_shlib
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: medical_workload_diagnosis_50_50_high_value
cron: *high-value-workload-cron
- name: ycsb_like_queryable_encrypt1_cfdefault_sharded_high_value
@ -2859,7 +2848,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64_with_mongocrypt_shlib
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: linkbench
@ -2883,7 +2871,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: smoke_test
- name: ycsb_60GB
@ -2909,7 +2896,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: smoke_test
- name: ycsb_60GB
@ -2935,7 +2921,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: smoke_test
- name: ycsb_60GB
@ -2961,7 +2946,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: crud_workloads_majority
- name: crud_workloads_w1
- name: smoke_test
@ -2989,7 +2973,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks: &linux_3_shard_tasks
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: industry_benchmarks_w1
- name: crud_workloads_majority
@ -3022,8 +3005,6 @@ buildvariants:
<<: *shard-baseline-expansions
mongodb_setup: shard-query-stats
tasks: &sharded-query-stats-tasks
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: mongos_workloads
- name: tsbs_query_sharded
@ -3060,7 +3041,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: industry_benchmarks_w1
@ -3104,9 +3084,8 @@ buildvariants:
run_on:
- "rhel70-perf-shard-lite"
depends_on: *_compile_amazon_linux2_arm64
tasks: &shardlitetasks
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: bestbuy_agg
- name: bestbuy_agg_merge_different_db
- name: bestbuy_agg_merge_same_db
@ -3147,9 +3126,8 @@ buildvariants:
run_on:
- "rhel70-perf-shard-lite"
depends_on: *_compile_amazon2
tasks: &shardlitetasks
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: mongos_workloads
- name: move_chunk_workloads
@ -3176,7 +3154,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: linkbench
@ -3200,7 +3177,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: change_streams_preimage_throughput
- name: change_streams_preimage_latency
- name: tsbs_query_sharded
@ -3217,7 +3193,6 @@ buildvariants:
mongodb_setup: shard-lite-query-stats
tasks: &shard-lite-query-stats-tasks
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: bestbuy_query
- name: industry_benchmarks
- name: linkbench
@ -3256,7 +3231,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: linux-3-node-replSet.2022-11
display_name: Linux 3-Node ReplSet 2022-11
@ -3278,7 +3252,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
cron: *high-value-workload-cron
- name: industry_benchmarks_w1
@ -3358,7 +3331,6 @@ buildvariants:
depends_on: *_compile_amazon2
tasks: &3node-intel-tasks
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
cron: &query-stats-cron "0 */4 * * 1-5" # runs every weekday, every 4 hours
- name: ycsb_60GB
@ -3431,7 +3403,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: ycsb_60GB
- name: ycsb_60GB.long
@ -3489,7 +3460,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: ycsb_60GB
- name: ycsb_60GB.long
@ -3547,7 +3517,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: ycsb_60GB
- name: ycsb_60GB.long
@ -3608,7 +3577,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: crud_workloads_majority
- name: industry_benchmarks
@ -3632,7 +3600,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks_stepdowns
- name: industry_benchmarks_rolling_restarts
- name: industry_benchmarks_non_retryable_writes_stepdowns
@ -3666,7 +3633,6 @@ buildvariants:
- "rhel70-perf-replset"
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: initialsync-large
- name: initialsync-large-fcbis
@ -3690,7 +3656,6 @@ buildvariants:
- "rhel70-perf-replset"
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: linux-replSet-initialsync-logkeeper.2022-11
display_name: Linux ReplSet Initial Sync LogKeeper 2022-11
@ -3714,7 +3679,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: initialsync-logkeeper
- name: initialsync-logkeeper-fcbis
@ -3741,7 +3705,6 @@ buildvariants:
# depends_on: *_compile_amazon_linux2_arm64
# tasks:
# - name: schedule_patch_auto_tasks
# - name: schedule_variant_auto_tasks
# - name: initialsync-logkeeper-snapshot-update
- name: linux-replSet-audit.2022-11
@ -3764,7 +3727,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
- name: linux-replSet-auth-delay.2022-11
@ -3787,7 +3749,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: linux-3-node-replSet-query-stats
display_name: Linux 3-Node ReplSet 2022-11 (QueryStats)
@ -3809,7 +3770,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: industry_benchmarks
cron: *high-value-workload-cron
- name: ycsb_60GB
@ -3833,7 +3793,6 @@ buildvariants:
cron: *high-value-workload-cron
- name: expressive_queries_high_value
cron: *high-value-workload-cron
- name: linux-3-node-replSet-250mbwtcache.2023-05
display_name: Linux 3-Node ReplSet 250 MB WiredTiger Cache 2023-05
cron: *linux-3-node-cron
@ -3854,7 +3813,6 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
# Template below used to create an anchor for linux microbenchmarks standalone arm without the high-value-workload cron override that would transfer to any variant that uses this
- &linux-microbenchmarks-standalone-arm
@ -4269,5 +4227,4 @@ buildvariants:
depends_on: *_compile_amazon_linux2_arm64
tasks:
- name: schedule_patch_auto_tasks
- name: schedule_variant_auto_tasks
- name: genny_resharding_withIndexes