From 788089a18d9e9534fb87c538a6fad5ecafe3f478 Mon Sep 17 00:00:00 2001 From: Ben Shteinfeld Date: Wed, 29 Oct 2025 12:49:05 -0400 Subject: [PATCH] SERVER-112851 Introduce knob to enerumate HashJoin plans from random join optimizer (#43217) GitOrigin-RevId: 193bfb1114523de7f3dcca45c71f498338048857 --- .../featureFlagSbeFull/basic_joins.md | 1058 ++++++++++++++++- .../sbeDisabled/basic_joins.md | 1058 ++++++++++++++++- .../expected_output/sbeFull/basic_joins.md | 1058 ++++++++++++++++- .../sbeRestricted/basic_joins.md | 1058 ++++++++++++++++- .../query_golden/join_opt/basic_joins_md.js | 62 +- .../compiler/optimizer/join/reorder_joins.cpp | 6 + src/mongo/db/query/query_knobs.idl | 9 + 7 files changed, 4256 insertions(+), 53 deletions(-) diff --git a/jstests/query_golden/expected_output/featureFlagSbeFull/basic_joins.md b/jstests/query_golden/expected_output/featureFlagSbeFull/basic_joins.md index fd400358f54..a04281327fa 100644 --- a/jstests/query_golden/expected_output/featureFlagSbeFull/basic_joins.md +++ b/jstests/query_golden/expected_output/featureFlagSbeFull/basic_joins.md @@ -71,7 +71,7 @@ Execution Engine: sbe } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -159,7 +159,95 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + } +] +``` +### Results +```json +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "2312073BEBB7A5E1754E2D03D9CB40B75C126DBF60867D4639E7E281C9E3DFAC", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 5, + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -247,6 +335,94 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + } +] +``` +### Results +```json +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "2312073BEBB7A5E1754E2D03D9CB40B75C126DBF60867D4639E7E281C9E3DFAC", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 5, + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + ## 2. Basic example with two joins and suffix ### No join opt ### Pipeline @@ -337,7 +513,7 @@ Execution Engine: sbe } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -455,7 +631,125 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + }, + { + "$sortByCount" : "$y.b" + } +] +``` +### Results +```json +{ "_id" : "bar", "count" : 6 } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "8E0DF14D3DA69D8B1CF0361E869369F9C5E5F2F2A5D81998E6800BDA4F997B57", + "stages" : [ + { + "$cursor" : { + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStage" : { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + "stage" : "PROJECTION_SIMPLE", + "transformBy" : { + "_id" : false, + "a" : true, + "b" : true + } + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 6, + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + } + } + }, + { + "$group" : { + "$willBeMerged" : false, + "_id" : "$y.b", + "count" : { + "$sum" : { + "$const" : 1 + } + } + } + }, + { + "$sort" : { + "sortKey" : { + "count" : -1 + } + } + } + ] +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -573,6 +867,124 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + }, + { + "$sortByCount" : "$y.b" + } +] +``` +### Results +```json +{ "_id" : "bar", "count" : 6 } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "8E0DF14D3DA69D8B1CF0361E869369F9C5E5F2F2A5D81998E6800BDA4F997B57", + "stages" : [ + { + "$cursor" : { + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStage" : { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + "stage" : "PROJECTION_SIMPLE", + "transformBy" : { + "_id" : false, + "a" : true, + "b" : true + } + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 6, + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + } + } + }, + { + "$group" : { + "$willBeMerged" : false, + "_id" : "$y.b", + "count" : { + "$sum" : { + "$const" : 1 + } + } + } + }, + { + "$sort" : { + "sortKey" : { + "count" : -1 + } + } + } + ] +} +``` + ## 3. Basic example with referencing field from previous lookup ### No join opt ### Pipeline @@ -644,7 +1056,7 @@ Execution Engine: sbe } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -730,7 +1142,93 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "z", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$z" + } +] +``` +### Results +```json +{ "_id" : 0, "a" : 1, "b" : "foo", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "DC5CAF413E9B6B8B5E8B2D3FB91E4D546524BF2707BBE30472D5CA8E12F6637E", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 5, + "rightEmbeddingField" : "z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -816,6 +1314,92 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "z", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$z" + } +] +``` +### Results +```json +{ "_id" : 0, "a" : 1, "b" : "foo", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "DC5CAF413E9B6B8B5E8B2D3FB91E4D546524BF2707BBE30472D5CA8E12F6637E", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 5, + "rightEmbeddingField" : "z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + ## 4. Basic example with 3 joins & subsequent join referencing fields from previous lookups ### No join opt ### Pipeline @@ -909,7 +1493,7 @@ Execution Engine: sbe } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -1025,7 +1609,123 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "z", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$z" + } +] +``` +### Results +```json +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "DD24FFAE1F16C6C1F5B03939E0C642A797120A2585058C72554C9649BEA427AB", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 7, + "rightEmbeddingField" : "z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -1141,6 +1841,122 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "z", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$z" + } +] +``` +### Results +```json +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "DD24FFAE1F16C6C1F5B03939E0C642A797120A2585058C72554C9649BEA427AB", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "z", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 7, + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + ## 5. Basic example with 3 joins & subsequent join referencing nested paths ### No join opt ### Pipeline @@ -1229,7 +2045,7 @@ Execution Engine: sbe } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -1340,7 +2156,118 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "x.y", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$x.y" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "x.y.z", + "localField" : "x.y.d", + "foreignField" : "d" + } + }, + { + "$unwind" : "$x.y.z" + } +] +``` +### Results +```json +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2, "y" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2, "z" : { "_id" : 0, "b" : "bar", "d" : 2 } } } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "4545A5343C4F418AA879D5C8E031D6B637DFEE78E362FFC2EE47775E87790625", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x.y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.y.d = d" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 7, + "rightEmbeddingField" : "x.y.z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -1451,3 +2378,114 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "x.y", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$x.y" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "x.y.z", + "localField" : "x.y.d", + "foreignField" : "d" + } + }, + { + "$unwind" : "$x.y.z" + } +] +``` +### Results +```json +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2, "y" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2, "z" : { "_id" : 0, "b" : "bar", "d" : 2 } } } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "4545A5343C4F418AA879D5C8E031D6B637DFEE78E362FFC2EE47775E87790625", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x.y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.y.d = d" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 7, + "rightEmbeddingField" : "x.y.z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + diff --git a/jstests/query_golden/expected_output/sbeDisabled/basic_joins.md b/jstests/query_golden/expected_output/sbeDisabled/basic_joins.md index bbc1fd272cb..e580395b2f5 100644 --- a/jstests/query_golden/expected_output/sbeDisabled/basic_joins.md +++ b/jstests/query_golden/expected_output/sbeDisabled/basic_joins.md @@ -78,7 +78,7 @@ Execution Engine: classic } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -166,7 +166,95 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + } +] +``` +### Results +```json +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "2312073BEBB7A5E1754E2D03D9CB40B75C126DBF60867D4639E7E281C9E3DFAC", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 5, + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -254,6 +342,94 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + } +] +``` +### Results +```json +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "2312073BEBB7A5E1754E2D03D9CB40B75C126DBF60867D4639E7E281C9E3DFAC", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 5, + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + ## 2. Basic example with two joins and suffix ### No join opt ### Pipeline @@ -358,7 +534,7 @@ Execution Engine: classic } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -476,7 +652,125 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + }, + { + "$sortByCount" : "$y.b" + } +] +``` +### Results +```json +{ "_id" : "bar", "count" : 6 } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "8E0DF14D3DA69D8B1CF0361E869369F9C5E5F2F2A5D81998E6800BDA4F997B57", + "stages" : [ + { + "$cursor" : { + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStage" : { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + "stage" : "PROJECTION_SIMPLE", + "transformBy" : { + "_id" : false, + "a" : true, + "b" : true + } + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 6, + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + } + } + }, + { + "$group" : { + "$willBeMerged" : false, + "_id" : "$y.b", + "count" : { + "$sum" : { + "$const" : 1 + } + } + } + }, + { + "$sort" : { + "sortKey" : { + "count" : -1 + } + } + } + ] +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -594,6 +888,124 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + }, + { + "$sortByCount" : "$y.b" + } +] +``` +### Results +```json +{ "_id" : "bar", "count" : 6 } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "8E0DF14D3DA69D8B1CF0361E869369F9C5E5F2F2A5D81998E6800BDA4F997B57", + "stages" : [ + { + "$cursor" : { + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStage" : { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + "stage" : "PROJECTION_SIMPLE", + "transformBy" : { + "_id" : false, + "a" : true, + "b" : true + } + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 6, + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + } + } + }, + { + "$group" : { + "$willBeMerged" : false, + "_id" : "$y.b", + "count" : { + "$sum" : { + "$const" : 1 + } + } + } + }, + { + "$sort" : { + "sortKey" : { + "count" : -1 + } + } + } + ] +} +``` + ## 3. Basic example with referencing field from previous lookup ### No join opt ### Pipeline @@ -672,7 +1084,7 @@ Execution Engine: classic } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -758,7 +1170,93 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "z", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$z" + } +] +``` +### Results +```json +{ "_id" : 0, "a" : 1, "b" : "foo", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "DC5CAF413E9B6B8B5E8B2D3FB91E4D546524BF2707BBE30472D5CA8E12F6637E", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 5, + "rightEmbeddingField" : "z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -844,6 +1342,92 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "z", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$z" + } +] +``` +### Results +```json +{ "_id" : 0, "a" : 1, "b" : "foo", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "DC5CAF413E9B6B8B5E8B2D3FB91E4D546524BF2707BBE30472D5CA8E12F6637E", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 5, + "rightEmbeddingField" : "z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + ## 4. Basic example with 3 joins & subsequent join referencing fields from previous lookups ### No join opt ### Pipeline @@ -946,7 +1530,7 @@ Execution Engine: classic } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -1062,7 +1646,123 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "z", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$z" + } +] +``` +### Results +```json +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "DD24FFAE1F16C6C1F5B03939E0C642A797120A2585058C72554C9649BEA427AB", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 7, + "rightEmbeddingField" : "z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -1178,6 +1878,122 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "z", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$z" + } +] +``` +### Results +```json +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "DD24FFAE1F16C6C1F5B03939E0C642A797120A2585058C72554C9649BEA427AB", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "z", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 7, + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + ## 5. Basic example with 3 joins & subsequent join referencing nested paths ### No join opt ### Pipeline @@ -1275,7 +2091,7 @@ Execution Engine: classic } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -1386,7 +2202,118 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "x.y", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$x.y" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "x.y.z", + "localField" : "x.y.d", + "foreignField" : "d" + } + }, + { + "$unwind" : "$x.y.z" + } +] +``` +### Results +```json +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2, "y" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2, "z" : { "_id" : 0, "b" : "bar", "d" : 2 } } } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "4545A5343C4F418AA879D5C8E031D6B637DFEE78E362FFC2EE47775E87790625", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x.y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.y.d = d" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 7, + "rightEmbeddingField" : "x.y.z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -1497,3 +2424,114 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "x.y", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$x.y" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "x.y.z", + "localField" : "x.y.d", + "foreignField" : "d" + } + }, + { + "$unwind" : "$x.y.z" + } +] +``` +### Results +```json +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2, "y" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2, "z" : { "_id" : 0, "b" : "bar", "d" : 2 } } } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "4545A5343C4F418AA879D5C8E031D6B637DFEE78E362FFC2EE47775E87790625", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x.y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.y.d = d" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 7, + "rightEmbeddingField" : "x.y.z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + diff --git a/jstests/query_golden/expected_output/sbeFull/basic_joins.md b/jstests/query_golden/expected_output/sbeFull/basic_joins.md index fd400358f54..a04281327fa 100644 --- a/jstests/query_golden/expected_output/sbeFull/basic_joins.md +++ b/jstests/query_golden/expected_output/sbeFull/basic_joins.md @@ -71,7 +71,7 @@ Execution Engine: sbe } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -159,7 +159,95 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + } +] +``` +### Results +```json +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "2312073BEBB7A5E1754E2D03D9CB40B75C126DBF60867D4639E7E281C9E3DFAC", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 5, + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -247,6 +335,94 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + } +] +``` +### Results +```json +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "2312073BEBB7A5E1754E2D03D9CB40B75C126DBF60867D4639E7E281C9E3DFAC", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 5, + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + ## 2. Basic example with two joins and suffix ### No join opt ### Pipeline @@ -337,7 +513,7 @@ Execution Engine: sbe } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -455,7 +631,125 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + }, + { + "$sortByCount" : "$y.b" + } +] +``` +### Results +```json +{ "_id" : "bar", "count" : 6 } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "8E0DF14D3DA69D8B1CF0361E869369F9C5E5F2F2A5D81998E6800BDA4F997B57", + "stages" : [ + { + "$cursor" : { + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStage" : { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + "stage" : "PROJECTION_SIMPLE", + "transformBy" : { + "_id" : false, + "a" : true, + "b" : true + } + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 6, + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + } + } + }, + { + "$group" : { + "$willBeMerged" : false, + "_id" : "$y.b", + "count" : { + "$sum" : { + "$const" : 1 + } + } + } + }, + { + "$sort" : { + "sortKey" : { + "count" : -1 + } + } + } + ] +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -573,6 +867,124 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + }, + { + "$sortByCount" : "$y.b" + } +] +``` +### Results +```json +{ "_id" : "bar", "count" : 6 } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "8E0DF14D3DA69D8B1CF0361E869369F9C5E5F2F2A5D81998E6800BDA4F997B57", + "stages" : [ + { + "$cursor" : { + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStage" : { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + "stage" : "PROJECTION_SIMPLE", + "transformBy" : { + "_id" : false, + "a" : true, + "b" : true + } + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 6, + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + } + } + }, + { + "$group" : { + "$willBeMerged" : false, + "_id" : "$y.b", + "count" : { + "$sum" : { + "$const" : 1 + } + } + } + }, + { + "$sort" : { + "sortKey" : { + "count" : -1 + } + } + } + ] +} +``` + ## 3. Basic example with referencing field from previous lookup ### No join opt ### Pipeline @@ -644,7 +1056,7 @@ Execution Engine: sbe } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -730,7 +1142,93 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "z", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$z" + } +] +``` +### Results +```json +{ "_id" : 0, "a" : 1, "b" : "foo", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "DC5CAF413E9B6B8B5E8B2D3FB91E4D546524BF2707BBE30472D5CA8E12F6637E", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 5, + "rightEmbeddingField" : "z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -816,6 +1314,92 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "z", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$z" + } +] +``` +### Results +```json +{ "_id" : 0, "a" : 1, "b" : "foo", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "DC5CAF413E9B6B8B5E8B2D3FB91E4D546524BF2707BBE30472D5CA8E12F6637E", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 5, + "rightEmbeddingField" : "z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + ## 4. Basic example with 3 joins & subsequent join referencing fields from previous lookups ### No join opt ### Pipeline @@ -909,7 +1493,7 @@ Execution Engine: sbe } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -1025,7 +1609,123 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "z", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$z" + } +] +``` +### Results +```json +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "DD24FFAE1F16C6C1F5B03939E0C642A797120A2585058C72554C9649BEA427AB", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 7, + "rightEmbeddingField" : "z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -1141,6 +1841,122 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "z", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$z" + } +] +``` +### Results +```json +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "DD24FFAE1F16C6C1F5B03939E0C642A797120A2585058C72554C9649BEA427AB", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "z", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 7, + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + ## 5. Basic example with 3 joins & subsequent join referencing nested paths ### No join opt ### Pipeline @@ -1229,7 +2045,7 @@ Execution Engine: sbe } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -1340,7 +2156,118 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "x.y", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$x.y" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "x.y.z", + "localField" : "x.y.d", + "foreignField" : "d" + } + }, + { + "$unwind" : "$x.y.z" + } +] +``` +### Results +```json +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2, "y" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2, "z" : { "_id" : 0, "b" : "bar", "d" : 2 } } } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "4545A5343C4F418AA879D5C8E031D6B637DFEE78E362FFC2EE47775E87790625", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x.y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.y.d = d" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 7, + "rightEmbeddingField" : "x.y.z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -1451,3 +2378,114 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "x.y", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$x.y" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "x.y.z", + "localField" : "x.y.d", + "foreignField" : "d" + } + }, + { + "$unwind" : "$x.y.z" + } +] +``` +### Results +```json +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2, "y" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2, "z" : { "_id" : 0, "b" : "bar", "d" : 2 } } } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "4545A5343C4F418AA879D5C8E031D6B637DFEE78E362FFC2EE47775E87790625", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x.y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.y.d = d" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 7, + "rightEmbeddingField" : "x.y.z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + diff --git a/jstests/query_golden/expected_output/sbeRestricted/basic_joins.md b/jstests/query_golden/expected_output/sbeRestricted/basic_joins.md index bbc1fd272cb..e580395b2f5 100644 --- a/jstests/query_golden/expected_output/sbeRestricted/basic_joins.md +++ b/jstests/query_golden/expected_output/sbeRestricted/basic_joins.md @@ -78,7 +78,7 @@ Execution Engine: classic } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -166,7 +166,95 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + } +] +``` +### Results +```json +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "2312073BEBB7A5E1754E2D03D9CB40B75C126DBF60867D4639E7E281C9E3DFAC", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 5, + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -254,6 +342,94 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + } +] +``` +### Results +```json +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "2312073BEBB7A5E1754E2D03D9CB40B75C126DBF60867D4639E7E281C9E3DFAC", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 5, + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + ## 2. Basic example with two joins and suffix ### No join opt ### Pipeline @@ -358,7 +534,7 @@ Execution Engine: classic } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -476,7 +652,125 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + }, + { + "$sortByCount" : "$y.b" + } +] +``` +### Results +```json +{ "_id" : "bar", "count" : 6 } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "8E0DF14D3DA69D8B1CF0361E869369F9C5E5F2F2A5D81998E6800BDA4F997B57", + "stages" : [ + { + "$cursor" : { + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStage" : { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + "stage" : "PROJECTION_SIMPLE", + "transformBy" : { + "_id" : false, + "a" : true, + "b" : true + } + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 6, + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + } + } + }, + { + "$group" : { + "$willBeMerged" : false, + "_id" : "$y.b", + "count" : { + "$sum" : { + "$const" : 1 + } + } + } + }, + { + "$sort" : { + "sortKey" : { + "count" : -1 + } + } + } + ] +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -594,6 +888,124 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + }, + { + "$sortByCount" : "$y.b" + } +] +``` +### Results +```json +{ "_id" : "bar", "count" : 6 } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "8E0DF14D3DA69D8B1CF0361E869369F9C5E5F2F2A5D81998E6800BDA4F997B57", + "stages" : [ + { + "$cursor" : { + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStage" : { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + "stage" : "PROJECTION_SIMPLE", + "transformBy" : { + "_id" : false, + "a" : true, + "b" : true + } + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 6, + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + } + } + }, + { + "$group" : { + "$willBeMerged" : false, + "_id" : "$y.b", + "count" : { + "$sum" : { + "$const" : 1 + } + } + } + }, + { + "$sort" : { + "sortKey" : { + "count" : -1 + } + } + } + ] +} +``` + ## 3. Basic example with referencing field from previous lookup ### No join opt ### Pipeline @@ -672,7 +1084,7 @@ Execution Engine: classic } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -758,7 +1170,93 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "z", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$z" + } +] +``` +### Results +```json +{ "_id" : 0, "a" : 1, "b" : "foo", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "DC5CAF413E9B6B8B5E8B2D3FB91E4D546524BF2707BBE30472D5CA8E12F6637E", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 5, + "rightEmbeddingField" : "z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -844,6 +1342,92 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "z", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$z" + } +] +``` +### Results +```json +{ "_id" : 0, "a" : 1, "b" : "foo", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "DC5CAF413E9B6B8B5E8B2D3FB91E4D546524BF2707BBE30472D5CA8E12F6637E", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 5, + "rightEmbeddingField" : "z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + ## 4. Basic example with 3 joins & subsequent join referencing fields from previous lookups ### No join opt ### Pipeline @@ -946,7 +1530,7 @@ Execution Engine: classic } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -1062,7 +1646,123 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "z", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$z" + } +] +``` +### Results +```json +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "DD24FFAE1F16C6C1F5B03939E0C642A797120A2585058C72554C9649BEA427AB", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 7, + "rightEmbeddingField" : "z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -1178,6 +1878,122 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "y", + "localField" : "b", + "foreignField" : "b" + } + }, + { + "$unwind" : "$y" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "z", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$z" + } +] +``` +### Results +```json +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 1, "a" : 1, "b" : "bar", "x" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 0, "a" : 1, "c" : "zoo", "d" : 1 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 0, "b" : "bar", "d" : 2 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 }, "y" : { "_id" : 1, "b" : "bar", "d" : 6 }, "z" : { "_id" : 2, "a" : 2, "c" : "x", "d" : 3 } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "DD24FFAE1F16C6C1F5B03939E0C642A797120A2585058C72554C9649BEA427AB", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "z", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "b = b" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 7, + "rightEmbeddingField" : "y", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + ## 5. Basic example with 3 joins & subsequent join referencing nested paths ### No join opt ### Pipeline @@ -1275,7 +2091,7 @@ Execution Engine: classic } ``` -### With random order, seed 44 +### With random order, seed 44, nested loop joins ### Pipeline ```json [ @@ -1386,7 +2202,118 @@ Execution Engine: sbe } ``` -### With random order, seed 420 +### With random order, seed 44, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "x.y", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$x.y" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "x.y.z", + "localField" : "x.y.d", + "foreignField" : "d" + } + }, + { + "$unwind" : "$x.y.z" + } +] +``` +### Results +```json +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2, "y" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2, "z" : { "_id" : 0, "b" : "bar", "d" : 2 } } } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "4545A5343C4F418AA879D5C8E031D6B637DFEE78E362FFC2EE47775E87790625", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x.y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.y.d = d" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 7, + "rightEmbeddingField" : "x.y.z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + +### With random order, seed 420, nested loop joins ### Pipeline ```json [ @@ -1497,3 +2424,114 @@ Execution Engine: sbe } ``` +### With random order, seed 420, hash join enabled +### Pipeline +```json +[ + { + "$lookup" : { + "from" : "basic_joins_md_foreign1", + "as" : "x", + "localField" : "a", + "foreignField" : "a" + } + }, + { + "$unwind" : "$x" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign3", + "as" : "x.y", + "localField" : "x.c", + "foreignField" : "c" + } + }, + { + "$unwind" : "$x.y" + }, + { + "$lookup" : { + "from" : "basic_joins_md_foreign2", + "as" : "x.y.z", + "localField" : "x.y.d", + "foreignField" : "d" + } + }, + { + "$unwind" : "$x.y.z" + } +] +``` +### Results +```json +{ "_id" : 2, "a" : 2, "b" : "bar", "x" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2, "y" : { "_id" : 1, "a" : 2, "c" : "blah", "d" : 2, "z" : { "_id" : 0, "b" : "bar", "d" : 2 } } } } +``` +### Summarized explain +Execution Engine: sbe +```json +{ + "queryShapeHash" : "4545A5343C4F418AA879D5C8E031D6B637DFEE78E362FFC2EE47775E87790625", + "rejectedPlans" : [ ], + "winningPlan" : { + "inputStages" : [ + { + "inputStages" : [ + { + "inputStages" : [ + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "a = a" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.c = c" + ], + "leftEmbeddingField" : "none", + "rightEmbeddingField" : "x.y", + "stage" : "HASH_JOIN_EMBEDDING" + }, + { + "direction" : "forward", + "filter" : { + + }, + "stage" : "COLLSCAN" + } + ], + "joinPredicates" : [ + "x.y.d = d" + ], + "leftEmbeddingField" : "none", + "planNodeId" : 7, + "rightEmbeddingField" : "x.y.z", + "stage" : "HASH_JOIN_EMBEDDING" + } +} +``` + diff --git a/jstests/query_golden/join_opt/basic_joins_md.js b/jstests/query_golden/join_opt/basic_joins_md.js index 39a896c64b4..12b5a261a18 100644 --- a/jstests/query_golden/join_opt/basic_joins_md.js +++ b/jstests/query_golden/join_opt/basic_joins_md.js @@ -38,22 +38,58 @@ assert.commandWorked( ); function runBasicJoinTest(pipeline) { - subSection("No join opt"); - assert.commandWorked(db.adminCommand({setParameter: 1, internalEnableJoinOptimization: false})); - outputAggregationPlanAndResults(coll, pipeline, {}, true, false); + try { + subSection("No join opt"); + assert.commandWorked(db.adminCommand({setParameter: 1, internalEnableJoinOptimization: false})); + outputAggregationPlanAndResults(coll, pipeline, {}, true, false); + const noJoinOptResults = coll.aggregate(pipeline).toArray(); - subSection("With random order, seed 44"); - assert.commandWorked(db.adminCommand({setParameter: 1, internalEnableJoinOptimization: true})); - assert.commandWorked(db.adminCommand({setParameter: 1, internalRandomJoinOrderSeed: 44})); - outputAggregationPlanAndResults(coll, pipeline, {}, true, false); + subSection("With random order, seed 44, nested loop joins"); + assert.commandWorked(db.adminCommand({setParameter: 1, internalEnableJoinOptimization: true})); + assert.commandWorked(db.adminCommand({setParameter: 1, internalRandomJoinOrderSeed: 44})); + outputAggregationPlanAndResults(coll, pipeline, {}, true, false); + const seed44NLJResults = coll.aggregate(pipeline).toArray(); - subSection("With random order, seed 420"); - assert.commandWorked(db.adminCommand({setParameter: 1, internalEnableJoinOptimization: true})); - assert.commandWorked(db.adminCommand({setParameter: 1, internalRandomJoinOrderSeed: 420})); - outputAggregationPlanAndResults(coll, pipeline, {}, true, false); + subSection("With random order, seed 44, hash join enabled"); + assert.commandWorked(db.adminCommand({setParameter: 1, internalRandomJoinReorderDefaultToHashJoin: true})); + outputAggregationPlanAndResults(coll, pipeline, {}, true, false); + const seed44HJResults = coll.aggregate(pipeline).toArray(); - // Reset flag. - assert.commandWorked(db.adminCommand({setParameter: 1, internalEnableJoinOptimization: false})); + assert.commandWorked(db.adminCommand({setParameter: 1, internalRandomJoinReorderDefaultToHashJoin: false})); + + subSection("With random order, seed 420, nested loop joins"); + assert.commandWorked(db.adminCommand({setParameter: 1, internalEnableJoinOptimization: true})); + assert.commandWorked(db.adminCommand({setParameter: 1, internalRandomJoinOrderSeed: 420})); + outputAggregationPlanAndResults(coll, pipeline, {}, true, false); + const seed420NLJResults = coll.aggregate(pipeline).toArray(); + + subSection("With random order, seed 420, hash join enabled"); + assert.commandWorked(db.adminCommand({setParameter: 1, internalRandomJoinReorderDefaultToHashJoin: true})); + outputAggregationPlanAndResults(coll, pipeline, {}, true, false); + const seed420HJResults = coll.aggregate(pipeline).toArray(); + + // Validate that all execution modes return the same results. + assert( + _resultSetsEqualUnordered(noJoinOptResults, seed44NLJResults), + "Results differ between no join opt and seed 44 NLJ", + ); + assert( + _resultSetsEqualUnordered(noJoinOptResults, seed44HJResults), + "Results differ between no join opt and seed 44 HJ", + ); + assert( + _resultSetsEqualUnordered(noJoinOptResults, seed420NLJResults), + "Results differ between no join opt and seed 420 NLJ", + ); + assert( + _resultSetsEqualUnordered(noJoinOptResults, seed420HJResults), + "Results differ between no join opt and seed 420 HJ", + ); + } finally { + // Reset flags. + assert.commandWorked(db.adminCommand({setParameter: 1, internalEnableJoinOptimization: false})); + assert.commandWorked(db.adminCommand({setParameter: 1, internalRandomJoinReorderDefaultToHashJoin: false})); + } } section("Basic example with two joins"); diff --git a/src/mongo/db/query/compiler/optimizer/join/reorder_joins.cpp b/src/mongo/db/query/compiler/optimizer/join/reorder_joins.cpp index 7fcec07e6e0..f94a640c5bd 100644 --- a/src/mongo/db/query/compiler/optimizer/join/reorder_joins.cpp +++ b/src/mongo/db/query/compiler/optimizer/join/reorder_joins.cpp @@ -243,6 +243,12 @@ std::unique_ptr constructSolutionWithRandomOrder( ctx.makeJoinPreds(edge), boost::none, currentNode.embedPath); + } else if (internalRandomJoinReorderDefaultToHashJoin.load()) { + soln = std::make_unique(std::move(soln), + std::move(rhs), + ctx.makeJoinPreds(edge), + boost::none, + currentNode.embedPath); } else { soln = std::make_unique(std::move(soln), std::move(rhs), diff --git a/src/mongo/db/query/query_knobs.idl b/src/mongo/db/query/query_knobs.idl index e3dad0daf50..cec3e952423 100644 --- a/src/mongo/db/query/query_knobs.idl +++ b/src/mongo/db/query/query_knobs.idl @@ -520,6 +520,15 @@ server_parameters: on_update: plan_cache_util::clearSbeCacheOnParameterChange redact: false + internalRandomJoinReorderDefaultToHashJoin: + description: "Change the default join algorithm produced from random reorderer from NestedLoopJoin to HashJoin." + set_at: [startup, runtime] + cpp_varname: "internalRandomJoinReorderDefaultToHashJoin" + cpp_vartype: AtomicWord + default: false + on_update: plan_cache_util::clearSbeCacheOnParameterChange + redact: false + internalQueryMaxPipelineRewrites: description: "Maximum number of pipeline rewrite rules to apply" set_at: [startup, runtime]