Skip to content

Navigation Menu

Sign in
Sign up

Commit 144a7b9

Browse files
authored
Merge pull request cockroachdb#172571 from harryfallows/opt-histogram-unconstrained-prefix
opt: use histograms for constant columns outside of the exact prefix
2 parents 32b032d + 8d5f388 commit 144a7b9

4 files changed

Lines changed: 326 additions & 3 deletions

File tree

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
exec-ddl
2+
CREATE TABLE t (
3+
k INT PRIMARY KEY,
4+
i INT,
5+
s STRING,
6+
j JSONB,
7+
INVERTED INDEX isj (i, s, j)
8+
)
9+
----
10+
11+
exec-ddl
12+
ALTER TABLE t INJECT STATISTICS '[
13+
{
14+
"columns": ["i"],
15+
"created_at": "2018年01月01日 1:00:00.00000+00:00",
16+
"row_count": 2000,
17+
"distinct_count": 41,
18+
"null_count": 0
19+
},
20+
{
21+
"columns": ["s"],
22+
"created_at": "2018年01月01日 1:00:00.00000+00:00",
23+
"row_count": 2000,
24+
"distinct_count": 40,
25+
"null_count": 100,
26+
"histo_col_type": "string",
27+
"histo_buckets": [
28+
{"num_eq": 0, "num_range": 0, "distinct_range": 0, "upper_bound": "apple"},
29+
{"num_eq": 100, "num_range": 200, "distinct_range": 9, "upper_bound": "banana"},
30+
{"num_eq": 100, "num_range": 300, "distinct_range": 9, "upper_bound": "cherry"},
31+
{"num_eq": 200, "num_range": 400, "distinct_range": 9, "upper_bound": "mango"},
32+
{"num_eq": 200, "num_range": 400, "distinct_range": 9, "upper_bound": "pineapple"}
33+
]
34+
},
35+
{
36+
"columns": ["j"],
37+
"created_at": "2018年01月01日 1:00:00.00000+00:00",
38+
"row_count": 2000,
39+
"distinct_count": 10,
40+
"null_count": 0,
41+
"histo_col_type": "BYTES",
42+
"histo_buckets": [
43+
{"distinct_range": 0, "num_eq": 10, "num_range": 0, "upper_bound": "\\x37000138"},
44+
{"distinct_range": 0, "num_eq": 10, "num_range": 0, "upper_bound": "\\x37000139"},
45+
{"distinct_range": 0, "num_eq": 990, "num_range": 0, "upper_bound": "\\x37000300012a0200"},
46+
{"distinct_range": 0, "num_eq": 100, "num_range": 0, "upper_bound": "\\x37000300012a0400"},
47+
{"distinct_range": 0, "num_eq": 10, "num_range": 0, "upper_bound": "\\x37000300012a0600"},
48+
{"distinct_range": 0, "num_eq": 990, "num_range": 0, "upper_bound": "\\x3761000112620001"},
49+
{"distinct_range": 0, "num_eq": 100, "num_range": 0, "upper_bound": "\\x3763000112640001"},
50+
{"distinct_range": 0, "num_eq": 10, "num_range": 0, "upper_bound": "\\x3765000112660001"}
51+
]
52+
}
53+
]'
54+
----
55+
56+
# Test a multi-column inverted index scan where the leading prefix column i
57+
# spans multiple values but the second prefix column s is held to a single
58+
# value. s's histogram should be used even though s is outside the exact prefix.
59+
opt
60+
SELECT k FROM t@isj WHERE i IN (200, 300) AND s = 'banana' AND j @> '{"a": "b"}'
61+
----
62+
project
63+
├── columns: k:1(int!null)
64+
├── immutable
65+
├── stats: [rows=1]
66+
├── key: (1)
67+
└── scan t@isj,inverted
68+
├── columns: k:1(int!null)
69+
├── constraint: /2/3
70+
│ ├── [/200/'banana' - /200/'banana']
71+
│ └── [/300/'banana' - /300/'banana']
72+
├── inverted constraint: /7/1
73+
│ └── spans: ["a"/"b", "a"/"b"]
74+
├── flags: force-index=isj
75+
├── stats: [rows=2.41463, distinct(2)=2, null(2)=0, distinct(3)=1, null(3)=0, distinct(7)=1, null(7)=0, distinct(3,7)=1, null(3,7)=0, distinct(2,3,7)=2, null(2,3,7)=0]
76+
│ histogram(3)= 0 2.4146
77+
│ <--- 'banana'
78+
│ histogram(7)= 0 2.4146 0 0
79+
│ <--- '\x3761000112620001' --- '\x3761000112620002'
80+
└── key: (1)
81+
82+
exec-ddl
83+
CREATE TABLE rbr (
84+
k INT PRIMARY KEY,
85+
s STRING,
86+
j JSONB,
87+
INVERTED INDEX sj (s, j)
88+
) LOCALITY REGIONAL BY ROW
89+
----
90+
91+
exec-ddl
92+
ALTER TABLE rbr INJECT STATISTICS '[
93+
{
94+
"columns": ["crdb_region"],
95+
"created_at": "2018年01月01日 1:00:00.00000+00:00",
96+
"row_count": 2000,
97+
"distinct_count": 3,
98+
"null_count": 0
99+
},
100+
{
101+
"columns": ["s"],
102+
"created_at": "2018年01月01日 1:00:00.00000+00:00",
103+
"row_count": 2000,
104+
"distinct_count": 40,
105+
"null_count": 100,
106+
"histo_col_type": "string",
107+
"histo_buckets": [
108+
{"num_eq": 0, "num_range": 0, "distinct_range": 0, "upper_bound": "apple"},
109+
{"num_eq": 100, "num_range": 200, "distinct_range": 9, "upper_bound": "banana"},
110+
{"num_eq": 100, "num_range": 300, "distinct_range": 9, "upper_bound": "cherry"},
111+
{"num_eq": 200, "num_range": 400, "distinct_range": 9, "upper_bound": "mango"},
112+
{"num_eq": 200, "num_range": 400, "distinct_range": 9, "upper_bound": "pineapple"}
113+
]
114+
},
115+
{
116+
"columns": ["j"],
117+
"created_at": "2018年01月01日 1:00:00.00000+00:00",
118+
"row_count": 2000,
119+
"distinct_count": 10,
120+
"null_count": 0,
121+
"histo_col_type": "BYTES",
122+
"histo_buckets": [
123+
{"distinct_range": 0, "num_eq": 10, "num_range": 0, "upper_bound": "\\x37000138"},
124+
{"distinct_range": 0, "num_eq": 10, "num_range": 0, "upper_bound": "\\x37000139"},
125+
{"distinct_range": 0, "num_eq": 990, "num_range": 0, "upper_bound": "\\x37000300012a0200"},
126+
{"distinct_range": 0, "num_eq": 100, "num_range": 0, "upper_bound": "\\x37000300012a0400"},
127+
{"distinct_range": 0, "num_eq": 10, "num_range": 0, "upper_bound": "\\x37000300012a0600"},
128+
{"distinct_range": 0, "num_eq": 990, "num_range": 0, "upper_bound": "\\x3761000112620001"},
129+
{"distinct_range": 0, "num_eq": 100, "num_range": 0, "upper_bound": "\\x3763000112640001"},
130+
{"distinct_range": 0, "num_eq": 10, "num_range": 0, "upper_bound": "\\x3765000112660001"}
131+
]
132+
}
133+
]'
134+
----
135+
136+
# Same scenario on a REGIONAL BY ROW table where the query does not constrain
137+
# crdb_region. The optimizer enumerates crdb_region into one span per region, all
138+
# pinning s='banana', so crdb_region varies while s is a constant column outside
139+
# the exact prefix.
140+
opt
141+
SELECT k FROM rbr@sj WHERE s = 'banana' AND j @> '{"a": "b"}'
142+
----
143+
project
144+
├── columns: k:1(int!null)
145+
├── immutable
146+
├── stats: [rows=11.1111]
147+
├── key: (1)
148+
└── scan rbr@sj,inverted
149+
├── columns: k:1(int!null) crdb_region:4(string!null)
150+
├── constraint: /4/2
151+
│ ├── [/'central'/'banana' - /'central'/'banana']
152+
│ ├── [/'east'/'banana' - /'east'/'banana']
153+
│ └── [/'west'/'banana' - /'west'/'banana']
154+
├── inverted constraint: /7/1
155+
│ └── spans: ["a"/"b", "a"/"b"]
156+
├── flags: force-index=sj
157+
├── stats: [rows=49.5, distinct(2)=1, null(2)=0, distinct(4)=3, null(4)=0, distinct(7)=1, null(7)=0, distinct(2,4,7)=3, null(2,4,7)=0]
158+
│ histogram(2)= 0 49.5
159+
│ <--- 'banana'
160+
│ histogram(7)= 0 49.5 0 0
161+
│ <--- '\x3761000112620001' --- '\x3761000112620002'
162+
├── key: (1)
163+
└── fd: (1)-->(4)

‎pkg/sql/opt/memo/testdata/stats/scan‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3466,3 +3466,119 @@ index-join stale
34663466
├── stats: [rows=0.00200002]
34673467
├── key: ()
34683468
└── fd: ()-->(1-3)
3469+
3470+
exec-ddl
3471+
CREATE TABLE hist_multi (
3472+
k INT PRIMARY KEY,
3473+
a INT NOT NULL,
3474+
b INT,
3475+
INDEX ab (a, b)
3476+
)
3477+
----
3478+
3479+
exec-ddl
3480+
ALTER TABLE hist_multi INJECT STATISTICS '[
3481+
{
3482+
"columns": ["a"],
3483+
"created_at": "2018年01月01日 1:00:00.00000+00:00",
3484+
"row_count": 1000,
3485+
"distinct_count": 5,
3486+
"null_count": 0
3487+
},
3488+
{
3489+
"columns": ["b"],
3490+
"created_at": "2018年01月01日 1:00:00.00000+00:00",
3491+
"row_count": 1000,
3492+
"distinct_count": 40,
3493+
"null_count": 0,
3494+
"histo_col_type": "int",
3495+
"histo_buckets": [
3496+
{"num_eq": 0, "num_range": 0, "distinct_range": 0, "upper_bound": "0"},
3497+
{"num_eq": 10, "num_range": 90, "distinct_range": 9, "upper_bound": "10"},
3498+
{"num_eq": 20, "num_range": 180, "distinct_range": 9, "upper_bound": "20"},
3499+
{"num_eq": 30, "num_range": 270, "distinct_range": 9, "upper_bound": "30"},
3500+
{"num_eq": 40, "num_range": 360, "distinct_range": 9, "upper_bound": "40"}
3501+
]
3502+
}
3503+
]'
3504+
----
3505+
3506+
# The leading index column a spans multiple values while b is held to a single
3507+
# value, so b's histogram is used for the estimate even though b falls outside
3508+
# the constraint's exact prefix.
3509+
opt
3510+
SELECT k FROM hist_multi@ab WHERE a IN (1, 2) AND b = 10
3511+
----
3512+
project
3513+
├── columns: k:1(int!null)
3514+
├── stats: [rows=4]
3515+
├── key: (1)
3516+
└── scan hist_multi@ab
3517+
├── columns: k:1(int!null) a:2(int!null) b:3(int!null)
3518+
├── constraint: /2/3/1
3519+
│ ├── [/1/10 - /1/10]
3520+
│ └── [/2/10 - /2/10]
3521+
├── flags: force-index=ab
3522+
├── stats: [rows=4, distinct(2)=2, null(2)=0, distinct(3)=1, null(3)=0, distinct(2,3)=2, null(2,3)=0]
3523+
│ histogram(3)= 0 4
3524+
│ <--- 10
3525+
├── key: (1)
3526+
└── fd: ()-->(3), (1)-->(2)
3527+
3528+
exec-ddl
3529+
CREATE TABLE hist_rbr (
3530+
k INT PRIMARY KEY,
3531+
a INT,
3532+
INDEX a_idx (a)
3533+
) LOCALITY REGIONAL BY ROW
3534+
----
3535+
3536+
exec-ddl
3537+
ALTER TABLE hist_rbr INJECT STATISTICS '[
3538+
{
3539+
"columns": ["crdb_region"],
3540+
"created_at": "2018年01月01日 1:00:00.00000+00:00",
3541+
"row_count": 1000,
3542+
"distinct_count": 3,
3543+
"null_count": 0
3544+
},
3545+
{
3546+
"columns": ["a"],
3547+
"created_at": "2018年01月01日 1:00:00.00000+00:00",
3548+
"row_count": 1000,
3549+
"distinct_count": 40,
3550+
"null_count": 0,
3551+
"histo_col_type": "int",
3552+
"histo_buckets": [
3553+
{"num_eq": 0, "num_range": 0, "distinct_range": 0, "upper_bound": "0"},
3554+
{"num_eq": 10, "num_range": 90, "distinct_range": 9, "upper_bound": "10"},
3555+
{"num_eq": 20, "num_range": 180, "distinct_range": 9, "upper_bound": "20"},
3556+
{"num_eq": 30, "num_range": 270, "distinct_range": 9, "upper_bound": "30"},
3557+
{"num_eq": 40, "num_range": 360, "distinct_range": 9, "upper_bound": "40"}
3558+
]
3559+
}
3560+
]'
3561+
----
3562+
3563+
# On a REGIONAL BY ROW table with crdb_region left unconstrained, the optimizer
3564+
# enumerates crdb_region into one span per region, so crdb_region varies while a
3565+
# is held to a single value outside the exact prefix. a's histogram is still used.
3566+
opt
3567+
SELECT k FROM hist_rbr@a_idx WHERE a = 10
3568+
----
3569+
project
3570+
├── columns: k:1(int!null)
3571+
├── stats: [rows=10]
3572+
├── key: (1)
3573+
└── scan hist_rbr@a_idx
3574+
├── columns: k:1(int!null) a:2(int!null)
3575+
├── constraint: /3/2/1
3576+
│ ├── [/'central'/10 - /'central'/10]
3577+
│ ├── [/'east'/10 - /'east'/10]
3578+
│ └── [/'west'/10 - /'west'/10]
3579+
├── flags: force-index=a_idx
3580+
├── stats: [rows=10, distinct(2)=1, null(2)=0]
3581+
│ histogram(2)= 0 10
3582+
│ <--- 10
3583+
├── key: (1)
3584+
└── fd: ()-->(2)

‎pkg/sql/opt/props/histogram.go‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -333,7 +333,8 @@ func maxDistinctValuesInRange(lowerBound, upperBound tree.Datum) (n float64, ok
333333

334334
// CanFilter returns true if the given constraint can filter the histogram.
335335
// This is the case if the histogram column matches one of the columns in
336-
// the exact prefix of c or the next column immediately after the exact prefix.
336+
// the exact prefix of c, the next column immediately after the exact prefix,
337+
// or a column constrained to a single value in every span (a constant column).
337338
// Returns the offset of the matching column in the constraint if found, as
338339
// well as the exact prefix.
339340
func (h *Histogram) CanFilter(
@@ -346,6 +347,17 @@ func (h *Histogram) CanFilter(
346347
return i, exactPrefix, true
347348
}
348349
}
350+
// A constant column (constrained to a single value in every span) can filter
351+
// the histogram even when an earlier unconstrained column pushes it past the
352+
// exact prefix, e.g. crdb_region on a REGIONAL BY ROW table. See Filter.
353+
for i := exactPrefix + 1; i < constrainedCols; i++ {
354+
if c.Columns.Get(i).ID() == h.col {
355+
if c.ExtractConstCols(ctx, h.evalCtx).Contains(h.col) {
356+
return i, exactPrefix, true
357+
}
358+
break
359+
}
360+
}
349361
return 0, exactPrefix, false
350362
}
351363

@@ -582,6 +594,25 @@ func (h *Histogram) Filter(ctx context.Context, c *constraint.Constraint) *Histo
582594
if !ok {
583595
panic(errors.AssertionFailedf("column mismatch"))
584596
}
597+
598+
// A column past the exact prefix was admitted as a constant column with value
599+
// V. The prefix-based path below assumes the columns before colOffset are
600+
// fixed to the first span's values, which does not hold when an earlier
601+
// column varies, so filter against a synthetic single-column [V - V]
602+
// constraint instead.
603+
if colOffset > exactPrefix {
604+
val := c.Spans.Get(0).StartKey().Value(colOffset)
605+
var cols constraint.Columns
606+
cols.InitSingle(opt.MakeOrderingColumn(h.col, false /* descending */))
607+
key := constraint.MakeKey(val)
608+
var span constraint.Span
609+
span.Init(key, constraint.IncludeBoundary, key, constraint.IncludeBoundary)
610+
return h.filter(
611+
ctx, 1 /* spanCount */, func(int) *constraint.Span { return &span },
612+
false /* desc */, 0 /* colOffset */, 1 /* exactPrefix */, nil /* prefix */, cols,
613+
)
614+
}
615+
585616
prefix := make([]tree.Datum, colOffset)
586617
for i := range prefix {
587618
prefix[i] = c.Spans.Get(0).StartKey().Value(i)

‎pkg/sql/opt/props/histogram_test.go‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ func TestCanFilter(t *testing.T) {
7979

8080
// The histogram column ID is 1 for all test cases. CanFilter should only
8181
// return true for constraints in which column ID 1 is part of the exact
82-
// prefix or the first column after.
82+
// prefix, the first column after, or a constant column.
8383
testData := []struct {
8484
constraint string
8585
canFilter bool
@@ -111,7 +111,8 @@ func TestCanFilter(t *testing.T) {
111111
},
112112
{
113113
constraint: "/2/-1: [/0/3 - /0/3] [/2/3 - /2/3]",
114-
canFilter: false,
114+
canFilter: true,
115+
colIdx: 1,
115116
},
116117
{
117118
constraint: "/2/1: [/0/3 - /0/3] [/0/5 - /0/5]",
@@ -367,6 +368,18 @@ func TestHistogram(t *testing.T) {
367368
distinct: 1,
368369
maxFrequency: 5.71,
369370
},
371+
{
372+
constraint: "/2/1: [/0/40 - /0/40] [/2/40 - /2/40]",
373+
// 0 5.7143
374+
// <---- 40 -
375+
buckets: []cat.HistogramBucket{
376+
{NumRange: 0, NumEq: 5.71, DistinctRange: 0, UpperBound: tree.NewDInt(40)},
377+
},
378+
count: 5.71,
379+
maxDistinct: 1,
380+
distinct: 1,
381+
maxFrequency: 5.71,
382+
},
370383
}
371384

372385
for i := range testData {

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /