-
Notifications
You must be signed in to change notification settings - Fork 60
/
0001-add-umonitor-umwait-C0.x-C-states.patch
265 lines (256 loc) · 7.64 KB
/
0001-add-umonitor-umwait-C0.x-C-states.patch
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
From 4a5c05e749fd5ab5523c52c76c875725e8bff3cf Mon Sep 17 00:00:00 2001
From: Arjan van de Ven <[email protected]>
Date: Thu, 19 Jan 2023 21:43:49 +0000
Subject: [PATCH] add umonitor/umwait C0.x C states
---
arch/x86/include/asm/mwait.h | 47 ++++++++++++++++
drivers/idle/intel_idle.c | 106 +++++++++++++++++++++++++++++++----
2 files changed, 141 insertions(+), 12 deletions(-)
diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h
index 3a8fdf881313..99934a44ab01 100644
--- a/arch/x86/include/asm/mwait.h
+++ b/arch/x86/include/asm/mwait.h
@@ -141,4 +141,51 @@ static inline void __tpause(u32 ecx, u32 edx, u32 eax)
#endif
}
+
+/*
+ * Monitor a memory address at 'rcx' using the 'umonitor' instruction.
+ */
+static inline void __umonitor(const void *rcx)
+{
+ /* "umonitor %rcx" */
+ asm volatile("umonitor %%rcx\n"
+ :
+ : "c"(rcx));
+}
+
+/*
+ * Same as '__tpause()', but uses the 'umwait' instruction, which is very
+ * similar to 'tpause', but also takes into account the address monitored with
+ * 'umonitor'.
+ */
+static inline void __umwait(u32 ecx, u32 edx, u32 eax)
+{
+ /* "umwait %ecx, %edx, %eax;" */
+ asm volatile("umwait %%ecx\n"
+ :
+ : "c"(ecx), "d"(edx), "a"(eax));
+}
+
+/*
+ * Enter low-latency C0.1 or C0.2 state and stays there until an event happens
+ * (an interrupt or the 'need_resched') or the deadline is reached. The
+ * deadline is the absolute TSC counter value to exist the idle state at.
+ * However, the deadline cannot exceed the global limit in the
+ * IA32_UMWAIT_CONTROL register.
+ */
+static inline void umwait_idle(u64 deadline, u32 state)
+{
+ if (!current_set_polling_and_test()) {
+ u32 eax, edx;
+
+ eax = lower_32_bits(deadline);
+ edx = upper_32_bits(deadline);
+
+ __umonitor((void *)¤t_thread_info()->flags);
+ mb();
+ if (!need_resched())
+ __umwait(state, edx, eax);
+ }
+ current_clr_polling();
+}
#endif /* _ASM_X86_MWAIT_H */
diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c
index cfeb24d40d37..48cc7eed81a6 100644
--- a/drivers/idle/intel_idle.c
+++ b/drivers/idle/intel_idle.c
@@ -204,6 +204,32 @@ static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev,
return __intel_idle(dev, drv, index);
}
+/**
+ * intel_idle_umwait - request C0.2 using the 'umwait' instruction.
+ * @dev: cpuidle device of the target CPU.
+ * @drv: cpuidle driver (assumed to point to intel_idle_driver).
+ * @index: Target idle state index.
+ *
+ * Enter an idle state using the 'umwait' CPU instruction. This instruction
+ * puts the CPU to C0.1 or C0.2 state.
+ */
+static __cpuidle int intel_idle_umwait(struct cpuidle_device *dev,
+ struct cpuidle_driver *drv, int index)
+{
+ struct cpuidle_state *state = &drv->states[index];
+
+ if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE)
+ local_irq_enable();
+
+ /*
+ * Request C0.2 with a large enough TSC quanta (50ms on a system with
+ * 2GHz TSC frequency). But we cannot exceed the global limits in
+ * 'MSR_IA32_UMWAIT_CONTROL'.
+ */
+ umwait_idle(__builtin_ia32_rdtsc() + 100000000, flg2MWAIT(state->flags));
+ return index;
+}
+
/**
* intel_idle_s2idle - Ask the processor to enter the given idle state.
* @dev: cpuidle device of the target CPU.
@@ -839,19 +865,33 @@ static struct cpuidle_state icx_cstates[] __initdata = {
* 'CPUIDLE_FLAG_UNUSABLE'.
*/
static struct cpuidle_state adl_cstates[] __initdata = {
+ {
+ .name = "C0.1",
+ .desc = "UMWAIT C0.1",
+ .flags = MWAIT2flg(TPAUSE_C01_STATE),
+ .exit_latency = 1,
+ .target_residency = 1,
+ .enter = &intel_idle_umwait, },
+ {
+ .name = "C0.2",
+ .desc = "UMWAIT C0.2",
+ .flags = MWAIT2flg(TPAUSE_C02_STATE),
+ .exit_latency = 1,
+ .target_residency = 2,
+ .enter = &intel_idle_umwait, },
{
.name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
- .exit_latency = 1,
- .target_residency = 1,
+ .exit_latency = 2,
+ .target_residency = 2,
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
.name = "C1E",
.desc = "MWAIT 0x01",
.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
- .exit_latency = 2,
+ .exit_latency = 3,
.target_residency = 4,
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
@@ -884,19 +924,33 @@ static struct cpuidle_state adl_cstates[] __initdata = {
};
static struct cpuidle_state adl_l_cstates[] __initdata = {
+ {
+ .name = "C0.1",
+ .desc = "UMWAIT C0.1",
+ .flags = MWAIT2flg(TPAUSE_C01_STATE),
+ .exit_latency = 1,
+ .target_residency = 1,
+ .enter = &intel_idle_umwait, },
+ {
+ .name = "C0.2",
+ .desc = "UMWAIT C0.2",
+ .flags = MWAIT2flg(TPAUSE_C02_STATE),
+ .exit_latency = 1,
+ .target_residency = 2,
+ .enter = &intel_idle_umwait, },
{
.name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
- .exit_latency = 1,
- .target_residency = 1,
+ .exit_latency = 2,
+ .target_residency = 2,
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
.name = "C1E",
.desc = "MWAIT 0x01",
.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
- .exit_latency = 2,
+ .exit_latency = 3,
.target_residency = 4,
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
@@ -929,19 +983,33 @@ static struct cpuidle_state adl_l_cstates[] __initdata = {
};
static struct cpuidle_state adl_n_cstates[] __initdata = {
+ {
+ .name = "C0.1",
+ .desc = "UMWAIT C0.1",
+ .flags = MWAIT2flg(TPAUSE_C01_STATE),
+ .exit_latency = 1,
+ .target_residency = 1,
+ .enter = &intel_idle_umwait, },
+ {
+ .name = "C0.2",
+ .desc = "UMWAIT C0.2",
+ .flags = MWAIT2flg(TPAUSE_C02_STATE),
+ .exit_latency = 1,
+ .target_residency = 2,
+ .enter = &intel_idle_umwait, },
{
.name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
- .exit_latency = 1,
- .target_residency = 1,
+ .exit_latency = 2,
+ .target_residency = 2,
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
.name = "C1E",
.desc = "MWAIT 0x01",
.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
- .exit_latency = 2,
+ .exit_latency = 3,
.target_residency = 40,
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
@@ -974,19 +1042,33 @@ static struct cpuidle_state adl_n_cstates[] __initdata = {
};
static struct cpuidle_state spr_cstates[] __initdata = {
+ {
+ .name = "C0.1",
+ .desc = "UMWAIT C0.1",
+ .flags = MWAIT2flg(TPAUSE_C01_STATE),
+ .exit_latency = 1,
+ .target_residency = 1,
+ .enter = &intel_idle_umwait, },
+ {
+ .name = "C0.2",
+ .desc = "UMWAIT C0.2",
+ .flags = MWAIT2flg(TPAUSE_C02_STATE),
+ .exit_latency = 1,
+ .target_residency = 2,
+ .enter = &intel_idle_umwait, },
{
.name = "C1",
.desc = "MWAIT 0x00",
.flags = MWAIT2flg(0x00),
- .exit_latency = 1,
- .target_residency = 1,
+ .exit_latency = 2,
+ .target_residency = 2,
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
.name = "C1E",
.desc = "MWAIT 0x01",
.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
- .exit_latency = 2,
+ .exit_latency = 3,
.target_residency = 4,
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
--
2.39.1
--- linux-6.1/arch/x86/kernel/cpu/umwait.c~ 2022-12-11 22:15:18.000000000 +0000
+++ linux-6.1/arch/x86/kernel/cpu/umwait.c 2023-01-30 15:23:52.999232486 +0000
@@ -16,7 +16,7 @@
* Cache IA32_UMWAIT_CONTROL MSR. This is a systemwide control. By default,
* umwait max time is 100000 in TSC-quanta and C0.2 is enabled
*/
-static u32 umwait_control_cached = UMWAIT_CTRL_VAL(100000, UMWAIT_C02_ENABLE);
+static u32 umwait_control_cached = UMWAIT_CTRL_VAL(10000000, UMWAIT_C02_ENABLE);
/*
* Cache the original IA32_UMWAIT_CONTROL MSR value which is configured by