1 /*
2 * SPDX-FileCopyrightText: Copyright 2019-2024 Arm Limited and/or its affiliates <open-source-office@arm.com>
3 * SPDX-License-Identifier: Apache-2.0
4 *
5 * Licensed under the Apache License, Version 2.0 (the License); you may
6 * not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an AS IS BASIS, WITHOUT
13 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18 /******************************************************************************
19 * Includes
20 ******************************************************************************/
21 #include "ethosu85_interface.h"
22
23 #include "ethosu_config_u85.h"
24 #include "ethosu_device.h"
25 #include "ethosu_log.h"
26
27 #include <assert.h>
28 #include <inttypes.h>
29 #include <stdbool.h>
30 #include <stddef.h>
31 #include <stdio.h>
32 #include <stdlib.h>
33
34 /******************************************************************************
35 * Defines
36 ******************************************************************************/
37
38 #define ETHOSU_PRODUCT_U85 2
39
40 #define BASEP_OFFSET 4
41
42 #define ADDRESS_BITS 40
43
44 #define ADDRESS_MASK ((1ull << ADDRESS_BITS) - 1)
45
46 #define NPU_CMD_PWR_CLK_MASK (0xC)
47 #define NPU_MAC_PWR_RAMP_CYCLES_MASK (0x3F)
48
49 /******************************************************************************
50 * Functions
51 ******************************************************************************/
52
ethosu_address_remap(uint64_t address,int index)53 uint64_t __attribute__((weak)) ethosu_address_remap(uint64_t address, int index)
54 {
55 (void)(index);
56 return address;
57 }
58
ethosu_dev_init(struct ethosu_device * dev,void * base_address,uint32_t secure_enable,uint32_t privilege_enable)59 bool ethosu_dev_init(struct ethosu_device *dev, void *base_address, uint32_t secure_enable, uint32_t privilege_enable)
60 {
61 dev->reg = (volatile struct NPU_REG *)base_address;
62 dev->secure = secure_enable;
63 dev->privileged = privilege_enable;
64
65 if (dev->reg->CONFIG.product != ETHOSU_PRODUCT_U85)
66 {
67 LOG_ERR("Failed to initialize device. Driver has not been compiled for this product");
68 return false;
69 }
70
71 // Make sure the NPU is in a known state
72 if (ethosu_dev_soft_reset(dev) != ETHOSU_SUCCESS)
73 {
74 return false;
75 }
76
77 return true;
78 }
79
ethosu_dev_axi_init(struct ethosu_device * dev)80 enum ethosu_error_codes ethosu_dev_axi_init(struct ethosu_device *dev)
81 {
82 struct regioncfg_r rcfg = {0};
83 struct axi_sram_r axi_s = {0};
84 struct axi_ext_r axi_e = {0};
85
86 // Configure MEM_ATTR array. These are user configurable,
87 // and each region will be set to use one of the entries
88 // as its config.
89 dev->reg->MEM_ATTR[0].word = NPU_MEM_ATTR_0;
90 dev->reg->MEM_ATTR[1].word = NPU_MEM_ATTR_1;
91 dev->reg->MEM_ATTR[2].word = NPU_MEM_ATTR_2;
92 dev->reg->MEM_ATTR[3].word = NPU_MEM_ATTR_3;
93
94 // Set MEM_ATTR entry for command stream
95 dev->reg->QCONFIG.word = NPU_QCONFIG;
96
97 // Set MEM_ATTR entries to use for regions 0-7
98 rcfg.region0 = NPU_REGIONCFG_0;
99 rcfg.region1 = NPU_REGIONCFG_1;
100 rcfg.region2 = NPU_REGIONCFG_2;
101 rcfg.region3 = NPU_REGIONCFG_3;
102 rcfg.region4 = NPU_REGIONCFG_4;
103 rcfg.region5 = NPU_REGIONCFG_5;
104 rcfg.region6 = NPU_REGIONCFG_6;
105 rcfg.region7 = NPU_REGIONCFG_7;
106 dev->reg->REGIONCFG.word = rcfg.word;
107
108 // Set AXI limits on SRAM AXI interfaces
109 axi_s.max_outstanding_read_m1 = AXI_LIMIT_SRAM_MAX_OUTSTANDING_READ_M1 - 1;
110 axi_s.max_outstanding_write_m1 = AXI_LIMIT_SRAM_MAX_OUTSTANDING_WRITE_M1 - 1;
111 axi_s.max_beats = AXI_LIMIT_SRAM_MAX_BEATS;
112 dev->reg->AXI_SRAM.word = axi_s.word;
113
114 // Set AXI limits on EXT AXI interface(s)
115 axi_e.max_outstanding_read_m1 = AXI_LIMIT_EXT_MAX_OUTSTANDING_READ_M1 - 1;
116 axi_e.max_outstanding_write_m1 = AXI_LIMIT_EXT_MAX_OUTSTANDING_WRITE_M1 - 1;
117 axi_e.max_beats = AXI_LIMIT_EXT_MAX_BEATS;
118 dev->reg->AXI_EXT.word = axi_e.word;
119
120 return ETHOSU_SUCCESS;
121 }
122
ethosu_dev_run_command_stream(struct ethosu_device * dev,const uint8_t * cmd_stream_ptr,uint32_t cms_length,const uint64_t * base_addr,int num_base_addr)123 void ethosu_dev_run_command_stream(struct ethosu_device *dev,
124 const uint8_t *cmd_stream_ptr,
125 uint32_t cms_length,
126 const uint64_t *base_addr,
127 int num_base_addr)
128 {
129 assert(num_base_addr <= NPU_REG_BASEP_ARRLEN);
130
131 struct cmd_r cmd;
132 uint64_t qbase = ethosu_address_remap((uintptr_t)cmd_stream_ptr, -1);
133 assert(qbase <= ADDRESS_MASK);
134 LOG_DEBUG("QBASE=0x%016llx, QSIZE=%" PRIu32 ", cmd_stream_ptr=%p", qbase, cms_length, cmd_stream_ptr);
135
136 dev->reg->QBASE.word[0] = qbase & 0xffffffff;
137 dev->reg->QBASE.word[1] = qbase >> 32;
138 dev->reg->QSIZE.word = cms_length;
139
140 for (int i = 0; i < num_base_addr; i++)
141 {
142 uint64_t addr = ethosu_address_remap(base_addr[i], i);
143 assert(addr <= ADDRESS_MASK);
144 LOG_DEBUG("BASEP%d=0x%016llx", i, addr);
145 dev->reg->BASEP[i].word[0] = addr & 0xffffffff;
146 dev->reg->BASEP[i].word[1] = addr >> 32;
147 }
148
149 cmd.word = dev->reg->CMD.word & NPU_CMD_PWR_CLK_MASK;
150 cmd.transition_to_running_state = 1;
151
152 dev->reg->CMD.word = cmd.word;
153 LOG_DEBUG("CMD=0x%08" PRIx32, cmd.word);
154 }
155
ethosu_dev_print_err_status(struct ethosu_device * dev)156 void ethosu_dev_print_err_status(struct ethosu_device *dev)
157 {
158 LOG_ERR("NPU status=0x%08" PRIx32 ", qread=%" PRIu32 ", cmd_end_reached=%u",
159 dev->reg->STATUS.word,
160 dev->reg->QREAD.word,
161 dev->reg->STATUS.cmd_end_reached);
162 }
163
ethosu_dev_handle_interrupt(struct ethosu_device * dev)164 bool ethosu_dev_handle_interrupt(struct ethosu_device *dev)
165 {
166 struct cmd_r cmd;
167
168 // Clear interrupt
169 cmd.word = dev->reg->CMD.word & NPU_CMD_PWR_CLK_MASK;
170 cmd.clear_irq = 1;
171 dev->reg->CMD.word = cmd.word;
172
173 // If a fault has occured, the NPU needs to be reset
174 if (dev->reg->STATUS.bus_status || dev->reg->STATUS.cmd_parse_error || dev->reg->STATUS.branch_fault ||
175 dev->reg->STATUS.ecc_fault || !dev->reg->STATUS.cmd_end_reached)
176 {
177 return false;
178 }
179
180 return true;
181 }
182
ethosu_dev_verify_access_state(struct ethosu_device * dev)183 bool ethosu_dev_verify_access_state(struct ethosu_device *dev)
184 {
185 if (dev->reg->PROT.active_CSL != (dev->secure ? SECURITY_LEVEL_SECURE : SECURITY_LEVEL_NON_SECURE) ||
186 dev->reg->PROT.active_CPL != (dev->privileged ? PRIVILEGE_LEVEL_PRIVILEGED : PRIVILEGE_LEVEL_USER))
187 {
188 return false;
189 }
190 return true;
191 }
192
ethosu_dev_soft_reset(struct ethosu_device * dev)193 enum ethosu_error_codes ethosu_dev_soft_reset(struct ethosu_device *dev)
194 {
195 struct reset_r reset;
196
197 reset.word = 0;
198 reset.pending_CPL = dev->privileged ? PRIVILEGE_LEVEL_PRIVILEGED : PRIVILEGE_LEVEL_USER;
199 reset.pending_CSL = dev->secure ? SECURITY_LEVEL_SECURE : SECURITY_LEVEL_NON_SECURE;
200
201 // Reset and set security level
202 LOG_INFO("Soft reset NPU");
203 dev->reg->RESET.word = reset.word;
204
205 // Wait until reset status indicates that reset has been completed
206 for (int i = 0; i < 100000 && dev->reg->STATUS.reset_status != 0; i++)
207 {
208 }
209
210 if (dev->reg->STATUS.reset_status != 0)
211 {
212 LOG_ERR("Soft reset timed out");
213 return ETHOSU_GENERIC_FAILURE;
214 }
215
216 // Verify that NPU has switched security state and privilege level
217 if (ethosu_dev_verify_access_state(dev) != true)
218 {
219 LOG_ERR("Failed to switch security state and privilege level");
220 return ETHOSU_GENERIC_FAILURE;
221 }
222
223 // Reinitialize AXI settings
224 ethosu_dev_axi_init(dev);
225
226 // MAC power ramping up/down control
227 dev->reg->POWER_CTRL.word = (NPU_MAC_PWR_RAMP_CYCLES & NPU_MAC_PWR_RAMP_CYCLES_MASK);
228
229 return ETHOSU_SUCCESS;
230 }
231
ethosu_dev_get_hw_info(struct ethosu_device * dev,struct ethosu_hw_info * hwinfo)232 void ethosu_dev_get_hw_info(struct ethosu_device *dev, struct ethosu_hw_info *hwinfo)
233 {
234 struct config_r cfg;
235 struct id_r id;
236
237 cfg.word = dev->reg->CONFIG.word;
238 id.word = dev->reg->ID.word;
239
240 hwinfo->cfg.cmd_stream_version = cfg.cmd_stream_version;
241 hwinfo->cfg.custom_dma = cfg.custom_dma;
242 hwinfo->cfg.macs_per_cc = cfg.macs_per_cc;
243
244 hwinfo->version.arch_major_rev = id.arch_major_rev;
245 hwinfo->version.arch_minor_rev = id.arch_minor_rev;
246 hwinfo->version.arch_patch_rev = id.arch_patch_rev;
247 hwinfo->version.product_major = id.product_major;
248 hwinfo->version.version_major = id.version_major;
249 hwinfo->version.version_minor = id.version_minor;
250 hwinfo->version.version_status = id.version_status;
251 }
252
ethosu_dev_set_clock_and_power(struct ethosu_device * dev,enum ethosu_clock_q_request clock_q,enum ethosu_power_q_request power_q)253 enum ethosu_error_codes ethosu_dev_set_clock_and_power(struct ethosu_device *dev,
254 enum ethosu_clock_q_request clock_q,
255 enum ethosu_power_q_request power_q)
256 {
257 struct cmd_r cmd = {0};
258 cmd.word = dev->reg->CMD.word & NPU_CMD_PWR_CLK_MASK;
259
260 if (power_q != ETHOSU_POWER_Q_UNCHANGED)
261 {
262 cmd.power_q_enable = power_q == ETHOSU_POWER_Q_ENABLE ? 1 : 0;
263 }
264 if (clock_q != ETHOSU_CLOCK_Q_UNCHANGED)
265 {
266 cmd.clock_q_enable = clock_q == ETHOSU_CLOCK_Q_ENABLE ? 1 : 0;
267 }
268
269 dev->reg->CMD.word = cmd.word;
270 LOG_DEBUG("CMD=0x%08" PRIx32, cmd.word);
271
272 return ETHOSU_SUCCESS;
273 }
274
ethosu_dev_verify_optimizer_config(struct ethosu_device * dev,uint32_t cfg_in,uint32_t id_in)275 bool ethosu_dev_verify_optimizer_config(struct ethosu_device *dev, uint32_t cfg_in, uint32_t id_in)
276 {
277 struct config_r *opt_cfg = (struct config_r *)&cfg_in;
278 struct config_r hw_cfg;
279 struct id_r *opt_id = (struct id_r *)&id_in;
280 struct id_r hw_id;
281 bool ret = true;
282
283 hw_cfg.word = dev->reg->CONFIG.word;
284 hw_id.word = dev->reg->ID.word;
285
286 LOG_INFO("Optimizer config. product=%u, cmd_stream_version=%u, macs_per_cc=%u, num_axi_ext=%u, num_axi_sram=%u, "
287 "custom_dma=%u",
288 opt_cfg->product,
289 opt_cfg->cmd_stream_version,
290 opt_cfg->macs_per_cc,
291 1U << opt_cfg->num_axi_ext,
292 1U << opt_cfg->num_axi_sram,
293 opt_cfg->custom_dma);
294
295 LOG_INFO("Optimizer config. arch version=%u.%u.%u",
296 opt_id->arch_major_rev,
297 opt_id->arch_minor_rev,
298 opt_id->arch_patch_rev);
299
300 LOG_INFO("Ethos-U config. product=%u, cmd_stream_version=%u, macs_per_cc=%u, num_axi_ext=%u, num_axi_sram=%u, "
301 "custom_dma=%u",
302 hw_cfg.product,
303 hw_cfg.cmd_stream_version,
304 hw_cfg.macs_per_cc,
305 1U << hw_cfg.num_axi_ext,
306 1U << hw_cfg.num_axi_sram,
307 hw_cfg.custom_dma);
308
309 LOG_INFO("Ethos-U. arch version=%u.%u.%u", hw_id.arch_major_rev, hw_id.arch_minor_rev, hw_id.arch_patch_rev);
310
311 if (opt_cfg->word != hw_cfg.word)
312 {
313 if (hw_cfg.product != opt_cfg->product)
314 {
315 LOG_ERR("NPU config mismatch. npu.product=%u, optimizer.product=%u", hw_cfg.product, opt_cfg->product);
316 ret = false;
317 }
318
319 if (hw_cfg.macs_per_cc != opt_cfg->macs_per_cc)
320 {
321 LOG_ERR("NPU config mismatch. npu.macs_per_cc=%u, optimizer.macs_per_cc=%u",
322 hw_cfg.macs_per_cc,
323 opt_cfg->macs_per_cc);
324 ret = false;
325 }
326
327 if (hw_cfg.num_axi_ext != opt_cfg->num_axi_ext)
328 {
329 LOG_ERR("NPU config mismatch. npu.num_axi_ext=%u, optimizer.num_axi_ext=%u",
330 1U << hw_cfg.num_axi_ext,
331 1U << opt_cfg->num_axi_ext);
332 ret = false;
333 }
334
335 if (hw_cfg.num_axi_sram != opt_cfg->num_axi_sram)
336 {
337 LOG_ERR("NPU config mismatch. npu.num_axi_sram=%u, optimizer.num_axi_sram=%u",
338 1U << hw_cfg.num_axi_sram,
339 1U << opt_cfg->num_axi_sram);
340 ret = false;
341 }
342
343 if (hw_cfg.cmd_stream_version != opt_cfg->cmd_stream_version)
344 {
345 LOG_ERR("NPU config mismatch. npu.cmd_stream_version=%u, optimizer.cmd_stream_version=%u",
346 hw_cfg.cmd_stream_version,
347 opt_cfg->cmd_stream_version);
348 ret = false;
349 }
350
351 if (!hw_cfg.custom_dma && opt_cfg->custom_dma)
352 {
353 LOG_ERR("NPU config mismatch. npu.custom_dma=%u, optimizer.custom_dma=%u",
354 hw_cfg.custom_dma,
355 opt_cfg->custom_dma);
356 ret = false;
357 }
358 }
359
360 if ((hw_id.arch_major_rev != opt_id->arch_major_rev) || (hw_id.arch_minor_rev < opt_id->arch_minor_rev))
361 {
362 LOG_ERR("NPU arch mismatch. npu.arch=%u.%u.%u, optimizer.arch=%u.%u.%u",
363 hw_id.arch_major_rev,
364 hw_id.arch_minor_rev,
365 hw_id.arch_patch_rev,
366 opt_id->arch_major_rev,
367 opt_id->arch_minor_rev,
368 opt_id->arch_patch_rev);
369 ret = false;
370 }
371
372 return ret;
373 }
374
