1 /*
2 * SPDX-FileCopyrightText: Copyright 2019-2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 *
6 * Licensed under the Apache License, Version 2.0 (the License); you may
7 * not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an AS IS BASIS, WITHOUT
14 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 /******************************************************************************
20 * Includes
21 ******************************************************************************/
22
23 #include "ethosu_interface.h"
24
25 #include "ethosu_device.h"
26 #include "ethosu_log.h"
27
28 #ifdef ETHOSU55
29 #include "ethosu_config_u55.h"
30 #else
31 #include "ethosu_config_u65.h"
32 #endif
33
34 #include <assert.h>
35 #include <inttypes.h>
36 #include <stdbool.h>
37 #include <stddef.h>
38 #include <stdio.h>
39 #include <stdlib.h>
40
41 /******************************************************************************
42 * Defines
43 ******************************************************************************/
44
45 #define ETHOSU_PRODUCT_U55 0
46 #define ETHOSU_PRODUCT_U65 1
47
48 #define BASEP_OFFSET 4
49
50 #ifdef ETHOSU65
51 #define ADDRESS_BITS 40
52 #else
53 #define ADDRESS_BITS 32
54 #endif
55
56 #define ADDRESS_MASK ((1ull << ADDRESS_BITS) - 1)
57
58 #define NPU_CMD_PWR_CLK_MASK (0xC)
59
60 /******************************************************************************
61 * Functions
62 ******************************************************************************/
63
ethosu_address_remap(uint64_t address,int index)64 uint64_t __attribute__((weak)) ethosu_address_remap(uint64_t address, int index)
65 {
66 (void)(index);
67 return address;
68 }
69
ethosu_dev_init(void * const base_address,uint32_t secure_enable,uint32_t privilege_enable)70 struct ethosu_device *ethosu_dev_init(void *const base_address, uint32_t secure_enable, uint32_t privilege_enable)
71 {
72 struct ethosu_device *dev = malloc(sizeof(struct ethosu_device));
73 if (!dev)
74 {
75 LOG_ERR("Failed to allocate memory for Ethos-U device");
76 return NULL;
77 }
78
79 dev->reg = (volatile struct NPU_REG *)base_address;
80 dev->secure = secure_enable;
81 dev->privileged = privilege_enable;
82
83 #ifdef ETHOSU55
84 if (dev->reg->CONFIG.product != ETHOSU_PRODUCT_U55)
85 #else
86 if (dev->reg->CONFIG.product != ETHOSU_PRODUCT_U65)
87 #endif
88 {
89 LOG_ERR("Failed to initialize device. Driver has not been compiled for this product");
90 goto err;
91 }
92
93 // Make sure the NPU is in a known state
94 if (ethosu_dev_soft_reset(dev) != ETHOSU_SUCCESS)
95 {
96 goto err;
97 }
98
99 return dev;
100
101 err:
102 free(dev);
103 return NULL;
104 }
105
ethosu_dev_deinit(struct ethosu_device * dev)106 void ethosu_dev_deinit(struct ethosu_device *dev)
107 {
108 free(dev);
109 }
110
ethosu_dev_axi_init(struct ethosu_device * dev)111 enum ethosu_error_codes ethosu_dev_axi_init(struct ethosu_device *dev)
112 {
113 struct regioncfg_r rcfg = {0};
114 struct axi_limit0_r l0 = {0};
115 struct axi_limit1_r l1 = {0};
116 struct axi_limit2_r l2 = {0};
117 struct axi_limit3_r l3 = {0};
118
119 dev->reg->QCONFIG.word = NPU_QCONFIG;
120
121 rcfg.region0 = NPU_REGIONCFG_0;
122 rcfg.region1 = NPU_REGIONCFG_1;
123 rcfg.region2 = NPU_REGIONCFG_2;
124 rcfg.region3 = NPU_REGIONCFG_3;
125 rcfg.region4 = NPU_REGIONCFG_4;
126 rcfg.region5 = NPU_REGIONCFG_5;
127 rcfg.region6 = NPU_REGIONCFG_6;
128 rcfg.region7 = NPU_REGIONCFG_7;
129 dev->reg->REGIONCFG.word = rcfg.word;
130
131 l0.max_beats = AXI_LIMIT0_MAX_BEATS_BYTES;
132 l0.memtype = AXI_LIMIT0_MEM_TYPE;
133 l0.max_outstanding_read_m1 = AXI_LIMIT0_MAX_OUTSTANDING_READS - 1;
134 l0.max_outstanding_write_m1 = AXI_LIMIT0_MAX_OUTSTANDING_WRITES - 1;
135
136 l1.max_beats = AXI_LIMIT1_MAX_BEATS_BYTES;
137 l1.memtype = AXI_LIMIT1_MEM_TYPE;
138 l1.max_outstanding_read_m1 = AXI_LIMIT1_MAX_OUTSTANDING_READS - 1;
139 l1.max_outstanding_write_m1 = AXI_LIMIT1_MAX_OUTSTANDING_WRITES - 1;
140
141 l2.max_beats = AXI_LIMIT2_MAX_BEATS_BYTES;
142 l2.memtype = AXI_LIMIT2_MEM_TYPE;
143 l2.max_outstanding_read_m1 = AXI_LIMIT2_MAX_OUTSTANDING_READS - 1;
144 l2.max_outstanding_write_m1 = AXI_LIMIT2_MAX_OUTSTANDING_WRITES - 1;
145
146 l3.max_beats = AXI_LIMIT3_MAX_BEATS_BYTES;
147 l3.memtype = AXI_LIMIT3_MEM_TYPE;
148 l3.max_outstanding_read_m1 = AXI_LIMIT3_MAX_OUTSTANDING_READS - 1;
149 l3.max_outstanding_write_m1 = AXI_LIMIT3_MAX_OUTSTANDING_WRITES - 1;
150
151 dev->reg->AXI_LIMIT0.word = l0.word;
152 dev->reg->AXI_LIMIT1.word = l1.word;
153 dev->reg->AXI_LIMIT2.word = l2.word;
154 dev->reg->AXI_LIMIT3.word = l3.word;
155
156 return ETHOSU_SUCCESS;
157 }
158
ethosu_dev_run_command_stream(struct ethosu_device * dev,const uint8_t * cmd_stream_ptr,uint32_t cms_length,const uint64_t * base_addr,int num_base_addr)159 void ethosu_dev_run_command_stream(struct ethosu_device *dev,
160 const uint8_t *cmd_stream_ptr,
161 uint32_t cms_length,
162 const uint64_t *base_addr,
163 int num_base_addr)
164 {
165 assert(num_base_addr <= NPU_REG_BASEP_ARRLEN);
166
167 struct cmd_r cmd;
168 uint64_t qbase = ethosu_address_remap((uintptr_t)cmd_stream_ptr, -1);
169 assert(qbase <= ADDRESS_MASK);
170 LOG_DEBUG("QBASE=0x%016llx, QSIZE=%" PRIu32 ", cmd_stream_ptr=%p", qbase, cms_length, cmd_stream_ptr);
171
172 dev->reg->QBASE.word[0] = qbase & 0xffffffff;
173 #ifdef ETHOSU65
174 dev->reg->QBASE.word[1] = qbase >> 32;
175 #endif
176 dev->reg->QSIZE.word = cms_length;
177
178 for (int i = 0; i < num_base_addr; i++)
179 {
180 uint64_t addr = ethosu_address_remap(base_addr[i], i);
181 assert(addr <= ADDRESS_MASK);
182 LOG_DEBUG("BASEP%d=0x%016llx", i, addr);
183 dev->reg->BASEP[i].word[0] = addr & 0xffffffff;
184 #ifdef ETHOSU65
185 dev->reg->BASEP[i].word[1] = addr >> 32;
186 #endif
187 }
188
189 cmd.word = dev->reg->CMD.word & NPU_CMD_PWR_CLK_MASK;
190 cmd.transition_to_running_state = 1;
191
192 dev->reg->CMD.word = cmd.word;
193 LOG_DEBUG("CMD=0x%08" PRIx32, cmd.word);
194 }
195
ethosu_dev_print_err_status(struct ethosu_device * dev)196 void ethosu_dev_print_err_status(struct ethosu_device *dev)
197 {
198 LOG_ERR("NPU status=0x%08" PRIx32 ", qread=%" PRIu32 ", cmd_end_reached=%u",
199 dev->reg->STATUS.word,
200 dev->reg->QREAD.word,
201 dev->reg->STATUS.cmd_end_reached);
202 }
203
ethosu_dev_handle_interrupt(struct ethosu_device * dev)204 bool ethosu_dev_handle_interrupt(struct ethosu_device *dev)
205 {
206 struct cmd_r cmd;
207
208 // Clear interrupt
209 cmd.word = dev->reg->CMD.word & NPU_CMD_PWR_CLK_MASK;
210 cmd.clear_irq = 1;
211 dev->reg->CMD.word = cmd.word;
212
213 // If a fault has occured, the NPU needs to be reset
214 if (dev->reg->STATUS.bus_status || dev->reg->STATUS.cmd_parse_error || dev->reg->STATUS.wd_fault ||
215 dev->reg->STATUS.ecc_fault || !dev->reg->STATUS.cmd_end_reached)
216 {
217 return false;
218 }
219
220 return true;
221 }
222
ethosu_dev_verify_access_state(struct ethosu_device * dev)223 bool ethosu_dev_verify_access_state(struct ethosu_device *dev)
224 {
225 if (dev->reg->PROT.active_CSL != (dev->secure ? SECURITY_LEVEL_SECURE : SECURITY_LEVEL_NON_SECURE) ||
226 dev->reg->PROT.active_CPL != (dev->privileged ? PRIVILEGE_LEVEL_PRIVILEGED : PRIVILEGE_LEVEL_USER))
227 {
228 return false;
229 }
230 return true;
231 }
232
ethosu_dev_soft_reset(struct ethosu_device * dev)233 enum ethosu_error_codes ethosu_dev_soft_reset(struct ethosu_device *dev)
234 {
235 // Note that after a soft-reset, the NPU is unconditionally
236 // powered until the next CMD gets written.
237
238 struct reset_r reset;
239
240 reset.word = 0;
241 reset.pending_CPL = dev->privileged ? PRIVILEGE_LEVEL_PRIVILEGED : PRIVILEGE_LEVEL_USER;
242 reset.pending_CSL = dev->secure ? SECURITY_LEVEL_SECURE : SECURITY_LEVEL_NON_SECURE;
243
244 // Reset and set security level
245 LOG_INFO("Soft reset NPU");
246 dev->reg->RESET.word = reset.word;
247
248 // Wait until reset status indicates that reset has been completed
249 for (int i = 0; i < 100000 && dev->reg->STATUS.reset_status != 0; i++)
250 {
251 }
252
253 if (dev->reg->STATUS.reset_status != 0)
254 {
255 LOG_ERR("Soft reset timed out");
256 return ETHOSU_GENERIC_FAILURE;
257 }
258
259 // Verify that NPU has switched security state and privilege level
260 if (ethosu_dev_verify_access_state(dev) != true)
261 {
262 LOG_ERR("Failed to switch security state and privilege level");
263 return ETHOSU_GENERIC_FAILURE;
264 }
265
266 // Reinitialize AXI settings
267 ethosu_dev_axi_init(dev);
268
269 return ETHOSU_SUCCESS;
270 }
271
ethosu_dev_get_hw_info(struct ethosu_device * dev,struct ethosu_hw_info * hwinfo)272 void ethosu_dev_get_hw_info(struct ethosu_device *dev, struct ethosu_hw_info *hwinfo)
273 {
274 struct config_r cfg;
275 struct id_r id;
276
277 cfg.word = dev->reg->CONFIG.word;
278 id.word = dev->reg->ID.word;
279
280 hwinfo->cfg.cmd_stream_version = cfg.cmd_stream_version;
281 hwinfo->cfg.custom_dma = cfg.custom_dma;
282 hwinfo->cfg.macs_per_cc = cfg.macs_per_cc;
283
284 hwinfo->version.arch_major_rev = id.arch_major_rev;
285 hwinfo->version.arch_minor_rev = id.arch_minor_rev;
286 hwinfo->version.arch_patch_rev = id.arch_patch_rev;
287 hwinfo->version.product_major = id.product_major;
288 hwinfo->version.version_major = id.version_major;
289 hwinfo->version.version_minor = id.version_minor;
290 hwinfo->version.version_status = id.version_status;
291 }
292
ethosu_dev_set_clock_and_power(struct ethosu_device * dev,enum ethosu_clock_q_request clock_q,enum ethosu_power_q_request power_q)293 enum ethosu_error_codes ethosu_dev_set_clock_and_power(struct ethosu_device *dev,
294 enum ethosu_clock_q_request clock_q,
295 enum ethosu_power_q_request power_q)
296 {
297 struct cmd_r cmd = {0};
298 cmd.word = dev->reg->CMD.word & NPU_CMD_PWR_CLK_MASK;
299
300 if (power_q != ETHOSU_POWER_Q_UNCHANGED)
301 {
302 cmd.power_q_enable = power_q == ETHOSU_POWER_Q_ENABLE ? 1 : 0;
303 }
304 if (clock_q != ETHOSU_CLOCK_Q_UNCHANGED)
305 {
306 cmd.clock_q_enable = clock_q == ETHOSU_CLOCK_Q_ENABLE ? 1 : 0;
307 }
308
309 dev->reg->CMD.word = cmd.word;
310 LOG_DEBUG("CMD=0x%08" PRIx32, cmd.word);
311
312 return ETHOSU_SUCCESS;
313 }
314
ethosu_dev_verify_optimizer_config(struct ethosu_device * dev,uint32_t cfg_in,uint32_t id_in)315 bool ethosu_dev_verify_optimizer_config(struct ethosu_device *dev, uint32_t cfg_in, uint32_t id_in)
316 {
317 struct config_r *opt_cfg = (struct config_r *)&cfg_in;
318 struct config_r hw_cfg;
319 struct id_r *opt_id = (struct id_r *)&id_in;
320 struct id_r hw_id;
321 bool ret = true;
322
323 hw_cfg.word = dev->reg->CONFIG.word;
324 hw_id.word = dev->reg->ID.word;
325
326 LOG_INFO("Optimizer config. product=%u, cmd_stream_version=%u, macs_per_cc=%u, shram_size=%u, custom_dma=%u",
327 opt_cfg->product,
328 opt_cfg->cmd_stream_version,
329 opt_cfg->macs_per_cc,
330 opt_cfg->shram_size,
331 opt_cfg->custom_dma);
332 LOG_INFO("Optimizer config. arch version: %u.%u.%u",
333 opt_id->arch_major_rev,
334 opt_id->arch_minor_rev,
335 opt_id->arch_patch_rev);
336 LOG_INFO("Ethos-U config. product=%u, cmd_stream_version=%u, macs_per_cc=%u, shram_size=%u, custom_dma=%u",
337 hw_cfg.product,
338 hw_cfg.cmd_stream_version,
339 hw_cfg.macs_per_cc,
340 hw_cfg.shram_size,
341 hw_cfg.custom_dma);
342 LOG_INFO("Ethos-U. arch version=%u.%u.%u", hw_id.arch_major_rev, hw_id.arch_minor_rev, hw_id.arch_patch_rev);
343
344 if (opt_cfg->word != hw_cfg.word)
345 {
346 if (hw_cfg.product != opt_cfg->product)
347 {
348 LOG_ERR("NPU config mismatch. npu.product=%u, optimizer.product=%u", hw_cfg.product, opt_cfg->product);
349 ret = false;
350 }
351
352 if (hw_cfg.macs_per_cc != opt_cfg->macs_per_cc)
353 {
354 LOG_ERR("NPU config mismatch. npu.macs_per_cc=%u, optimizer.macs_per_cc=%u",
355 hw_cfg.macs_per_cc,
356 opt_cfg->macs_per_cc);
357 ret = false;
358 }
359
360 if (hw_cfg.cmd_stream_version != opt_cfg->cmd_stream_version)
361 {
362 LOG_ERR("NPU config mismatch. npu.cmd_stream_version=%u, optimizer.cmd_stream_version=%u",
363 hw_cfg.cmd_stream_version,
364 opt_cfg->cmd_stream_version);
365 ret = false;
366 }
367
368 if (!hw_cfg.custom_dma && opt_cfg->custom_dma)
369 {
370 LOG_ERR("NPU config mismatch. npu.custom_dma=%u, optimizer.custom_dma=%u",
371 hw_cfg.custom_dma,
372 opt_cfg->custom_dma);
373 ret = false;
374 }
375 }
376
377 if ((hw_id.arch_major_rev != opt_id->arch_major_rev) || (hw_id.arch_minor_rev < opt_id->arch_minor_rev))
378 {
379 LOG_ERR("NPU arch mismatch. npu.arch=%u.%u.%u, optimizer.arch=%u.%u.%u",
380 hw_id.arch_major_rev,
381 hw_id.arch_minor_rev,
382 hw_id.arch_patch_rev,
383 opt_id->arch_major_rev,
384 opt_id->arch_minor_rev,
385 opt_id->arch_patch_rev);
386 ret = false;
387 }
388
389 return ret;
390 }
391
