1 /*
2 * Copyright (c) 2019-2022 Arm Limited. All rights reserved.
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 *
6 * Licensed under the Apache License, Version 2.0 (the License); you may
7 * not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an AS IS BASIS, WITHOUT
14 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 #include "ethosu_interface.h"
20
21 #include "ethosu_device.h"
22 #include "ethosu_log.h"
23
24 #ifdef ETHOSU55
25 #include "ethosu_config_u55.h"
26 #else
27 #include "ethosu_config_u65.h"
28 #endif
29
30 #include <assert.h>
31 #include <inttypes.h>
32 #include <stdbool.h>
33 #include <stddef.h>
34 #include <stdio.h>
35 #include <stdlib.h>
36
37 #define ETHOSU_PRODUCT_U55 0
38 #define ETHOSU_PRODUCT_U65 1
39
40 #define BASEP_OFFSET 4
41
42 #ifdef ETHOSU65
43 #define ADDRESS_BITS 40
44 #else
45 #define ADDRESS_BITS 32
46 #endif
47
48 #define ADDRESS_MASK ((1ull << ADDRESS_BITS) - 1)
49
50 #define NPU_CMD_PWR_CLK_MASK (0xC)
51
ethosu_dev_init(const void * base_address,uint32_t secure_enable,uint32_t privilege_enable)52 struct ethosu_device *ethosu_dev_init(const void *base_address, uint32_t secure_enable, uint32_t privilege_enable)
53 {
54 struct ethosu_device *dev = malloc(sizeof(struct ethosu_device));
55 if (!dev)
56 {
57 LOG_ERR("Failed to allocate memory for Ethos-U device");
58 return NULL;
59 }
60
61 dev->reg = (volatile struct NPU_REG *)base_address;
62 dev->secure = secure_enable;
63 dev->privileged = privilege_enable;
64
65 #ifdef ETHOSU55
66 if (dev->reg->CONFIG.product != ETHOSU_PRODUCT_U55)
67 #else
68 if (dev->reg->CONFIG.product != ETHOSU_PRODUCT_U65)
69 #endif
70 {
71 LOG_ERR("Failed to initialize device. Driver has not been compiled for this product");
72 goto err;
73 }
74
75 // Make sure the NPU is in a known state
76 if (ethosu_dev_soft_reset(dev) != ETHOSU_SUCCESS)
77 {
78 goto err;
79 }
80
81 return dev;
82
83 err:
84 free(dev);
85 return NULL;
86 }
87
ethosu_dev_deinit(struct ethosu_device * dev)88 void ethosu_dev_deinit(struct ethosu_device *dev)
89 {
90 free(dev);
91 }
92
ethosu_dev_axi_init(struct ethosu_device * dev)93 enum ethosu_error_codes ethosu_dev_axi_init(struct ethosu_device *dev)
94 {
95 struct regioncfg_r rcfg = {0};
96 struct axi_limit0_r l0 = {0};
97 struct axi_limit1_r l1 = {0};
98 struct axi_limit2_r l2 = {0};
99 struct axi_limit3_r l3 = {0};
100
101 dev->reg->QCONFIG.word = NPU_QCONFIG;
102
103 rcfg.region0 = NPU_REGIONCFG_0;
104 rcfg.region1 = NPU_REGIONCFG_1;
105 rcfg.region2 = NPU_REGIONCFG_2;
106 rcfg.region3 = NPU_REGIONCFG_3;
107 rcfg.region4 = NPU_REGIONCFG_4;
108 rcfg.region5 = NPU_REGIONCFG_5;
109 rcfg.region6 = NPU_REGIONCFG_6;
110 rcfg.region7 = NPU_REGIONCFG_7;
111 dev->reg->REGIONCFG.word = rcfg.word;
112
113 l0.max_beats = AXI_LIMIT0_MAX_BEATS_BYTES;
114 l0.memtype = AXI_LIMIT0_MEM_TYPE;
115 l0.max_outstanding_read_m1 = AXI_LIMIT0_MAX_OUTSTANDING_READS - 1;
116 l0.max_outstanding_write_m1 = AXI_LIMIT0_MAX_OUTSTANDING_WRITES - 1;
117
118 l1.max_beats = AXI_LIMIT1_MAX_BEATS_BYTES;
119 l1.memtype = AXI_LIMIT1_MEM_TYPE;
120 l1.max_outstanding_read_m1 = AXI_LIMIT1_MAX_OUTSTANDING_READS - 1;
121 l1.max_outstanding_write_m1 = AXI_LIMIT1_MAX_OUTSTANDING_WRITES - 1;
122
123 l2.max_beats = AXI_LIMIT2_MAX_BEATS_BYTES;
124 l2.memtype = AXI_LIMIT2_MEM_TYPE;
125 l2.max_outstanding_read_m1 = AXI_LIMIT2_MAX_OUTSTANDING_READS - 1;
126 l2.max_outstanding_write_m1 = AXI_LIMIT2_MAX_OUTSTANDING_WRITES - 1;
127
128 l3.max_beats = AXI_LIMIT3_MAX_BEATS_BYTES;
129 l3.memtype = AXI_LIMIT3_MEM_TYPE;
130 l3.max_outstanding_read_m1 = AXI_LIMIT3_MAX_OUTSTANDING_READS - 1;
131 l3.max_outstanding_write_m1 = AXI_LIMIT3_MAX_OUTSTANDING_WRITES - 1;
132
133 dev->reg->AXI_LIMIT0.word = l0.word;
134 dev->reg->AXI_LIMIT1.word = l1.word;
135 dev->reg->AXI_LIMIT2.word = l2.word;
136 dev->reg->AXI_LIMIT3.word = l3.word;
137
138 return ETHOSU_SUCCESS;
139 }
140
ethosu_dev_run_command_stream(struct ethosu_device * dev,const uint8_t * cmd_stream_ptr,uint32_t cms_length,const uint64_t * base_addr,int num_base_addr)141 void ethosu_dev_run_command_stream(struct ethosu_device *dev,
142 const uint8_t *cmd_stream_ptr,
143 uint32_t cms_length,
144 const uint64_t *base_addr,
145 int num_base_addr)
146 {
147 assert(num_base_addr <= NPU_REG_BASEP_ARRLEN);
148
149 struct cmd_r cmd;
150 uint64_t qbase = (uintptr_t)cmd_stream_ptr + BASE_POINTER_OFFSET;
151 assert(qbase <= ADDRESS_MASK);
152 LOG_DEBUG("QBASE=0x%016llx, QSIZE=%u, base_pointer_offset=0x%08x", qbase, cms_length, BASE_POINTER_OFFSET);
153
154 dev->reg->QBASE.word[0] = qbase & 0xffffffff;
155 #ifdef ETHOSU65
156 dev->reg->QBASE.word[1] = qbase >> 32;
157 #endif
158 dev->reg->QSIZE.word = cms_length;
159
160 for (int i = 0; i < num_base_addr; i++)
161 {
162 uint64_t addr = base_addr[i] + BASE_POINTER_OFFSET;
163 assert(addr <= ADDRESS_MASK);
164 LOG_DEBUG("BASEP%d=0x%016llx", i, addr);
165 dev->reg->BASEP[i].word[0] = addr & 0xffffffff;
166 #ifdef ETHOSU65
167 dev->reg->BASEP[i].word[1] = addr >> 32;
168 #endif
169 }
170
171 cmd.word = dev->reg->CMD.word & NPU_CMD_PWR_CLK_MASK;
172 cmd.transition_to_running_state = 1;
173
174 dev->reg->CMD.word = cmd.word;
175 LOG_DEBUG("CMD=0x%08x", cmd.word);
176 }
177
ethosu_dev_print_err_status(struct ethosu_device * dev)178 void ethosu_dev_print_err_status(struct ethosu_device *dev)
179 {
180 LOG_ERR("NPU status=0x%08" PRIx32 ", qread=%" PRIu32 ", cmd_end_reached=%d",
181 dev->reg->STATUS.word,
182 dev->reg->QREAD.word,
183 dev->reg->STATUS.cmd_end_reached);
184 }
185
ethosu_dev_handle_interrupt(struct ethosu_device * dev)186 bool ethosu_dev_handle_interrupt(struct ethosu_device *dev)
187 {
188 struct cmd_r cmd;
189
190 // Clear interrupt
191 cmd.word = dev->reg->CMD.word & NPU_CMD_PWR_CLK_MASK;
192 cmd.clear_irq = 1;
193 dev->reg->CMD.word = cmd.word;
194
195 // If a fault has occured, the NPU needs to be reset
196 if (dev->reg->STATUS.bus_status || dev->reg->STATUS.cmd_parse_error || dev->reg->STATUS.wd_fault ||
197 dev->reg->STATUS.ecc_fault || !dev->reg->STATUS.cmd_end_reached)
198 {
199 return false;
200 }
201
202 return true;
203 }
204
ethosu_dev_verify_access_state(struct ethosu_device * dev)205 bool ethosu_dev_verify_access_state(struct ethosu_device *dev)
206 {
207 if (dev->reg->PROT.active_CSL != (dev->secure ? SECURITY_LEVEL_SECURE : SECURITY_LEVEL_NON_SECURE) ||
208 dev->reg->PROT.active_CPL != (dev->privileged ? PRIVILEGE_LEVEL_PRIVILEGED : PRIVILEGE_LEVEL_USER))
209 {
210 return false;
211 }
212 return true;
213 }
214
ethosu_dev_soft_reset(struct ethosu_device * dev)215 enum ethosu_error_codes ethosu_dev_soft_reset(struct ethosu_device *dev)
216 {
217 // Note that after a soft-reset, the NPU is unconditionally
218 // powered until the next CMD gets written.
219
220 struct reset_r reset;
221
222 reset.word = 0;
223 reset.pending_CPL = dev->privileged ? PRIVILEGE_LEVEL_PRIVILEGED : PRIVILEGE_LEVEL_USER;
224 reset.pending_CSL = dev->secure ? SECURITY_LEVEL_SECURE : SECURITY_LEVEL_NON_SECURE;
225
226 // Reset and set security level
227 LOG_INFO("Soft reset NPU");
228 dev->reg->RESET.word = reset.word;
229
230 // Wait until reset status indicates that reset has been completed
231 for (int i = 0; i < 100000 && dev->reg->STATUS.reset_status != 0; i++)
232 {
233 }
234
235 if (dev->reg->STATUS.reset_status != 0)
236 {
237 LOG_ERR("Soft reset timed out");
238 return ETHOSU_GENERIC_FAILURE;
239 }
240
241 // Verify that NPU has switched security state and privilege level
242 if (ethosu_dev_verify_access_state(dev) != true)
243 {
244 LOG_ERR("Failed to switch security state and privilege level");
245 return ETHOSU_GENERIC_FAILURE;
246 }
247
248 // Reinitialize AXI settings
249 ethosu_dev_axi_init(dev);
250
251 return ETHOSU_SUCCESS;
252 }
253
ethosu_dev_get_hw_info(struct ethosu_device * dev,struct ethosu_hw_info * hwinfo)254 void ethosu_dev_get_hw_info(struct ethosu_device *dev, struct ethosu_hw_info *hwinfo)
255 {
256 struct config_r cfg;
257 struct id_r id;
258
259 cfg.word = dev->reg->CONFIG.word;
260 id.word = dev->reg->ID.word;
261
262 hwinfo->cfg.cmd_stream_version = cfg.cmd_stream_version;
263 hwinfo->cfg.custom_dma = cfg.custom_dma;
264 hwinfo->cfg.macs_per_cc = cfg.macs_per_cc;
265
266 hwinfo->version.arch_major_rev = id.arch_major_rev;
267 hwinfo->version.arch_minor_rev = id.arch_minor_rev;
268 hwinfo->version.arch_patch_rev = id.arch_patch_rev;
269 hwinfo->version.product_major = id.product_major;
270 hwinfo->version.version_major = id.version_major;
271 hwinfo->version.version_minor = id.version_minor;
272 hwinfo->version.version_status = id.version_status;
273 }
274
ethosu_dev_set_clock_and_power(struct ethosu_device * dev,enum ethosu_clock_q_request clock_q,enum ethosu_power_q_request power_q)275 enum ethosu_error_codes ethosu_dev_set_clock_and_power(struct ethosu_device *dev,
276 enum ethosu_clock_q_request clock_q,
277 enum ethosu_power_q_request power_q)
278 {
279 struct cmd_r cmd = {0};
280 cmd.word = dev->reg->CMD.word & NPU_CMD_PWR_CLK_MASK;
281
282 if (power_q != ETHOSU_POWER_Q_UNCHANGED)
283 {
284 cmd.power_q_enable = power_q == ETHOSU_POWER_Q_ENABLE ? 1 : 0;
285 }
286 if (clock_q != ETHOSU_CLOCK_Q_UNCHANGED)
287 {
288 cmd.clock_q_enable = clock_q == ETHOSU_CLOCK_Q_ENABLE ? 1 : 0;
289 }
290
291 dev->reg->CMD.word = cmd.word;
292 LOG_DEBUG("CMD=0x%08x", cmd.word);
293
294 return ETHOSU_SUCCESS;
295 }
296
ethosu_dev_verify_optimizer_config(struct ethosu_device * dev,uint32_t cfg_in,uint32_t id_in)297 bool ethosu_dev_verify_optimizer_config(struct ethosu_device *dev, uint32_t cfg_in, uint32_t id_in)
298 {
299 struct config_r *opt_cfg = (struct config_r *)&cfg_in;
300 struct config_r hw_cfg;
301 struct id_r *opt_id = (struct id_r *)&id_in;
302 struct id_r hw_id;
303 bool ret = true;
304
305 hw_cfg.word = dev->reg->CONFIG.word;
306 hw_id.word = dev->reg->ID.word;
307
308 LOG_INFO("Optimizer config. product=%d, cmd_stream_version=%d, macs_per_cc=%d, shram_size=%d, custom_dma=%d",
309 opt_cfg->product,
310 opt_cfg->cmd_stream_version,
311 opt_cfg->macs_per_cc,
312 opt_cfg->shram_size,
313 opt_cfg->custom_dma);
314 LOG_INFO("Optimizer config. arch version: %d.%d.%d",
315 opt_id->arch_major_rev,
316 opt_id->arch_minor_rev,
317 opt_id->arch_patch_rev);
318 LOG_INFO("Ethos-U config. product=%d, cmd_stream_version=%d, macs_per_cc=%d, shram_size=%d, custom_dma=%d",
319 hw_cfg.product,
320 hw_cfg.cmd_stream_version,
321 hw_cfg.macs_per_cc,
322 hw_cfg.shram_size,
323 hw_cfg.custom_dma);
324 LOG_INFO("Ethos-U. arch version=%d.%d.%d", hw_id.arch_major_rev, hw_id.arch_minor_rev, hw_id.arch_patch_rev);
325
326 if (opt_cfg->word != hw_cfg.word)
327 {
328 if (hw_cfg.product != opt_cfg->product)
329 {
330 LOG_ERR("NPU config mismatch. npu.product=%d, optimizer.product=%d", hw_cfg.product, opt_cfg->product);
331 ret = false;
332 }
333
334 if (hw_cfg.macs_per_cc != opt_cfg->macs_per_cc)
335 {
336 LOG_ERR("NPU config mismatch. npu.macs_per_cc=%d, optimizer.macs_per_cc=%d",
337 hw_cfg.macs_per_cc,
338 opt_cfg->macs_per_cc);
339 ret = false;
340 }
341
342 if (hw_cfg.cmd_stream_version != opt_cfg->cmd_stream_version)
343 {
344 LOG_ERR("NPU config mismatch. npu.cmd_stream_version=%d, optimizer.cmd_stream_version=%d",
345 hw_cfg.cmd_stream_version,
346 opt_cfg->cmd_stream_version);
347 ret = false;
348 }
349
350 if (!hw_cfg.custom_dma && opt_cfg->custom_dma)
351 {
352 LOG_ERR("NPU config mismatch. npu.custom_dma=%d, optimizer.custom_dma=%d",
353 hw_cfg.custom_dma,
354 opt_cfg->custom_dma);
355 ret = false;
356 }
357 }
358
359 if ((hw_id.arch_major_rev != opt_id->arch_major_rev) || (hw_id.arch_minor_rev < opt_id->arch_minor_rev))
360 {
361 LOG_ERR("NPU arch mismatch. npu.arch=%d.%d.%d, optimizer.arch=%d.%d.%d",
362 hw_id.arch_major_rev,
363 hw_id.arch_minor_rev,
364 hw_id.arch_patch_rev,
365 opt_id->arch_major_rev,
366 opt_id->arch_minor_rev,
367 opt_id->arch_patch_rev);
368 ret = false;
369 }
370
371 return ret;
372 }
373
