41 static inline size_t buffer_size(struct persistent_ram_zone *prz)  in buffer_size()  argument
43 	return atomic_read(&prz->buffer->size);  in buffer_size()
46 static inline size_t buffer_start(struct persistent_ram_zone *prz)  in buffer_start()  argument
48 	return atomic_read(&prz->buffer->start);  in buffer_start()
52 static size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a)  in buffer_start_add()  argument
58 	if (!(prz->flags & PRZ_FLAG_NO_LOCK))  in buffer_start_add()
59 		raw_spin_lock_irqsave(&prz->buffer_lock, flags);  in buffer_start_add()
61 	old = atomic_read(&prz->buffer->start);  in buffer_start_add()
63 	while (unlikely(new >= prz->buffer_size))  in buffer_start_add()
64 		new -= prz->buffer_size;  in buffer_start_add()
65 	atomic_set(&prz->buffer->start, new);  in buffer_start_add()
67 	if (!(prz->flags & PRZ_FLAG_NO_LOCK))  in buffer_start_add()
68 		raw_spin_unlock_irqrestore(&prz->buffer_lock, flags);  in buffer_start_add()
74 static void buffer_size_add(struct persistent_ram_zone *prz, size_t a)  in buffer_size_add()  argument
80 	if (!(prz->flags & PRZ_FLAG_NO_LOCK))  in buffer_size_add()
81 		raw_spin_lock_irqsave(&prz->buffer_lock, flags);  in buffer_size_add()
83 	old = atomic_read(&prz->buffer->size);  in buffer_size_add()
84 	if (old == prz->buffer_size)  in buffer_size_add()
88 	if (new > prz->buffer_size)  in buffer_size_add()
89 		new = prz->buffer_size;  in buffer_size_add()
90 	atomic_set(&prz->buffer->size, new);  in buffer_size_add()
93 	if (!(prz->flags & PRZ_FLAG_NO_LOCK))  in buffer_size_add()
94 		raw_spin_unlock_irqrestore(&prz->buffer_lock, flags);  in buffer_size_add()
97 static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz,  in persistent_ram_encode_rs8()  argument
103 	memset(prz->ecc_info.par, 0,  in persistent_ram_encode_rs8()
104 	       prz->ecc_info.ecc_size * sizeof(prz->ecc_info.par[0]));  in persistent_ram_encode_rs8()
105 	encode_rs8(prz->rs_decoder, data, len, prz->ecc_info.par, 0);  in persistent_ram_encode_rs8()
106 	for (i = 0; i < prz->ecc_info.ecc_size; i++)  in persistent_ram_encode_rs8()
107 		ecc[i] = prz->ecc_info.par[i];  in persistent_ram_encode_rs8()
110 static int persistent_ram_decode_rs8(struct persistent_ram_zone *prz,  in persistent_ram_decode_rs8()  argument
115 	for (i = 0; i < prz->ecc_info.ecc_size; i++)  in persistent_ram_decode_rs8()
116 		prz->ecc_info.par[i] = ecc[i];  in persistent_ram_decode_rs8()
117 	return decode_rs8(prz->rs_decoder, data, prz->ecc_info.par, len,  in persistent_ram_decode_rs8()
121 static void notrace persistent_ram_update_ecc(struct persistent_ram_zone *prz,  in persistent_ram_update_ecc()  argument
124 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_update_ecc()
125 	uint8_t *buffer_end = buffer->data + prz->buffer_size;  in persistent_ram_update_ecc()
128 	int ecc_block_size = prz->ecc_info.block_size;  in persistent_ram_update_ecc()
129 	int ecc_size = prz->ecc_info.ecc_size;  in persistent_ram_update_ecc()
136 	par = prz->par_buffer + (start / ecc_block_size) * ecc_size;  in persistent_ram_update_ecc()
141 		persistent_ram_encode_rs8(prz, block, size, par);  in persistent_ram_update_ecc()
147 static void persistent_ram_update_header_ecc(struct persistent_ram_zone *prz)  in persistent_ram_update_header_ecc()  argument
149 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_update_header_ecc()
151 	if (!prz->ecc_info.ecc_size)  in persistent_ram_update_header_ecc()
154 	persistent_ram_encode_rs8(prz, (uint8_t *)buffer, sizeof(*buffer),  in persistent_ram_update_header_ecc()
155 				  prz->par_header);  in persistent_ram_update_header_ecc()
158 static void persistent_ram_ecc_old(struct persistent_ram_zone *prz)  in persistent_ram_ecc_old()  argument
160 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_ecc_old()
164 	if (!prz->ecc_info.ecc_size)  in persistent_ram_ecc_old()
168 	par = prz->par_buffer;  in persistent_ram_ecc_old()
169 	while (block < buffer->data + buffer_size(prz)) {  in persistent_ram_ecc_old()
171 		int size = prz->ecc_info.block_size;  in persistent_ram_ecc_old()
172 		if (block + size > buffer->data + prz->buffer_size)  in persistent_ram_ecc_old()
173 			size = buffer->data + prz->buffer_size - block;  in persistent_ram_ecc_old()
174 		numerr = persistent_ram_decode_rs8(prz, block, size, par);  in persistent_ram_ecc_old()
177 			prz->corrected_bytes += numerr;  in persistent_ram_ecc_old()
180 			prz->bad_blocks++;  in persistent_ram_ecc_old()
182 		block += prz->ecc_info.block_size;  in persistent_ram_ecc_old()
183 		par += prz->ecc_info.ecc_size;  in persistent_ram_ecc_old()
187 static int persistent_ram_init_ecc(struct persistent_ram_zone *prz,  in persistent_ram_init_ecc()  argument
191 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_init_ecc()
198 	prz->ecc_info.block_size = ecc_info->block_size ?: 128;  in persistent_ram_init_ecc()
199 	prz->ecc_info.ecc_size = ecc_info->ecc_size ?: 16;  in persistent_ram_init_ecc()
200 	prz->ecc_info.symsize = ecc_info->symsize ?: 8;  in persistent_ram_init_ecc()
201 	prz->ecc_info.poly = ecc_info->poly ?: 0x11d;  in persistent_ram_init_ecc()
203 	ecc_blocks = DIV_ROUND_UP(prz->buffer_size - prz->ecc_info.ecc_size,  in persistent_ram_init_ecc()
204 				  prz->ecc_info.block_size +  in persistent_ram_init_ecc()
205 				  prz->ecc_info.ecc_size);  in persistent_ram_init_ecc()
206 	ecc_total = (ecc_blocks + 1) * prz->ecc_info.ecc_size;  in persistent_ram_init_ecc()
207 	if (ecc_total >= prz->buffer_size) {  in persistent_ram_init_ecc()
209 		       __func__, prz->ecc_info.ecc_size,  in persistent_ram_init_ecc()
210 		       ecc_total, prz->buffer_size);  in persistent_ram_init_ecc()
214 	prz->buffer_size -= ecc_total;  in persistent_ram_init_ecc()
215 	prz->par_buffer = buffer->data + prz->buffer_size;  in persistent_ram_init_ecc()
216 	prz->par_header = prz->par_buffer +  in persistent_ram_init_ecc()
217 			  ecc_blocks * prz->ecc_info.ecc_size;  in persistent_ram_init_ecc()
223 	prz->rs_decoder = init_rs(prz->ecc_info.symsize, prz->ecc_info.poly,  in persistent_ram_init_ecc()
224 				  0, 1, prz->ecc_info.ecc_size);  in persistent_ram_init_ecc()
225 	if (prz->rs_decoder == NULL) {  in persistent_ram_init_ecc()
231 	prz->ecc_info.par = kmalloc_array(prz->ecc_info.ecc_size,  in persistent_ram_init_ecc()
232 					  sizeof(*prz->ecc_info.par),  in persistent_ram_init_ecc()
234 	if (!prz->ecc_info.par) {  in persistent_ram_init_ecc()
239 	prz->corrected_bytes = 0;  in persistent_ram_init_ecc()
240 	prz->bad_blocks = 0;  in persistent_ram_init_ecc()
242 	numerr = persistent_ram_decode_rs8(prz, buffer, sizeof(*buffer),  in persistent_ram_init_ecc()
243 					   prz->par_header);  in persistent_ram_init_ecc()
246 		prz->corrected_bytes += numerr;  in persistent_ram_init_ecc()
249 		prz->bad_blocks++;  in persistent_ram_init_ecc()
255 ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz,  in persistent_ram_ecc_string()  argument
260 	if (!prz->ecc_info.ecc_size)  in persistent_ram_ecc_string()
263 	if (prz->corrected_bytes || prz->bad_blocks)  in persistent_ram_ecc_string()
266 			prz->corrected_bytes, prz->bad_blocks);  in persistent_ram_ecc_string()
273 static void notrace persistent_ram_update(struct persistent_ram_zone *prz,  in persistent_ram_update()  argument
276 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_update()
278 	persistent_ram_update_ecc(prz, start, count);  in persistent_ram_update()
281 static int notrace persistent_ram_update_user(struct persistent_ram_zone *prz,  in persistent_ram_update_user()  argument
284 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_update_user()
287 	persistent_ram_update_ecc(prz, start, count);  in persistent_ram_update_user()
291 void persistent_ram_save_old(struct persistent_ram_zone *prz)  in persistent_ram_save_old()  argument
293 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_save_old()
294 	size_t size = buffer_size(prz);  in persistent_ram_save_old()
295 	size_t start = buffer_start(prz);  in persistent_ram_save_old()
300 	if (!prz->old_log) {  in persistent_ram_save_old()
301 		persistent_ram_ecc_old(prz);  in persistent_ram_save_old()
302 		prz->old_log = kmalloc(size, GFP_KERNEL);  in persistent_ram_save_old()
304 	if (!prz->old_log) {  in persistent_ram_save_old()
309 	prz->old_log_size = size;  in persistent_ram_save_old()
310 	memcpy_fromio(prz->old_log, &buffer->data[start], size - start);  in persistent_ram_save_old()
311 	memcpy_fromio(prz->old_log + size - start, &buffer->data[0], start);  in persistent_ram_save_old()
314 int notrace persistent_ram_write(struct persistent_ram_zone *prz,  in persistent_ram_write()  argument
321 	if (unlikely(c > prz->buffer_size)) {  in persistent_ram_write()
322 		s += c - prz->buffer_size;  in persistent_ram_write()
323 		c = prz->buffer_size;  in persistent_ram_write()
326 	buffer_size_add(prz, c);  in persistent_ram_write()
328 	start = buffer_start_add(prz, c);  in persistent_ram_write()
330 	rem = prz->buffer_size - start;  in persistent_ram_write()
332 		persistent_ram_update(prz, s, start, rem);  in persistent_ram_write()
337 	persistent_ram_update(prz, s, start, c);  in persistent_ram_write()
339 	persistent_ram_update_header_ecc(prz);  in persistent_ram_write()
344 int notrace persistent_ram_write_user(struct persistent_ram_zone *prz,  in persistent_ram_write_user()  argument
352 	if (unlikely(c > prz->buffer_size)) {  in persistent_ram_write_user()
353 		s += c - prz->buffer_size;  in persistent_ram_write_user()
354 		c = prz->buffer_size;  in persistent_ram_write_user()
357 	buffer_size_add(prz, c);  in persistent_ram_write_user()
359 	start = buffer_start_add(prz, c);  in persistent_ram_write_user()
361 	rem = prz->buffer_size - start;  in persistent_ram_write_user()
363 		ret = persistent_ram_update_user(prz, s, start, rem);  in persistent_ram_write_user()
369 		ret = persistent_ram_update_user(prz, s, start, c);  in persistent_ram_write_user()
371 	persistent_ram_update_header_ecc(prz);  in persistent_ram_write_user()
376 size_t persistent_ram_old_size(struct persistent_ram_zone *prz)  in persistent_ram_old_size()  argument
378 	return prz->old_log_size;  in persistent_ram_old_size()
381 void *persistent_ram_old(struct persistent_ram_zone *prz)  in persistent_ram_old()  argument
383 	return prz->old_log;  in persistent_ram_old()
386 void persistent_ram_free_old(struct persistent_ram_zone *prz)  in persistent_ram_free_old()  argument
388 	kfree(prz->old_log);  in persistent_ram_free_old()
389 	prz->old_log = NULL;  in persistent_ram_free_old()
390 	prz->old_log_size = 0;  in persistent_ram_free_old()
393 void persistent_ram_zap(struct persistent_ram_zone *prz)  in persistent_ram_zap()  argument
395 	atomic_set(&prz->buffer->start, 0);  in persistent_ram_zap()
396 	atomic_set(&prz->buffer->size, 0);  in persistent_ram_zap()
397 	persistent_ram_update_header_ecc(prz);  in persistent_ram_zap()
465 		struct persistent_ram_zone *prz, int memtype)  in persistent_ram_buffer_map()  argument
467 	prz->paddr = start;  in persistent_ram_buffer_map()
468 	prz->size = size;  in persistent_ram_buffer_map()
471 		prz->vaddr = persistent_ram_vmap(start, size, memtype);  in persistent_ram_buffer_map()
473 		prz->vaddr = persistent_ram_iomap(start, size, memtype);  in persistent_ram_buffer_map()
475 	if (!prz->vaddr) {  in persistent_ram_buffer_map()
481 	prz->buffer = prz->vaddr;  in persistent_ram_buffer_map()
482 	prz->buffer_size = size - sizeof(struct persistent_ram_buffer);  in persistent_ram_buffer_map()
487 static int persistent_ram_post_init(struct persistent_ram_zone *prz, u32 sig,  in persistent_ram_post_init()  argument
492 	ret = persistent_ram_init_ecc(prz, ecc_info);  in persistent_ram_post_init()
498 	if (prz->buffer->sig == sig) {  in persistent_ram_post_init()
499 		if (buffer_size(prz) > prz->buffer_size ||  in persistent_ram_post_init()
500 		    buffer_start(prz) > buffer_size(prz))  in persistent_ram_post_init()
502 				buffer_size(prz), buffer_start(prz));  in persistent_ram_post_init()
505 				 buffer_size(prz), buffer_start(prz));  in persistent_ram_post_init()
506 			persistent_ram_save_old(prz);  in persistent_ram_post_init()
511 			 prz->buffer->sig);  in persistent_ram_post_init()
515 	prz->buffer->sig = sig;  in persistent_ram_post_init()
516 	persistent_ram_zap(prz);  in persistent_ram_post_init()
521 void persistent_ram_free(struct persistent_ram_zone *prz)  in persistent_ram_free()  argument
523 	if (!prz)  in persistent_ram_free()
526 	if (prz->vaddr) {  in persistent_ram_free()
527 		if (pfn_valid(prz->paddr >> PAGE_SHIFT)) {  in persistent_ram_free()
529 			vunmap(prz->vaddr - offset_in_page(prz->paddr));  in persistent_ram_free()
531 			iounmap(prz->vaddr);  in persistent_ram_free()
532 			release_mem_region(prz->paddr, prz->size);  in persistent_ram_free()
534 		prz->vaddr = NULL;  in persistent_ram_free()
536 	if (prz->rs_decoder) {  in persistent_ram_free()
537 		free_rs(prz->rs_decoder);  in persistent_ram_free()
538 		prz->rs_decoder = NULL;  in persistent_ram_free()
540 	kfree(prz->ecc_info.par);  in persistent_ram_free()
541 	prz->ecc_info.par = NULL;  in persistent_ram_free()
543 	persistent_ram_free_old(prz);  in persistent_ram_free()
544 	kfree(prz);  in persistent_ram_free()
551 	struct persistent_ram_zone *prz;  in persistent_ram_new()  local
554 	prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL);  in persistent_ram_new()
555 	if (!prz) {  in persistent_ram_new()
561 	raw_spin_lock_init(&prz->buffer_lock);  in persistent_ram_new()
562 	prz->flags = flags;  in persistent_ram_new()
564 	ret = persistent_ram_buffer_map(start, size, prz, memtype);  in persistent_ram_new()
568 	ret = persistent_ram_post_init(prz, sig, ecc_info);  in persistent_ram_new()
572 	return prz;  in persistent_ram_new()
574 	persistent_ram_free(prz);  in persistent_ram_new()