1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2018 HUAWEI, Inc.
4 * https://www.huawei.com/
5 */
6 #ifndef __EROFS_FS_ZPVEC_H
7 #define __EROFS_FS_ZPVEC_H
8
9 #include "tagptr.h"
10
11 /* page type in pagevec for decompress subsystem */
12 enum z_erofs_page_type {
13 /* including Z_EROFS_VLE_PAGE_TAIL_EXCLUSIVE */
14 Z_EROFS_PAGE_TYPE_EXCLUSIVE,
15
16 Z_EROFS_VLE_PAGE_TYPE_TAIL_SHARED,
17
18 Z_EROFS_VLE_PAGE_TYPE_HEAD,
19 Z_EROFS_VLE_PAGE_TYPE_MAX
20 };
21
22 extern void __compiletime_error("Z_EROFS_PAGE_TYPE_EXCLUSIVE != 0")
23 __bad_page_type_exclusive(void);
24
25 /* pagevec tagged pointer */
26 typedef tagptr2_t erofs_vtptr_t;
27
28 /* pagevec collector */
29 struct z_erofs_pagevec_ctor {
30 struct page *curr, *next;
31 erofs_vtptr_t *pages;
32
33 unsigned int nr, index;
34 };
35
z_erofs_pagevec_ctor_exit(struct z_erofs_pagevec_ctor * ctor,bool atomic)36 static inline void z_erofs_pagevec_ctor_exit(struct z_erofs_pagevec_ctor *ctor,
37 bool atomic)
38 {
39 if (!ctor->curr)
40 return;
41
42 if (atomic)
43 kunmap_atomic(ctor->pages);
44 else
45 kunmap(ctor->curr);
46 }
47
48 static inline struct page *
z_erofs_pagevec_ctor_next_page(struct z_erofs_pagevec_ctor * ctor,unsigned int nr)49 z_erofs_pagevec_ctor_next_page(struct z_erofs_pagevec_ctor *ctor,
50 unsigned int nr)
51 {
52 unsigned int index;
53
54 /* keep away from occupied pages */
55 if (ctor->next)
56 return ctor->next;
57
58 for (index = 0; index < nr; ++index) {
59 const erofs_vtptr_t t = ctor->pages[index];
60 const unsigned int tags = tagptr_unfold_tags(t);
61
62 if (tags == Z_EROFS_PAGE_TYPE_EXCLUSIVE)
63 return tagptr_unfold_ptr(t);
64 }
65 DBG_BUGON(nr >= ctor->nr);
66 return NULL;
67 }
68
69 static inline void
z_erofs_pagevec_ctor_pagedown(struct z_erofs_pagevec_ctor * ctor,bool atomic)70 z_erofs_pagevec_ctor_pagedown(struct z_erofs_pagevec_ctor *ctor,
71 bool atomic)
72 {
73 struct page *next = z_erofs_pagevec_ctor_next_page(ctor, ctor->nr);
74
75 z_erofs_pagevec_ctor_exit(ctor, atomic);
76
77 ctor->curr = next;
78 ctor->next = NULL;
79 ctor->pages = atomic ?
80 kmap_atomic(ctor->curr) : kmap(ctor->curr);
81
82 ctor->nr = PAGE_SIZE / sizeof(struct page *);
83 ctor->index = 0;
84 }
85
z_erofs_pagevec_ctor_init(struct z_erofs_pagevec_ctor * ctor,unsigned int nr,erofs_vtptr_t * pages,unsigned int i)86 static inline void z_erofs_pagevec_ctor_init(struct z_erofs_pagevec_ctor *ctor,
87 unsigned int nr,
88 erofs_vtptr_t *pages,
89 unsigned int i)
90 {
91 ctor->nr = nr;
92 ctor->curr = ctor->next = NULL;
93 ctor->pages = pages;
94
95 if (i >= nr) {
96 i -= nr;
97 z_erofs_pagevec_ctor_pagedown(ctor, false);
98 while (i > ctor->nr) {
99 i -= ctor->nr;
100 z_erofs_pagevec_ctor_pagedown(ctor, false);
101 }
102 }
103 ctor->next = z_erofs_pagevec_ctor_next_page(ctor, i);
104 ctor->index = i;
105 }
106
z_erofs_pagevec_enqueue(struct z_erofs_pagevec_ctor * ctor,struct page * page,enum z_erofs_page_type type)107 static inline bool z_erofs_pagevec_enqueue(struct z_erofs_pagevec_ctor *ctor,
108 struct page *page,
109 enum z_erofs_page_type type)
110 {
111 if (!ctor->next && type)
112 if (ctor->index + 1 == ctor->nr)
113 return false;
114
115 if (ctor->index >= ctor->nr)
116 z_erofs_pagevec_ctor_pagedown(ctor, false);
117
118 /* exclusive page type must be 0 */
119 if (Z_EROFS_PAGE_TYPE_EXCLUSIVE != (uintptr_t)NULL)
120 __bad_page_type_exclusive();
121
122 /* should remind that collector->next never equal to 1, 2 */
123 if (type == (uintptr_t)ctor->next) {
124 ctor->next = page;
125 }
126 ctor->pages[ctor->index++] = tagptr_fold(erofs_vtptr_t, page, type);
127 return true;
128 }
129
130 static inline struct page *
z_erofs_pagevec_dequeue(struct z_erofs_pagevec_ctor * ctor,enum z_erofs_page_type * type)131 z_erofs_pagevec_dequeue(struct z_erofs_pagevec_ctor *ctor,
132 enum z_erofs_page_type *type)
133 {
134 erofs_vtptr_t t;
135
136 if (ctor->index >= ctor->nr) {
137 DBG_BUGON(!ctor->next);
138 z_erofs_pagevec_ctor_pagedown(ctor, true);
139 }
140
141 t = ctor->pages[ctor->index];
142
143 *type = tagptr_unfold_tags(t);
144
145 /* should remind that collector->next never equal to 1, 2 */
146 if (*type == (uintptr_t)ctor->next)
147 ctor->next = tagptr_unfold_ptr(t);
148
149 ctor->pages[ctor->index++] = tagptr_fold(erofs_vtptr_t, NULL, 0);
150 return tagptr_unfold_ptr(t);
151 }
152 #endif
153
