root / linux-aio.c @ 2542bfd5
History | View | Annotate | Download (5.4 kB)
1 | 5c6c3a6c | Christoph Hellwig | /*
|
---|---|---|---|
2 | 5c6c3a6c | Christoph Hellwig | * Linux native AIO support.
|
3 | 5c6c3a6c | Christoph Hellwig | *
|
4 | 5c6c3a6c | Christoph Hellwig | * Copyright (C) 2009 IBM, Corp.
|
5 | 5c6c3a6c | Christoph Hellwig | * Copyright (C) 2009 Red Hat, Inc.
|
6 | 5c6c3a6c | Christoph Hellwig | *
|
7 | 5c6c3a6c | Christoph Hellwig | * This work is licensed under the terms of the GNU GPL, version 2 or later.
|
8 | 5c6c3a6c | Christoph Hellwig | * See the COPYING file in the top-level directory.
|
9 | 5c6c3a6c | Christoph Hellwig | */
|
10 | 5c6c3a6c | Christoph Hellwig | #include "qemu-common.h" |
11 | 5c6c3a6c | Christoph Hellwig | #include "qemu-aio.h" |
12 | 5c6c3a6c | Christoph Hellwig | #include "block_int.h" |
13 | 5c6c3a6c | Christoph Hellwig | #include "block/raw-posix-aio.h" |
14 | 5c6c3a6c | Christoph Hellwig | |
15 | 5c6c3a6c | Christoph Hellwig | #include <sys/eventfd.h> |
16 | 5c6c3a6c | Christoph Hellwig | #include <libaio.h> |
17 | 5c6c3a6c | Christoph Hellwig | |
18 | 5c6c3a6c | Christoph Hellwig | /*
|
19 | 5c6c3a6c | Christoph Hellwig | * Queue size (per-device).
|
20 | 5c6c3a6c | Christoph Hellwig | *
|
21 | 5c6c3a6c | Christoph Hellwig | * XXX: eventually we need to communicate this to the guest and/or make it
|
22 | 5c6c3a6c | Christoph Hellwig | * tunable by the guest. If we get more outstanding requests at a time
|
23 | 5c6c3a6c | Christoph Hellwig | * than this we will get EAGAIN from io_submit which is communicated to
|
24 | 5c6c3a6c | Christoph Hellwig | * the guest as an I/O error.
|
25 | 5c6c3a6c | Christoph Hellwig | */
|
26 | 5c6c3a6c | Christoph Hellwig | #define MAX_EVENTS 128 |
27 | 5c6c3a6c | Christoph Hellwig | |
28 | 5c6c3a6c | Christoph Hellwig | struct qemu_laiocb {
|
29 | 5c6c3a6c | Christoph Hellwig | BlockDriverAIOCB common; |
30 | 5c6c3a6c | Christoph Hellwig | struct qemu_laio_state *ctx;
|
31 | 5c6c3a6c | Christoph Hellwig | struct iocb iocb;
|
32 | 5c6c3a6c | Christoph Hellwig | ssize_t ret; |
33 | 5c6c3a6c | Christoph Hellwig | size_t nbytes; |
34 | db0ffc24 | Kevin Wolf | QLIST_ENTRY(qemu_laiocb) node; |
35 | 5c6c3a6c | Christoph Hellwig | }; |
36 | 5c6c3a6c | Christoph Hellwig | |
37 | 5c6c3a6c | Christoph Hellwig | struct qemu_laio_state {
|
38 | 5c6c3a6c | Christoph Hellwig | io_context_t ctx; |
39 | 5c6c3a6c | Christoph Hellwig | int efd;
|
40 | 5c6c3a6c | Christoph Hellwig | int count;
|
41 | 5c6c3a6c | Christoph Hellwig | }; |
42 | 5c6c3a6c | Christoph Hellwig | |
43 | 5c6c3a6c | Christoph Hellwig | static inline ssize_t io_event_ret(struct io_event *ev) |
44 | 5c6c3a6c | Christoph Hellwig | { |
45 | 5c6c3a6c | Christoph Hellwig | return (ssize_t)(((uint64_t)ev->res2 << 32) | ev->res); |
46 | 5c6c3a6c | Christoph Hellwig | } |
47 | 5c6c3a6c | Christoph Hellwig | |
48 | db0ffc24 | Kevin Wolf | /*
|
49 | db0ffc24 | Kevin Wolf | * Completes an AIO request (calls the callback and frees the ACB).
|
50 | db0ffc24 | Kevin Wolf | */
|
51 | db0ffc24 | Kevin Wolf | static void qemu_laio_process_completion(struct qemu_laio_state *s, |
52 | db0ffc24 | Kevin Wolf | struct qemu_laiocb *laiocb)
|
53 | db0ffc24 | Kevin Wolf | { |
54 | db0ffc24 | Kevin Wolf | int ret;
|
55 | db0ffc24 | Kevin Wolf | |
56 | db0ffc24 | Kevin Wolf | s->count--; |
57 | db0ffc24 | Kevin Wolf | |
58 | db0ffc24 | Kevin Wolf | ret = laiocb->ret; |
59 | db0ffc24 | Kevin Wolf | if (ret != -ECANCELED) {
|
60 | db0ffc24 | Kevin Wolf | if (ret == laiocb->nbytes)
|
61 | db0ffc24 | Kevin Wolf | ret = 0;
|
62 | db0ffc24 | Kevin Wolf | else if (ret >= 0) |
63 | db0ffc24 | Kevin Wolf | ret = -EINVAL; |
64 | db0ffc24 | Kevin Wolf | |
65 | db0ffc24 | Kevin Wolf | laiocb->common.cb(laiocb->common.opaque, ret); |
66 | db0ffc24 | Kevin Wolf | } |
67 | db0ffc24 | Kevin Wolf | |
68 | db0ffc24 | Kevin Wolf | qemu_aio_release(laiocb); |
69 | db0ffc24 | Kevin Wolf | } |
70 | db0ffc24 | Kevin Wolf | |
71 | db0ffc24 | Kevin Wolf | /*
|
72 | 384acbf4 | Kevin Wolf | * All requests are directly processed when they complete, so there's nothing
|
73 | 384acbf4 | Kevin Wolf | * left to do during qemu_aio_wait().
|
74 | db0ffc24 | Kevin Wolf | */
|
75 | db0ffc24 | Kevin Wolf | static int qemu_laio_process_requests(void *opaque) |
76 | db0ffc24 | Kevin Wolf | { |
77 | 384acbf4 | Kevin Wolf | return 0; |
78 | db0ffc24 | Kevin Wolf | } |
79 | db0ffc24 | Kevin Wolf | |
80 | 5c6c3a6c | Christoph Hellwig | static void qemu_laio_completion_cb(void *opaque) |
81 | 5c6c3a6c | Christoph Hellwig | { |
82 | 5c6c3a6c | Christoph Hellwig | struct qemu_laio_state *s = opaque;
|
83 | 5c6c3a6c | Christoph Hellwig | |
84 | 5c6c3a6c | Christoph Hellwig | while (1) { |
85 | 5c6c3a6c | Christoph Hellwig | struct io_event events[MAX_EVENTS];
|
86 | 5c6c3a6c | Christoph Hellwig | uint64_t val; |
87 | 5c6c3a6c | Christoph Hellwig | ssize_t ret; |
88 | 5c6c3a6c | Christoph Hellwig | struct timespec ts = { 0 }; |
89 | 5c6c3a6c | Christoph Hellwig | int nevents, i;
|
90 | 5c6c3a6c | Christoph Hellwig | |
91 | 5c6c3a6c | Christoph Hellwig | do {
|
92 | 5c6c3a6c | Christoph Hellwig | ret = read(s->efd, &val, sizeof(val));
|
93 | 2be50649 | Stefan Hajnoczi | } while (ret == -1 && errno == EINTR); |
94 | 5c6c3a6c | Christoph Hellwig | |
95 | 5c6c3a6c | Christoph Hellwig | if (ret == -1 && errno == EAGAIN) |
96 | 5c6c3a6c | Christoph Hellwig | break;
|
97 | 5c6c3a6c | Christoph Hellwig | |
98 | 5c6c3a6c | Christoph Hellwig | if (ret != 8) |
99 | 5c6c3a6c | Christoph Hellwig | break;
|
100 | 5c6c3a6c | Christoph Hellwig | |
101 | 5c6c3a6c | Christoph Hellwig | do {
|
102 | 5c6c3a6c | Christoph Hellwig | nevents = io_getevents(s->ctx, val, MAX_EVENTS, events, &ts); |
103 | 5c6c3a6c | Christoph Hellwig | } while (nevents == -EINTR);
|
104 | 5c6c3a6c | Christoph Hellwig | |
105 | 5c6c3a6c | Christoph Hellwig | for (i = 0; i < nevents; i++) { |
106 | 5c6c3a6c | Christoph Hellwig | struct iocb *iocb = events[i].obj;
|
107 | 5c6c3a6c | Christoph Hellwig | struct qemu_laiocb *laiocb =
|
108 | 5c6c3a6c | Christoph Hellwig | container_of(iocb, struct qemu_laiocb, iocb);
|
109 | 5c6c3a6c | Christoph Hellwig | |
110 | db0ffc24 | Kevin Wolf | laiocb->ret = io_event_ret(&events[i]); |
111 | 384acbf4 | Kevin Wolf | qemu_laio_process_completion(s, laiocb); |
112 | 5c6c3a6c | Christoph Hellwig | } |
113 | 5c6c3a6c | Christoph Hellwig | } |
114 | 5c6c3a6c | Christoph Hellwig | } |
115 | 5c6c3a6c | Christoph Hellwig | |
116 | 5c6c3a6c | Christoph Hellwig | static int qemu_laio_flush_cb(void *opaque) |
117 | 5c6c3a6c | Christoph Hellwig | { |
118 | 5c6c3a6c | Christoph Hellwig | struct qemu_laio_state *s = opaque;
|
119 | 5c6c3a6c | Christoph Hellwig | |
120 | 5c6c3a6c | Christoph Hellwig | return (s->count > 0) ? 1 : 0; |
121 | 5c6c3a6c | Christoph Hellwig | } |
122 | 5c6c3a6c | Christoph Hellwig | |
123 | 5c6c3a6c | Christoph Hellwig | static void laio_cancel(BlockDriverAIOCB *blockacb) |
124 | 5c6c3a6c | Christoph Hellwig | { |
125 | 5c6c3a6c | Christoph Hellwig | struct qemu_laiocb *laiocb = (struct qemu_laiocb *)blockacb; |
126 | 5c6c3a6c | Christoph Hellwig | struct io_event event;
|
127 | 5c6c3a6c | Christoph Hellwig | int ret;
|
128 | 5c6c3a6c | Christoph Hellwig | |
129 | 5c6c3a6c | Christoph Hellwig | if (laiocb->ret != -EINPROGRESS)
|
130 | 5c6c3a6c | Christoph Hellwig | return;
|
131 | 5c6c3a6c | Christoph Hellwig | |
132 | 5c6c3a6c | Christoph Hellwig | /*
|
133 | 5c6c3a6c | Christoph Hellwig | * Note that as of Linux 2.6.31 neither the block device code nor any
|
134 | 5c6c3a6c | Christoph Hellwig | * filesystem implements cancellation of AIO request.
|
135 | 5c6c3a6c | Christoph Hellwig | * Thus the polling loop below is the normal code path.
|
136 | 5c6c3a6c | Christoph Hellwig | */
|
137 | 5c6c3a6c | Christoph Hellwig | ret = io_cancel(laiocb->ctx->ctx, &laiocb->iocb, &event); |
138 | 5c6c3a6c | Christoph Hellwig | if (ret == 0) { |
139 | 5c6c3a6c | Christoph Hellwig | laiocb->ret = -ECANCELED; |
140 | 5c6c3a6c | Christoph Hellwig | return;
|
141 | 5c6c3a6c | Christoph Hellwig | } |
142 | 5c6c3a6c | Christoph Hellwig | |
143 | 5c6c3a6c | Christoph Hellwig | /*
|
144 | 5c6c3a6c | Christoph Hellwig | * We have to wait for the iocb to finish.
|
145 | 5c6c3a6c | Christoph Hellwig | *
|
146 | 5c6c3a6c | Christoph Hellwig | * The only way to get the iocb status update is by polling the io context.
|
147 | 5c6c3a6c | Christoph Hellwig | * We might be able to do this slightly more optimal by removing the
|
148 | 5c6c3a6c | Christoph Hellwig | * O_NONBLOCK flag.
|
149 | 5c6c3a6c | Christoph Hellwig | */
|
150 | 5c6c3a6c | Christoph Hellwig | while (laiocb->ret == -EINPROGRESS)
|
151 | 5c6c3a6c | Christoph Hellwig | qemu_laio_completion_cb(laiocb->ctx); |
152 | 5c6c3a6c | Christoph Hellwig | } |
153 | 5c6c3a6c | Christoph Hellwig | |
154 | 5c6c3a6c | Christoph Hellwig | static AIOPool laio_pool = {
|
155 | 5c6c3a6c | Christoph Hellwig | .aiocb_size = sizeof(struct qemu_laiocb), |
156 | 5c6c3a6c | Christoph Hellwig | .cancel = laio_cancel, |
157 | 5c6c3a6c | Christoph Hellwig | }; |
158 | 5c6c3a6c | Christoph Hellwig | |
159 | 5c6c3a6c | Christoph Hellwig | BlockDriverAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd, |
160 | 5c6c3a6c | Christoph Hellwig | int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
|
161 | 5c6c3a6c | Christoph Hellwig | BlockDriverCompletionFunc *cb, void *opaque, int type) |
162 | 5c6c3a6c | Christoph Hellwig | { |
163 | 5c6c3a6c | Christoph Hellwig | struct qemu_laio_state *s = aio_ctx;
|
164 | 5c6c3a6c | Christoph Hellwig | struct qemu_laiocb *laiocb;
|
165 | 5c6c3a6c | Christoph Hellwig | struct iocb *iocbs;
|
166 | 5c6c3a6c | Christoph Hellwig | off_t offset = sector_num * 512;
|
167 | 5c6c3a6c | Christoph Hellwig | |
168 | 5c6c3a6c | Christoph Hellwig | laiocb = qemu_aio_get(&laio_pool, bs, cb, opaque); |
169 | 5c6c3a6c | Christoph Hellwig | if (!laiocb)
|
170 | 5c6c3a6c | Christoph Hellwig | return NULL; |
171 | 5c6c3a6c | Christoph Hellwig | laiocb->nbytes = nb_sectors * 512;
|
172 | 5c6c3a6c | Christoph Hellwig | laiocb->ctx = s; |
173 | 5c6c3a6c | Christoph Hellwig | laiocb->ret = -EINPROGRESS; |
174 | 5c6c3a6c | Christoph Hellwig | |
175 | 5c6c3a6c | Christoph Hellwig | iocbs = &laiocb->iocb; |
176 | 5c6c3a6c | Christoph Hellwig | |
177 | 5c6c3a6c | Christoph Hellwig | switch (type) {
|
178 | 5c6c3a6c | Christoph Hellwig | case QEMU_AIO_WRITE:
|
179 | 5c6c3a6c | Christoph Hellwig | io_prep_pwritev(iocbs, fd, qiov->iov, qiov->niov, offset); |
180 | 5c6c3a6c | Christoph Hellwig | break;
|
181 | 5c6c3a6c | Christoph Hellwig | case QEMU_AIO_READ:
|
182 | 5c6c3a6c | Christoph Hellwig | io_prep_preadv(iocbs, fd, qiov->iov, qiov->niov, offset); |
183 | 5c6c3a6c | Christoph Hellwig | break;
|
184 | 5c6c3a6c | Christoph Hellwig | default:
|
185 | 5c6c3a6c | Christoph Hellwig | fprintf(stderr, "%s: invalid AIO request type 0x%x.\n",
|
186 | 5c6c3a6c | Christoph Hellwig | __func__, type); |
187 | 5c6c3a6c | Christoph Hellwig | goto out_free_aiocb;
|
188 | 5c6c3a6c | Christoph Hellwig | } |
189 | 5c6c3a6c | Christoph Hellwig | io_set_eventfd(&laiocb->iocb, s->efd); |
190 | 5c6c3a6c | Christoph Hellwig | s->count++; |
191 | 5c6c3a6c | Christoph Hellwig | |
192 | 5c6c3a6c | Christoph Hellwig | if (io_submit(s->ctx, 1, &iocbs) < 0) |
193 | 5c6c3a6c | Christoph Hellwig | goto out_dec_count;
|
194 | 5c6c3a6c | Christoph Hellwig | return &laiocb->common;
|
195 | 5c6c3a6c | Christoph Hellwig | |
196 | 5c6c3a6c | Christoph Hellwig | out_free_aiocb:
|
197 | 5c6c3a6c | Christoph Hellwig | qemu_aio_release(laiocb); |
198 | 5c6c3a6c | Christoph Hellwig | out_dec_count:
|
199 | 5c6c3a6c | Christoph Hellwig | s->count--; |
200 | 5c6c3a6c | Christoph Hellwig | return NULL; |
201 | 5c6c3a6c | Christoph Hellwig | } |
202 | 5c6c3a6c | Christoph Hellwig | |
203 | 5c6c3a6c | Christoph Hellwig | void *laio_init(void) |
204 | 5c6c3a6c | Christoph Hellwig | { |
205 | 5c6c3a6c | Christoph Hellwig | struct qemu_laio_state *s;
|
206 | 5c6c3a6c | Christoph Hellwig | |
207 | 7267c094 | Anthony Liguori | s = g_malloc0(sizeof(*s));
|
208 | 5c6c3a6c | Christoph Hellwig | s->efd = eventfd(0, 0); |
209 | 5c6c3a6c | Christoph Hellwig | if (s->efd == -1) |
210 | 5c6c3a6c | Christoph Hellwig | goto out_free_state;
|
211 | 5c6c3a6c | Christoph Hellwig | fcntl(s->efd, F_SETFL, O_NONBLOCK); |
212 | 5c6c3a6c | Christoph Hellwig | |
213 | 5c6c3a6c | Christoph Hellwig | if (io_setup(MAX_EVENTS, &s->ctx) != 0) |
214 | 5c6c3a6c | Christoph Hellwig | goto out_close_efd;
|
215 | 5c6c3a6c | Christoph Hellwig | |
216 | db0ffc24 | Kevin Wolf | qemu_aio_set_fd_handler(s->efd, qemu_laio_completion_cb, NULL,
|
217 | db0ffc24 | Kevin Wolf | qemu_laio_flush_cb, qemu_laio_process_requests, s); |
218 | 5c6c3a6c | Christoph Hellwig | |
219 | 5c6c3a6c | Christoph Hellwig | return s;
|
220 | 5c6c3a6c | Christoph Hellwig | |
221 | 5c6c3a6c | Christoph Hellwig | out_close_efd:
|
222 | 5c6c3a6c | Christoph Hellwig | close(s->efd); |
223 | 5c6c3a6c | Christoph Hellwig | out_free_state:
|
224 | 7267c094 | Anthony Liguori | g_free(s); |
225 | 5c6c3a6c | Christoph Hellwig | return NULL; |
226 | 5c6c3a6c | Christoph Hellwig | } |