Commit 114a4406 authored by Antoine Kaufmann's avatar Antoine Kaufmann
Browse files

initial commit

parents
CFLAGS += -std=gnu99 -O0 -g -Wall -fno-omit-frame-pointer
LDFLAGS += -pthread -g
guest-proxy: guest-proxy.o
host-proxy: host-proxy.o
clean:
rm -f *.o guest-proxy host-proxy
Simplest design:
- tas running on host unmodified
- host proxy connecting to TAS as app and to qemu for ivshmem
- qemu ivshmem
- guest proxy mapping PCI device
Host Proxy:
- peer id 0
- N + 1 vectors for N TAS cores
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <sys/epoll.h>
#include <sys/eventfd.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <unistd.h>
#include <linux/vfio.h>
struct vfio_dev {
int containerfd;
int groupfd;
int devfd;
struct vfio_device_info info;
};
static int vfio_dev_open(struct vfio_dev *dev, const char *groupname,
const char *pci_dev)
{
int container, group, device, i;
struct vfio_group_status group_status =
{ .argsz = sizeof(group_status) };
/* Create a new container */
if ((container = open("/dev/vfio/vfio", O_RDWR)) < 0) {
perror("open vfio failed");
goto out;
}
if (ioctl(container, VFIO_GET_API_VERSION) != VFIO_API_VERSION) {
perror("unexpected vfio api version");
goto out_container;
}
/* Open the group */
group = open(groupname, O_RDWR);
if (group < 0) {
perror("open group failed");
goto out_container;
}
/* Test the group is viable and available */
if (ioctl(group, VFIO_GROUP_GET_STATUS, &group_status) < 0) {
perror("ioctl get status failed");
goto out_group;
}
if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
fprintf(stderr, "vfio group not viable\n");
goto out_group;
}
/* Add the group to the container */
if (ioctl(group, VFIO_GROUP_SET_CONTAINER, &container) < 0) {
fprintf(stderr, "set container failed\n");
goto out_group;
}
/* Enable the IOMMU model we want */
if (ioctl(container, VFIO_SET_IOMMU, VFIO_NOIOMMU_IOMMU) < 0) {
fprintf(stderr, "set iommu model failed\n");
goto out_group;
}
/* Get a file descriptor for the device */
if ((device = ioctl(group, VFIO_GROUP_GET_DEVICE_FD, pci_dev)) < 0) {
perror("getting device failed");
goto out_group;
}
/* Test and setup the device */
memset(&dev->info, 0, sizeof(dev->info));
dev->info.argsz = sizeof(dev->info);
if ((i = ioctl(device, VFIO_DEVICE_GET_INFO, &dev->info)) < 0) {
perror("getting device info failed");
goto out_dev;
}
dev->containerfd = container;
dev->groupfd = group;
dev->devfd = device;
return 0;
out_dev:
close(device);
out_group:
close(group);
out_container:
close(container);
out:
return -1;
}
static int vfio_dev_reset(struct vfio_dev *dev)
{
if (ioctl(dev->devfd, VFIO_DEVICE_RESET) < 0) {
perror("vfio_dev_reset: reset failed");
return -1;
}
return 0;
}
static int vfio_irq_info(struct vfio_dev *dev, uint32_t index,
struct vfio_irq_info *irq)
{
memset(irq, 0, sizeof(*irq));
irq->argsz = sizeof(*irq);
irq->index = index;
if (ioctl(dev->devfd, VFIO_DEVICE_GET_IRQ_INFO, irq) < 0) {
perror("vfio_irq_info: get info failed");
return -1;
}
return 0;
}
static int vfio_irq_eventfd(struct vfio_dev *dev, uint32_t index,
uint32_t count, int *fds)
{
uint32_t i;
size_t sz;
int *pfd;
void *alloc;
struct vfio_irq_set *info;
sz = sizeof(struct vfio_irq_set) + count * sizeof(int);
if ((alloc = calloc(1, sz)) == NULL) {
perror("calloc failed");
return -1;
}
pfd = (int *) ((uint8_t *) alloc + sizeof(struct vfio_irq_set));
for (i = 0; i < count; i++) {
if ((fds[i] = eventfd(0, EFD_NONBLOCK)) < 0) {
perror("vfio_irq_eventfd: eventfd failed");
free(alloc);
return -1;
}
pfd[i] = fds[i];
}
info = alloc;
info->argsz = sz;
info->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;;
info->index = index;
info->start = 0;
info->count = count;
if (ioctl(dev->devfd, VFIO_DEVICE_SET_IRQS, info) < 0) {
perror("vfio_irq_eventfd: set failed");
return -1;
}
free(alloc);
return 0;
}
static int vfio_region_info(struct vfio_dev *dev, uint32_t index,
struct vfio_region_info *reg)
{
memset(reg, 0, sizeof(*reg));
reg->argsz = sizeof(*reg);
reg->index = index;
if (ioctl(dev->devfd, VFIO_DEVICE_GET_REGION_INFO, reg) < 0) {
perror("vfio_region_info: get info failed");
return -1;
}
return 0;
}
static int vfio_region_map(struct vfio_dev *dev, uint32_t index,
void **addr, size_t *len)
{
void *ret;
struct vfio_region_info reg;
if (vfio_region_info(dev, index, &reg) != 0) {
return -1;
}
if ((ret = mmap(NULL, reg.size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, dev->devfd, reg.offset))
== MAP_FAILED)
{
perror("vfio_region_map: mmap failed");
return -1;
}
*addr = ret;
*len = reg.size;
return 0;
}
int main(int argc, char *argv[])
{
int i, n, efd;
int *intfds;
ssize_t sz;
struct vfio_dev dev;
struct vfio_irq_info irq;
struct vfio_region_info reg;
void *mem_regs, *mem_data;
size_t len_regs, len_data;
if (vfio_dev_open(&dev, "/dev/vfio/noiommu-0", "0000:00:04.0") != 0) {
return EXIT_FAILURE;
}
printf("num flags: %x\n", dev.info.flags);
printf("num regions: %u\n", dev.info.num_regions);
printf("num irqs: %u\n", dev.info.num_irqs);
for (i = 0; i < dev.info.num_regions; i++) {
if (vfio_region_info(&dev, i, &reg) != 0) {
fprintf(stderr, " region %d: getting region info failed\n", i);
continue;
}
printf(" region %d: flags=%x index=%x size=%llx offset=%llx\n", i, reg.flags,
reg.index, reg.size, reg.offset);
}
for (i = 0; i < dev.info.num_irqs; i++) {
if (vfio_irq_info(&dev, i, &irq) != 0) {
fprintf(stderr, " irq %d: getting irq info failed\n", i);
continue;
}
printf(" irq %d: flags=%x index=%u count=%u\n", i, irq.flags, irq.index, irq.count);
}
vfio_dev_reset(&dev);
vfio_region_map(&dev, 0, &mem_regs, &len_regs);
vfio_region_map(&dev, 2, &mem_data, &len_data);
if ((efd = epoll_create1(0)) < 0) {
perror("epoll_create1 failed");
return EXIT_FAILURE;
}
vfio_irq_info(&dev, 2, &irq);
intfds = calloc(irq.count, sizeof(*intfds));
if (vfio_irq_eventfd(&dev, 2, irq.count, intfds) != 0) {
fprintf(stderr, "setting up irqs failed\n");
exit(EXIT_FAILURE);
}
for (i = 0; i < irq.count; i++) {
struct epoll_event evt;
evt.events = EPOLLIN;
evt.data.fd = i;
if (epoll_ctl(efd, EPOLL_CTL_ADD, intfds[i], &evt) != 0) {
perror("epoll_ctl failed");
}
}
volatile uint32_t *shmp = mem_data;
shmp[0] = 4;
shmp[0] = 8;
volatile uint32_t *dbp = (volatile uint32_t *) ((uint8_t *) mem_regs + 12);
*dbp = 0;
*dbp = 1;
struct epoll_event evts[8];
while (1) {
if ((n = epoll_wait(efd, evts, 8, -1)) < 0) {
perror("epoll_wait failed");
exit(0);
}
for (i = 0; i < n; i++) {
printf("interrupt: %d\n", evts[i].data.fd);
uint64_t x;
if ((sz = read(intfds[evts[i].data.fd], &x, sizeof(x))) < 0) {
perror("read failed");
}
}
}
return 0;
}
/*
* Copyright 2019 University of Washington, Max Planck Institute for
* Software Systems, and The University of Texas at Austin
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <stdlib.h>
#include <stdio.h>
#include <stdbool.h>
#include <sys/un.h>
#include <sys/socket.h>
#include <sys/epoll.h>
#include <sys/eventfd.h>
#include <sys/mman.h>
#include <unistd.h>
#define PEERID_LOCAL 0
#define PEERID_VM 1
#define NOTIFY_FDS 5
#define INT_FDS 32
static int epfd;
void *shmptr;
static int uxfd;
static int shmfd;
static int nfd[NOTIFY_FDS];
static int intfd[INT_FDS];
static int uxsocket_init(const char *path)
{
int fd;
struct sockaddr_un saun;
struct epoll_event ev;
if ((fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
perror("uxsocket_init: socket failed");
goto error_exit;
}
memset(&saun, 0, sizeof(saun));
saun.sun_family = AF_UNIX;
memcpy(saun.sun_path, path, strlen(path));
if (bind(fd, (struct sockaddr *) &saun, sizeof(saun))) {
perror("uxsocket_init: bind failed");
goto error_close;
}
if (listen(fd, 5)) {
perror("uxsocket_init: listen failed");
goto error_close;
}
ev.events = EPOLLIN;
ev.data.fd = -1;
if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev) != 0) {
perror("uxsocket_init: epoll_ctl listen failed");
goto error_close;
}
uxfd = fd;
return 0;
error_close:
close(fd);
error_exit:
return -1;
}
static int uxsocket_send(int connfd, int64_t payload, int fd)
{
ssize_t tx;
struct iovec iov = {
.iov_base = &payload,
.iov_len = sizeof(payload),
};
union {
char buf[CMSG_SPACE(sizeof(int))];
struct cmsghdr align;
} u;
struct msghdr msg = {
.msg_name = NULL,
.msg_namelen = 0,
.msg_iov = &iov,
.msg_iovlen = 1,
.msg_control = u.buf,
.msg_controllen = 0,
.msg_flags = 0,
};
struct cmsghdr *cmsg = &u.align;
if (fd >= 0) {
msg.msg_controllen = sizeof(u.buf);
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_RIGHTS;
cmsg->cmsg_len = CMSG_LEN(sizeof(int));
*(int *) CMSG_DATA(cmsg) = fd;
}
if((tx = sendmsg(connfd, &msg, 0)) != sizeof(int64_t)) {
fprintf(stderr, "tx == %zd\n", tx);
return -1;
}
return 0;
}
static int uxsocket_accept(int lfd)
{
int cfd, i;
/* new connection on unix socket */
if ((cfd = accept(lfd, NULL, NULL)) < 0) {
fprintf(stderr, "uxsocket_accept: accept failed\n");
return -1;
}
printf("connection accepted: %d\n", cfd);
/* send protocol version, peer id, and shared memory fd */
uxsocket_send(cfd, 0, -1);
uxsocket_send(cfd, PEERID_VM, -1);
uxsocket_send(cfd, -1, shmfd);
printf("initial message sent: %d\n", cfd);
/* send our notify fds to peer */
for (i = 0; i < NOTIFY_FDS; i++) {
fprintf(stderr, "send notify fd: %d\n", i);
uxsocket_send(cfd, PEERID_LOCAL, nfd[i]);
}
printf("notify fds sent: %d\n", cfd);
/* send interrupt notify fds to peer */
for (i = 0; i < INT_FDS; i++) {
fprintf(stderr, "send int fd: %d\n", i);
uxsocket_send(cfd, PEERID_VM, intfd[i]);
}
printf("interrupt fds sent: %d\n", cfd);
return 0;
}
static int shm_create(const char *name, size_t size, void **addr)
{
int fd;
void *p;
char path[128];
snprintf(path, sizeof(path), "/dev/hugepages/%s", name);
if ((fd = open(path, O_CREAT | O_RDWR, 0666)) == -1) {
perror("util_create_shmsiszed: open failed");
goto error_out;
}
if (ftruncate(fd, size) != 0) {
perror("util_create_shmsiszed: ftruncate failed");
goto error_remove;
}
if ((p = mmap(NULL, size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, fd, 0)) == (void *) -1)
{
perror("util_create_shmsiszed: mmap failed");
goto error_remove;
}
memset(p, 0, size);
*addr = p;
return fd;
error_remove:
close(fd);
unlink(path);
error_out:
return -1;
}
int main(int argc, char *argv[])
{
int n, i;
ssize_t ret;
uint64_t x;
struct epoll_event evs[32];
volatile uint32_t *up;
if ((shmfd = shm_create("tas-vm", 32 * 1024 * 1024, &shmptr)) < 0) {
return EXIT_FAILURE;
}
up = (volatile uint32_t *) shmptr;
if ((epfd = epoll_create1(0)) == -1) {
perror("epoll_create1 failed");
return EXIT_FAILURE;
}
if (uxsocket_init("/tmp/tas-vm") != 0) {
return EXIT_FAILURE;
}
/* outgoing fds */
for (i = 0; i < NOTIFY_FDS; i++) {
if ((nfd[i] = eventfd(0, EFD_NONBLOCK)) < 0) {
perror("eventfd failed");
return EXIT_FAILURE;
}
struct epoll_event ev;
ev.events = EPOLLIN;
ev.data.fd = i;
if (epoll_ctl(epfd, EPOLL_CTL_ADD, nfd[i], &ev) != 0) {
perror("uxsocket_init: epoll_ctl listen failed");
return EXIT_FAILURE;
}
}
/* incoming fds */
for (i = 0; i < INT_FDS; i++) {
if ((intfd[i] = eventfd(0, EFD_NONBLOCK)) < 0) {
perror("eventfd failed");
return EXIT_FAILURE;
}
}
printf("starting event loop\n");
while (true) {
n = epoll_wait(epfd, evs, 32, -1);
for (i = 0; i < n; i++) {
if (evs[i].data.fd == -1) {
uxsocket_accept(uxfd);
} else {
printf("notification: %d\n", evs[i].data.fd);
ret = read(nfd[evs[i].data.fd], &x, sizeof(x));
uint32_t y = up[evs[i].data.fd];
up[evs[i].data.fd] = 0;
printf(" ret=%zd x=%lu mem[%d]=%u\n", ret, x, evs[i].data.fd, y);
if (y < INT_FDS) {
x = 1;
write(intfd[y], &x, sizeof(x));
}
}
}
}
close(uxfd);
close(epfd);
return 0;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment