read PCI memory and config spyce through /dev/mem

[Warlich, Christof]

Hi,

I'd like to look at some of my PCI devices through /dev/mem.
My kernel config has "Filter access to /dev/mem" (STRICT_DEVMEM)
switched off, so as root, I should have unrestricted access.

I get my PCI device's BAR address through /proc/iomem, e.g.:

$ cat /proc/iomem
...
f7600000-f7603fff : r8169
...

Ok, so there should be 16K of PCI memory from my ethernet card
starting at f7600000, which is 4150263808 in decimal to be used
With dd. But:

$ dd if=/dev/mem bs=1 count=1024 skip=4150263808
dd: reading `/dev/mem': Bad address
0+0 records in
0+0 records out
0 bytes (0 B) copied, 6.6658e-05 s, 0.0 kB/s

The kernel config help says that _PCI_ memory access is even
possible with STRICT_DEVMEM enabled. Can anyone give me a hint
what I may do wrong?

Thanks a lot,

Chris

[V***@vt.edu]

Post by Warlich, Christof
$ dd if=/dev/mem bs=1 count=1024 skip=4150263808
dd: reading `/dev/mem': Bad address
0+0 records in
0+0 records out
0 bytes (0 B) copied, 6.6658e-05 s, 0.0 kB/s
The kernel config help says that _PCI_ memory access is even
possible with STRICT_DEVMEM enabled. Can anyone give me a hint
what I may do wrong?

You *do* realize that doing a byte access on a 64-bit PCI register
will probably fail, right? And changing it to bs=8 won't fix it either,
because some registers are 32-bit, and some are write-only.

Bottom line - what you're doing wrong is not accessing the PCI address
space in the exact manner that the PCI spec requires. You're lucky it
just threw "bad address" - it *is* possible to end up wiping your entire
system this way if you screw up the PCI config for your disk controller or
similar. Be glad you didn't have to recover from your backups this time.

You *do* have backups, right?

[Warlich, Christof]

Post by V***@vt.edu

Post by Warlich, Christof
$ dd if=/dev/mem bs=1 count=1024 skip=4150263808
dd: reading `/dev/mem': Bad address
0+0 records in
0+0 records out
0 bytes (0 B) copied, 6.6658e-05 s, 0.0 kB/s
The kernel config help says that _PCI_ memory access is even
possible with STRICT_DEVMEM enabled. Can anyone give me a hint
what I may do wrong?

You *do* realize that doing a byte access on a 64-bit PCI register
will probably fail, right? And changing it to bs=8 won't fix it either,
because some registers are 32-bit, and some are write-only.

The real target that I need to explore is a custom FPGA device on an
embedded (i386) system. One of its BARs (at 0x81423000) point to memory
that _does_ allow unrestricted byte access and has a size of two pages.
But anyhow, all of the following commands show the same error:

$ dd if=/dev/mem bs=1 count=1 skip=2168598528 | hexdump
$ dd if=/dev/mem bs=2 count=1 skip=1084299264 | hexdump
$ dd if=/dev/mem bs=4 count=1 skip=542149632 | hexdump
$ dd if=/dev/mem bs=8 count=1 skip=271074816 | hexdump
$ dd if=/dev/mem bs=4096 count=1 skip=529443 | hexdump # 4096 being the system's page size

Furthermore, read / write access through a simple program using /dev/mem
_does_ work as expected:

#include <unistd.h>
#include <stdio.h>
#include <fcntl.h>
#include <assert.h>
#include <sys/mman.h>
int main( int argc, char *argv[]) {
char *mem;
int fd;
fd = open ("/dev/mem", O_RDWR);
assert(fd >= 0);
mem = mmap(NULL, getpagesize(), PROT_READ|PROT_WRITE, MAP_SHARED, fd, (off_t) 0x81423000);
assert(mem != MAP_FAILED);
printf("Memory pointer: %p\n", mem);
printf("The PCI memory is : %#x\n", *mem);
*mem = *argv[1];
printf("The PCI memory is : %#x\n", *mem);
munmap(mem, getpagesize());
close(fd);
return 0;
}

Calling it, yields:

$ led 4
Memory pointer: 0xb7678000
The PCI memory is : 0x33
The PCI memory is : 0x34

And the LED matrix display being behind that address switches from 3 to
4 as expected.

Thus, I still cannot see why the access through /dev/mem using dd doesn't
work.

Post by V***@vt.edu
Bottom line - what you're doing wrong is not accessing the PCI address
space in the exact manner that the PCI spec requires. You're lucky it
just threw "bad address" - it *is* possible to end up wiping your entire
system this way if you screw up the PCI config for your disk controller or
similar. Be glad you didn't have to recover from your backups this time.
You *do* have backups, right?

Don't worry about that, I've restored the system in less than a minute.

[Jonathan Neuschäfer]

On Tue, May 07, 2013 at 06:35:43AM +0000, Warlich, Christof wrote:
[...]

Post by Warlich, Christof
Furthermore, read / write access through a simple program using /dev/mem
#include <unistd.h>
#include <stdio.h>
#include <fcntl.h>
#include <assert.h>
#include <sys/mman.h>
int main( int argc, char *argv[]) {
char *mem;
int fd;
fd = open ("/dev/mem", O_RDWR);
assert(fd >= 0);
mem = mmap(NULL, getpagesize(), PROT_READ|PROT_WRITE, MAP_SHARED, fd, (off_t) 0x81423000);
assert(mem != MAP_FAILED);
printf("Memory pointer: %p\n", mem);
printf("The PCI memory is : %#x\n", *mem);
*mem = *argv[1];
printf("The PCI memory is : %#x\n", *mem);
munmap(mem, getpagesize());
close(fd);
return 0;
}
$ led 4
Memory pointer: 0xb7678000
The PCI memory is : 0x33
The PCI memory is : 0x34
And the LED matrix display being behind that address switches from 3 to
4 as expected.
Thus, I still cannot see why the access through /dev/mem using dd doesn't
work.

I suspect dd uses read instead of mmap.


Thanks,
Jonathan Neuschäfer

[Warlich, Christof]

...

It's the lseek that fails: Obviously, /dev/mem is not seekable.
What really puzzles me is that dd _does_ work on /dev/mem when
looking at ordinary RAM, but I'm reluctant to further debug why.

But anyhow, as I think it may at times be also useful for others
to have a simple tool that allows to read (and write!) PCI memory,
I share the few lines of code that I wrote to fulfill my needs. I
hope that the included "documentation" is verbose enough. And I'm
certainly interested if anyone finds bugs :-):

#include <unistd.h>
#include <stdio.h>
#include <fcntl.h>
#include <stdlib.h>
#include <stdint.h>
#include <assert.h>
#include <sys/mman.h>
void usage(const char *name) {
fprintf(stderr, "Usage: %s address access [size]\033[2m\n", name);
fprintf(stderr, " Either - prints \"size\" bytes \033[1mfrom\033[2m physical memory,\n");
fprintf(stderr, " starting at \"address\" and using \"access\"\n");
fprintf(stderr, " bytes per access \033[1mto\033[2m stdout\n");
fprintf(stderr, " or - writes as many bytes as are available \033\[1mfrom\033[2m\n");
fprintf(stderr, " stdin \033[1mto\033[2m physical memory starting at \"address\"\n");
fprintf(stderr, " using \"access\" bytes per access.\n");
fprintf(stderr, " Note that both \"address\" and either \"size\" or the\n");
fprintf(stderr, " number of available bytes from stdin must be a multiple of\n");
fprintf(stderr, " \"access\", and \"access\" must either be %d, %d, %d or %d.\033[0m\n",
sizeof(uint8_t), sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t));
exit(-1);
}
template<typename T> void readOrWrite(uint8_t *src, uint8_t *mem) {
if(src) *(T *) mem = *(T *) src;
else {
T t = *(T *) mem;
assert(write(1, &t, sizeof(T)) == sizeof(T));
}
}
int main( int argc, char *argv[]) {
int fd, pageSize = getpagesize(), size = 0, n;
uint8_t *mem, *b = 0, *buffer = 0;
// Paranoia check to ensure that page size is a power of 2.
assert((pageSize != 0) && !(pageSize & (pageSize - 1)));
if (argc == 3) do {
#define CHUNK_SIZE 1000
assert(b = buffer = (uint8_t *) realloc(buffer, size + CHUNK_SIZE));
assert((n = read(0, buffer + size, CHUNK_SIZE)) >= 0);
size += n;
} while(n);
else if(argc == 4) size = strtoul(argv[3], 0, 0);
else usage(argv[0]);
off_t address = strtoul(argv[1], 0, 0);
size_t access = strtoul(argv[2], 0, 0);
if(address % access || size % access) usage(argv[0]);
assert((fd = open ("/dev/mem", O_RDWR)) >= 0);
size_t s = (((address & (pageSize - 1)) + size - 1) / pageSize + 1) * pageSize;
mem = (uint8_t *) mmap(NULL, s, PROT_READ | PROT_WRITE, MAP_SHARED, fd, address & ~(pageSize - 1));
assert(mem != MAP_FAILED);
for(uint8_t *i = mem; i < mem + size; i += access) {
#define ROW(t) sizeof(t): readOrWrite<t>(b, i); break
switch(access) {
case ROW(uint8_t);
case ROW(uint16_t);
case ROW(uint32_t);
case ROW(uint64_t);
default: usage(argv[0]);
}
if(b) b += access;
}
free(buffer);
munmap(mem, getpagesize());
close(fd);
return 0;
}