x86: --- arch/x86_64/mm/ioremap.c | 21 ++++++++-- arch/x86_64/mm/pat.c | 94 +++++++++++++++++++++++++++++++++++++++++++++++ include/asm-x86_64/pat.h | 12 ++++++ 3 files changed, 122 insertions(+), 5 deletions(-) Index: linux/arch/x86_64/mm/pat.c =================================================================== --- linux.orig/arch/x86_64/mm/pat.c +++ linux/arch/x86_64/mm/pat.c @@ -6,6 +6,8 @@ #include #include #include +#include +#include static u64 boot_pat_state; @@ -52,3 +54,95 @@ void pat_shutdown(void) wrmsrl(MSR_IA32_CR_PAT, boot_pat_state); } +/* The global memattr list keeps track of caching attributes for specific + physical memory areas. Conflicting caching attributes in different + mappings can cause CPU cache corruption. To avoid this we keep track. + + The list is sorted and can contain multiple entries for each address + (this allows reference counting for overlapping areas). All the aliases + have the same cache attributes of course. Zero attributes are represente + as holes. + + Currently the data structure is a list because the number of mappings + are right now expected to be relatively small. If this should be a problem + it could be changed to a rbtree or similar. + + mattr_lock protects the whole list. */ + +struct memattr { + struct list_head nd; + u64 start; + u64 end; + unsigned long attr; +}; + +static LIST_HEAD(mattr_list); +static DEFINE_SPINLOCK(mattr_lock); /* protects memattr list */ + +int reserve_mattr(u64 start, u64 end, unsigned long attr, unsigned long *fattr) +{ + struct memattr *ma = NULL, *ml; + int err = 0; + if (attr) { + ma = kmalloc(sizeof(struct memattr), GFP_KERNEL); + if (!ma) + return -ENOMEM; + ma->start = start; + ma->end = end; + ma->attr = attr; + } + if (fattr) + *fattr = attr; + spin_lock(&mattr_lock); + list_for_each_entry(ml, &mattr_list, nd) { + if (ml->start <= start && ml->end >= end) { + if (fattr) { + attr = ml->attr; + *fattr = attr; + } + if (attr != ml->attr) { + printk( + KERN_ERR "%s:%d conflicting cache attribute %Lx-%Lx %x<->%x\n", + current->comm, current->pid, + start, end, attr, ml->attr); + err = -EBUSY; + break; + } + } else if (ml->start >= end) { + if (ma) { + list_add(&ma->nd, ml->nd.prev); + ma = NULL; + } + break; + } + } + if (ma) + list_add_tail(&ma->nd, &mattr_list); + spin_unlock(&mattr_lock); + return 0; +} + +int free_mattr(u64 start, u64 end, unsigned long attr) +{ + struct memattr *ml; + int err = attr ? -EBUSY : 0; + spin_lock(&mattr_lock); + list_for_each_entry(ml, &mattr_list, nd) { + if (ml->start == start && ml->end == end) { + if (ml->attr != attr) + printk(KERN_ERR + "%s:%d conflicting cache attributes on free %Lx-%Lx %x<->%x\n", + current->comm, current->pid, start, end, attr,ml->attr); + list_del(&ml->nd); + err = 0; + break; + } + } + spin_unlock(&mattr_lock); + if (err) + printk(KERN_ERR "%s:%d freeing invalid mattr %Lx-%Lx %x\n", + current->comm, current->pid, + start, end, attr); + return err; +} + Index: linux/include/asm-x86_64/pat.h =================================================================== --- /dev/null +++ linux/include/asm-x86_64/pat.h @@ -0,0 +1,12 @@ +#ifndef _ASM_PAT_H +#define _ASM_PAT_H 1 + +#include + +/* Handle the page attribute table (PAT) of the CPU */ + +int reserve_mattr(u64 start, u64 end, unsigned long attr, unsigned long *fattr); +int free_mattr(u64 start, u64 end, unsigned long attr); + +#endif + Index: linux/arch/x86_64/mm/ioremap.c =================================================================== --- linux.orig/arch/x86_64/mm/ioremap.c +++ linux/arch/x86_64/mm/ioremap.c @@ -18,6 +18,7 @@ #include #include #include +#include #define ISA_START_ADDRESS 0xa0000 #define ISA_END_ADDRESS 0x100000 @@ -213,12 +214,20 @@ void __iomem * __ioremap(unsigned long p remove_vm_area((void *)(PAGE_MASK & (unsigned long) addr)); return NULL; } - if (flags && ioremap_change_attr(phys_addr, size, flags) < 0) { - area->flags &= 0xffffff; - vunmap(addr); - return NULL; + if (flags) { + if (reserve_mattr(phys_addr, phys_addr + size, flags, NULL) < 0) + goto out; + if (ioremap_change_attr(phys_addr, size, flags) < 0) { + free_mattr(phys_addr, phys_addr + size, flags); + goto out; + } } return (__force void __iomem *) (offset + (char *)addr); + +out: + area->flags &= 0xffffff; + vunmap(addr); + return NULL; } EXPORT_SYMBOL(__ioremap); @@ -286,8 +295,10 @@ void iounmap(volatile void __iomem *addr } /* Reset the direct mapping. Can block */ - if (p->flags >> 20) + if (p->flags >> 20) { + free_mattr(p->phys_addr, p->phys_addr + p->size, p->flags>>20); ioremap_change_attr(p->phys_addr, p->size, 0); + } /* Finally remove it */ o = remove_vm_area((void *)addr); Index: linux/include/asm-i386/pat.h =================================================================== --- /dev/null +++ linux/include/asm-i386/pat.h @@ -0,0 +1 @@ +#include