95e861db3e
The file_lock spinlock sits close to mostly read fields of 'struct files_struct' In SMP (and NUMA) environments, each time a thread wants to open or close a file, it has to acquire the spinlock, thus invalidating the cache line containing this spinlock on other CPUS. So other threads doing read()/write()/... calls that use RCU to access the file table are going to ask further memory (possibly NUMA) transactions to read again this memory line. Move the spinlock to another cache line, so that concurrent threads can share the cache line containing 'count' and 'fdt' fields. It's worth up to 9% on a microbenchmark using a 4-thread 2-package x86 machine. See http://marc.theaimsgroup.com/?l=linux-kernel&m=112680448713342&w=2 Signed-off-by: Eric Dumazet <dada1@cosmosbay.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
99 lines
2.6 KiB
C
99 lines
2.6 KiB
C
/*
|
|
* Wrapper functions for accessing the file_struct fd array.
|
|
*/
|
|
|
|
#ifndef __LINUX_FILE_H
|
|
#define __LINUX_FILE_H
|
|
|
|
#include <asm/atomic.h>
|
|
#include <linux/posix_types.h>
|
|
#include <linux/compiler.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/rcupdate.h>
|
|
|
|
/*
|
|
* The default fd array needs to be at least BITS_PER_LONG,
|
|
* as this is the granularity returned by copy_fdset().
|
|
*/
|
|
#define NR_OPEN_DEFAULT BITS_PER_LONG
|
|
|
|
struct fdtable {
|
|
unsigned int max_fds;
|
|
int max_fdset;
|
|
int next_fd;
|
|
struct file ** fd; /* current fd array */
|
|
fd_set *close_on_exec;
|
|
fd_set *open_fds;
|
|
struct rcu_head rcu;
|
|
struct files_struct *free_files;
|
|
struct fdtable *next;
|
|
};
|
|
|
|
/*
|
|
* Open file table structure
|
|
*/
|
|
struct files_struct {
|
|
atomic_t count;
|
|
struct fdtable *fdt;
|
|
struct fdtable fdtab;
|
|
fd_set close_on_exec_init;
|
|
fd_set open_fds_init;
|
|
struct file * fd_array[NR_OPEN_DEFAULT];
|
|
spinlock_t file_lock; /* Protects concurrent writers. Nests inside tsk->alloc_lock */
|
|
};
|
|
|
|
#define files_fdtable(files) (rcu_dereference((files)->fdt))
|
|
|
|
extern void FASTCALL(__fput(struct file *));
|
|
extern void FASTCALL(fput(struct file *));
|
|
|
|
static inline void fput_light(struct file *file, int fput_needed)
|
|
{
|
|
if (unlikely(fput_needed))
|
|
fput(file);
|
|
}
|
|
|
|
extern struct file * FASTCALL(fget(unsigned int fd));
|
|
extern struct file * FASTCALL(fget_light(unsigned int fd, int *fput_needed));
|
|
extern void FASTCALL(set_close_on_exec(unsigned int fd, int flag));
|
|
extern void put_filp(struct file *);
|
|
extern int get_unused_fd(void);
|
|
extern void FASTCALL(put_unused_fd(unsigned int fd));
|
|
struct kmem_cache;
|
|
extern void filp_ctor(void * objp, struct kmem_cache *cachep, unsigned long cflags);
|
|
extern void filp_dtor(void * objp, struct kmem_cache *cachep, unsigned long dflags);
|
|
|
|
extern struct file ** alloc_fd_array(int);
|
|
extern void free_fd_array(struct file **, int);
|
|
|
|
extern fd_set *alloc_fdset(int);
|
|
extern void free_fdset(fd_set *, int);
|
|
|
|
extern int expand_files(struct files_struct *, int nr);
|
|
extern void free_fdtable(struct fdtable *fdt);
|
|
extern void __init files_defer_init(void);
|
|
|
|
static inline struct file * fcheck_files(struct files_struct *files, unsigned int fd)
|
|
{
|
|
struct file * file = NULL;
|
|
struct fdtable *fdt = files_fdtable(files);
|
|
|
|
if (fd < fdt->max_fds)
|
|
file = rcu_dereference(fdt->fd[fd]);
|
|
return file;
|
|
}
|
|
|
|
/*
|
|
* Check whether the specified fd has an open file.
|
|
*/
|
|
#define fcheck(fd) fcheck_files(current->files, fd)
|
|
|
|
extern void FASTCALL(fd_install(unsigned int fd, struct file * file));
|
|
|
|
struct task_struct;
|
|
|
|
struct files_struct *get_files_struct(struct task_struct *);
|
|
void FASTCALL(put_files_struct(struct files_struct *fs));
|
|
|
|
#endif /* __LINUX_FILE_H */
|