Error Trace

[Home]

Bug # 124

Show/hide error trace
Error trace
Function bodies
Blocks
  • Others...
    Function bodies without model function calls
    Initialization function calls
    Initialization function bodies
    Entry point
    Entry point body
    Function calls
    Skipped function calls
    Formal parameter names
    Declarations
    Assumes
    Assume conditions
    Returns
    Return values
    DEG initialization
    DEG function calls
    Model function calls
    Model function bodies
    Model asserts
    Model state changes
    Model function function calls
    Model function function bodies
    Model returns
    Model others
    Identation
    Line numbers
    Expand signs
-__CPAchecker_initialize()
{
20 typedef unsigned char __u8;
23 typedef unsigned short __u16;
25 typedef int __s32;
26 typedef unsigned int __u32;
30 typedef unsigned long long __u64;
15 typedef signed char s8;
16 typedef unsigned char u8;
19 typedef unsigned short u16;
21 typedef int s32;
22 typedef unsigned int u32;
24 typedef long long s64;
25 typedef unsigned long long u64;
14 typedef long __kernel_long_t;
15 typedef unsigned long __kernel_ulong_t;
27 typedef int __kernel_pid_t;
48 typedef unsigned int __kernel_uid32_t;
49 typedef unsigned int __kernel_gid32_t;
71 typedef __kernel_ulong_t __kernel_size_t;
72 typedef __kernel_long_t __kernel_ssize_t;
87 typedef long long __kernel_loff_t;
88 typedef __kernel_long_t __kernel_time_t;
89 typedef __kernel_long_t __kernel_clock_t;
90 typedef int __kernel_timer_t;
91 typedef int __kernel_clockid_t;
12 typedef __u32 __kernel_dev_t;
15 typedef __kernel_dev_t dev_t;
18 typedef unsigned short umode_t;
21 typedef __kernel_pid_t pid_t;
26 typedef __kernel_clockid_t clockid_t;
29 typedef _Bool bool;
31 typedef __kernel_uid32_t uid_t;
32 typedef __kernel_gid32_t gid_t;
45 typedef __kernel_loff_t loff_t;
54 typedef __kernel_size_t size_t;
59 typedef __kernel_ssize_t ssize_t;
69 typedef __kernel_time_t time_t;
102 typedef __s32 int32_t;
108 typedef __u32 uint32_t;
133 typedef unsigned long sector_t;
134 typedef unsigned long blkcnt_t;
152 typedef u64 dma_addr_t;
157 typedef unsigned int gfp_t;
158 typedef unsigned int fmode_t;
159 typedef unsigned int oom_flags_t;
162 typedef u64 phys_addr_t;
167 typedef phys_addr_t resource_size_t;
177 struct __anonstruct_atomic_t_6 { int counter; } ;
177 typedef struct __anonstruct_atomic_t_6 atomic_t;
182 struct __anonstruct_atomic64_t_7 { long counter; } ;
182 typedef struct __anonstruct_atomic64_t_7 atomic64_t;
183 struct list_head { struct list_head *next; struct list_head *prev; } ;
188 struct hlist_node ;
188 struct hlist_head { struct hlist_node *first; } ;
192 struct hlist_node { struct hlist_node *next; struct hlist_node **pprev; } ;
203 struct callback_head { struct callback_head *next; void (*func)(struct callback_head *); } ;
234 enum dma_data_direction { DMA_BIDIRECTIONAL = 0, DMA_TO_DEVICE = 1, DMA_FROM_DEVICE = 2, DMA_NONE = 3 } ;
5 struct device ;
5 struct page ;
7 struct dma_attrs ;
13 typedef unsigned long kernel_ulong_t;
14 struct pci_device_id { __u32 vendor; __u32 device; __u32 subvendor; __u32 subdevice; __u32 class; __u32 class_mask; kernel_ulong_t driver_data; } ;
186 struct acpi_device_id { __u8 id[9U]; kernel_ulong_t driver_data; __u32 cls; __u32 cls_msk; } ;
229 struct of_device_id { char name[32U]; char type[32U]; char compatible[128U]; const void *data; } ;
131 typedef void (*ctor_fn_t)();
148 struct resource { resource_size_t start; resource_size_t end; const char *name; unsigned long flags; struct resource *parent; struct resource *sibling; struct resource *child; } ;
250 struct kernel_symbol { unsigned long value; const char *name; } ;
33 struct module ;
72 struct pt_regs { unsigned long r15; unsigned long r14; unsigned long r13; unsigned long r12; unsigned long bp; unsigned long bx; unsigned long r11; unsigned long r10; unsigned long r9; unsigned long r8; unsigned long ax; unsigned long cx; unsigned long dx; unsigned long si; unsigned long di; unsigned long orig_ax; unsigned long ip; unsigned long cs; unsigned long flags; unsigned long sp; unsigned long ss; } ;
66 struct __anonstruct____missing_field_name_12 { unsigned int a; unsigned int b; } ;
66 struct __anonstruct____missing_field_name_13 { u16 limit0; u16 base0; unsigned char base1; unsigned char type; unsigned char s; unsigned char dpl; unsigned char p; unsigned char limit; unsigned char avl; unsigned char l; unsigned char d; unsigned char g; unsigned char base2; } ;
66 union __anonunion____missing_field_name_11 { struct __anonstruct____missing_field_name_12 __annonCompField4; struct __anonstruct____missing_field_name_13 __annonCompField5; } ;
66 struct desc_struct { union __anonunion____missing_field_name_11 __annonCompField6; } ;
12 typedef unsigned long pteval_t;
13 typedef unsigned long pmdval_t;
15 typedef unsigned long pgdval_t;
16 typedef unsigned long pgprotval_t;
18 struct __anonstruct_pte_t_14 { pteval_t pte; } ;
18 typedef struct __anonstruct_pte_t_14 pte_t;
20 struct pgprot { pgprotval_t pgprot; } ;
221 typedef struct pgprot pgprot_t;
223 struct __anonstruct_pgd_t_15 { pgdval_t pgd; } ;
223 typedef struct __anonstruct_pgd_t_15 pgd_t;
262 struct __anonstruct_pmd_t_17 { pmdval_t pmd; } ;
262 typedef struct __anonstruct_pmd_t_17 pmd_t;
390 typedef struct page *pgtable_t;
401 struct file ;
414 struct seq_file ;
452 struct thread_struct ;
454 struct mm_struct ;
455 struct task_struct ;
456 struct cpumask ;
20 struct qspinlock { atomic_t val; } ;
33 typedef struct qspinlock arch_spinlock_t;
34 struct qrwlock { atomic_t cnts; arch_spinlock_t wait_lock; } ;
14 typedef struct qrwlock arch_rwlock_t;
683 struct cpuinfo_x86 ;
234 struct _ddebug { const char *modname; const char *function; const char *filename; const char *format; unsigned int lineno; unsigned char flags; } ;
420 struct file_operations ;
432 struct completion ;
27 union __anonunion___u_19 { struct list_head *__val; char __c[1U]; } ;
17 struct lockdep_map ;
26 union __anonunion___u_35 { int __val; char __c[1U]; } ;
38 union __anonunion___u_37 { int __val; char __c[1U]; } ;
23 typedef atomic64_t atomic_long_t;
186 struct bug_entry { int bug_addr_disp; int file_disp; unsigned short line; unsigned short flags; } ;
55 struct stack_trace { unsigned int nr_entries; unsigned int max_entries; unsigned long *entries; int skip; } ;
28 struct lockdep_subclass_key { char __one_byte; } ;
53 struct lock_class_key { struct lockdep_subclass_key subkeys[8U]; } ;
59 struct lock_class { struct list_head hash_entry; struct list_head lock_entry; struct lockdep_subclass_key *key; unsigned int subclass; unsigned int dep_gen_id; unsigned long usage_mask; struct stack_trace usage_traces[13U]; struct list_head locks_after; struct list_head locks_before; unsigned int version; unsigned long ops; const char *name; int name_version; unsigned long contention_point[4U]; unsigned long contending_point[4U]; } ;
144 struct lockdep_map { struct lock_class_key *key; struct lock_class *class_cache[2U]; const char *name; int cpu; unsigned long ip; } ;
205 struct held_lock { u64 prev_chain_key; unsigned long acquire_ip; struct lockdep_map *instance; struct lockdep_map *nest_lock; u64 waittime_stamp; u64 holdtime_stamp; unsigned short class_idx; unsigned char irq_context; unsigned char trylock; unsigned char read; unsigned char check; unsigned char hardirqs_off; unsigned short references; unsigned int pin_count; } ;
546 struct raw_spinlock { arch_spinlock_t raw_lock; unsigned int magic; unsigned int owner_cpu; void *owner; struct lockdep_map dep_map; } ;
32 typedef struct raw_spinlock raw_spinlock_t;
33 struct __anonstruct____missing_field_name_47 { u8 __padding[24U]; struct lockdep_map dep_map; } ;
33 union __anonunion____missing_field_name_46 { struct raw_spinlock rlock; struct __anonstruct____missing_field_name_47 __annonCompField7; } ;
33 struct spinlock { union __anonunion____missing_field_name_46 __annonCompField8; } ;
76 typedef struct spinlock spinlock_t;
23 struct __anonstruct_rwlock_t_48 { arch_rwlock_t raw_lock; unsigned int magic; unsigned int owner_cpu; void *owner; struct lockdep_map dep_map; } ;
23 typedef struct __anonstruct_rwlock_t_48 rwlock_t;
24 struct math_emu_info { long ___orig_eip; struct pt_regs *regs; } ;
14 struct __anonstruct_pfn_t_52 { unsigned long val; } ;
14 typedef struct __anonstruct_pfn_t_52 pfn_t;
328 struct cpumask { unsigned long bits[128U]; } ;
15 typedef struct cpumask cpumask_t;
656 typedef struct cpumask *cpumask_var_t;
82 struct static_key { atomic_t enabled; } ;
264 struct tracepoint_func { void *func; void *data; int prio; } ;
18 struct tracepoint { const char *name; struct static_key key; void (*regfunc)(); void (*unregfunc)(); struct tracepoint_func *funcs; } ;
260 struct fregs_state { u32 cwd; u32 swd; u32 twd; u32 fip; u32 fcs; u32 foo; u32 fos; u32 st_space[20U]; u32 status; } ;
26 struct __anonstruct____missing_field_name_61 { u64 rip; u64 rdp; } ;
26 struct __anonstruct____missing_field_name_62 { u32 fip; u32 fcs; u32 foo; u32 fos; } ;
26 union __anonunion____missing_field_name_60 { struct __anonstruct____missing_field_name_61 __annonCompField14; struct __anonstruct____missing_field_name_62 __annonCompField15; } ;
26 union __anonunion____missing_field_name_63 { u32 padding1[12U]; u32 sw_reserved[12U]; } ;
26 struct fxregs_state { u16 cwd; u16 swd; u16 twd; u16 fop; union __anonunion____missing_field_name_60 __annonCompField16; u32 mxcsr; u32 mxcsr_mask; u32 st_space[32U]; u32 xmm_space[64U]; u32 padding[12U]; union __anonunion____missing_field_name_63 __annonCompField17; } ;
66 struct swregs_state { u32 cwd; u32 swd; u32 twd; u32 fip; u32 fcs; u32 foo; u32 fos; u32 st_space[20U]; u8 ftop; u8 changed; u8 lookahead; u8 no_update; u8 rm; u8 alimit; struct math_emu_info *info; u32 entry_eip; } ;
214 struct xstate_header { u64 xfeatures; u64 xcomp_bv; u64 reserved[6U]; } ;
220 struct xregs_state { struct fxregs_state i387; struct xstate_header header; u8 extended_state_area[0U]; } ;
235 union fpregs_state { struct fregs_state fsave; struct fxregs_state fxsave; struct swregs_state soft; struct xregs_state xsave; u8 __padding[4096U]; } ;
252 struct fpu { unsigned int last_cpu; unsigned char fpstate_active; unsigned char fpregs_active; unsigned char counter; union fpregs_state state; } ;
81 struct cpuinfo_x86 { __u8 x86; __u8 x86_vendor; __u8 x86_model; __u8 x86_mask; int x86_tlbsize; __u8 x86_virt_bits; __u8 x86_phys_bits; __u8 x86_coreid_bits; __u32 extended_cpuid_level; int cpuid_level; __u32 x86_capability[17U]; char x86_vendor_id[16U]; char x86_model_id[64U]; int x86_cache_size; int x86_cache_alignment; int x86_cache_max_rmid; int x86_cache_occ_scale; int x86_power; unsigned long loops_per_jiffy; u16 x86_max_cores; u16 apicid; u16 initial_apicid; u16 x86_clflush_size; u16 booted_cores; u16 phys_proc_id; u16 cpu_core_id; u8 compute_unit_id; u16 cpu_index; u32 microcode; } ;
170 struct seq_operations ;
369 struct perf_event ;
370 struct thread_struct { struct desc_struct tls_array[3U]; unsigned long sp0; unsigned long sp; unsigned short es; unsigned short ds; unsigned short fsindex; unsigned short gsindex; unsigned long fs; unsigned long gs; struct perf_event *ptrace_bps[4U]; unsigned long debugreg6; unsigned long ptrace_dr7; unsigned long cr2; unsigned long trap_nr; unsigned long error_code; unsigned long *io_bitmap_ptr; unsigned long iopl; unsigned int io_bitmap_max; struct fpu fpu; } ;
13 struct optimistic_spin_queue { atomic_t tail; } ;
39 struct mutex { atomic_t count; spinlock_t wait_lock; struct list_head wait_list; struct task_struct *owner; void *magic; struct lockdep_map dep_map; } ;
67 struct mutex_waiter { struct list_head list; struct task_struct *task; void *magic; } ;
177 struct timespec ;
178 struct compat_timespec ;
179 struct __anonstruct_futex_77 { u32 *uaddr; u32 val; u32 flags; u32 bitset; u64 time; u32 *uaddr2; } ;
179 struct __anonstruct_nanosleep_78 { clockid_t clockid; struct timespec *rmtp; struct compat_timespec *compat_rmtp; u64 expires; } ;
179 struct pollfd ;
179 struct __anonstruct_poll_79 { struct pollfd *ufds; int nfds; int has_timeout; unsigned long tv_sec; unsigned long tv_nsec; } ;
179 union __anonunion____missing_field_name_76 { struct __anonstruct_futex_77 futex; struct __anonstruct_nanosleep_78 nanosleep; struct __anonstruct_poll_79 poll; } ;
179 struct restart_block { long int (*fn)(struct restart_block *); union __anonunion____missing_field_name_76 __annonCompField20; } ;
416 struct seqcount { unsigned int sequence; struct lockdep_map dep_map; } ;
52 typedef struct seqcount seqcount_t;
404 struct __anonstruct_seqlock_t_92 { struct seqcount seqcount; spinlock_t lock; } ;
404 typedef struct __anonstruct_seqlock_t_92 seqlock_t;
12 struct __wait_queue ;
12 typedef struct __wait_queue wait_queue_t;
15 struct __wait_queue { unsigned int flags; void *private; int (*func)(wait_queue_t *, unsigned int, int, void *); struct list_head task_list; } ;
38 struct __wait_queue_head { spinlock_t lock; struct list_head task_list; } ;
43 typedef struct __wait_queue_head wait_queue_head_t;
1221 struct completion { unsigned int done; wait_queue_head_t wait; } ;
105 struct timespec { __kernel_time_t tv_sec; long tv_nsec; } ;
446 union ktime { s64 tv64; } ;
41 typedef union ktime ktime_t;
1147 union __anonunion____missing_field_name_93 { unsigned long bitmap[4U]; struct callback_head callback_head; } ;
1147 struct idr_layer { int prefix; int layer; struct idr_layer *ary[256U]; int count; union __anonunion____missing_field_name_93 __annonCompField21; } ;
41 struct idr { struct idr_layer *hint; struct idr_layer *top; int layers; int cur; spinlock_t lock; int id_free_cnt; struct idr_layer *id_free; } ;
124 struct ida_bitmap { long nr_busy; unsigned long bitmap[15U]; } ;
167 struct ida { struct idr idr; struct ida_bitmap *free_bitmap; } ;
199 struct rb_node { unsigned long __rb_parent_color; struct rb_node *rb_right; struct rb_node *rb_left; } ;
41 struct rb_root { struct rb_node *rb_node; } ;
98 struct dentry ;
99 struct iattr ;
100 struct vm_area_struct ;
101 struct super_block ;
102 struct file_system_type ;
103 struct kernfs_open_node ;
104 struct kernfs_iattrs ;
127 struct kernfs_root ;
127 struct kernfs_elem_dir { unsigned long subdirs; struct rb_root children; struct kernfs_root *root; } ;
85 struct kernfs_node ;
85 struct kernfs_elem_symlink { struct kernfs_node *target_kn; } ;
89 struct kernfs_ops ;
89 struct kernfs_elem_attr { const struct kernfs_ops *ops; struct kernfs_open_node *open; loff_t size; struct kernfs_node *notify_next; } ;
96 union __anonunion____missing_field_name_100 { struct kernfs_elem_dir dir; struct kernfs_elem_symlink symlink; struct kernfs_elem_attr attr; } ;
96 struct kernfs_node { atomic_t count; atomic_t active; struct lockdep_map dep_map; struct kernfs_node *parent; const char *name; struct rb_node rb; const void *ns; unsigned int hash; union __anonunion____missing_field_name_100 __annonCompField22; void *priv; unsigned short flags; umode_t mode; unsigned int ino; struct kernfs_iattrs *iattr; } ;
138 struct kernfs_syscall_ops { int (*remount_fs)(struct kernfs_root *, int *, char *); int (*show_options)(struct seq_file *, struct kernfs_root *); int (*mkdir)(struct kernfs_node *, const char *, umode_t ); int (*rmdir)(struct kernfs_node *); int (*rename)(struct kernfs_node *, struct kernfs_node *, const char *); } ;
155 struct kernfs_root { struct kernfs_node *kn; unsigned int flags; struct ida ino_ida; struct kernfs_syscall_ops *syscall_ops; struct list_head supers; wait_queue_head_t deactivate_waitq; } ;
171 struct vm_operations_struct ;
171 struct kernfs_open_file { struct kernfs_node *kn; struct file *file; void *priv; struct mutex mutex; int event; struct list_head list; char *prealloc_buf; size_t atomic_write_len; bool mmapped; const struct vm_operations_struct *vm_ops; } ;
188 struct kernfs_ops { int (*seq_show)(struct seq_file *, void *); void * (*seq_start)(struct seq_file *, loff_t *); void * (*seq_next)(struct seq_file *, void *, loff_t *); void (*seq_stop)(struct seq_file *, void *); ssize_t (*read)(struct kernfs_open_file *, char *, size_t , loff_t ); size_t atomic_write_len; bool prealloc; ssize_t (*write)(struct kernfs_open_file *, char *, size_t , loff_t ); int (*mmap)(struct kernfs_open_file *, struct vm_area_struct *); struct lock_class_key lockdep_key; } ;
284 struct inode ;
493 struct sock ;
494 struct kobject ;
495 enum kobj_ns_type { KOBJ_NS_TYPE_NONE = 0, KOBJ_NS_TYPE_NET = 1, KOBJ_NS_TYPES = 2 } ;
501 struct kobj_ns_type_operations { enum kobj_ns_type type; bool (*current_may_mount)(); void * (*grab_current_ns)(); const void * (*netlink_ns)(struct sock *); const void * (*initial_ns)(); void (*drop_ns)(void *); } ;
83 struct user_namespace ;
22 struct __anonstruct_kuid_t_103 { uid_t val; } ;
22 typedef struct __anonstruct_kuid_t_103 kuid_t;
27 struct __anonstruct_kgid_t_104 { gid_t val; } ;
27 typedef struct __anonstruct_kgid_t_104 kgid_t;
139 struct kstat { u64 ino; dev_t dev; umode_t mode; unsigned int nlink; kuid_t uid; kgid_t gid; dev_t rdev; loff_t size; struct timespec atime; struct timespec mtime; struct timespec ctime; unsigned long blksize; unsigned long long blocks; } ;
36 struct bin_attribute ;
37 struct attribute { const char *name; umode_t mode; bool ignore_lockdep; struct lock_class_key *key; struct lock_class_key skey; } ;
37 struct attribute_group { const char *name; umode_t (*is_visible)(struct kobject *, struct attribute *, int); umode_t (*is_bin_visible)(struct kobject *, struct bin_attribute *, int); struct attribute **attrs; struct bin_attribute **bin_attrs; } ;
92 struct bin_attribute { struct attribute attr; size_t size; void *private; ssize_t (*read)(struct file *, struct kobject *, struct bin_attribute *, char *, loff_t , size_t ); ssize_t (*write)(struct file *, struct kobject *, struct bin_attribute *, char *, loff_t , size_t ); int (*mmap)(struct file *, struct kobject *, struct bin_attribute *, struct vm_area_struct *); } ;
165 struct sysfs_ops { ssize_t (*show)(struct kobject *, struct attribute *, char *); ssize_t (*store)(struct kobject *, struct attribute *, const char *, size_t ); } ;
530 struct kref { atomic_t refcount; } ;
139 struct timer_list { struct hlist_node entry; unsigned long expires; void (*function)(unsigned long); unsigned long data; u32 flags; int slack; int start_pid; void *start_site; char start_comm[16U]; struct lockdep_map lockdep_map; } ;
238 struct hrtimer ;
239 enum hrtimer_restart ;
838 struct nsproxy ;
259 struct workqueue_struct ;
260 struct work_struct ;
54 struct work_struct { atomic_long_t data; struct list_head entry; void (*func)(struct work_struct *); struct lockdep_map lockdep_map; } ;
107 struct delayed_work { struct work_struct work; struct timer_list timer; struct workqueue_struct *wq; int cpu; } ;
52 struct kset ;
52 struct kobj_type ;
52 struct kobject { const char *name; struct list_head entry; struct kobject *parent; struct kset *kset; struct kobj_type *ktype; struct kernfs_node *sd; struct kref kref; struct delayed_work release; unsigned char state_initialized; unsigned char state_in_sysfs; unsigned char state_add_uevent_sent; unsigned char state_remove_uevent_sent; unsigned char uevent_suppress; } ;
115 struct kobj_type { void (*release)(struct kobject *); const struct sysfs_ops *sysfs_ops; struct attribute **default_attrs; const struct kobj_ns_type_operations * (*child_ns_type)(struct kobject *); const void * (*namespace)(struct kobject *); } ;
123 struct kobj_uevent_env { char *argv[3U]; char *envp[32U]; int envp_idx; char buf[2048U]; int buflen; } ;
131 struct kset_uevent_ops { const int (*filter)(struct kset *, struct kobject *); const const char * (*name)(struct kset *, struct kobject *); const int (*uevent)(struct kset *, struct kobject *, struct kobj_uevent_env *); } ;
148 struct kset { struct list_head list; spinlock_t list_lock; struct kobject kobj; const struct kset_uevent_ops *uevent_ops; } ;
223 struct klist_node ;
37 struct klist_node { void *n_klist; struct list_head n_node; struct kref n_ref; } ;
95 struct __anonstruct_nodemask_t_107 { unsigned long bits[16U]; } ;
95 typedef struct __anonstruct_nodemask_t_107 nodemask_t;
357 enum node_states { N_POSSIBLE = 0, N_ONLINE = 1, N_NORMAL_MEMORY = 2, N_HIGH_MEMORY = 2, N_MEMORY = 3, N_CPU = 4, NR_NODE_STATES = 5 } ;
520 struct path ;
521 struct seq_file { char *buf; size_t size; size_t from; size_t count; size_t pad_until; loff_t index; loff_t read_pos; u64 version; struct mutex lock; const struct seq_operations *op; int poll_event; struct user_namespace *user_ns; void *private; } ;
35 struct seq_operations { void * (*start)(struct seq_file *, loff_t *); void (*stop)(struct seq_file *, void *); void * (*next)(struct seq_file *, void *, loff_t *); int (*show)(struct seq_file *, void *); } ;
227 struct pinctrl ;
228 struct pinctrl_state ;
194 struct dev_pin_info { struct pinctrl *p; struct pinctrl_state *default_state; struct pinctrl_state *init_state; struct pinctrl_state *sleep_state; struct pinctrl_state *idle_state; } ;
58 struct pm_message { int event; } ;
64 typedef struct pm_message pm_message_t;
65 struct dev_pm_ops { int (*prepare)(struct device *); void (*complete)(struct device *); int (*suspend)(struct device *); int (*resume)(struct device *); int (*freeze)(struct device *); int (*thaw)(struct device *); int (*poweroff)(struct device *); int (*restore)(struct device *); int (*suspend_late)(struct device *); int (*resume_early)(struct device *); int (*freeze_late)(struct device *); int (*thaw_early)(struct device *); int (*poweroff_late)(struct device *); int (*restore_early)(struct device *); int (*suspend_noirq)(struct device *); int (*resume_noirq)(struct device *); int (*freeze_noirq)(struct device *); int (*thaw_noirq)(struct device *); int (*poweroff_noirq)(struct device *); int (*restore_noirq)(struct device *); int (*runtime_suspend)(struct device *); int (*runtime_resume)(struct device *); int (*runtime_idle)(struct device *); } ;
320 enum rpm_status { RPM_ACTIVE = 0, RPM_RESUMING = 1, RPM_SUSPENDED = 2, RPM_SUSPENDING = 3 } ;
327 enum rpm_request { RPM_REQ_NONE = 0, RPM_REQ_IDLE = 1, RPM_REQ_SUSPEND = 2, RPM_REQ_AUTOSUSPEND = 3, RPM_REQ_RESUME = 4 } ;
335 struct wakeup_source ;
336 struct wake_irq ;
337 struct pm_domain_data ;
338 struct pm_subsys_data { spinlock_t lock; unsigned int refcount; struct list_head clock_list; struct pm_domain_data *domain_data; } ;
556 struct dev_pm_qos ;
556 struct dev_pm_info { pm_message_t power_state; unsigned char can_wakeup; unsigned char async_suspend; bool is_prepared; bool is_suspended; bool is_noirq_suspended; bool is_late_suspended; bool ignore_children; bool early_init; bool direct_complete; spinlock_t lock; struct list_head entry; struct completion completion; struct wakeup_source *wakeup; bool wakeup_path; bool syscore; bool no_pm_callbacks; struct timer_list suspend_timer; unsigned long timer_expires; struct work_struct work; wait_queue_head_t wait_queue; struct wake_irq *wakeirq; atomic_t usage_count; atomic_t child_count; unsigned char disable_depth; unsigned char idle_notification; unsigned char request_pending; unsigned char deferred_resume; unsigned char run_wake; unsigned char runtime_auto; unsigned char no_callbacks; unsigned char irq_safe; unsigned char use_autosuspend; unsigned char timer_autosuspends; unsigned char memalloc_noio; enum rpm_request request; enum rpm_status runtime_status; int runtime_error; int autosuspend_delay; unsigned long last_busy; unsigned long active_jiffies; unsigned long suspended_jiffies; unsigned long accounting_timestamp; struct pm_subsys_data *subsys_data; void (*set_latency_tolerance)(struct device *, s32 ); struct dev_pm_qos *qos; } ;
616 struct dev_pm_domain { struct dev_pm_ops ops; void (*detach)(struct device *, bool ); int (*activate)(struct device *); void (*sync)(struct device *); void (*dismiss)(struct device *); } ;
726 struct rw_semaphore ;
727 struct rw_semaphore { long count; struct list_head wait_list; raw_spinlock_t wait_lock; struct optimistic_spin_queue osq; struct task_struct *owner; struct lockdep_map dep_map; } ;
169 struct pci_dev ;
133 struct pci_bus ;
25 struct ldt_struct ;
25 struct __anonstruct_mm_context_t_172 { struct ldt_struct *ldt; unsigned short ia32_compat; struct mutex lock; void *vdso; atomic_t perf_rdpmc_allowed; } ;
25 typedef struct __anonstruct_mm_context_t_172 mm_context_t;
22 struct bio_vec ;
1211 struct llist_node ;
64 struct llist_node { struct llist_node *next; } ;
17 struct call_single_data { struct llist_node llist; void (*func)(void *); void *info; unsigned int flags; } ;
559 struct dma_map_ops ;
559 struct dev_archdata { struct dma_map_ops *dma_ops; void *iommu; } ;
24 struct device_private ;
25 struct device_driver ;
26 struct driver_private ;
27 struct class ;
28 struct subsys_private ;
29 struct bus_type ;
30 struct device_node ;
31 struct fwnode_handle ;
32 struct iommu_ops ;
33 struct iommu_group ;
61 struct device_attribute ;
61 struct bus_type { const char *name; const char *dev_name; struct device *dev_root; struct device_attribute *dev_attrs; const struct attribute_group **bus_groups; const struct attribute_group **dev_groups; const struct attribute_group **drv_groups; int (*match)(struct device *, struct device_driver *); int (*uevent)(struct device *, struct kobj_uevent_env *); int (*probe)(struct device *); int (*remove)(struct device *); void (*shutdown)(struct device *); int (*online)(struct device *); int (*offline)(struct device *); int (*suspend)(struct device *, pm_message_t ); int (*resume)(struct device *); const struct dev_pm_ops *pm; const struct iommu_ops *iommu_ops; struct subsys_private *p; struct lock_class_key lock_key; } ;
139 struct device_type ;
198 enum probe_type { PROBE_DEFAULT_STRATEGY = 0, PROBE_PREFER_ASYNCHRONOUS = 1, PROBE_FORCE_SYNCHRONOUS = 2 } ;
204 struct device_driver { const char *name; struct bus_type *bus; struct module *owner; const char *mod_name; bool suppress_bind_attrs; enum probe_type probe_type; const struct of_device_id *of_match_table; const struct acpi_device_id *acpi_match_table; int (*probe)(struct device *); int (*remove)(struct device *); void (*shutdown)(struct device *); int (*suspend)(struct device *, pm_message_t ); int (*resume)(struct device *); const struct attribute_group **groups; const struct dev_pm_ops *pm; struct driver_private *p; } ;
354 struct class_attribute ;
354 struct class { const char *name; struct module *owner; struct class_attribute *class_attrs; const struct attribute_group **dev_groups; struct kobject *dev_kobj; int (*dev_uevent)(struct device *, struct kobj_uevent_env *); char * (*devnode)(struct device *, umode_t *); void (*class_release)(struct class *); void (*dev_release)(struct device *); int (*suspend)(struct device *, pm_message_t ); int (*resume)(struct device *); const struct kobj_ns_type_operations *ns_type; const void * (*namespace)(struct device *); const struct dev_pm_ops *pm; struct subsys_private *p; } ;
447 struct class_attribute { struct attribute attr; ssize_t (*show)(struct class *, struct class_attribute *, char *); ssize_t (*store)(struct class *, struct class_attribute *, const char *, size_t ); } ;
515 struct device_type { const char *name; const struct attribute_group **groups; int (*uevent)(struct device *, struct kobj_uevent_env *); char * (*devnode)(struct device *, umode_t *, kuid_t *, kgid_t *); void (*release)(struct device *); const struct dev_pm_ops *pm; } ;
543 struct device_attribute { struct attribute attr; ssize_t (*show)(struct device *, struct device_attribute *, char *); ssize_t (*store)(struct device *, struct device_attribute *, const char *, size_t ); } ;
684 struct device_dma_parameters { unsigned int max_segment_size; unsigned long segment_boundary_mask; } ;
693 struct irq_domain ;
693 struct dma_coherent_mem ;
693 struct cma ;
693 struct device { struct device *parent; struct device_private *p; struct kobject kobj; const char *init_name; const struct device_type *type; struct mutex mutex; struct bus_type *bus; struct device_driver *driver; void *platform_data; void *driver_data; struct dev_pm_info power; struct dev_pm_domain *pm_domain; struct irq_domain *msi_domain; struct dev_pin_info *pins; struct list_head msi_list; int numa_node; u64 *dma_mask; u64 coherent_dma_mask; unsigned long dma_pfn_offset; struct device_dma_parameters *dma_parms; struct list_head dma_pools; struct dma_coherent_mem *dma_mem; struct cma *cma_area; struct dev_archdata archdata; struct device_node *of_node; struct fwnode_handle *fwnode; dev_t devt; u32 id; spinlock_t devres_lock; struct list_head devres_head; struct klist_node knode_class; struct class *class; const struct attribute_group **groups; void (*release)(struct device *); struct iommu_group *iommu_group; bool offline_disabled; bool offline; } ;
847 struct wakeup_source { const char *name; struct list_head entry; spinlock_t lock; struct wake_irq *wakeirq; struct timer_list timer; unsigned long timer_expires; ktime_t total_time; ktime_t max_time; ktime_t last_time; ktime_t start_prevent_time; ktime_t prevent_sleep_time; unsigned long event_count; unsigned long active_count; unsigned long relax_count; unsigned long expire_count; unsigned long wakeup_count; bool active; bool autosleep_enabled; } ;
60 struct kmem_cache ;
58 struct arch_uprobe_task { unsigned long saved_scratch_register; unsigned int saved_trap_nr; unsigned int saved_tf; } ;
66 enum uprobe_task_state { UTASK_RUNNING = 0, UTASK_SSTEP = 1, UTASK_SSTEP_ACK = 2, UTASK_SSTEP_TRAPPED = 3 } ;
73 struct __anonstruct____missing_field_name_207 { struct arch_uprobe_task autask; unsigned long vaddr; } ;
73 struct __anonstruct____missing_field_name_208 { struct callback_head dup_xol_work; unsigned long dup_xol_addr; } ;
73 union __anonunion____missing_field_name_206 { struct __anonstruct____missing_field_name_207 __annonCompField37; struct __anonstruct____missing_field_name_208 __annonCompField38; } ;
73 struct uprobe ;
73 struct return_instance ;
73 struct uprobe_task { enum uprobe_task_state state; union __anonunion____missing_field_name_206 __annonCompField39; struct uprobe *active_uprobe; unsigned long xol_vaddr; struct return_instance *return_instances; unsigned int depth; } ;
94 struct return_instance { struct uprobe *uprobe; unsigned long func; unsigned long stack; unsigned long orig_ret_vaddr; bool chained; struct return_instance *next; } ;
110 struct xol_area ;
111 struct uprobes_state { struct xol_area *xol_area; } ;
150 struct address_space ;
151 struct mem_cgroup ;
152 union __anonunion____missing_field_name_209 { struct address_space *mapping; void *s_mem; atomic_t compound_mapcount; } ;
152 union __anonunion____missing_field_name_211 { unsigned long index; void *freelist; } ;
152 struct __anonstruct____missing_field_name_215 { unsigned short inuse; unsigned short objects; unsigned char frozen; } ;
152 union __anonunion____missing_field_name_214 { atomic_t _mapcount; struct __anonstruct____missing_field_name_215 __annonCompField42; int units; } ;
152 struct __anonstruct____missing_field_name_213 { union __anonunion____missing_field_name_214 __annonCompField43; atomic_t _count; } ;
152 union __anonunion____missing_field_name_212 { unsigned long counters; struct __anonstruct____missing_field_name_213 __annonCompField44; unsigned int active; } ;
152 struct __anonstruct____missing_field_name_210 { union __anonunion____missing_field_name_211 __annonCompField41; union __anonunion____missing_field_name_212 __annonCompField45; } ;
152 struct dev_pagemap ;
152 struct __anonstruct____missing_field_name_217 { struct page *next; int pages; int pobjects; } ;
152 struct __anonstruct____missing_field_name_218 { unsigned long compound_head; unsigned int compound_dtor; unsigned int compound_order; } ;
152 struct __anonstruct____missing_field_name_219 { unsigned long __pad; pgtable_t pmd_huge_pte; } ;
152 union __anonunion____missing_field_name_216 { struct list_head lru; struct dev_pagemap *pgmap; struct __anonstruct____missing_field_name_217 __annonCompField47; struct callback_head callback_head; struct __anonstruct____missing_field_name_218 __annonCompField48; struct __anonstruct____missing_field_name_219 __annonCompField49; } ;
152 union __anonunion____missing_field_name_220 { unsigned long private; spinlock_t *ptl; struct kmem_cache *slab_cache; } ;
152 struct page { unsigned long flags; union __anonunion____missing_field_name_209 __annonCompField40; struct __anonstruct____missing_field_name_210 __annonCompField46; union __anonunion____missing_field_name_216 __annonCompField50; union __anonunion____missing_field_name_220 __annonCompField51; struct mem_cgroup *mem_cgroup; } ;
191 struct page_frag { struct page *page; __u32 offset; __u32 size; } ;
276 struct userfaultfd_ctx ;
276 struct vm_userfaultfd_ctx { struct userfaultfd_ctx *ctx; } ;
283 struct __anonstruct_shared_221 { struct rb_node rb; unsigned long rb_subtree_last; } ;
283 struct anon_vma ;
283 struct mempolicy ;
283 struct vm_area_struct { unsigned long vm_start; unsigned long vm_end; struct vm_area_struct *vm_next; struct vm_area_struct *vm_prev; struct rb_node vm_rb; unsigned long rb_subtree_gap; struct mm_struct *vm_mm; pgprot_t vm_page_prot; unsigned long vm_flags; struct __anonstruct_shared_221 shared; struct list_head anon_vma_chain; struct anon_vma *anon_vma; const struct vm_operations_struct *vm_ops; unsigned long vm_pgoff; struct file *vm_file; void *vm_private_data; struct mempolicy *vm_policy; struct vm_userfaultfd_ctx vm_userfaultfd_ctx; } ;
356 struct core_thread { struct task_struct *task; struct core_thread *next; } ;
361 struct core_state { atomic_t nr_threads; struct core_thread dumper; struct completion startup; } ;
375 struct task_rss_stat { int events; int count[4U]; } ;
383 struct mm_rss_stat { atomic_long_t count[4U]; } ;
388 struct kioctx_table ;
389 struct linux_binfmt ;
389 struct mmu_notifier_mm ;
389 struct mm_struct { struct vm_area_struct *mmap; struct rb_root mm_rb; u32 vmacache_seqnum; unsigned long int (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); unsigned long mmap_base; unsigned long mmap_legacy_base; unsigned long task_size; unsigned long highest_vm_end; pgd_t *pgd; atomic_t mm_users; atomic_t mm_count; atomic_long_t nr_ptes; atomic_long_t nr_pmds; int map_count; spinlock_t page_table_lock; struct rw_semaphore mmap_sem; struct list_head mmlist; unsigned long hiwater_rss; unsigned long hiwater_vm; unsigned long total_vm; unsigned long locked_vm; unsigned long pinned_vm; unsigned long data_vm; unsigned long exec_vm; unsigned long stack_vm; unsigned long def_flags; unsigned long start_code; unsigned long end_code; unsigned long start_data; unsigned long end_data; unsigned long start_brk; unsigned long brk; unsigned long start_stack; unsigned long arg_start; unsigned long arg_end; unsigned long env_start; unsigned long env_end; unsigned long saved_auxv[46U]; struct mm_rss_stat rss_stat; struct linux_binfmt *binfmt; cpumask_var_t cpu_vm_mask_var; mm_context_t context; unsigned long flags; struct core_state *core_state; spinlock_t ioctx_lock; struct kioctx_table *ioctx_table; struct task_struct *owner; struct file *exe_file; struct mmu_notifier_mm *mmu_notifier_mm; struct cpumask cpumask_allocation; unsigned long numa_next_scan; unsigned long numa_scan_offset; int numa_scan_seq; bool tlb_flush_pending; struct uprobes_state uprobes_state; void *bd_addr; atomic_long_t hugetlb_usage; } ;
22 struct kernel_cap_struct { __u32 cap[2U]; } ;
25 typedef struct kernel_cap_struct kernel_cap_t;
84 struct plist_node { int prio; struct list_head prio_list; struct list_head node_list; } ;
4 typedef unsigned long cputime_t;
25 struct sem_undo_list ;
25 struct sysv_sem { struct sem_undo_list *undo_list; } ;
78 struct user_struct ;
26 struct sysv_shm { struct list_head shm_clist; } ;
24 struct __anonstruct_sigset_t_224 { unsigned long sig[1U]; } ;
24 typedef struct __anonstruct_sigset_t_224 sigset_t;
25 struct siginfo ;
17 typedef void __signalfn_t(int);
18 typedef __signalfn_t *__sighandler_t;
20 typedef void __restorefn_t();
21 typedef __restorefn_t *__sigrestore_t;
34 union sigval { int sival_int; void *sival_ptr; } ;
10 typedef union sigval sigval_t;
11 struct __anonstruct__kill_226 { __kernel_pid_t _pid; __kernel_uid32_t _uid; } ;
11 struct __anonstruct__timer_227 { __kernel_timer_t _tid; int _overrun; char _pad[0U]; sigval_t _sigval; int _sys_private; } ;
11 struct __anonstruct__rt_228 { __kernel_pid_t _pid; __kernel_uid32_t _uid; sigval_t _sigval; } ;
11 struct __anonstruct__sigchld_229 { __kernel_pid_t _pid; __kernel_uid32_t _uid; int _status; __kernel_clock_t _utime; __kernel_clock_t _stime; } ;
11 struct __anonstruct__addr_bnd_231 { void *_lower; void *_upper; } ;
11 struct __anonstruct__sigfault_230 { void *_addr; short _addr_lsb; struct __anonstruct__addr_bnd_231 _addr_bnd; } ;
11 struct __anonstruct__sigpoll_232 { long _band; int _fd; } ;
11 struct __anonstruct__sigsys_233 { void *_call_addr; int _syscall; unsigned int _arch; } ;
11 union __anonunion__sifields_225 { int _pad[28U]; struct __anonstruct__kill_226 _kill; struct __anonstruct__timer_227 _timer; struct __anonstruct__rt_228 _rt; struct __anonstruct__sigchld_229 _sigchld; struct __anonstruct__sigfault_230 _sigfault; struct __anonstruct__sigpoll_232 _sigpoll; struct __anonstruct__sigsys_233 _sigsys; } ;
11 struct siginfo { int si_signo; int si_errno; int si_code; union __anonunion__sifields_225 _sifields; } ;
113 typedef struct siginfo siginfo_t;
22 struct sigpending { struct list_head list; sigset_t signal; } ;
242 struct sigaction { __sighandler_t sa_handler; unsigned long sa_flags; __sigrestore_t sa_restorer; sigset_t sa_mask; } ;
256 struct k_sigaction { struct sigaction sa; } ;
442 enum pid_type { PIDTYPE_PID = 0, PIDTYPE_PGID = 1, PIDTYPE_SID = 2, PIDTYPE_MAX = 3 } ;
449 struct pid_namespace ;
449 struct upid { int nr; struct pid_namespace *ns; struct hlist_node pid_chain; } ;
56 struct pid { atomic_t count; unsigned int level; struct hlist_head tasks[3U]; struct callback_head rcu; struct upid numbers[1U]; } ;
68 struct pid_link { struct hlist_node node; struct pid *pid; } ;
174 struct percpu_counter { raw_spinlock_t lock; s64 count; struct list_head list; s32 *counters; } ;
53 struct seccomp_filter ;
54 struct seccomp { int mode; struct seccomp_filter *filter; } ;
40 struct rt_mutex_waiter ;
41 struct rlimit { __kernel_ulong_t rlim_cur; __kernel_ulong_t rlim_max; } ;
11 struct timerqueue_node { struct rb_node node; ktime_t expires; } ;
12 struct timerqueue_head { struct rb_root head; struct timerqueue_node *next; } ;
50 struct hrtimer_clock_base ;
51 struct hrtimer_cpu_base ;
60 enum hrtimer_restart { HRTIMER_NORESTART = 0, HRTIMER_RESTART = 1 } ;
65 struct hrtimer { struct timerqueue_node node; ktime_t _softexpires; enum hrtimer_restart (*function)(struct hrtimer *); struct hrtimer_clock_base *base; unsigned long state; int start_pid; void *start_site; char start_comm[16U]; } ;
123 struct hrtimer_clock_base { struct hrtimer_cpu_base *cpu_base; int index; clockid_t clockid; struct timerqueue_head active; ktime_t (*get_time)(); ktime_t offset; } ;
156 struct hrtimer_cpu_base { raw_spinlock_t lock; seqcount_t seq; struct hrtimer *running; unsigned int cpu; unsigned int active_bases; unsigned int clock_was_set_seq; bool migration_enabled; bool nohz_active; unsigned char in_hrtirq; unsigned char hres_active; unsigned char hang_detected; ktime_t expires_next; struct hrtimer *next_timer; unsigned int nr_events; unsigned int nr_retries; unsigned int nr_hangs; unsigned int max_hang_time; struct hrtimer_clock_base clock_base[4U]; } ;
466 struct task_io_accounting { u64 rchar; u64 wchar; u64 syscr; u64 syscw; u64 read_bytes; u64 write_bytes; u64 cancelled_write_bytes; } ;
45 struct latency_record { unsigned long backtrace[12U]; unsigned int count; unsigned long time; unsigned long max; } ;
39 struct assoc_array_ptr ;
39 struct assoc_array { struct assoc_array_ptr *root; unsigned long nr_leaves_on_tree; } ;
31 typedef int32_t key_serial_t;
34 typedef uint32_t key_perm_t;
35 struct key ;
36 struct signal_struct ;
37 struct cred ;
38 struct key_type ;
42 struct keyring_index_key { struct key_type *type; const char *description; size_t desc_len; } ;
91 union key_payload { void *rcu_data0; void *data[4U]; } ;
128 union __anonunion____missing_field_name_252 { struct list_head graveyard_link; struct rb_node serial_node; } ;
128 struct key_user ;
128 union __anonunion____missing_field_name_253 { time_t expiry; time_t revoked_at; } ;
128 struct __anonstruct____missing_field_name_255 { struct key_type *type; char *description; } ;
128 union __anonunion____missing_field_name_254 { struct keyring_index_key index_key; struct __anonstruct____missing_field_name_255 __annonCompField54; } ;
128 struct __anonstruct____missing_field_name_257 { struct list_head name_link; struct assoc_array keys; } ;
128 union __anonunion____missing_field_name_256 { union key_payload payload; struct __anonstruct____missing_field_name_257 __annonCompField56; int reject_error; } ;
128 struct key { atomic_t usage; key_serial_t serial; union __anonunion____missing_field_name_252 __annonCompField52; struct rw_semaphore sem; struct key_user *user; void *security; union __anonunion____missing_field_name_253 __annonCompField53; time_t last_used_at; kuid_t uid; kgid_t gid; key_perm_t perm; unsigned short quotalen; unsigned short datalen; unsigned long flags; union __anonunion____missing_field_name_254 __annonCompField55; union __anonunion____missing_field_name_256 __annonCompField57; } ;
354 struct audit_context ;
27 struct group_info { atomic_t usage; int ngroups; int nblocks; kgid_t small_block[32U]; kgid_t *blocks[0U]; } ;
90 struct cred { atomic_t usage; atomic_t subscribers; void *put_addr; unsigned int magic; kuid_t uid; kgid_t gid; kuid_t suid; kgid_t sgid; kuid_t euid; kgid_t egid; kuid_t fsuid; kgid_t fsgid; unsigned int securebits; kernel_cap_t cap_inheritable; kernel_cap_t cap_permitted; kernel_cap_t cap_effective; kernel_cap_t cap_bset; kernel_cap_t cap_ambient; unsigned char jit_keyring; struct key *session_keyring; struct key *process_keyring; struct key *thread_keyring; struct key *request_key_auth; void *security; struct user_struct *user; struct user_namespace *user_ns; struct group_info *group_info; struct callback_head rcu; } ;
377 struct percpu_ref ;
55 typedef void percpu_ref_func_t(struct percpu_ref *);
68 struct percpu_ref { atomic_long_t count; unsigned long percpu_count_ptr; percpu_ref_func_t *release; percpu_ref_func_t *confirm_switch; bool force_atomic; struct callback_head rcu; } ;
327 enum rcu_sync_type { RCU_SYNC = 0, RCU_SCHED_SYNC = 1, RCU_BH_SYNC = 2 } ;
333 struct rcu_sync { int gp_state; int gp_count; wait_queue_head_t gp_wait; int cb_state; struct callback_head cb_head; enum rcu_sync_type gp_type; } ;
65 struct percpu_rw_semaphore { struct rcu_sync rss; unsigned int *fast_read_ctr; struct rw_semaphore rw_sem; atomic_t slow_read_ctr; wait_queue_head_t write_waitq; } ;
54 struct cgroup ;
55 struct cgroup_root ;
56 struct cgroup_subsys ;
57 struct cgroup_taskset ;
100 struct cgroup_file { struct kernfs_node *kn; } ;
89 struct cgroup_subsys_state { struct cgroup *cgroup; struct cgroup_subsys *ss; struct percpu_ref refcnt; struct cgroup_subsys_state *parent; struct list_head sibling; struct list_head children; int id; unsigned int flags; u64 serial_nr; struct callback_head callback_head; struct work_struct destroy_work; } ;
134 struct css_set { atomic_t refcount; struct hlist_node hlist; struct list_head tasks; struct list_head mg_tasks; struct list_head cgrp_links; struct cgroup *dfl_cgrp; struct cgroup_subsys_state *subsys[13U]; struct list_head mg_preload_node; struct list_head mg_node; struct cgroup *mg_src_cgrp; struct css_set *mg_dst_cset; struct list_head e_cset_node[13U]; struct list_head task_iters; struct callback_head callback_head; } ;
210 struct cgroup { struct cgroup_subsys_state self; unsigned long flags; int id; int level; int populated_cnt; struct kernfs_node *kn; struct cgroup_file procs_file; struct cgroup_file events_file; unsigned int subtree_control; unsigned int child_subsys_mask; struct cgroup_subsys_state *subsys[13U]; struct cgroup_root *root; struct list_head cset_links; struct list_head e_csets[13U]; struct list_head pidlists; struct mutex pidlist_mutex; wait_queue_head_t offline_waitq; struct work_struct release_agent_work; int ancestor_ids[]; } ;
294 struct cgroup_root { struct kernfs_root *kf_root; unsigned int subsys_mask; int hierarchy_id; struct cgroup cgrp; int cgrp_ancestor_id_storage; atomic_t nr_cgrps; struct list_head root_list; unsigned int flags; struct idr cgroup_idr; char release_agent_path[4096U]; char name[64U]; } ;
333 struct cftype { char name[64U]; unsigned long private; size_t max_write_len; unsigned int flags; unsigned int file_offset; struct cgroup_subsys *ss; struct list_head node; struct kernfs_ops *kf_ops; u64 (*read_u64)(struct cgroup_subsys_state *, struct cftype *); s64 (*read_s64)(struct cgroup_subsys_state *, struct cftype *); int (*seq_show)(struct seq_file *, void *); void * (*seq_start)(struct seq_file *, loff_t *); void * (*seq_next)(struct seq_file *, void *, loff_t *); void (*seq_stop)(struct seq_file *, void *); int (*write_u64)(struct cgroup_subsys_state *, struct cftype *, u64 ); int (*write_s64)(struct cgroup_subsys_state *, struct cftype *, s64 ); ssize_t (*write)(struct kernfs_open_file *, char *, size_t , loff_t ); struct lock_class_key lockdep_key; } ;
418 struct cgroup_subsys { struct cgroup_subsys_state * (*css_alloc)(struct cgroup_subsys_state *); int (*css_online)(struct cgroup_subsys_state *); void (*css_offline)(struct cgroup_subsys_state *); void (*css_released)(struct cgroup_subsys_state *); void (*css_free)(struct cgroup_subsys_state *); void (*css_reset)(struct cgroup_subsys_state *); void (*css_e_css_changed)(struct cgroup_subsys_state *); int (*can_attach)(struct cgroup_taskset *); void (*cancel_attach)(struct cgroup_taskset *); void (*attach)(struct cgroup_taskset *); int (*can_fork)(struct task_struct *); void (*cancel_fork)(struct task_struct *); void (*fork)(struct task_struct *); void (*exit)(struct task_struct *); void (*free)(struct task_struct *); void (*bind)(struct cgroup_subsys_state *); int early_init; bool broken_hierarchy; bool warned_broken_hierarchy; int id; const char *name; const char *legacy_name; struct cgroup_root *root; struct idr css_idr; struct list_head cfts; struct cftype *dfl_cftypes; struct cftype *legacy_cftypes; unsigned int depends_on; } ;
128 struct futex_pi_state ;
129 struct robust_list_head ;
130 struct bio_list ;
131 struct fs_struct ;
132 struct perf_event_context ;
133 struct blk_plug ;
135 struct nameidata ;
188 struct cfs_rq ;
189 struct task_group ;
482 struct sighand_struct { atomic_t count; struct k_sigaction action[64U]; spinlock_t siglock; wait_queue_head_t signalfd_wqh; } ;
523 struct pacct_struct { int ac_flag; long ac_exitcode; unsigned long ac_mem; cputime_t ac_utime; cputime_t ac_stime; unsigned long ac_minflt; unsigned long ac_majflt; } ;
531 struct cpu_itimer { cputime_t expires; cputime_t incr; u32 error; u32 incr_error; } ;
538 struct prev_cputime { cputime_t utime; cputime_t stime; raw_spinlock_t lock; } ;
563 struct task_cputime { cputime_t utime; cputime_t stime; unsigned long long sum_exec_runtime; } ;
579 struct task_cputime_atomic { atomic64_t utime; atomic64_t stime; atomic64_t sum_exec_runtime; } ;
601 struct thread_group_cputimer { struct task_cputime_atomic cputime_atomic; bool running; bool checking_timer; } ;
646 struct autogroup ;
647 struct tty_struct ;
647 struct taskstats ;
647 struct tty_audit_buf ;
647 struct signal_struct { atomic_t sigcnt; atomic_t live; int nr_threads; struct list_head thread_head; wait_queue_head_t wait_chldexit; struct task_struct *curr_target; struct sigpending shared_pending; int group_exit_code; int notify_count; struct task_struct *group_exit_task; int group_stop_count; unsigned int flags; unsigned char is_child_subreaper; unsigned char has_child_subreaper; int posix_timer_id; struct list_head posix_timers; struct hrtimer real_timer; struct pid *leader_pid; ktime_t it_real_incr; struct cpu_itimer it[2U]; struct thread_group_cputimer cputimer; struct task_cputime cputime_expires; struct list_head cpu_timers[3U]; struct pid *tty_old_pgrp; int leader; struct tty_struct *tty; struct autogroup *autogroup; seqlock_t stats_lock; cputime_t utime; cputime_t stime; cputime_t cutime; cputime_t cstime; cputime_t gtime; cputime_t cgtime; struct prev_cputime prev_cputime; unsigned long nvcsw; unsigned long nivcsw; unsigned long cnvcsw; unsigned long cnivcsw; unsigned long min_flt; unsigned long maj_flt; unsigned long cmin_flt; unsigned long cmaj_flt; unsigned long inblock; unsigned long oublock; unsigned long cinblock; unsigned long coublock; unsigned long maxrss; unsigned long cmaxrss; struct task_io_accounting ioac; unsigned long long sum_sched_runtime; struct rlimit rlim[16U]; struct pacct_struct pacct; struct taskstats *stats; unsigned int audit_tty; unsigned int audit_tty_log_passwd; struct tty_audit_buf *tty_audit_buf; oom_flags_t oom_flags; short oom_score_adj; short oom_score_adj_min; struct mutex cred_guard_mutex; } ;
814 struct user_struct { atomic_t __count; atomic_t processes; atomic_t sigpending; atomic_t inotify_watches; atomic_t inotify_devs; atomic_t fanotify_listeners; atomic_long_t epoll_watches; unsigned long mq_bytes; unsigned long locked_shm; unsigned long unix_inflight; atomic_long_t pipe_bufs; struct key *uid_keyring; struct key *session_keyring; struct hlist_node uidhash_node; kuid_t uid; atomic_long_t locked_vm; } ;
859 struct backing_dev_info ;
860 struct reclaim_state ;
861 struct sched_info { unsigned long pcount; unsigned long long run_delay; unsigned long long last_arrival; unsigned long long last_queued; } ;
875 struct task_delay_info { spinlock_t lock; unsigned int flags; u64 blkio_start; u64 blkio_delay; u64 swapin_delay; u32 blkio_count; u32 swapin_count; u64 freepages_start; u64 freepages_delay; u32 freepages_count; } ;
923 struct wake_q_node { struct wake_q_node *next; } ;
1150 struct io_context ;
1184 struct pipe_inode_info ;
1186 struct load_weight { unsigned long weight; u32 inv_weight; } ;
1193 struct sched_avg { u64 last_update_time; u64 load_sum; u32 util_sum; u32 period_contrib; unsigned long load_avg; unsigned long util_avg; } ;
1213 struct sched_statistics { u64 wait_start; u64 wait_max; u64 wait_count; u64 wait_sum; u64 iowait_count; u64 iowait_sum; u64 sleep_start; u64 sleep_max; s64 sum_sleep_runtime; u64 block_start; u64 block_max; u64 exec_max; u64 slice_max; u64 nr_migrations_cold; u64 nr_failed_migrations_affine; u64 nr_failed_migrations_running; u64 nr_failed_migrations_hot; u64 nr_forced_migrations; u64 nr_wakeups; u64 nr_wakeups_sync; u64 nr_wakeups_migrate; u64 nr_wakeups_local; u64 nr_wakeups_remote; u64 nr_wakeups_affine; u64 nr_wakeups_affine_attempts; u64 nr_wakeups_passive; u64 nr_wakeups_idle; } ;
1248 struct sched_entity { struct load_weight load; struct rb_node run_node; struct list_head group_node; unsigned int on_rq; u64 exec_start; u64 sum_exec_runtime; u64 vruntime; u64 prev_sum_exec_runtime; u64 nr_migrations; struct sched_statistics statistics; int depth; struct sched_entity *parent; struct cfs_rq *cfs_rq; struct cfs_rq *my_q; struct sched_avg avg; } ;
1285 struct rt_rq ;
1285 struct sched_rt_entity { struct list_head run_list; unsigned long timeout; unsigned long watchdog_stamp; unsigned int time_slice; struct sched_rt_entity *back; struct sched_rt_entity *parent; struct rt_rq *rt_rq; struct rt_rq *my_q; } ;
1301 struct sched_dl_entity { struct rb_node rb_node; u64 dl_runtime; u64 dl_deadline; u64 dl_period; u64 dl_bw; s64 runtime; u64 deadline; unsigned int flags; int dl_throttled; int dl_new; int dl_boosted; int dl_yielded; struct hrtimer dl_timer; } ;
1369 struct tlbflush_unmap_batch { struct cpumask cpumask; bool flush_required; bool writable; } ;
1388 struct sched_class ;
1388 struct files_struct ;
1388 struct compat_robust_list_head ;
1388 struct numa_group ;
1388 struct task_struct { volatile long state; void *stack; atomic_t usage; unsigned int flags; unsigned int ptrace; struct llist_node wake_entry; int on_cpu; unsigned int wakee_flips; unsigned long wakee_flip_decay_ts; struct task_struct *last_wakee; int wake_cpu; int on_rq; int prio; int static_prio; int normal_prio; unsigned int rt_priority; const struct sched_class *sched_class; struct sched_entity se; struct sched_rt_entity rt; struct task_group *sched_task_group; struct sched_dl_entity dl; struct hlist_head preempt_notifiers; unsigned int policy; int nr_cpus_allowed; cpumask_t cpus_allowed; unsigned long rcu_tasks_nvcsw; bool rcu_tasks_holdout; struct list_head rcu_tasks_holdout_list; int rcu_tasks_idle_cpu; struct sched_info sched_info; struct list_head tasks; struct plist_node pushable_tasks; struct rb_node pushable_dl_tasks; struct mm_struct *mm; struct mm_struct *active_mm; u32 vmacache_seqnum; struct vm_area_struct *vmacache[4U]; struct task_rss_stat rss_stat; int exit_state; int exit_code; int exit_signal; int pdeath_signal; unsigned long jobctl; unsigned int personality; unsigned char sched_reset_on_fork; unsigned char sched_contributes_to_load; unsigned char sched_migrated; unsigned char; unsigned char in_execve; unsigned char in_iowait; unsigned char memcg_may_oom; unsigned char memcg_kmem_skip_account; unsigned char brk_randomized; unsigned long atomic_flags; struct restart_block restart_block; pid_t pid; pid_t tgid; struct task_struct *real_parent; struct task_struct *parent; struct list_head children; struct list_head sibling; struct task_struct *group_leader; struct list_head ptraced; struct list_head ptrace_entry; struct pid_link pids[3U]; struct list_head thread_group; struct list_head thread_node; struct completion *vfork_done; int *set_child_tid; int *clear_child_tid; cputime_t utime; cputime_t stime; cputime_t utimescaled; cputime_t stimescaled; cputime_t gtime; struct prev_cputime prev_cputime; unsigned long nvcsw; unsigned long nivcsw; u64 start_time; u64 real_start_time; unsigned long min_flt; unsigned long maj_flt; struct task_cputime cputime_expires; struct list_head cpu_timers[3U]; const struct cred *real_cred; const struct cred *cred; char comm[16U]; struct nameidata *nameidata; struct sysv_sem sysvsem; struct sysv_shm sysvshm; unsigned long last_switch_count; struct fs_struct *fs; struct files_struct *files; struct nsproxy *nsproxy; struct signal_struct *signal; struct sighand_struct *sighand; sigset_t blocked; sigset_t real_blocked; sigset_t saved_sigmask; struct sigpending pending; unsigned long sas_ss_sp; size_t sas_ss_size; struct callback_head *task_works; struct audit_context *audit_context; kuid_t loginuid; unsigned int sessionid; struct seccomp seccomp; u32 parent_exec_id; u32 self_exec_id; spinlock_t alloc_lock; raw_spinlock_t pi_lock; struct wake_q_node wake_q; struct rb_root pi_waiters; struct rb_node *pi_waiters_leftmost; struct rt_mutex_waiter *pi_blocked_on; struct mutex_waiter *blocked_on; unsigned int irq_events; unsigned long hardirq_enable_ip; unsigned long hardirq_disable_ip; unsigned int hardirq_enable_event; unsigned int hardirq_disable_event; int hardirqs_enabled; int hardirq_context; unsigned long softirq_disable_ip; unsigned long softirq_enable_ip; unsigned int softirq_disable_event; unsigned int softirq_enable_event; int softirqs_enabled; int softirq_context; u64 curr_chain_key; int lockdep_depth; unsigned int lockdep_recursion; struct held_lock held_locks[48U]; gfp_t lockdep_reclaim_gfp; unsigned int in_ubsan; void *journal_info; struct bio_list *bio_list; struct blk_plug *plug; struct reclaim_state *reclaim_state; struct backing_dev_info *backing_dev_info; struct io_context *io_context; unsigned long ptrace_message; siginfo_t *last_siginfo; struct task_io_accounting ioac; u64 acct_rss_mem1; u64 acct_vm_mem1; cputime_t acct_timexpd; nodemask_t mems_allowed; seqcount_t mems_allowed_seq; int cpuset_mem_spread_rotor; int cpuset_slab_spread_rotor; struct css_set *cgroups; struct list_head cg_list; struct robust_list_head *robust_list; struct compat_robust_list_head *compat_robust_list; struct list_head pi_state_list; struct futex_pi_state *pi_state_cache; struct perf_event_context *perf_event_ctxp[2U]; struct mutex perf_event_mutex; struct list_head perf_event_list; struct mempolicy *mempolicy; short il_next; short pref_node_fork; int numa_scan_seq; unsigned int numa_scan_period; unsigned int numa_scan_period_max; int numa_preferred_nid; unsigned long numa_migrate_retry; u64 node_stamp; u64 last_task_numa_placement; u64 last_sum_exec_runtime; struct callback_head numa_work; struct list_head numa_entry; struct numa_group *numa_group; unsigned long *numa_faults; unsigned long total_numa_faults; unsigned long numa_faults_locality[3U]; unsigned long numa_pages_migrated; struct tlbflush_unmap_batch tlb_ubc; struct callback_head rcu; struct pipe_inode_info *splice_pipe; struct page_frag task_frag; struct task_delay_info *delays; int make_it_fail; int nr_dirtied; int nr_dirtied_pause; unsigned long dirty_paused_when; int latency_record_count; struct latency_record latency_record[32U]; unsigned long timer_slack_ns; unsigned long default_timer_slack_ns; unsigned int kasan_depth; unsigned long trace; unsigned long trace_recursion; struct mem_cgroup *memcg_in_oom; gfp_t memcg_oom_gfp_mask; int memcg_oom_order; unsigned int memcg_nr_pages_over_high; struct uprobe_task *utask; unsigned int sequential_io; unsigned int sequential_io_avg; unsigned long task_state_change; int pagefault_disabled; struct thread_struct thread; } ;
70 struct hotplug_slot ;
70 struct pci_slot { struct pci_bus *bus; struct list_head list; struct hotplug_slot *hotplug; unsigned char number; struct kobject kobj; } ;
110 typedef int pci_power_t;
137 typedef unsigned int pci_channel_state_t;
138 enum pci_channel_state { pci_channel_io_normal = 1, pci_channel_io_frozen = 2, pci_channel_io_perm_failure = 3 } ;
163 typedef unsigned short pci_dev_flags_t;
192 typedef unsigned short pci_bus_flags_t;
249 struct pcie_link_state ;
250 struct pci_vpd ;
251 struct pci_sriov ;
253 struct proc_dir_entry ;
253 struct pci_driver ;
253 union __anonunion____missing_field_name_282 { struct pci_sriov *sriov; struct pci_dev *physfn; } ;
253 struct pci_dev { struct list_head bus_list; struct pci_bus *bus; struct pci_bus *subordinate; void *sysdata; struct proc_dir_entry *procent; struct pci_slot *slot; unsigned int devfn; unsigned short vendor; unsigned short device; unsigned short subsystem_vendor; unsigned short subsystem_device; unsigned int class; u8 revision; u8 hdr_type; u8 pcie_cap; u8 msi_cap; u8 msix_cap; unsigned char pcie_mpss; u8 rom_base_reg; u8 pin; u16 pcie_flags_reg; u8 dma_alias_devfn; struct pci_driver *driver; u64 dma_mask; struct device_dma_parameters dma_parms; pci_power_t current_state; u8 pm_cap; unsigned char pme_support; unsigned char pme_interrupt; unsigned char pme_poll; unsigned char d1_support; unsigned char d2_support; unsigned char no_d1d2; unsigned char no_d3cold; unsigned char d3cold_allowed; unsigned char mmio_always_on; unsigned char wakeup_prepared; unsigned char runtime_d3cold; unsigned char ignore_hotplug; unsigned int d3_delay; unsigned int d3cold_delay; struct pcie_link_state *link_state; pci_channel_state_t error_state; struct device dev; int cfg_size; unsigned int irq; struct resource resource[17U]; bool match_driver; unsigned char transparent; unsigned char multifunction; unsigned char is_added; unsigned char is_busmaster; unsigned char no_msi; unsigned char no_64bit_msi; unsigned char block_cfg_access; unsigned char broken_parity_status; unsigned char irq_reroute_variant; unsigned char msi_enabled; unsigned char msix_enabled; unsigned char ari_enabled; unsigned char ats_enabled; unsigned char is_managed; unsigned char needs_freset; unsigned char state_saved; unsigned char is_physfn; unsigned char is_virtfn; unsigned char reset_fn; unsigned char is_hotplug_bridge; unsigned char __aer_firmware_first_valid; unsigned char __aer_firmware_first; unsigned char broken_intx_masking; unsigned char io_window_1k; unsigned char irq_managed; unsigned char has_secondary_link; pci_dev_flags_t dev_flags; atomic_t enable_cnt; u32 saved_config_space[16U]; struct hlist_head saved_cap_space; struct bin_attribute *rom_attr; int rom_attr_enabled; struct bin_attribute *res_attr[17U]; struct bin_attribute *res_attr_wc[17U]; const struct attribute_group **msi_irq_groups; struct pci_vpd *vpd; union __anonunion____missing_field_name_282 __annonCompField63; u16 ats_cap; u8 ats_stu; atomic_t ats_ref_cnt; phys_addr_t rom; size_t romlen; char *driver_override; } ;
453 struct pci_ops ;
453 struct msi_controller ;
453 struct pci_bus { struct list_head node; struct pci_bus *parent; struct list_head children; struct list_head devices; struct pci_dev *self; struct list_head slots; struct resource *resource[4U]; struct list_head resources; struct resource busn_res; struct pci_ops *ops; struct msi_controller *msi; void *sysdata; struct proc_dir_entry *procdir; unsigned char number; unsigned char primary; unsigned char max_bus_speed; unsigned char cur_bus_speed; char name[48U]; unsigned short bridge_ctl; pci_bus_flags_t bus_flags; struct device *bridge; struct device dev; struct bin_attribute *legacy_io; struct bin_attribute *legacy_mem; unsigned char is_added; } ;
577 struct pci_ops { void * (*map_bus)(struct pci_bus *, unsigned int, int); int (*read)(struct pci_bus *, unsigned int, int, int, u32 *); int (*write)(struct pci_bus *, unsigned int, int, int, u32 ); } ;
605 struct pci_dynids { spinlock_t lock; struct list_head list; } ;
619 typedef unsigned int pci_ers_result_t;
629 struct pci_error_handlers { pci_ers_result_t (*error_detected)(struct pci_dev *, enum pci_channel_state ); pci_ers_result_t (*mmio_enabled)(struct pci_dev *); pci_ers_result_t (*link_reset)(struct pci_dev *); pci_ers_result_t (*slot_reset)(struct pci_dev *); void (*reset_notify)(struct pci_dev *, bool ); void (*resume)(struct pci_dev *); } ;
662 struct pci_driver { struct list_head node; const char *name; const struct pci_device_id *id_table; int (*probe)(struct pci_dev *, const struct pci_device_id *); void (*remove)(struct pci_dev *); int (*suspend)(struct pci_dev *, pm_message_t ); int (*suspend_late)(struct pci_dev *, pm_message_t ); int (*resume_early)(struct pci_dev *); int (*resume)(struct pci_dev *); void (*shutdown)(struct pci_dev *); int (*sriov_configure)(struct pci_dev *, int); const struct pci_error_handlers *err_handler; struct device_driver driver; struct pci_dynids dynids; } ;
93 struct shrink_control { gfp_t gfp_mask; unsigned long nr_to_scan; int nid; struct mem_cgroup *memcg; } ;
27 struct shrinker { unsigned long int (*count_objects)(struct shrinker *, struct shrink_control *); unsigned long int (*scan_objects)(struct shrinker *, struct shrink_control *); int seeks; long batch; unsigned long flags; struct list_head list; atomic_long_t *nr_deferred; } ;
68 struct file_ra_state ;
69 struct writeback_control ;
70 struct bdi_writeback ;
226 struct vm_fault { unsigned int flags; gfp_t gfp_mask; unsigned long pgoff; void *virtual_address; struct page *cow_page; struct page *page; unsigned long max_pgoff; pte_t *pte; } ;
262 struct vm_operations_struct { void (*open)(struct vm_area_struct *); void (*close)(struct vm_area_struct *); int (*mremap)(struct vm_area_struct *); int (*fault)(struct vm_area_struct *, struct vm_fault *); int (*pmd_fault)(struct vm_area_struct *, unsigned long, pmd_t *, unsigned int); void (*map_pages)(struct vm_area_struct *, struct vm_fault *); int (*page_mkwrite)(struct vm_area_struct *, struct vm_fault *); int (*pfn_mkwrite)(struct vm_area_struct *, struct vm_fault *); int (*access)(struct vm_area_struct *, unsigned long, void *, int, int); const char * (*name)(struct vm_area_struct *); int (*set_policy)(struct vm_area_struct *, struct mempolicy *); struct mempolicy * (*get_policy)(struct vm_area_struct *, unsigned long); struct page * (*find_special_page)(struct vm_area_struct *, unsigned long); } ;
1285 struct kvec ;
2365 struct scatterlist { unsigned long sg_magic; unsigned long page_link; unsigned int offset; unsigned int length; dma_addr_t dma_address; unsigned int dma_length; } ;
21 struct sg_table { struct scatterlist *sgl; unsigned int nents; unsigned int orig_nents; } ;
1405 struct acpi_device ;
1405 struct pci_sysdata { int domain; int node; struct acpi_device *companion; void *iommu; } ;
34 struct dma_attrs { unsigned long flags[1U]; } ;
158 struct dma_map_ops { void * (*alloc)(struct device *, size_t , dma_addr_t *, gfp_t , struct dma_attrs *); void (*free)(struct device *, size_t , void *, dma_addr_t , struct dma_attrs *); int (*mmap)(struct device *, struct vm_area_struct *, void *, dma_addr_t , size_t , struct dma_attrs *); int (*get_sgtable)(struct device *, struct sg_table *, void *, dma_addr_t , size_t , struct dma_attrs *); dma_addr_t (*map_page)(struct device *, struct page *, unsigned long, size_t , enum dma_data_direction , struct dma_attrs *); void (*unmap_page)(struct device *, dma_addr_t , size_t , enum dma_data_direction , struct dma_attrs *); int (*map_sg)(struct device *, struct scatterlist *, int, enum dma_data_direction , struct dma_attrs *); void (*unmap_sg)(struct device *, struct scatterlist *, int, enum dma_data_direction , struct dma_attrs *); void (*sync_single_for_cpu)(struct device *, dma_addr_t , size_t , enum dma_data_direction ); void (*sync_single_for_device)(struct device *, dma_addr_t , size_t , enum dma_data_direction ); void (*sync_sg_for_cpu)(struct device *, struct scatterlist *, int, enum dma_data_direction ); void (*sync_sg_for_device)(struct device *, struct scatterlist *, int, enum dma_data_direction ); int (*mapping_error)(struct device *, dma_addr_t ); int (*dma_supported)(struct device *, u64 ); int (*set_dma_mask)(struct device *, u64 ); int is_phys; } ;
1992 enum irqreturn { IRQ_NONE = 0, IRQ_HANDLED = 1, IRQ_WAKE_THREAD = 2 } ;
16 typedef enum irqreturn irqreturn_t;
62 struct exception_table_entry { int insn; int fixup; } ;
54 struct __anonstruct_b_300 { unsigned char data; unsigned char error_feature; unsigned char sector; unsigned char nsector; unsigned char lcyl; unsigned char hcyl; unsigned char select; unsigned char status_command; unsigned char data_hob; unsigned char error_feature_hob; unsigned char sector_hob; unsigned char nsector_hob; unsigned char lcyl_hob; unsigned char hcyl_hob; unsigned char select_hob; unsigned char control_hob; } ;
54 union ide_reg_valid_s { unsigned short all; struct __anonstruct_b_300 b; } ;
57 typedef union ide_reg_valid_s ide_reg_valid_t;
58 struct ide_task_request_s { __u8 io_ports[8U]; __u8 hob_ports[8U]; ide_reg_valid_t out_flags; ide_reg_valid_t in_flags; int data_phase; int req_cmd; unsigned long out_size; unsigned long in_size; } ;
68 typedef struct ide_task_request_s ide_task_request_t;
82 struct hd_geometry { unsigned char heads; unsigned char sectors; unsigned short cylinders; unsigned long start; } ;
129 struct iovec { void *iov_base; __kernel_size_t iov_len; } ;
21 struct kvec { void *iov_base; size_t iov_len; } ;
27 union __anonunion____missing_field_name_301 { const struct iovec *iov; const struct kvec *kvec; const struct bio_vec *bvec; } ;
27 struct iov_iter { int type; size_t iov_offset; size_t count; union __anonunion____missing_field_name_301 __annonCompField72; unsigned long nr_segs; } ;
38 struct kiocb ;
81 struct hlist_bl_node ;
81 struct hlist_bl_head { struct hlist_bl_node *first; } ;
36 struct hlist_bl_node { struct hlist_bl_node *next; struct hlist_bl_node **pprev; } ;
114 struct __anonstruct____missing_field_name_322 { spinlock_t lock; int count; } ;
114 union __anonunion____missing_field_name_321 { struct __anonstruct____missing_field_name_322 __annonCompField73; } ;
114 struct lockref { union __anonunion____missing_field_name_321 __annonCompField74; } ;
50 struct vfsmount ;
51 struct __anonstruct____missing_field_name_324 { u32 hash; u32 len; } ;
51 union __anonunion____missing_field_name_323 { struct __anonstruct____missing_field_name_324 __annonCompField75; u64 hash_len; } ;
51 struct qstr { union __anonunion____missing_field_name_323 __annonCompField76; const unsigned char *name; } ;
90 struct dentry_operations ;
90 union __anonunion_d_u_325 { struct hlist_node d_alias; struct callback_head d_rcu; } ;
90 struct dentry { unsigned int d_flags; seqcount_t d_seq; struct hlist_bl_node d_hash; struct dentry *d_parent; struct qstr d_name; struct inode *d_inode; unsigned char d_iname[32U]; struct lockref d_lockref; const struct dentry_operations *d_op; struct super_block *d_sb; unsigned long d_time; void *d_fsdata; struct list_head d_lru; struct list_head d_child; struct list_head d_subdirs; union __anonunion_d_u_325 d_u; } ;
142 struct dentry_operations { int (*d_revalidate)(struct dentry *, unsigned int); int (*d_weak_revalidate)(struct dentry *, unsigned int); int (*d_hash)(const struct dentry *, struct qstr *); int (*d_compare)(const struct dentry *, const struct dentry *, unsigned int, const char *, const struct qstr *); int (*d_delete)(const struct dentry *); void (*d_release)(struct dentry *); void (*d_prune)(struct dentry *); void (*d_iput)(struct dentry *, struct inode *); char * (*d_dname)(struct dentry *, char *, int); struct vfsmount * (*d_automount)(struct path *); int (*d_manage)(struct dentry *, bool ); struct inode * (*d_select_inode)(struct dentry *, unsigned int); } ;
586 struct path { struct vfsmount *mnt; struct dentry *dentry; } ;
27 struct list_lru_one { struct list_head list; long nr_items; } ;
32 struct list_lru_memcg { struct list_lru_one *lru[0U]; } ;
37 struct list_lru_node { spinlock_t lock; struct list_lru_one lru; struct list_lru_memcg *memcg_lrus; } ;
47 struct list_lru { struct list_lru_node *node; struct list_head list; } ;
67 struct __anonstruct____missing_field_name_329 { struct radix_tree_node *parent; void *private_data; } ;
67 union __anonunion____missing_field_name_328 { struct __anonstruct____missing_field_name_329 __annonCompField77; struct callback_head callback_head; } ;
67 struct radix_tree_node { unsigned int path; unsigned int count; union __anonunion____missing_field_name_328 __annonCompField78; struct list_head private_list; void *slots[64U]; unsigned long tags[3U][1U]; } ;
114 struct radix_tree_root { unsigned int height; gfp_t gfp_mask; struct radix_tree_node *rnode; } ;
437 struct semaphore { raw_spinlock_t lock; unsigned int count; struct list_head wait_list; } ;
45 struct fiemap_extent { __u64 fe_logical; __u64 fe_physical; __u64 fe_length; __u64 fe_reserved64[2U]; __u32 fe_flags; __u32 fe_reserved[3U]; } ;
38 enum migrate_mode { MIGRATE_ASYNC = 0, MIGRATE_SYNC_LIGHT = 1, MIGRATE_SYNC = 2 } ;
44 struct bio_set ;
45 struct bio ;
46 struct bio_integrity_payload ;
47 struct block_device ;
17 typedef void bio_end_io_t(struct bio *);
19 struct bio_vec { struct page *bv_page; unsigned int bv_len; unsigned int bv_offset; } ;
28 struct bvec_iter { sector_t bi_sector; unsigned int bi_size; unsigned int bi_idx; unsigned int bi_bvec_done; } ;
40 union __anonunion____missing_field_name_334 { struct bio_integrity_payload *bi_integrity; } ;
40 struct bio { struct bio *bi_next; struct block_device *bi_bdev; unsigned int bi_flags; int bi_error; unsigned long bi_rw; struct bvec_iter bi_iter; unsigned int bi_phys_segments; unsigned int bi_seg_front_size; unsigned int bi_seg_back_size; atomic_t __bi_remaining; bio_end_io_t *bi_end_io; void *bi_private; struct io_context *bi_ioc; struct cgroup_subsys_state *bi_css; union __anonunion____missing_field_name_334 __annonCompField79; unsigned short bi_vcnt; unsigned short bi_max_vecs; atomic_t __bi_cnt; struct bio_vec *bi_io_vec; struct bio_set *bi_pool; struct bio_vec bi_inline_vecs[0U]; } ;
245 typedef unsigned int blk_qc_t;
268 struct delayed_call { void (*fn)(void *); void *arg; } ;
162 struct export_operations ;
163 struct poll_table_struct ;
164 struct kstatfs ;
165 struct swap_info_struct ;
76 struct iattr { unsigned int ia_valid; umode_t ia_mode; kuid_t ia_uid; kgid_t ia_gid; loff_t ia_size; struct timespec ia_atime; struct timespec ia_mtime; struct timespec ia_ctime; struct file *ia_file; } ;
212 struct dquot ;
19 typedef __kernel_uid32_t projid_t;
23 struct __anonstruct_kprojid_t_335 { projid_t val; } ;
23 typedef struct __anonstruct_kprojid_t_335 kprojid_t;
166 enum quota_type { USRQUOTA = 0, GRPQUOTA = 1, PRJQUOTA = 2 } ;
66 typedef long long qsize_t;
67 union __anonunion____missing_field_name_336 { kuid_t uid; kgid_t gid; kprojid_t projid; } ;
67 struct kqid { union __anonunion____missing_field_name_336 __annonCompField80; enum quota_type type; } ;
184 struct mem_dqblk { qsize_t dqb_bhardlimit; qsize_t dqb_bsoftlimit; qsize_t dqb_curspace; qsize_t dqb_rsvspace; qsize_t dqb_ihardlimit; qsize_t dqb_isoftlimit; qsize_t dqb_curinodes; time_t dqb_btime; time_t dqb_itime; } ;
206 struct quota_format_type ;
207 struct mem_dqinfo { struct quota_format_type *dqi_format; int dqi_fmt_id; struct list_head dqi_dirty_list; unsigned long dqi_flags; unsigned int dqi_bgrace; unsigned int dqi_igrace; qsize_t dqi_max_spc_limit; qsize_t dqi_max_ino_limit; void *dqi_priv; } ;
272 struct dquot { struct hlist_node dq_hash; struct list_head dq_inuse; struct list_head dq_free; struct list_head dq_dirty; struct mutex dq_lock; atomic_t dq_count; wait_queue_head_t dq_wait_unused; struct super_block *dq_sb; struct kqid dq_id; loff_t dq_off; unsigned long dq_flags; struct mem_dqblk dq_dqb; } ;
299 struct quota_format_ops { int (*check_quota_file)(struct super_block *, int); int (*read_file_info)(struct super_block *, int); int (*write_file_info)(struct super_block *, int); int (*free_file_info)(struct super_block *, int); int (*read_dqblk)(struct dquot *); int (*commit_dqblk)(struct dquot *); int (*release_dqblk)(struct dquot *); } ;
310 struct dquot_operations { int (*write_dquot)(struct dquot *); struct dquot * (*alloc_dquot)(struct super_block *, int); void (*destroy_dquot)(struct dquot *); int (*acquire_dquot)(struct dquot *); int (*release_dquot)(struct dquot *); int (*mark_dirty)(struct dquot *); int (*write_info)(struct super_block *, int); qsize_t * (*get_reserved_space)(struct inode *); int (*get_projid)(struct inode *, kprojid_t *); } ;
325 struct qc_dqblk { int d_fieldmask; u64 d_spc_hardlimit; u64 d_spc_softlimit; u64 d_ino_hardlimit; u64 d_ino_softlimit; u64 d_space; u64 d_ino_count; s64 d_ino_timer; s64 d_spc_timer; int d_ino_warns; int d_spc_warns; u64 d_rt_spc_hardlimit; u64 d_rt_spc_softlimit; u64 d_rt_space; s64 d_rt_spc_timer; int d_rt_spc_warns; } ;
348 struct qc_type_state { unsigned int flags; unsigned int spc_timelimit; unsigned int ino_timelimit; unsigned int rt_spc_timelimit; unsigned int spc_warnlimit; unsigned int ino_warnlimit; unsigned int rt_spc_warnlimit; unsigned long long ino; blkcnt_t blocks; blkcnt_t nextents; } ;
394 struct qc_state { unsigned int s_incoredqs; struct qc_type_state s_state[3U]; } ;
405 struct qc_info { int i_fieldmask; unsigned int i_flags; unsigned int i_spc_timelimit; unsigned int i_ino_timelimit; unsigned int i_rt_spc_timelimit; unsigned int i_spc_warnlimit; unsigned int i_ino_warnlimit; unsigned int i_rt_spc_warnlimit; } ;
418 struct quotactl_ops { int (*quota_on)(struct super_block *, int, int, struct path *); int (*quota_off)(struct super_block *, int); int (*quota_enable)(struct super_block *, unsigned int); int (*quota_disable)(struct super_block *, unsigned int); int (*quota_sync)(struct super_block *, int); int (*set_info)(struct super_block *, int, struct qc_info *); int (*get_dqblk)(struct super_block *, struct kqid , struct qc_dqblk *); int (*set_dqblk)(struct super_block *, struct kqid , struct qc_dqblk *); int (*get_state)(struct super_block *, struct qc_state *); int (*rm_xquota)(struct super_block *, unsigned int); } ;
432 struct quota_format_type { int qf_fmt_id; const struct quota_format_ops *qf_ops; struct module *qf_owner; struct quota_format_type *qf_next; } ;
496 struct quota_info { unsigned int flags; struct mutex dqio_mutex; struct mutex dqonoff_mutex; struct inode *files[3U]; struct mem_dqinfo info[3U]; const struct quota_format_ops *ops[3U]; } ;
526 struct kiocb { struct file *ki_filp; loff_t ki_pos; void (*ki_complete)(struct kiocb *, long, long); void *private; int ki_flags; } ;
367 struct address_space_operations { int (*writepage)(struct page *, struct writeback_control *); int (*readpage)(struct file *, struct page *); int (*writepages)(struct address_space *, struct writeback_control *); int (*set_page_dirty)(struct page *); int (*readpages)(struct file *, struct address_space *, struct list_head *, unsigned int); int (*write_begin)(struct file *, struct address_space *, loff_t , unsigned int, unsigned int, struct page **, void **); int (*write_end)(struct file *, struct address_space *, loff_t , unsigned int, unsigned int, struct page *, void *); sector_t (*bmap)(struct address_space *, sector_t ); void (*invalidatepage)(struct page *, unsigned int, unsigned int); int (*releasepage)(struct page *, gfp_t ); void (*freepage)(struct page *); ssize_t (*direct_IO)(struct kiocb *, struct iov_iter *, loff_t ); int (*migratepage)(struct address_space *, struct page *, struct page *, enum migrate_mode ); int (*launder_page)(struct page *); int (*is_partially_uptodate)(struct page *, unsigned long, unsigned long); void (*is_dirty_writeback)(struct page *, bool *, bool *); int (*error_remove_page)(struct address_space *, struct page *); int (*swap_activate)(struct swap_info_struct *, struct file *, sector_t *); void (*swap_deactivate)(struct file *); } ;
424 struct address_space { struct inode *host; struct radix_tree_root page_tree; spinlock_t tree_lock; atomic_t i_mmap_writable; struct rb_root i_mmap; struct rw_semaphore i_mmap_rwsem; unsigned long nrpages; unsigned long nrexceptional; unsigned long writeback_index; const struct address_space_operations *a_ops; unsigned long flags; spinlock_t private_lock; struct list_head private_list; void *private_data; } ;
445 struct request_queue ;
446 struct hd_struct ;
446 struct gendisk ;
446 struct block_device { dev_t bd_dev; int bd_openers; struct inode *bd_inode; struct super_block *bd_super; struct mutex bd_mutex; struct list_head bd_inodes; void *bd_claiming; void *bd_holder; int bd_holders; bool bd_write_holder; struct list_head bd_holder_disks; struct block_device *bd_contains; unsigned int bd_block_size; struct hd_struct *bd_part; unsigned int bd_part_count; int bd_invalidated; struct gendisk *bd_disk; struct request_queue *bd_queue; struct list_head bd_list; unsigned long bd_private; int bd_fsfreeze_count; struct mutex bd_fsfreeze_mutex; int bd_map_count; } ;
565 struct posix_acl ;
566 struct inode_operations ;
566 union __anonunion____missing_field_name_341 { const unsigned int i_nlink; unsigned int __i_nlink; } ;
566 union __anonunion____missing_field_name_342 { struct hlist_head i_dentry; struct callback_head i_rcu; } ;
566 struct file_lock_context ;
566 struct cdev ;
566 union __anonunion____missing_field_name_343 { struct pipe_inode_info *i_pipe; struct block_device *i_bdev; struct cdev *i_cdev; char *i_link; } ;
566 struct inode { umode_t i_mode; unsigned short i_opflags; kuid_t i_uid; kgid_t i_gid; unsigned int i_flags; struct posix_acl *i_acl; struct posix_acl *i_default_acl; const struct inode_operations *i_op; struct super_block *i_sb; struct address_space *i_mapping; void *i_security; unsigned long i_ino; union __anonunion____missing_field_name_341 __annonCompField81; dev_t i_rdev; loff_t i_size; struct timespec i_atime; struct timespec i_mtime; struct timespec i_ctime; spinlock_t i_lock; unsigned short i_bytes; unsigned int i_blkbits; blkcnt_t i_blocks; unsigned long i_state; struct mutex i_mutex; unsigned long dirtied_when; unsigned long dirtied_time_when; struct hlist_node i_hash; struct list_head i_io_list; struct bdi_writeback *i_wb; int i_wb_frn_winner; u16 i_wb_frn_avg_time; u16 i_wb_frn_history; struct list_head i_lru; struct list_head i_sb_list; union __anonunion____missing_field_name_342 __annonCompField82; u64 i_version; atomic_t i_count; atomic_t i_dio_count; atomic_t i_writecount; atomic_t i_readcount; const struct file_operations *i_fop; struct file_lock_context *i_flctx; struct address_space i_data; struct list_head i_devices; union __anonunion____missing_field_name_343 __annonCompField83; __u32 i_generation; __u32 i_fsnotify_mask; struct hlist_head i_fsnotify_marks; void *i_private; } ;
837 struct fown_struct { rwlock_t lock; struct pid *pid; enum pid_type pid_type; kuid_t uid; kuid_t euid; int signum; } ;
845 struct file_ra_state { unsigned long start; unsigned int size; unsigned int async_size; unsigned int ra_pages; unsigned int mmap_miss; loff_t prev_pos; } ;
868 union __anonunion_f_u_344 { struct llist_node fu_llist; struct callback_head fu_rcuhead; } ;
868 struct file { union __anonunion_f_u_344 f_u; struct path f_path; struct inode *f_inode; const struct file_operations *f_op; spinlock_t f_lock; atomic_long_t f_count; unsigned int f_flags; fmode_t f_mode; struct mutex f_pos_lock; loff_t f_pos; struct fown_struct f_owner; const struct cred *f_cred; struct file_ra_state f_ra; u64 f_version; void *f_security; void *private_data; struct list_head f_ep_links; struct list_head f_tfile_llink; struct address_space *f_mapping; } ;
953 typedef void *fl_owner_t;
954 struct file_lock ;
955 struct file_lock_operations { void (*fl_copy_lock)(struct file_lock *, struct file_lock *); void (*fl_release_private)(struct file_lock *); } ;
961 struct lock_manager_operations { int (*lm_compare_owner)(struct file_lock *, struct file_lock *); unsigned long int (*lm_owner_key)(struct file_lock *); fl_owner_t (*lm_get_owner)(fl_owner_t ); void (*lm_put_owner)(fl_owner_t ); void (*lm_notify)(struct file_lock *); int (*lm_grant)(struct file_lock *, int); bool (*lm_break)(struct file_lock *); int (*lm_change)(struct file_lock *, int, struct list_head *); void (*lm_setup)(struct file_lock *, void **); } ;
988 struct nlm_lockowner ;
989 struct nfs_lock_info { u32 state; struct nlm_lockowner *owner; struct list_head list; } ;
14 struct nfs4_lock_state ;
15 struct nfs4_lock_info { struct nfs4_lock_state *owner; } ;
19 struct fasync_struct ;
19 struct __anonstruct_afs_346 { struct list_head link; int state; } ;
19 union __anonunion_fl_u_345 { struct nfs_lock_info nfs_fl; struct nfs4_lock_info nfs4_fl; struct __anonstruct_afs_346 afs; } ;
19 struct file_lock { struct file_lock *fl_next; struct list_head fl_list; struct hlist_node fl_link; struct list_head fl_block; fl_owner_t fl_owner; unsigned int fl_flags; unsigned char fl_type; unsigned int fl_pid; int fl_link_cpu; struct pid *fl_nspid; wait_queue_head_t fl_wait; struct file *fl_file; loff_t fl_start; loff_t fl_end; struct fasync_struct *fl_fasync; unsigned long fl_break_time; unsigned long fl_downgrade_time; const struct file_lock_operations *fl_ops; const struct lock_manager_operations *fl_lmops; union __anonunion_fl_u_345 fl_u; } ;
1041 struct file_lock_context { spinlock_t flc_lock; struct list_head flc_flock; struct list_head flc_posix; struct list_head flc_lease; } ;
1244 struct fasync_struct { spinlock_t fa_lock; int magic; int fa_fd; struct fasync_struct *fa_next; struct file *fa_file; struct callback_head fa_rcu; } ;
1279 struct sb_writers { int frozen; wait_queue_head_t wait_unfrozen; struct percpu_rw_semaphore rw_sem[3U]; } ;
1305 struct super_operations ;
1305 struct xattr_handler ;
1305 struct mtd_info ;
1305 struct super_block { struct list_head s_list; dev_t s_dev; unsigned char s_blocksize_bits; unsigned long s_blocksize; loff_t s_maxbytes; struct file_system_type *s_type; const struct super_operations *s_op; const struct dquot_operations *dq_op; const struct quotactl_ops *s_qcop; const struct export_operations *s_export_op; unsigned long s_flags; unsigned long s_iflags; unsigned long s_magic; struct dentry *s_root; struct rw_semaphore s_umount; int s_count; atomic_t s_active; void *s_security; const struct xattr_handler **s_xattr; struct hlist_bl_head s_anon; struct list_head s_mounts; struct block_device *s_bdev; struct backing_dev_info *s_bdi; struct mtd_info *s_mtd; struct hlist_node s_instances; unsigned int s_quota_types; struct quota_info s_dquot; struct sb_writers s_writers; char s_id[32U]; u8 s_uuid[16U]; void *s_fs_info; unsigned int s_max_links; fmode_t s_mode; u32 s_time_gran; struct mutex s_vfs_rename_mutex; char *s_subtype; char *s_options; const struct dentry_operations *s_d_op; int cleancache_poolid; struct shrinker s_shrink; atomic_long_t s_remove_count; int s_readonly_remount; struct workqueue_struct *s_dio_done_wq; struct hlist_head s_pins; struct list_lru s_dentry_lru; struct list_lru s_inode_lru; struct callback_head rcu; struct work_struct destroy_work; struct mutex s_sync_lock; int s_stack_depth; spinlock_t s_inode_list_lock; struct list_head s_inodes; } ;
1554 struct fiemap_extent_info { unsigned int fi_flags; unsigned int fi_extents_mapped; unsigned int fi_extents_max; struct fiemap_extent *fi_extents_start; } ;
1568 struct dir_context ;
1593 struct dir_context { int (*actor)(struct dir_context *, const char *, int, loff_t , u64 , unsigned int); loff_t pos; } ;
1599 struct block_device_operations ;
1600 struct file_operations { struct module *owner; loff_t (*llseek)(struct file *, loff_t , int); ssize_t (*read)(struct file *, char *, size_t , loff_t *); ssize_t (*write)(struct file *, const char *, size_t , loff_t *); ssize_t (*read_iter)(struct kiocb *, struct iov_iter *); ssize_t (*write_iter)(struct kiocb *, struct iov_iter *); int (*iterate)(struct file *, struct dir_context *); unsigned int (*poll)(struct file *, struct poll_table_struct *); long int (*unlocked_ioctl)(struct file *, unsigned int, unsigned long); long int (*compat_ioctl)(struct file *, unsigned int, unsigned long); int (*mmap)(struct file *, struct vm_area_struct *); int (*open)(struct inode *, struct file *); int (*flush)(struct file *, fl_owner_t ); int (*release)(struct inode *, struct file *); int (*fsync)(struct file *, loff_t , loff_t , int); int (*aio_fsync)(struct kiocb *, int); int (*fasync)(int, struct file *, int); int (*lock)(struct file *, int, struct file_lock *); ssize_t (*sendpage)(struct file *, struct page *, int, size_t , loff_t *, int); unsigned long int (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); int (*check_flags)(int); int (*flock)(struct file *, int, struct file_lock *); ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, loff_t *, size_t , unsigned int); ssize_t (*splice_read)(struct file *, loff_t *, struct pipe_inode_info *, size_t , unsigned int); int (*setlease)(struct file *, long, struct file_lock **, void **); long int (*fallocate)(struct file *, int, loff_t , loff_t ); void (*show_fdinfo)(struct seq_file *, struct file *); ssize_t (*copy_file_range)(struct file *, loff_t , struct file *, loff_t , size_t , unsigned int); int (*clone_file_range)(struct file *, loff_t , struct file *, loff_t , u64 ); ssize_t (*dedupe_file_range)(struct file *, u64 , u64 , struct file *, u64 ); } ;
1668 struct inode_operations { struct dentry * (*lookup)(struct inode *, struct dentry *, unsigned int); const char * (*get_link)(struct dentry *, struct inode *, struct delayed_call *); int (*permission)(struct inode *, int); struct posix_acl * (*get_acl)(struct inode *, int); int (*readlink)(struct dentry *, char *, int); int (*create)(struct inode *, struct dentry *, umode_t , bool ); int (*link)(struct dentry *, struct inode *, struct dentry *); int (*unlink)(struct inode *, struct dentry *); int (*symlink)(struct inode *, struct dentry *, const char *); int (*mkdir)(struct inode *, struct dentry *, umode_t ); int (*rmdir)(struct inode *, struct dentry *); int (*mknod)(struct inode *, struct dentry *, umode_t , dev_t ); int (*rename)(struct inode *, struct dentry *, struct inode *, struct dentry *); int (*rename2)(struct inode *, struct dentry *, struct inode *, struct dentry *, unsigned int); int (*setattr)(struct dentry *, struct iattr *); int (*getattr)(struct vfsmount *, struct dentry *, struct kstat *); int (*setxattr)(struct dentry *, const char *, const void *, size_t , int); ssize_t (*getxattr)(struct dentry *, const char *, void *, size_t ); ssize_t (*listxattr)(struct dentry *, char *, size_t ); int (*removexattr)(struct dentry *, const char *); int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 , u64 ); int (*update_time)(struct inode *, struct timespec *, int); int (*atomic_open)(struct inode *, struct dentry *, struct file *, unsigned int, umode_t , int *); int (*tmpfile)(struct inode *, struct dentry *, umode_t ); int (*set_acl)(struct inode *, struct posix_acl *, int); } ;
1723 struct super_operations { struct inode * (*alloc_inode)(struct super_block *); void (*destroy_inode)(struct inode *); void (*dirty_inode)(struct inode *, int); int (*write_inode)(struct inode *, struct writeback_control *); int (*drop_inode)(struct inode *); void (*evict_inode)(struct inode *); void (*put_super)(struct super_block *); int (*sync_fs)(struct super_block *, int); int (*freeze_super)(struct super_block *); int (*freeze_fs)(struct super_block *); int (*thaw_super)(struct super_block *); int (*unfreeze_fs)(struct super_block *); int (*statfs)(struct dentry *, struct kstatfs *); int (*remount_fs)(struct super_block *, int *, char *); void (*umount_begin)(struct super_block *); int (*show_options)(struct seq_file *, struct dentry *); int (*show_devname)(struct seq_file *, struct dentry *); int (*show_path)(struct seq_file *, struct dentry *); int (*show_stats)(struct seq_file *, struct dentry *); ssize_t (*quota_read)(struct super_block *, int, char *, size_t , loff_t ); ssize_t (*quota_write)(struct super_block *, int, const char *, size_t , loff_t ); struct dquot ** (*get_dquots)(struct inode *); int (*bdev_try_to_free_page)(struct super_block *, struct page *, gfp_t ); long int (*nr_cached_objects)(struct super_block *, struct shrink_control *); long int (*free_cached_objects)(struct super_block *, struct shrink_control *); } ;
1962 struct file_system_type { const char *name; int fs_flags; struct dentry * (*mount)(struct file_system_type *, int, const char *, void *); void (*kill_sb)(struct super_block *); struct module *owner; struct file_system_type *next; struct hlist_head fs_supers; struct lock_class_key s_lock_key; struct lock_class_key s_umount_key; struct lock_class_key s_vfs_rename_key; struct lock_class_key s_writers_key[3U]; struct lock_class_key i_lock_key; struct lock_class_key i_mutex_key; struct lock_class_key i_mutex_dir_key; } ;
18 typedef s32 compat_time_t;
39 typedef s32 compat_long_t;
42 typedef u32 compat_ulong_t;
44 typedef u32 compat_uptr_t;
45 struct compat_timespec { compat_time_t tv_sec; s32 tv_nsec; } ;
276 struct compat_robust_list { compat_uptr_t next; } ;
280 struct compat_robust_list_head { struct compat_robust_list list; compat_long_t futex_offset; compat_uptr_t list_op_pending; } ;
15 typedef __u64 Elf64_Addr;
16 typedef __u16 Elf64_Half;
20 typedef __u32 Elf64_Word;
21 typedef __u64 Elf64_Xword;
190 struct elf64_sym { Elf64_Word st_name; unsigned char st_info; unsigned char st_other; Elf64_Half st_shndx; Elf64_Addr st_value; Elf64_Xword st_size; } ;
198 typedef struct elf64_sym Elf64_Sym;
53 struct kernel_param ;
58 struct kernel_param_ops { unsigned int flags; int (*set)(const char *, const struct kernel_param *); int (*get)(char *, const struct kernel_param *); void (*free)(void *); } ;
62 struct kparam_string ;
62 struct kparam_array ;
62 union __anonunion____missing_field_name_365 { void *arg; const struct kparam_string *str; const struct kparam_array *arr; } ;
62 struct kernel_param { const char *name; struct module *mod; const struct kernel_param_ops *ops; const u16 perm; s8 level; u8 flags; union __anonunion____missing_field_name_365 __annonCompField84; } ;
83 struct kparam_string { unsigned int maxlen; char *string; } ;
89 struct kparam_array { unsigned int max; unsigned int elemsize; unsigned int *num; const struct kernel_param_ops *ops; void *elem; } ;
470 struct latch_tree_node { struct rb_node node[2U]; } ;
211 struct mod_arch_specific { } ;
38 struct module_param_attrs ;
38 struct module_kobject { struct kobject kobj; struct module *mod; struct kobject *drivers_dir; struct module_param_attrs *mp; struct completion *kobj_completion; } ;
48 struct module_attribute { struct attribute attr; ssize_t (*show)(struct module_attribute *, struct module_kobject *, char *); ssize_t (*store)(struct module_attribute *, struct module_kobject *, const char *, size_t ); void (*setup)(struct module *, const char *); int (*test)(struct module *); void (*free)(struct module *); } ;
290 enum module_state { MODULE_STATE_LIVE = 0, MODULE_STATE_COMING = 1, MODULE_STATE_GOING = 2, MODULE_STATE_UNFORMED = 3 } ;
297 struct mod_tree_node { struct module *mod; struct latch_tree_node node; } ;
304 struct module_layout { void *base; unsigned int size; unsigned int text_size; unsigned int ro_size; struct mod_tree_node mtn; } ;
318 struct module_sect_attrs ;
318 struct module_notes_attrs ;
318 struct trace_event_call ;
318 struct trace_enum_map ;
318 struct module { enum module_state state; struct list_head list; char name[56U]; struct module_kobject mkobj; struct module_attribute *modinfo_attrs; const char *version; const char *srcversion; struct kobject *holders_dir; const struct kernel_symbol *syms; const unsigned long *crcs; unsigned int num_syms; struct mutex param_lock; struct kernel_param *kp; unsigned int num_kp; unsigned int num_gpl_syms; const struct kernel_symbol *gpl_syms; const unsigned long *gpl_crcs; const struct kernel_symbol *unused_syms; const unsigned long *unused_crcs; unsigned int num_unused_syms; unsigned int num_unused_gpl_syms; const struct kernel_symbol *unused_gpl_syms; const unsigned long *unused_gpl_crcs; bool sig_ok; bool async_probe_requested; const struct kernel_symbol *gpl_future_syms; const unsigned long *gpl_future_crcs; unsigned int num_gpl_future_syms; unsigned int num_exentries; struct exception_table_entry *extable; int (*init)(); struct module_layout core_layout; struct module_layout init_layout; struct mod_arch_specific arch; unsigned int taints; unsigned int num_bugs; struct list_head bug_list; struct bug_entry *bug_table; Elf64_Sym *symtab; Elf64_Sym *core_symtab; unsigned int num_symtab; unsigned int core_num_syms; char *strtab; char *core_strtab; struct module_sect_attrs *sect_attrs; struct module_notes_attrs *notes_attrs; char *args; void *percpu; unsigned int percpu_size; unsigned int num_tracepoints; const struct tracepoint **tracepoints_ptrs; unsigned int num_trace_bprintk_fmt; const char **trace_bprintk_fmt_start; struct trace_event_call **trace_events; unsigned int num_trace_events; struct trace_enum_map **trace_enums; unsigned int num_trace_enums; bool klp_alive; struct list_head source_list; struct list_head target_list; void (*exit)(); atomic_t refcnt; ctor_fn_t (**ctors)(); unsigned int num_ctors; } ;
81 struct disk_stats { unsigned long sectors[2U]; unsigned long ios[2U]; unsigned long merges[2U]; unsigned long ticks[2U]; unsigned long io_ticks; unsigned long time_in_queue; } ;
90 struct partition_meta_info { char uuid[37U]; u8 volname[64U]; } ;
102 struct hd_struct { sector_t start_sect; sector_t nr_sects; seqcount_t nr_sects_seq; sector_t alignment_offset; unsigned int discard_alignment; struct device __dev; struct kobject *holder_dir; int policy; int partno; struct partition_meta_info *info; int make_it_fail; unsigned long stamp; atomic_t in_flight[2U]; struct disk_stats *dkstats; struct percpu_ref ref; struct callback_head callback_head; } ;
156 struct disk_part_tbl { struct callback_head callback_head; int len; struct hd_struct *last_lookup; struct hd_struct *part[]; } ;
163 struct disk_events ;
164 struct badblocks ;
165 struct blk_integrity_profile ;
165 struct blk_integrity { struct blk_integrity_profile *profile; unsigned char flags; unsigned char tuple_size; unsigned char interval_exp; unsigned char tag_size; } ;
176 struct timer_rand_state ;
176 struct gendisk { int major; int first_minor; int minors; char disk_name[32U]; char * (*devnode)(struct gendisk *, umode_t *); unsigned int events; unsigned int async_events; struct disk_part_tbl *part_tbl; struct hd_struct part0; const struct block_device_operations *fops; struct request_queue *queue; void *private_data; int flags; struct device *driverfs_dev; struct kobject *slave_dir; struct timer_rand_state *random; atomic_t sync_io; struct disk_events *ev; struct kobject integrity_kobj; int node_id; struct badblocks *bb; } ;
72 struct fprop_local_percpu { struct percpu_counter events; unsigned int period; raw_spinlock_t lock; } ;
32 typedef int congested_fn(void *, int);
41 struct bdi_writeback_congested { unsigned long state; atomic_t refcnt; struct backing_dev_info *bdi; int blkcg_id; struct rb_node rb_node; } ;
60 union __anonunion____missing_field_name_372 { struct work_struct release_work; struct callback_head rcu; } ;
60 struct bdi_writeback { struct backing_dev_info *bdi; unsigned long state; unsigned long last_old_flush; struct list_head b_dirty; struct list_head b_io; struct list_head b_more_io; struct list_head b_dirty_time; spinlock_t list_lock; struct percpu_counter stat[4U]; struct bdi_writeback_congested *congested; unsigned long bw_time_stamp; unsigned long dirtied_stamp; unsigned long written_stamp; unsigned long write_bandwidth; unsigned long avg_write_bandwidth; unsigned long dirty_ratelimit; unsigned long balanced_dirty_ratelimit; struct fprop_local_percpu completions; int dirty_exceeded; spinlock_t work_lock; struct list_head work_list; struct delayed_work dwork; struct list_head bdi_node; struct percpu_ref refcnt; struct fprop_local_percpu memcg_completions; struct cgroup_subsys_state *memcg_css; struct cgroup_subsys_state *blkcg_css; struct list_head memcg_node; struct list_head blkcg_node; union __anonunion____missing_field_name_372 __annonCompField85; } ;
134 struct backing_dev_info { struct list_head bdi_list; unsigned long ra_pages; unsigned int capabilities; congested_fn *congested_fn; void *congested_data; char *name; unsigned int min_ratio; unsigned int max_ratio; unsigned int max_prop_frac; atomic_long_t tot_write_bandwidth; struct bdi_writeback wb; struct list_head wb_list; struct radix_tree_root cgwb_tree; struct rb_root cgwb_congested_tree; atomic_t usage_cnt; wait_queue_head_t wb_waitq; struct device *dev; struct timer_list laptop_mode_wb_timer; struct dentry *debug_dir; struct dentry *debug_stats; } ;
11 typedef void * mempool_alloc_t(gfp_t , void *);
12 typedef void mempool_free_t(void *, void *);
13 struct mempool_s { spinlock_t lock; int min_nr; int curr_nr; void **elements; void *pool_data; mempool_alloc_t *alloc; mempool_free_t *free; wait_queue_head_t wait; } ;
24 typedef struct mempool_s mempool_t;
78 union __anonunion____missing_field_name_373 { struct list_head q_node; struct kmem_cache *__rcu_icq_cache; } ;
78 union __anonunion____missing_field_name_374 { struct hlist_node ioc_node; struct callback_head __rcu_head; } ;
78 struct io_cq { struct request_queue *q; struct io_context *ioc; union __anonunion____missing_field_name_373 __annonCompField86; union __anonunion____missing_field_name_374 __annonCompField87; unsigned int flags; } ;
92 struct io_context { atomic_long_t refcount; atomic_t active_ref; atomic_t nr_tasks; spinlock_t lock; unsigned short ioprio; int nr_batch_requests; unsigned long last_waited; struct radix_tree_root icq_tree; struct io_cq *icq_hint; struct hlist_head icq_list; struct work_struct release_work; } ;
320 struct bio_integrity_payload { struct bio *bip_bio; struct bvec_iter bip_iter; bio_end_io_t *bip_end_io; unsigned short bip_slab; unsigned short bip_vcnt; unsigned short bip_max_vcnt; unsigned short bip_flags; struct work_struct bip_work; struct bio_vec *bip_vec; struct bio_vec bip_inline_vecs[0U]; } ;
551 struct bio_list { struct bio *head; struct bio *tail; } ;
672 struct bio_set { struct kmem_cache *bio_slab; unsigned int front_pad; mempool_t *bio_pool; mempool_t *bvec_pool; mempool_t *bio_integrity_pool; mempool_t *bvec_integrity_pool; spinlock_t rescue_lock; struct bio_list rescue_list; struct work_struct rescue_work; struct workqueue_struct *rescue_workqueue; } ;
64 struct bsg_class_device { struct device *class_dev; struct device *parent; int minor; struct request_queue *queue; struct kref ref; void (*release)(struct device *); } ;
22 struct elevator_queue ;
24 struct request ;
26 struct bsg_job ;
27 struct blkcg_gq ;
28 struct blk_flush_queue ;
29 struct pr_ops ;
51 typedef void rq_end_io_fn(struct request *, int);
52 struct request_list { struct request_queue *q; struct blkcg_gq *blkg; int count[2U]; int starved[2U]; mempool_t *rq_pool; wait_queue_head_t wait[2U]; unsigned int flags; } ;
77 union __anonunion____missing_field_name_375 { struct call_single_data csd; unsigned long fifo_time; } ;
77 struct blk_mq_ctx ;
77 union __anonunion____missing_field_name_376 { struct hlist_node hash; struct list_head ipi_list; } ;
77 union __anonunion____missing_field_name_377 { struct rb_node rb_node; void *completion_data; } ;
77 struct __anonstruct_elv_379 { struct io_cq *icq; void *priv[2U]; } ;
77 struct __anonstruct_flush_380 { unsigned int seq; struct list_head list; rq_end_io_fn *saved_end_io; } ;
77 union __anonunion____missing_field_name_378 { struct __anonstruct_elv_379 elv; struct __anonstruct_flush_380 flush; } ;
77 struct request { struct list_head queuelist; union __anonunion____missing_field_name_375 __annonCompField88; struct request_queue *q; struct blk_mq_ctx *mq_ctx; u64 cmd_flags; unsigned int cmd_type; unsigned long atomic_flags; int cpu; unsigned int __data_len; sector_t __sector; struct bio *bio; struct bio *biotail; union __anonunion____missing_field_name_376 __annonCompField89; union __anonunion____missing_field_name_377 __annonCompField90; union __anonunion____missing_field_name_378 __annonCompField91; struct gendisk *rq_disk; struct hd_struct *part; unsigned long start_time; struct request_list *rl; unsigned long long start_time_ns; unsigned long long io_start_time_ns; unsigned short nr_phys_segments; unsigned short nr_integrity_segments; unsigned short ioprio; void *special; int tag; int errors; unsigned char __cmd[16U]; unsigned char *cmd; unsigned short cmd_len; unsigned int extra_len; unsigned int sense_len; unsigned int resid_len; void *sense; unsigned long deadline; struct list_head timeout_list; unsigned int timeout; int retries; rq_end_io_fn *end_io; void *end_io_data; struct request *next_rq; } ;
117 struct elevator_type ;
12 typedef int elevator_merge_fn(struct request_queue *, struct request **, struct bio *);
15 typedef void elevator_merge_req_fn(struct request_queue *, struct request *, struct request *);
17 typedef void elevator_merged_fn(struct request_queue *, struct request *, int);
19 typedef int elevator_allow_merge_fn(struct request_queue *, struct request *, struct bio *);
21 typedef void elevator_bio_merged_fn(struct request_queue *, struct request *, struct bio *);
24 typedef int elevator_dispatch_fn(struct request_queue *, int);
26 typedef void elevator_add_req_fn(struct request_queue *, struct request *);
27 typedef struct request * elevator_request_list_fn(struct request_queue *, struct request *);
28 typedef void elevator_completed_req_fn(struct request_queue *, struct request *);
29 typedef int elevator_may_queue_fn(struct request_queue *, int);
31 typedef void elevator_init_icq_fn(struct io_cq *);
32 typedef void elevator_exit_icq_fn(struct io_cq *);
33 typedef int elevator_set_req_fn(struct request_queue *, struct request *, struct bio *, gfp_t );
35 typedef void elevator_put_req_fn(struct request *);
36 typedef void elevator_activate_req_fn(struct request_queue *, struct request *);
37 typedef void elevator_deactivate_req_fn(struct request_queue *, struct request *);
39 typedef int elevator_init_fn(struct request_queue *, struct elevator_type *);
41 typedef void elevator_exit_fn(struct elevator_queue *);
42 typedef void elevator_registered_fn(struct request_queue *);
43 struct elevator_ops { elevator_merge_fn *elevator_merge_fn; elevator_merged_fn *elevator_merged_fn; elevator_merge_req_fn *elevator_merge_req_fn; elevator_allow_merge_fn *elevator_allow_merge_fn; elevator_bio_merged_fn *elevator_bio_merged_fn; elevator_dispatch_fn *elevator_dispatch_fn; elevator_add_req_fn *elevator_add_req_fn; elevator_activate_req_fn *elevator_activate_req_fn; elevator_deactivate_req_fn *elevator_deactivate_req_fn; elevator_completed_req_fn *elevator_completed_req_fn; elevator_request_list_fn *elevator_former_req_fn; elevator_request_list_fn *elevator_latter_req_fn; elevator_init_icq_fn *elevator_init_icq_fn; elevator_exit_icq_fn *elevator_exit_icq_fn; elevator_set_req_fn *elevator_set_req_fn; elevator_put_req_fn *elevator_put_req_fn; elevator_may_queue_fn *elevator_may_queue_fn; elevator_init_fn *elevator_init_fn; elevator_exit_fn *elevator_exit_fn; elevator_registered_fn *elevator_registered_fn; } ;
74 struct elv_fs_entry { struct attribute attr; ssize_t (*show)(struct elevator_queue *, char *); ssize_t (*store)(struct elevator_queue *, const char *, size_t ); } ;
82 struct elevator_type { struct kmem_cache *icq_cache; struct elevator_ops ops; size_t icq_size; size_t icq_align; struct elv_fs_entry *elevator_attrs; char elevator_name[16U]; struct module *elevator_owner; char icq_cache_name[21U]; struct list_head list; } ;
103 struct elevator_queue { struct elevator_type *type; void *elevator_data; struct kobject kobj; struct mutex sysfs_lock; unsigned char registered; struct hlist_head hash[64U]; } ;
212 typedef void request_fn_proc(struct request_queue *);
213 typedef blk_qc_t make_request_fn(struct request_queue *, struct bio *);
214 typedef int prep_rq_fn(struct request_queue *, struct request *);
215 typedef void unprep_rq_fn(struct request_queue *, struct request *);
218 typedef void softirq_done_fn(struct request *);
219 typedef int dma_drain_needed_fn(struct request *);
220 typedef int lld_busy_fn(struct request_queue *);
221 typedef int bsg_job_fn(struct bsg_job *);
222 enum blk_eh_timer_return { BLK_EH_NOT_HANDLED = 0, BLK_EH_HANDLED = 1, BLK_EH_RESET_TIMER = 2 } ;
229 typedef enum blk_eh_timer_return rq_timed_out_fn(struct request *);
235 struct blk_queue_tag { struct request **tag_index; unsigned long *tag_map; int busy; int max_depth; int real_max_depth; atomic_t refcnt; int alloc_policy; int next_tag; } ;
246 struct queue_limits { unsigned long bounce_pfn; unsigned long seg_boundary_mask; unsigned long virt_boundary_mask; unsigned int max_hw_sectors; unsigned int max_dev_sectors; unsigned int chunk_sectors; unsigned int max_sectors; unsigned int max_segment_size; unsigned int physical_block_size; unsigned int alignment_offset; unsigned int io_min; unsigned int io_opt; unsigned int max_discard_sectors; unsigned int max_hw_discard_sectors; unsigned int max_write_same_sectors; unsigned int discard_granularity; unsigned int discard_alignment; unsigned short logical_block_size; unsigned short max_segments; unsigned short max_integrity_segments; unsigned char misaligned; unsigned char discard_misaligned; unsigned char cluster; unsigned char discard_zeroes_data; unsigned char raid_partial_stripes_expensive; } ;
282 struct blk_mq_ops ;
282 struct blk_mq_hw_ctx ;
282 struct throtl_data ;
282 struct blk_mq_tag_set ;
282 struct request_queue { struct list_head queue_head; struct request *last_merge; struct elevator_queue *elevator; int nr_rqs[2U]; int nr_rqs_elvpriv; struct request_list root_rl; request_fn_proc *request_fn; make_request_fn *make_request_fn; prep_rq_fn *prep_rq_fn; unprep_rq_fn *unprep_rq_fn; softirq_done_fn *softirq_done_fn; rq_timed_out_fn *rq_timed_out_fn; dma_drain_needed_fn *dma_drain_needed; lld_busy_fn *lld_busy_fn; struct blk_mq_ops *mq_ops; unsigned int *mq_map; struct blk_mq_ctx *queue_ctx; unsigned int nr_queues; struct blk_mq_hw_ctx **queue_hw_ctx; unsigned int nr_hw_queues; sector_t end_sector; struct request *boundary_rq; struct delayed_work delay_work; struct backing_dev_info backing_dev_info; void *queuedata; unsigned long queue_flags; int id; gfp_t bounce_gfp; spinlock_t __queue_lock; spinlock_t *queue_lock; struct kobject kobj; struct kobject mq_kobj; struct blk_integrity integrity; struct device *dev; int rpm_status; unsigned int nr_pending; unsigned long nr_requests; unsigned int nr_congestion_on; unsigned int nr_congestion_off; unsigned int nr_batching; unsigned int dma_drain_size; void *dma_drain_buffer; unsigned int dma_pad_mask; unsigned int dma_alignment; struct blk_queue_tag *queue_tags; struct list_head tag_busy_list; unsigned int nr_sorted; unsigned int in_flight[2U]; unsigned int request_fn_active; unsigned int rq_timeout; struct timer_list timeout; struct work_struct timeout_work; struct list_head timeout_list; struct list_head icq_list; unsigned long blkcg_pols[1U]; struct blkcg_gq *root_blkg; struct list_head blkg_list; struct queue_limits limits; unsigned int sg_timeout; unsigned int sg_reserved_size; int node; unsigned int flush_flags; unsigned char flush_not_queueable; struct blk_flush_queue *fq; struct list_head requeue_list; spinlock_t requeue_lock; struct work_struct requeue_work; struct mutex sysfs_lock; int bypass_depth; atomic_t mq_freeze_depth; bsg_job_fn *bsg_job_fn; int bsg_job_size; struct bsg_class_device bsg_dev; struct throtl_data *td; struct callback_head callback_head; wait_queue_head_t mq_freeze_wq; struct percpu_ref q_usage_counter; struct list_head all_q_node; struct blk_mq_tag_set *tag_set; struct list_head tag_set_list; struct bio_set *bio_split; bool mq_sysfs_init_done; } ;
1029 struct blk_plug { struct list_head list; struct list_head mq_list; struct list_head cb_list; } ;
1446 struct blk_integrity_iter { void *prot_buf; void *data_buf; sector_t seed; unsigned int data_size; unsigned short interval; const char *disk_name; } ;
1475 typedef int integrity_processing_fn(struct blk_integrity_iter *);
1476 struct blk_integrity_profile { integrity_processing_fn *generate_fn; integrity_processing_fn *verify_fn; const char *name; } ;
1635 struct block_device_operations { int (*open)(struct block_device *, fmode_t ); void (*release)(struct gendisk *, fmode_t ); int (*rw_page)(struct block_device *, sector_t , struct page *, int); int (*ioctl)(struct block_device *, fmode_t , unsigned int, unsigned long); int (*compat_ioctl)(struct block_device *, fmode_t , unsigned int, unsigned long); long int (*direct_access)(struct block_device *, sector_t , void **, pfn_t *); unsigned int (*check_events)(struct gendisk *, unsigned int); int (*media_changed)(struct gendisk *); void (*unlock_native_capacity)(struct gendisk *); int (*revalidate_disk)(struct gendisk *); int (*getgeo)(struct block_device *, struct hd_geometry *); void (*swap_slot_free_notify)(struct block_device *, unsigned long); struct module *owner; const struct pr_ops *pr_ops; } ;
1663 struct blk_mq_tags ;
1664 struct blk_mq_cpu_notifier { struct list_head list; void *data; int (*notify)(void *, unsigned long, unsigned int); } ;
14 struct blk_align_bitmap ;
14 struct blk_mq_ctxmap { unsigned int size; unsigned int bits_per_word; struct blk_align_bitmap *map; } ;
20 struct __anonstruct____missing_field_name_382 { spinlock_t lock; struct list_head dispatch; } ;
20 struct blk_mq_hw_ctx { struct __anonstruct____missing_field_name_382 __annonCompField92; unsigned long state; struct delayed_work run_work; struct delayed_work delay_work; cpumask_var_t cpumask; int next_cpu; int next_cpu_batch; unsigned long flags; struct request_queue *queue; struct blk_flush_queue *fq; void *driver_data; struct blk_mq_ctxmap ctx_map; unsigned int nr_ctx; struct blk_mq_ctx **ctxs; atomic_t wait_index; struct blk_mq_tags *tags; unsigned long queued; unsigned long run; unsigned long dispatched[10U]; unsigned int numa_node; unsigned int queue_num; atomic_t nr_active; struct blk_mq_cpu_notifier cpu_notifier; struct kobject kobj; unsigned long poll_invoked; unsigned long poll_success; } ;
66 struct blk_mq_tag_set { struct blk_mq_ops *ops; unsigned int nr_hw_queues; unsigned int queue_depth; unsigned int reserved_tags; unsigned int cmd_size; int numa_node; unsigned int timeout; unsigned int flags; void *driver_data; struct blk_mq_tags **tags; struct mutex tag_list_lock; struct list_head tag_list; } ;
83 struct blk_mq_queue_data { struct request *rq; struct list_head *list; bool last; } ;
90 typedef int queue_rq_fn(struct blk_mq_hw_ctx *, const struct blk_mq_queue_data *);
91 typedef struct blk_mq_hw_ctx * map_queue_fn(struct request_queue *, const int);
92 typedef enum blk_eh_timer_return timeout_fn(struct request *, bool );
93 typedef int init_hctx_fn(struct blk_mq_hw_ctx *, void *, unsigned int);
94 typedef void exit_hctx_fn(struct blk_mq_hw_ctx *, unsigned int);
95 typedef int init_request_fn(void *, struct request *, unsigned int, unsigned int, unsigned int);
97 typedef void exit_request_fn(void *, struct request *, unsigned int, unsigned int);
103 typedef int poll_fn(struct blk_mq_hw_ctx *, unsigned int);
104 struct blk_mq_ops { queue_rq_fn *queue_rq; map_queue_fn *map_queue; timeout_fn *timeout; poll_fn *poll; softirq_done_fn *complete; init_hctx_fn *init_hctx; exit_hctx_fn *exit_hctx; init_request_fn *init_request; exit_request_fn *exit_request; } ;
933 enum fwnode_type { FWNODE_INVALID = 0, FWNODE_OF = 1, FWNODE_ACPI = 2, FWNODE_ACPI_DATA = 3, FWNODE_PDATA = 4, FWNODE_IRQCHIP = 5 } ;
942 struct fwnode_handle { enum fwnode_type type; struct fwnode_handle *secondary; } ;
162 typedef u64 acpi_io_address;
450 typedef void *acpi_handle;
644 typedef u32 acpi_object_type;
901 struct __anonstruct_integer_389 { acpi_object_type type; u64 value; } ;
901 struct __anonstruct_string_390 { acpi_object_type type; u32 length; char *pointer; } ;
901 struct __anonstruct_buffer_391 { acpi_object_type type; u32 length; u8 *pointer; } ;
901 struct __anonstruct_package_392 { acpi_object_type type; u32 count; union acpi_object *elements; } ;
901 struct __anonstruct_reference_393 { acpi_object_type type; acpi_object_type actual_type; acpi_handle handle; } ;
901 struct __anonstruct_processor_394 { acpi_object_type type; u32 proc_id; acpi_io_address pblk_address; u32 pblk_length; } ;
901 struct __anonstruct_power_resource_395 { acpi_object_type type; u32 system_level; u32 resource_order; } ;
901 union acpi_object { acpi_object_type type; struct __anonstruct_integer_389 integer; struct __anonstruct_string_390 string; struct __anonstruct_buffer_391 buffer; struct __anonstruct_package_392 package; struct __anonstruct_reference_393 reference; struct __anonstruct_processor_394 processor; struct __anonstruct_power_resource_395 power_resource; } ;
108 struct acpi_driver ;
109 struct acpi_hotplug_profile { struct kobject kobj; int (*scan_dependent)(struct acpi_device *); void (*notify_online)(struct acpi_device *); bool enabled; bool demand_offline; } ;
131 struct acpi_scan_handler { const struct acpi_device_id *ids; struct list_head list_node; bool (*match)(const char *, const struct acpi_device_id **); int (*attach)(struct acpi_device *, const struct acpi_device_id *); void (*detach)(struct acpi_device *); void (*bind)(struct device *); void (*unbind)(struct device *); struct acpi_hotplug_profile hotplug; } ;
141 struct acpi_hotplug_context { struct acpi_device *self; int (*notify)(struct acpi_device *, u32 ); void (*uevent)(struct acpi_device *, u32 ); void (*fixup)(struct acpi_device *); } ;
162 struct acpi_device_ops { int (*add)(struct acpi_device *); int (*remove)(struct acpi_device *); void (*notify)(struct acpi_device *, u32 ); } ;
168 struct acpi_driver { char name[80U]; char class[80U]; const struct acpi_device_id *ids; unsigned int flags; struct acpi_device_ops ops; struct device_driver drv; struct module *owner; } ;
180 struct acpi_device_status { unsigned char present; unsigned char enabled; unsigned char show_in_ui; unsigned char functional; unsigned char battery_present; unsigned int reserved; } ;
196 struct acpi_device_flags { unsigned char dynamic_status; unsigned char removable; unsigned char ejectable; unsigned char power_manageable; unsigned char match_driver; unsigned char initialized; unsigned char visited; unsigned char hotplug_notify; unsigned char is_dock_station; unsigned char of_compatible_ok; unsigned char coherent_dma; unsigned char cca_seen; unsigned int reserved; } ;
214 struct acpi_device_dir { struct proc_dir_entry *entry; } ;
225 typedef char acpi_bus_id[8U];
226 typedef unsigned long acpi_bus_address;
227 typedef char acpi_device_name[40U];
228 typedef char acpi_device_class[20U];
234 struct acpi_pnp_type { unsigned char hardware_id; unsigned char bus_address; unsigned char platform_id; unsigned int reserved; } ;
241 struct acpi_device_pnp { acpi_bus_id bus_id; struct acpi_pnp_type type; acpi_bus_address bus_address; char *unique_id; struct list_head ids; acpi_device_name device_name; acpi_device_class device_class; union acpi_object *str_obj; } ;
256 struct acpi_device_power_flags { unsigned char explicit_get; unsigned char power_resources; unsigned char inrush_current; unsigned char power_removed; unsigned char ignore_parent; unsigned char dsw_present; unsigned int reserved; } ;
271 struct __anonstruct_flags_396 { unsigned char valid; unsigned char explicit_set; unsigned char reserved; } ;
271 struct acpi_device_power_state { struct __anonstruct_flags_396 flags; int power; int latency; struct list_head resources; } ;
282 struct acpi_device_power { int state; struct acpi_device_power_flags flags; struct acpi_device_power_state states[5U]; } ;
288 struct acpi_device_perf_flags { u8 reserved; } ;
294 struct __anonstruct_flags_397 { unsigned char valid; unsigned char reserved; } ;
294 struct acpi_device_perf_state { struct __anonstruct_flags_397 flags; u8 power; u8 performance; int latency; } ;
304 struct acpi_device_perf { int state; struct acpi_device_perf_flags flags; int state_count; struct acpi_device_perf_state *states; } ;
311 struct acpi_device_wakeup_flags { unsigned char valid; unsigned char run_wake; unsigned char notifier_present; unsigned char enabled; } ;
319 struct acpi_device_wakeup_context { struct work_struct work; struct device *dev; } ;
324 struct acpi_device_wakeup { acpi_handle gpe_device; u64 gpe_number; u64 sleep_state; struct list_head resources; struct acpi_device_wakeup_flags flags; struct acpi_device_wakeup_context context; struct wakeup_source *ws; int prepare_count; } ;
342 struct acpi_device_data { const union acpi_object *pointer; const union acpi_object *properties; const union acpi_object *of_compatible; struct list_head subnodes; } ;
350 struct acpi_gpio_mapping ;
351 struct acpi_device { int device_type; acpi_handle handle; struct fwnode_handle fwnode; struct acpi_device *parent; struct list_head children; struct list_head node; struct list_head wakeup_list; struct list_head del_list; struct acpi_device_status status; struct acpi_device_flags flags; struct acpi_device_pnp pnp; struct acpi_device_power power; struct acpi_device_wakeup wakeup; struct acpi_device_perf performance; struct acpi_device_dir dir; struct acpi_device_data data; struct acpi_scan_handler *handler; struct acpi_hotplug_context *hp; struct acpi_driver *driver; const struct acpi_gpio_mapping *driver_gpios; void *driver_data; struct device dev; unsigned int physical_node_count; unsigned int dep_unmet; struct list_head physical_node_list; struct mutex physical_node_lock; void (*remove)(struct acpi_device *); } ;
769 struct acpi_gpio_params { unsigned int crs_entry_index; unsigned int line_index; bool active_low; } ;
822 struct acpi_gpio_mapping { const char *name; const struct acpi_gpio_params *data; unsigned int size; } ;
32 typedef u32 phandle;
34 struct property { char *name; int length; void *value; struct property *next; unsigned long _flags; unsigned int unique_id; struct bin_attribute attr; } ;
44 struct device_node { const char *name; const char *type; phandle phandle; const char *full_name; struct fwnode_handle fwnode; struct property *properties; struct property *deadprops; struct device_node *parent; struct device_node *child; struct device_node *sibling; struct kobject kobj; unsigned long _flags; void *data; } ;
77 struct smart_attr { u8 attr_id; u16 flags; u8 cur; u8 worst; u32 data; u8 res[3U]; } ;
173 struct mtip_work { struct work_struct work; void *port; int cpu_binding; u32 completed; } ;
180 struct mtip_trim_entry { u32 lba; u16 rsvd; u16 range; } ;
202 union __anonunion____missing_field_name_407 { unsigned char lba_low; unsigned char sector; } ;
202 union __anonunion____missing_field_name_408 { unsigned char lba_mid; unsigned char cyl_low; } ;
202 union __anonunion____missing_field_name_409 { unsigned char lba_hi; unsigned char cyl_hi; } ;
202 union __anonunion____missing_field_name_410 { unsigned char device; unsigned char head; } ;
202 union __anonunion____missing_field_name_411 { unsigned char lba_low_ex; unsigned char sector_ex; } ;
202 union __anonunion____missing_field_name_412 { unsigned char lba_mid_ex; unsigned char cyl_low_ex; } ;
202 union __anonunion____missing_field_name_413 { unsigned char lba_hi_ex; unsigned char cyl_hi_ex; } ;
202 struct host_to_dev_fis { unsigned char type; unsigned char opts; unsigned char command; unsigned char features; union __anonunion____missing_field_name_407 __annonCompField97; union __anonunion____missing_field_name_408 __annonCompField98; union __anonunion____missing_field_name_409 __annonCompField99; union __anonunion____missing_field_name_410 __annonCompField100; union __anonunion____missing_field_name_411 __annonCompField101; union __anonunion____missing_field_name_412 __annonCompField102; union __anonunion____missing_field_name_413 __annonCompField103; unsigned char features_ex; unsigned char sect_count; unsigned char sect_cnt_ex; unsigned char res2; unsigned char control; unsigned int res3; } ;
259 union __anonunion____missing_field_name_414 { unsigned int byte_count; unsigned int status; } ;
259 struct mtip_cmd_hdr { unsigned int opts; union __anonunion____missing_field_name_414 __annonCompField104; unsigned int ctba; unsigned int ctbau; unsigned int res[4U]; } ;
290 struct mtip_cmd_sg { unsigned int dba; unsigned int dba_upper; unsigned int reserved; unsigned int info; } ;
314 struct mtip_port ;
315 struct mtip_cmd { struct mtip_cmd_hdr *command_header; dma_addr_t command_header_dma; void *command; dma_addr_t command_dma; void *comp_data; void (*comp_func)(struct mtip_port *, int, struct mtip_cmd *, int); int scatter_ents; int unaligned; struct scatterlist sg[504U]; int retries; int direction; } ;
347 struct driver_data ;
347 struct mtip_port { struct driver_data *dd; unsigned long identify_valid; void *mmio; void *s_active[8U]; void *completed[8U]; void *cmd_issue[8U]; void *command_list; dma_addr_t command_list_dma; void *rxfis; dma_addr_t rxfis_dma; void *block1; dma_addr_t block1_dma; u16 *identify; dma_addr_t identify_dma; u16 *sector_buffer; dma_addr_t sector_buffer_dma; u16 *log_buf; dma_addr_t log_buf_dma; u8 *smart_buf; dma_addr_t smart_buf_dma; unsigned long cmds_to_issue[4U]; wait_queue_head_t svc_wait; unsigned long flags; unsigned long ic_pause_timer; struct semaphore cmd_slot_unal; spinlock_t cmd_issue_lock[8U]; } ;
443 struct driver_data { void *mmio; int major; int instance; struct gendisk *disk; struct pci_dev *pdev; struct request_queue *queue; struct blk_mq_tag_set tags; struct mtip_port *port; unsigned int product_type; unsigned int slot_groups; unsigned long index; unsigned long dd_flag; struct task_struct *mtip_svc_handler; struct dentry *dfs_node; bool trim_supp; bool sr; int numa_node; char workq_name[32U]; struct workqueue_struct *isr_workq; atomic_t irq_workers_active; struct mtip_work work[8U]; int isr_binding; struct block_device *bdev; struct list_head online_list; struct list_head remove_list; int unal_qdepth; } ;
126 struct mtip_compat_ide_task_request_s { __u8 io_ports[8U]; __u8 hob_ports[8U]; ide_reg_valid_t out_flags; ide_reg_valid_t in_flags; int data_phase; int req_cmd; compat_ulong_t out_size; compat_ulong_t in_size; } ;
38 typedef int Set;
1 long int __builtin_expect(long, long);
1 void __builtin_prefetch(const void *, ...);
1 void * __builtin_memcpy(void *, const void *, unsigned long);
1 unsigned long int __builtin_object_size(void *, int);
216 void __read_once_size(const volatile void *p, void *res, int size);
241 void __write_once_size(volatile void *p, void *res, int size);
7 dma_addr_t ldv_dma_map_single_attrs(struct device *dev, void *ptr, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs);
12 void * memdup_user(const void *, size_t );
31 void * __memcpy(void *, const void *, size_t );
56 void * __memset(void *, int, size_t );
60 void * __memmove(void *, const void *, size_t );
63 size_t strlen(const char *);
27 size_t strlcpy(char *, const char *, size_t );
33 extern struct module __this_module;
72 void set_bit(long nr, volatile unsigned long *addr);
110 void clear_bit(long nr, volatile unsigned long *addr);
308 int constant_test_bit(long nr, const volatile unsigned long *addr);
314 int variable_test_bit(long nr, const volatile unsigned long *addr);
14 unsigned long int find_next_bit(const unsigned long *, unsigned long, unsigned long);
42 unsigned long int find_first_bit(const unsigned long *, unsigned long);
46 __u16 __fswab16(__u16 val);
154 __u16 __swab16p(const __u16 *p);
223 void __swab16s(__u16 *p);
142 int printk(const char *, ...);
53 void __dynamic_dev_dbg(struct _ddebug *, const struct device *, const char *, ...);
178 void __might_sleep(const char *, int, int);
248 void __might_fault(const char *, int);
403 int sprintf(char *, const char *, ...);
406 int snprintf(char *, size_t , const char *, ...);
25 void INIT_LIST_HEAD(struct list_head *list);
48 void __list_add(struct list_head *, struct list_head *, struct list_head *);
61 void list_add(struct list_head *new, struct list_head *head);
112 void __list_del_entry(struct list_head *);
154 void list_del_init(struct list_head *entry);
28 long int PTR_ERR(const void *ptr);
33 bool IS_ERR(const void *ptr);
38 bool IS_ERR_OR_NULL(const void *ptr);
87 void __bad_percpu_size();
295 void __bad_size_call_parameter();
18 extern unsigned long __per_cpu_offset[8192U];
15 void __xadd_wrong_size();
24 int atomic_read(const atomic_t *v);
36 void atomic_set(atomic_t *v, int i);
154 int atomic_add_return(int i, atomic_t *v);
166 int atomic_sub_return(int i, atomic_t *v);
71 void warn_slowpath_null(const char *, const int);
280 void lockdep_init_map(struct lockdep_map *, const char *, struct lock_class_key *, int);
112 int __bitmap_weight(const unsigned long *, unsigned int);
282 int bitmap_empty(const unsigned long *src, unsigned int nbits);
298 int bitmap_weight(const unsigned long *src, unsigned int nbits);
37 extern int nr_cpu_ids;
90 extern struct cpumask __cpu_present_mask;
118 unsigned int cpumask_check(unsigned int cpu);
176 unsigned int cpumask_first(const struct cpumask *srcp);
188 unsigned int cpumask_next(int n, const struct cpumask *srcp);
460 bool cpumask_empty(const struct cpumask *srcp);
478 unsigned int cpumask_weight(const struct cpumask *srcp);
793 extern const unsigned long cpu_bit_bitmap[65U][128U];
795 const struct cpumask * get_cpu_mask(unsigned int cpu);
163 extern struct cpuinfo_x86 cpu_info;
93 void __raw_spin_lock_init(raw_spinlock_t *, const char *, struct lock_class_key *);
22 void _raw_spin_lock(raw_spinlock_t *);
34 unsigned long int _raw_spin_lock_irqsave(raw_spinlock_t *);
41 void _raw_spin_unlock(raw_spinlock_t *);
45 void _raw_spin_unlock_irqrestore(raw_spinlock_t *, unsigned long);
289 raw_spinlock_t * spinlock_check(spinlock_t *lock);
300 void spin_lock(spinlock_t *lock);
345 void spin_unlock(spinlock_t *lock);
360 void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags);
72 void __init_waitqueue_head(wait_queue_head_t *, const char *, struct lock_class_key *);
200 void __wake_up(wait_queue_head_t *, unsigned int, int, void *);
977 long int prepare_to_wait_event(wait_queue_head_t *, wait_queue_t *, int);
978 void finish_wait(wait_queue_head_t *, wait_queue_t *);
73 void init_completion(struct completion *x);
99 long int wait_for_completion_interruptible_timeout(struct completion *, unsigned long);
106 void complete(struct completion *);
78 extern volatile unsigned long jiffies;
284 unsigned int jiffies_to_msecs(const unsigned long);
292 unsigned long int __msecs_to_jiffies(const unsigned int);
354 unsigned long int msecs_to_jiffies(const unsigned int m);
176 int ida_pre_get(struct ida *, gfp_t );
177 int ida_get_new_above(struct ida *, int, int *);
178 void ida_remove(struct ida *, int);
193 int ida_get_new(struct ida *ida, int *p_id);
228 int sysfs_create_file_ns(struct kobject *, const struct attribute *, const void *);
235 void sysfs_remove_file_ns(struct kobject *, const struct attribute *, const void *);
491 int sysfs_create_file(struct kobject *kobj, const struct attribute *attr);
497 void sysfs_remove_file(struct kobject *kobj, const struct attribute *attr);
181 void __init_work(struct work_struct *, int);
361 struct workqueue_struct * __alloc_workqueue_key(const char *, unsigned int, int, struct lock_class_key *, const char *, ...);
421 void destroy_workqueue(struct workqueue_struct *);
429 bool queue_work_on(int, struct workqueue_struct *, struct work_struct *);
436 void flush_workqueue(struct workqueue_struct *);
87 const char * kobject_name(const struct kobject *kobj);
110 struct kobject * kobject_get(struct kobject *);
111 void kobject_put(struct kobject *);
251 int __first_node(const nodemask_t *srcp);
257 int __next_node(int n, const nodemask_t *srcp);
395 extern nodemask_t node_states[5U];
398 int node_state(int node, enum node_states state);
423 int next_online_node(int nid);
57 int __cpu_to_node(int);
76 const struct cpumask * cpumask_of_node(int);
58 unsigned int readl(const volatile void *addr);
66 void writel(unsigned int val, volatile void *addr);
29 extern int cpu_number;
850 const char * dev_name(const struct device *dev);
863 int dev_to_node(struct device *dev);
897 void * dev_get_drvdata(const struct device *dev);
902 void dev_set_drvdata(struct device *dev, void *data);
1099 const char * dev_driver_string(const struct device *);
1120 void dev_err(const struct device *, const char *, ...);
1122 void dev_warn(const struct device *, const char *, ...);
1126 void _dev_info(const struct device *, const char *, ...);
209 bool capable(int);
430 void schedule();
2470 int wake_up_process(struct task_struct *);
148 void kfree(const void *);
312 void * __kmalloc(size_t , gfp_t );
327 void * __kmalloc_node(size_t , gfp_t , int);
451 void * kmalloc(size_t size, gfp_t flags);
491 void * kmalloc_node(size_t size, gfp_t flags, int node);
605 void * kzalloc(size_t size, gfp_t flags);
616 void * kzalloc_node(size_t size, gfp_t flags, int node);
858 int pci_find_capability(struct pci_dev *, int);
884 int pci_bus_read_config_word(struct pci_bus *, unsigned int, int, u16 *);
890 int pci_bus_write_config_word(struct pci_bus *, unsigned int, int, u16 );
910 int pci_read_config_word(const struct pci_dev *dev, int where, u16 *val);
923 int pci_write_config_word(const struct pci_dev *dev, int where, u16 val);
978 int pcim_enable_device(struct pci_dev *);
1008 void pci_disable_device(struct pci_dev *);
1011 void pci_set_master(struct pci_dev *);
1066 int pci_save_state(struct pci_dev *);
1067 void pci_restore_state(struct pci_dev *);
1080 int pci_set_power_state(struct pci_dev *, pci_power_t );
1182 int __pci_register_driver(struct pci_driver *, struct module *, const char *);
1191 void pci_unregister_driver(struct pci_driver *);
926 void * lowmem_page_address(const struct page *page);
120 struct page * sg_page(struct scatterlist *sg);
239 void * sg_virt(struct scatterlist *sg);
246 struct scatterlist * sg_next(struct scatterlist *);
248 void sg_init_table(struct scatterlist *, unsigned int);
1263 void pci_disable_msi(struct pci_dev *);
1270 int pci_enable_msi_range(struct pci_dev *, int, int);
1271 int pci_enable_msi_exact(struct pci_dev *dev, int nvec);
44 void debug_dma_unmap_page(struct device *, dma_addr_t , size_t , int, bool );
47 void debug_dma_map_sg(struct device *, struct scatterlist *, int, int, int);
50 void debug_dma_unmap_sg(struct device *, struct scatterlist *, int, int);
131 void kmemcheck_mark_initialized(void *address, unsigned int n);
77 int valid_dma_direction(int dma_direction);
29 extern struct dma_map_ops *dma_ops;
31 struct dma_map_ops * get_dma_ops(struct device *dev);
47 int dma_supported(struct device *, u64 );
143 dma_addr_t ldv_dma_map_single_attrs_1(struct device *dev, void *ptr, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs);
144 void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs);
161 int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, struct dma_attrs *attrs);
179 void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, struct dma_attrs *attrs);
458 int dma_set_mask(struct device *dev, u64 mask);
482 int dma_set_coherent_mask(struct device *dev, u64 mask);
617 void * dmam_alloc_coherent(struct device *, size_t , dma_addr_t *, gfp_t );
619 void dmam_free_coherent(struct device *, size_t , void *, dma_addr_t );
32 dma_addr_t pci_map_single(struct pci_dev *hwdev, void *ptr, size_t size, int direction);
38 void pci_unmap_single(struct pci_dev *hwdev, dma_addr_t dma_addr, size_t size, int direction);
107 int pci_set_dma_mask(struct pci_dev *dev, u64 mask);
112 int pci_set_consistent_dma_mask(struct pci_dev *dev, u64 mask);
116 int __pcibus_to_node(const struct pci_bus *bus);
1535 void * pci_get_drvdata(struct pci_dev *pdev);
1540 void pci_set_drvdata(struct pci_dev *pdev, void *data);
1684 const void ** pcim_iomap_table(struct pci_dev *);
1685 int pcim_iomap_regions(struct pci_dev *, int, const char *);
1688 void pcim_iounmap_regions(struct pci_dev *, int);
667 unsigned long int _copy_from_user(void *, const void *, unsigned int);
669 unsigned long int _copy_to_user(void *, const void *, unsigned int);
689 void __copy_from_user_overflow();
694 void __copy_to_user_overflow();
710 unsigned long int copy_from_user(void *to, const void *from, unsigned long n);
745 unsigned long int copy_to_user(void *to, const void *from, unsigned long n);
11 void synchronize_irq(unsigned int);
154 int devm_request_threaded_irq(struct device *, unsigned int, irqreturn_t (*)(int, void *), irqreturn_t (*)(int, void *), unsigned long, const char *, void *);
160 int devm_request_irq(struct device *dev, unsigned int irq, irqreturn_t (*handler)(int, void *), unsigned long irqflags, const char *devname, void *dev_id);
172 void devm_free_irq(struct device *, unsigned int, void *);
266 int irq_set_affinity_hint(unsigned int, const struct cpumask *);
10 void __const_udelay(unsigned long);
46 void msleep(unsigned int);
50 void ssleep(unsigned int seconds);
32 void sema_init(struct semaphore *sem, int val);
42 int down_trylock(struct semaphore *);
44 void up(struct semaphore *);
2297 int register_blkdev(unsigned int, const char *);
2298 void unregister_blkdev(unsigned int, const char *);
2303 void bdput(struct block_device *);
2709 loff_t no_llseek(struct file *, loff_t , int);
2849 int simple_open(struct inode *, struct file *);
434 void add_disk(struct gendisk *);
435 void del_gendisk(struct gendisk *);
437 struct block_device * bdget_disk(struct gendisk *, int);
464 void set_capacity(struct gendisk *disk, sector_t size);
633 struct gendisk * alloc_disk_node(int, int);
636 void put_disk(struct gendisk *);
770 void blk_put_request(struct request *);
838 sector_t blk_rq_pos(const struct request *rq);
843 unsigned int blk_rq_bytes(const struct request *rq);
855 unsigned int blk_rq_sectors(const struct request *rq);
963 void blk_cleanup_queue(struct request_queue *);
965 void blk_queue_bounce_limit(struct request_queue *, u64 );
966 void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
968 void blk_queue_max_segments(struct request_queue *, unsigned short);
969 void blk_queue_max_segment_size(struct request_queue *, unsigned int);
970 void blk_queue_max_discard_sectors(struct request_queue *, unsigned int);
975 void blk_queue_physical_block_size(struct request_queue *, unsigned int);
979 void blk_queue_io_min(struct request_queue *, unsigned int);
1006 void blk_queue_flush(struct request_queue *, unsigned int);
1010 int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
176 struct request_queue * blk_mq_init_queue(struct blk_mq_tag_set *);
182 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *);
183 void blk_mq_free_tag_set(struct blk_mq_tag_set *);
197 struct request * blk_mq_alloc_request(struct request_queue *, int, unsigned int);
199 struct request * blk_mq_tag_to_rq(struct blk_mq_tags *, unsigned int);
219 struct blk_mq_hw_ctx * blk_mq_map_queue(struct request_queue *, const int);
223 void blk_mq_start_request(struct request *);
224 void blk_mq_end_request(struct request *, int);
236 void blk_mq_stop_hw_queues(struct request_queue *);
238 void blk_mq_start_stopped_hw_queues(struct request_queue *, bool );
251 struct request * blk_mq_rq_from_pdu(void *pdu);
255 void * blk_mq_rq_to_pdu(struct request *rq);
8 struct task_struct * kthread_create_on_node(int (*)(void *), void *, int, const char *, ...);
42 int kthread_stop(struct task_struct *);
43 bool kthread_should_stop();
49 struct dentry * debugfs_create_file(const char *, umode_t , struct dentry *, void *, const struct file_operations *);
58 struct dentry * debugfs_create_dir(const char *, struct dentry *);
68 void debugfs_remove(struct dentry *);
69 void debugfs_remove_recursive(struct dentry *);
106 int instance = 0;
108 struct list_head online_list = { };
109 struct list_head removing_list = { };
110 struct spinlock dev_lock = { };
116 int mtip_major = 0;
117 struct dentry *dfs_parent = 0;
118 struct dentry *dfs_device_status = 0;
120 u32 cpu_use[8192U] = { };
122 struct spinlock rssd_index_lock = { { { { { 0 } }, 3735899821U, 4294967295U, (void *)-1, { 0, { 0, 0 }, "rssd_index_lock", 0, 0UL } } } };
123 struct ida rssd_index_ida = { { 0, 0, 0, 0, { { { { { 0 } }, 3735899821U, 4294967295U, (void *)-1, { 0, { 0, 0 }, "(rssd_index_ida).idr.lock", 0, 0UL } } } }, 0, 0 }, (struct ida_bitmap *)0 };
125 int mtip_block_initialize(struct driver_data *dd);
150 bool mtip_check_surprise_removal(struct pci_dev *pdev);
173 struct mtip_cmd * mtip_get_int_command(struct driver_data *dd);
181 void mtip_put_int_command(struct driver_data *dd, struct mtip_cmd *cmd);
189 struct request * mtip_rq_from_tag(struct driver_data *dd, unsigned int tag);
197 struct mtip_cmd * mtip_cmd_from_tag(struct driver_data *dd, unsigned int tag);
223 void mtip_async_complete(struct mtip_port *port, int tag, struct mtip_cmd *cmd, int status);
257 int mtip_hba_reset(struct driver_data *dd);
298 void mtip_issue_ncq_command(struct mtip_port *port, int tag);
320 int mtip_enable_fis(struct mtip_port *port, int enable);
346 int mtip_enable_engine(struct mtip_port *port, int enable);
367 void mtip_start_port(struct mtip_port *port);
385 void mtip_deinit_port(struct mtip_port *port);
410 void mtip_init_port(struct mtip_port *port);
453 void mtip_restart_port(struct mtip_port *port);
523 int mtip_device_reset(struct driver_data *dd);
546 void print_tags(struct driver_data *dd, char *msg, unsigned long *tagbits, int cnt);
577 void mtip_completion(struct mtip_port *port, int tag, struct mtip_cmd *command, int status);
588 void mtip_null_completion(struct mtip_port *port, int tag, struct mtip_cmd *command, int status);
593 int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer, dma_addr_t buffer_dma, unsigned int sectors);
595 int mtip_get_smart_attr(struct mtip_port *port, unsigned int id, struct smart_attr *attrib);
605 void mtip_handle_tfe(struct driver_data *dd);
785 void mtip_workq_sdbfx(struct mtip_port *port, int group, u32 completed);
833 void mtip_process_legacy(struct driver_data *dd, u32 port_stat);
853 void mtip_process_errors(struct driver_data *dd, u32 port_stat);
881 irqreturn_t mtip_handle_irq(struct driver_data *data);
973 irqreturn_t mtip_irq_handler(int irq, void *instance___0);
980 void mtip_issue_non_ncq_command(struct mtip_port *port, int tag);
986 bool mtip_pause_ncq(struct mtip_port *port, struct host_to_dev_fis *fis);
1030 int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout);
1089 int mtip_exec_internal_command(struct mtip_port *port, struct host_to_dev_fis *fis, int fis_len, dma_addr_t buffer, int buf_len, u32 opts, gfp_t atomic, unsigned long timeout);
1276 void ata_swap_string(u16 *buf, unsigned int len);
1283 void mtip_set_timeout(struct driver_data *dd, struct host_to_dev_fis *fis, unsigned int *timeout, u8 erasemode);
1333 int mtip_get_identify(struct mtip_port *port, void *user_buffer);
1424 int mtip_standby_immediate(struct mtip_port *port);
1506 int mtip_get_smart_data(struct mtip_port *port, u8 *buffer, dma_addr_t buffer_dma);
1599 int mtip_send_trim(struct driver_data *dd, unsigned int lba, unsigned int len);
1673 bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors);
1694 void mtip_dump_identify(struct mtip_port *port);
1749 void fill_command_sg(struct driver_data *dd, struct mtip_cmd *command, int nents);
1782 int exec_drive_task(struct mtip_port *port, u8 *command);
1852 int exec_drive_command(struct mtip_port *port, u8 *command, void *user_buffer);
1962 unsigned int implicit_sector(unsigned char command, unsigned char features);
2001 int exec_drive_taskfile(struct driver_data *dd, void *buf, ide_task_request_t *req_task, int outtotal);
2259 int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd, unsigned long arg);
2363 void mtip_hw_submit_io(struct driver_data *dd, struct request *rq, struct mtip_cmd *command, int nents, struct blk_mq_hw_ctx *hctx);
2451 ssize_t mtip_hw_show_status(struct device *dev, struct device_attribute *attr, char *buf);
2468 struct device_attribute dev_attr_status = { { "status", 292U, (_Bool)0, 0, { { { 0 }, { 0 }, { 0 }, { 0 }, { 0 }, { 0 }, { 0 }, { 0 } } } }, &mtip_hw_show_status, (ssize_t (*)(struct device *, struct device_attribute *, const char *, size_t ))0 };
2472 ssize_t show_device_status(struct device_driver *drv, char *buf);
2545 ssize_t mtip_hw_read_device_status(struct file *f, char *ubuf, size_t len, loff_t *offset);
2574 ssize_t mtip_hw_read_registers(struct file *f, char *ubuf, size_t len, loff_t *offset);
2641 ssize_t mtip_hw_read_flags(struct file *f, char *ubuf, size_t len, loff_t *offset);
2673 const struct file_operations mtip_device_status_fops = { &__this_module, &no_llseek, &mtip_hw_read_device_status, 0, 0, 0, 0, 0, 0, 0, 0, &simple_open, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
2680 const struct file_operations mtip_regs_fops = { &__this_module, &no_llseek, &mtip_hw_read_registers, 0, 0, 0, 0, 0, 0, 0, 0, &simple_open, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
2687 const struct file_operations mtip_flags_fops = { &__this_module, &no_llseek, &mtip_hw_read_flags, 0, 0, 0, 0, 0, 0, 0, 0, &simple_open, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
2704 int mtip_hw_sysfs_init(struct driver_data *dd, struct kobject *kobj);
2725 int mtip_hw_sysfs_exit(struct driver_data *dd, struct kobject *kobj);
2735 int mtip_hw_debugfs_init(struct driver_data *dd);
2757 void mtip_hw_debugfs_exit(struct driver_data *dd);
2771 void hba_setup(struct driver_data *dd);
2783 int mtip_device_unaligned_constrained(struct driver_data *dd);
2798 void mtip_detect_product(struct driver_data *dd);
2848 int mtip_ftl_rebuild_poll(struct driver_data *dd);
2903 int mtip_service_thread(void *data);
2994 void mtip_dma_free(struct driver_data *dd);
3016 int mtip_dma_alloc(struct driver_data *dd);
3054 int mtip_hw_get_identify(struct driver_data *dd);
3118 int mtip_hw_init(struct driver_data *dd);
3267 void mtip_standby_drive(struct driver_data *dd);
3291 int mtip_hw_exit(struct driver_data *dd);
3332 int mtip_hw_shutdown(struct driver_data *dd);
3356 int mtip_hw_suspend(struct driver_data *dd);
3388 int mtip_hw_resume(struct driver_data *dd);
3418 int rssd_disk_name_format(char *prefix, int index, char *buf, int buflen);
3458 int mtip_block_ioctl(struct block_device *dev, fmode_t mode, unsigned int cmd, unsigned long arg);
3496 int mtip_block_compat_ioctl(struct block_device *dev, fmode_t mode, unsigned int cmd, unsigned long arg);
3578 int mtip_block_getgeo(struct block_device *dev, struct hd_geometry *geo);
3606 const struct block_device_operations mtip_block_ops = { 0, 0, 0, &mtip_block_ioctl, &mtip_block_compat_ioctl, 0, 0, 0, 0, 0, &mtip_block_getgeo, 0, &__this_module, 0 };
3615 bool is_se_active(struct driver_data *dd);
3646 int mtip_submit_request(struct blk_mq_hw_ctx *hctx, struct request *rq);
3690 bool mtip_check_unal_depth(struct blk_mq_hw_ctx *hctx, struct request *rq);
3714 int mtip_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd);
3733 void mtip_free_cmd(void *data, struct request *rq, unsigned int hctx_idx, unsigned int request_idx);
3746 int mtip_init_cmd(void *data, struct request *rq, unsigned int hctx_idx, unsigned int request_idx, unsigned int numa_node___0);
3783 struct blk_mq_ops mtip_mq_ops = { &mtip_queue_rq, &blk_mq_map_queue, 0, 0, 0, 0, 0, &mtip_init_cmd, &mtip_free_cmd };
4008 int mtip_block_remove(struct driver_data *dd);
4076 int mtip_block_shutdown(struct driver_data *dd);
4101 int mtip_block_suspend(struct driver_data *dd);
4109 int mtip_block_resume(struct driver_data *dd);
4117 void drop_cpu(int cpu);
4122 int get_least_used_cpu_on_node(int node);
4143 int mtip_get_next_rr_node();
4158 void mtip_workq_sdbf0(struct work_struct *work);
4159 void mtip_workq_sdbf1(struct work_struct *work);
4160 void mtip_workq_sdbf2(struct work_struct *work);
4161 void mtip_workq_sdbf3(struct work_struct *work);
4162 void mtip_workq_sdbf4(struct work_struct *work);
4163 void mtip_workq_sdbf5(struct work_struct *work);
4164 void mtip_workq_sdbf6(struct work_struct *work);
4165 void mtip_workq_sdbf7(struct work_struct *work);
4167 void mtip_disable_link_opts(struct driver_data *dd, struct pci_dev *pdev);
4191 void mtip_fix_ero_nosnoop(struct driver_data *dd, struct pci_dev *pdev);
4227 int mtip_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
4425 void mtip_pci_remove(struct pci_dev *pdev);
4483 int mtip_pci_suspend(struct pci_dev *pdev, pm_message_t mesg);
4524 int mtip_pci_resume(struct pci_dev *pdev);
4571 void mtip_pci_shutdown(struct pci_dev *pdev);
4579 const struct pci_device_id mtip_pci_tbl[8U] = { { 4932U, 20816U, 4294967295U, 4294967295U, 0U, 0U, 0UL }, { 4932U, 20817U, 4294967295U, 4294967295U, 0U, 0U, 0UL }, { 4932U, 20818U, 4294967295U, 4294967295U, 0U, 0U, 0UL }, { 4932U, 20819U, 4294967295U, 4294967295U, 0U, 0U, 0UL }, { 4932U, 20832U, 4294967295U, 4294967295U, 0U, 0U, 0UL }, { 4932U, 20833U, 4294967295U, 4294967295U, 0U, 0U, 0UL }, { 4932U, 20835U, 4294967295U, 4294967295U, 0U, 0U, 0UL }, { 0U, 0U, 0U, 0U, 0U, 0U, 0UL } };
4591 struct pci_driver mtip_pci_driver = { { 0, 0 }, "mtip32xx", (const struct pci_device_id *)(&mtip_pci_tbl), &mtip_pci_probe, &mtip_pci_remove, &mtip_pci_suspend, 0, 0, &mtip_pci_resume, &mtip_pci_shutdown, 0, 0, { 0, 0, 0, 0, (_Bool)0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, { { { { { { 0 } }, 0U, 0U, 0, { 0, { 0, 0 }, 0, 0, 0UL } } } }, { 0, 0 } } };
4601 const struct pci_device_id __mod_pci__mtip_pci_tbl_device_table[8U] = { };
4613 int mtip_init();
4668 void mtip_exit();
4703 void ldv_check_final_state();
4706 void ldv_check_return_value(int);
4709 void ldv_check_return_value_probe(int);
4712 void ldv_initialize();
4715 void ldv_handler_precall();
4718 int nondet_int();
4721 int LDV_IN_INTERRUPT = 0;
4724 void ldv_main0_sequence_infinite_withcheck_stateful();
10 void ldv_error();
25 int ldv_undef_int();
26 void * ldv_undef_ptr();
8 int LDV_DMA_MAP_CALLS = 0;
11 dma_addr_t ldv_dma_map_page(struct device *dev, struct page *page, size_t offset, size_t size, enum dma_data_direction dir);
25 int ldv_dma_mapping_error(struct device *dev, dma_addr_t dma_addr);
41 dma_addr_t ldv_dma_map_single(struct device *dev, void *cpu_addr, size_t size, enum dma_data_direction dir);
return ;
}
-entry_point
{
4726 struct file *var_group1;
4727 char *var_mtip_hw_read_device_status_47_p1;
4728 unsigned long var_mtip_hw_read_device_status_47_p2;
4729 loff_t *var_mtip_hw_read_device_status_47_p3;
4730 long res_mtip_hw_read_device_status_47;
4731 char *var_mtip_hw_read_registers_48_p1;
4732 unsigned long var_mtip_hw_read_registers_48_p2;
4733 loff_t *var_mtip_hw_read_registers_48_p3;
4734 long res_mtip_hw_read_registers_48;
4735 char *var_mtip_hw_read_flags_49_p1;
4736 unsigned long var_mtip_hw_read_flags_49_p2;
4737 loff_t *var_mtip_hw_read_flags_49_p3;
4738 long res_mtip_hw_read_flags_49;
4739 struct block_device *var_group2;
4740 unsigned int var_mtip_block_ioctl_69_p1;
4741 unsigned int var_mtip_block_ioctl_69_p2;
4742 unsigned long var_mtip_block_ioctl_69_p3;
4743 unsigned int var_mtip_block_compat_ioctl_70_p1;
4744 unsigned int var_mtip_block_compat_ioctl_70_p2;
4745 unsigned long var_mtip_block_compat_ioctl_70_p3;
4746 struct hd_geometry *var_group3;
4747 struct blk_mq_hw_ctx *var_group4;
4748 const struct blk_mq_queue_data *var_mtip_queue_rq_75_p1;
4749 void *var_mtip_init_cmd_77_p0;
4750 struct request *var_group5;
4751 unsigned int var_mtip_init_cmd_77_p2;
4752 unsigned int var_mtip_init_cmd_77_p3;
4753 unsigned int var_mtip_init_cmd_77_p4;
4754 void *var_mtip_free_cmd_76_p0;
4755 unsigned int var_mtip_free_cmd_76_p2;
4756 unsigned int var_mtip_free_cmd_76_p3;
4757 struct pci_dev *var_group6;
4758 const struct pci_device_id *var_mtip_pci_probe_88_p1;
4759 int res_mtip_pci_probe_88;
4760 struct pm_message var_mtip_pci_suspend_90_p1;
4761 int var_mtip_irq_handler_23_p0;
4762 void *var_mtip_irq_handler_23_p1;
4763 int ldv_s_mtip_device_status_fops_file_operations;
4764 int ldv_s_mtip_regs_fops_file_operations;
4765 int ldv_s_mtip_flags_fops_file_operations;
4766 int ldv_s_mtip_pci_driver_pci_driver;
4767 int tmp;
4768 int tmp___0;
4769 int tmp___1;
5782 ldv_s_mtip_device_status_fops_file_operations = 0;
5784 ldv_s_mtip_regs_fops_file_operations = 0;
5786 ldv_s_mtip_flags_fops_file_operations = 0;
5792 ldv_s_mtip_pci_driver_pci_driver = 0;
5708 LDV_IN_INTERRUPT = 1;
5717 -ldv_initialize()
{
71 LDV_DMA_MAP_CALLS = 0;
72 return ;;
}
5779 ldv_handler_precall() { /* Function call is skipped due to function is undefined */}
5780 -mtip_init()
{
4615 int error;
4616 struct lock_class_key __key;
4617 _Bool tmp;
4618 _Bool tmp___0;
4617 printk("\016mtip32xx Version 1.3.1\n") { /* Function call is skipped due to function is undefined */}
4619 -spinlock_check(&dev_lock)
{
291 return &(lock->__annonCompField8.rlock);;
}
4619 __raw_spin_lock_init(&(dev_lock.__annonCompField8.rlock), "&(&dev_lock)->rlock", &__key) { /* Function call is skipped due to function is undefined */}
4621 -INIT_LIST_HEAD(&online_list)
{
27 union __anonunion___u_19 __u;
27 __u.__val = list;
27 -__write_once_size((volatile void *)(&(list->next)), (void *)(&(__u.__c)), 8)
{
243 switch (size);
244 assume(!(size == 1));
245 assume(!(size == 2));
246 assume(!(size == 4));
247 assume(size == 8);
247 *((volatile __u64 *)p) = *((__u64 *)res);
247 goto ldv_852;
254 return ;;
}
28 list->prev = list;
29 return ;;
}
4622 -INIT_LIST_HEAD(&removing_list)
{
27 union __anonunion___u_19 __u;
27 __u.__val = list;
27 -__write_once_size((volatile void *)(&(list->next)), (void *)(&(__u.__c)), 8)
{
243 switch (size);
244 assume(!(size == 1));
245 assume(!(size == 2));
246 assume(!(size == 4));
247 assume(size == 8);
247 *((volatile __u64 *)p) = *((__u64 *)res);
247 goto ldv_852;
254 return ;;
}
28 list->prev = list;
29 return ;;
}
4625 error = register_blkdev(0U, "mtip32xx") { /* Function call is skipped due to function is undefined */}
4626 assume(!(error <= 0));
4631 mtip_major = error;
4633 dfs_parent = debugfs_create_dir("rssd", (struct dentry *)0) { /* Function call is skipped due to function is undefined */}
4634 -IS_ERR_OR_NULL((const void *)dfs_parent)
{
40 long tmp;
41 long tmp___0;
42 int tmp___1;
40 tmp = __builtin_expect(((unsigned long)ptr) == ((unsigned long)((const void *)0)), 0L) { /* Function call is skipped due to function is undefined */}
40 assume(tmp != 0L);
40 tmp___1 = 1;
40 return (bool )tmp___1;;
}
4634 assume(!(((int)tmp) == 0));
4635 printk("\fError creating debugfs parent\n") { /* Function call is skipped due to function is undefined */}
4636 dfs_parent = (struct dentry *)0;
4638 assume(!(((unsigned long)dfs_parent) != ((unsigned long)((struct dentry *)0))));
4649 error = __pci_register_driver(&mtip_pci_driver, &__this_module, "mtip32xx") { /* Function call is skipped due to function is undefined */}
4650 assume(!(error != 0));
4655 return error;;
}
5780 assume(!(tmp != 0));
5798 goto ldv_48047;
5798 tmp___1 = nondet_int() { /* Function call is skipped due to function is undefined */}
5798 assume(tmp___1 != 0);
5804 goto ldv_48046;
5799 ldv_48046:;
5805 tmp___0 = nondet_int() { /* Function call is skipped due to function is undefined */}
5805 switch (tmp___0);
5806 assume(!(tmp___0 == 0));
5885 assume(!(tmp___0 == 1));
5963 assume(!(tmp___0 == 2));
6041 assume(!(tmp___0 == 3));
6116 assume(tmp___0 == 4);
6177 ldv_handler_precall() { /* Function call is skipped due to function is undefined */}
6178 -mtip_block_compat_ioctl(var_group2, var_mtip_block_compat_ioctl_70_p1, var_mtip_block_compat_ioctl_70_p2, var_mtip_block_compat_ioctl_70_p3)
{
3499 struct driver_data *dd;
3500 _Bool tmp;
3501 int tmp___0;
3502 int tmp___1;
3503 long tmp___2;
3504 struct mtip_compat_ide_task_request_s *compat_req_task;
3505 struct ide_task_request_s req_task;
3506 int compat_tasksize;
3507 int outtotal;
3508 int ret;
3509 unsigned long tmp___3;
3510 int __ret_gu;
3511 unsigned long __val_gu;
3512 long tmp___4;
3513 int __ret_gu___0;
3514 unsigned long __val_gu___0;
3515 long tmp___5;
3516 unsigned long tmp___6;
3517 int __ret_pu;
3518 unsigned int __pu_val;
3519 long tmp___7;
3520 int __ret_pu___0;
3521 unsigned int __pu_val___0;
3522 long tmp___8;
3523 int tmp___9;
3501 struct driver_data *__CPAchecker_TMP_0 = (struct driver_data *)(dev->bd_disk->private_data);
3501 dd = __CPAchecker_TMP_0;
3503 tmp = capable(21) { /* Function call is skipped due to function is undefined */}
3503 assume(!(tmp == 0));
3503 tmp___0 = 0;
3503 assume(tmp___0 == 0);
3506 assume(!(((unsigned long)dd) == ((unsigned long)((struct driver_data *)0))));
3509 -constant_test_bit(1L, (const volatile unsigned long *)(&(dd->dd_flag)))
{
310 return ((int)(((unsigned long)(*(addr + ((unsigned long)(nr >> 6))))) >> (((int)nr) & 63))) & 1;;
}
3509 tmp___2 = __builtin_expect(tmp___1 != 0, 0L) { /* Function call is skipped due to function is undefined */}
3509 assume(!(tmp___2 != 0L));
3512 switch (cmd);
3513 assume(!(cmd == 4705U));
3515 assume(cmd == 797U);
3520 compat_tasksize = 40;
3523 compat_req_task = (struct mtip_compat_ide_task_request_s *)arg;
3526 -copy_from_user((void *)(&req_task), (const void *)arg, ((unsigned long)compat_tasksize) - 8UL)
{
712 int sz;
713 unsigned long tmp;
714 long tmp___0;
715 long tmp___1;
712 tmp = __builtin_object_size((const void *)to, 0) { /* Function call is skipped due to function is undefined */}
712 sz = (int)tmp;
714 __might_fault("./arch/x86/include/asm/uaccess.h", 714) { /* Function call is skipped due to function is undefined */}
734 tmp___0 = __builtin_expect(sz < 0, 1L) { /* Function call is skipped due to function is undefined */}
734 assume(tmp___0 != 0L);
735 n = _copy_from_user(to, from, (unsigned int)n) { /* Function call is skipped due to function is undefined */}
741 return n;;
}
3526 assume(!(tmp___3 != 0UL));
3530 __might_fault("/home/cluser/ldv/ref_launch/work/current--X--drivers--X--defaultlinux-4.5-rc1.tar.xz--X--331_1a--X--cpachecker/linux-4.5-rc1.tar.xz/csd_deg_dscv/11248/dscv_tempdir/dscv/ri/331_1a/drivers/block/mtip32xx/mtip32xx.c", 3530) { /* Function call is skipped due to function is undefined */}
3530 Ignored inline assembler code
3530 req_task.out_size = (unsigned long)((unsigned int)__val_gu);
3530 tmp___4 = __builtin_expect((long)__ret_gu, 0L) { /* Function call is skipped due to function is undefined */}
3530 assume(!(tmp___4 != 0L));
3533 __might_fault("/home/cluser/ldv/ref_launch/work/current--X--drivers--X--defaultlinux-4.5-rc1.tar.xz--X--331_1a--X--cpachecker/linux-4.5-rc1.tar.xz/csd_deg_dscv/11248/dscv_tempdir/dscv/ri/331_1a/drivers/block/mtip32xx/mtip32xx.c", 3533) { /* Function call is skipped due to function is undefined */}
3533 Ignored inline assembler code
3533 req_task.in_size = (unsigned long)((unsigned int)__val_gu___0);
3533 tmp___5 = __builtin_expect((long)__ret_gu___0, 0L) { /* Function call is skipped due to function is undefined */}
3533 assume(!(tmp___5 != 0L));
3536 outtotal = 40;
3538 -exec_drive_taskfile(dd, (void *)arg, &req_task, outtotal)
{
2004 struct host_to_dev_fis fis;
2005 struct host_to_dev_fis *reply;
2006 u8 *outbuf;
2007 u8 *inbuf;
2008 unsigned long long outbuf_dma;
2009 unsigned long long inbuf_dma;
2010 unsigned long long dma_buffer;
2011 int err;
2012 unsigned int taskin;
2013 unsigned int taskout;
2014 unsigned char nsect;
2015 unsigned int timeout;
2016 unsigned int force_single_sector;
2017 unsigned int transfer_size;
2018 unsigned long task_file_data;
2019 int intotal;
2020 int erasemode;
2021 void *tmp;
2022 long tmp___0;
2023 _Bool tmp___1;
2024 void *tmp___2;
2025 long tmp___3;
2026 _Bool tmp___4;
2027 int tmp___5;
2028 unsigned int tmp___6;
2029 unsigned long tmp___7;
2030 unsigned long tmp___8;
2008 outbuf = (u8 *)0U;
2009 inbuf = (u8 *)0U;
2010 outbuf_dma = 0ULL;
2011 inbuf_dma = 0ULL;
2012 dma_buffer = 0ULL;
2013 err = 0;
2014 taskin = 0U;
2015 taskout = 0U;
2016 nsect = 0U;
2021 unsigned int __CPAchecker_TMP_0 = (unsigned int)(req_task->out_size);
2021 intotal = (int)(__CPAchecker_TMP_0 + ((unsigned int)outtotal));
2022 erasemode = 0;
2024 unsigned int __CPAchecker_TMP_1 = (unsigned int)(req_task->out_size);
2024 taskout = __CPAchecker_TMP_1;
2025 unsigned int __CPAchecker_TMP_2 = (unsigned int)(req_task->in_size);
2025 taskin = __CPAchecker_TMP_2;
2027 assume(!(taskin > 130560U));
2027 assume(!(taskout > 130560U));
2032 assume(taskout != 0U);
2033 tmp = memdup_user(((const void *)buf) + ((unsigned long)outtotal), (size_t )taskout) { /* Function call is skipped due to function is undefined */}
2033 outbuf = (u8 *)tmp;
2034 -IS_ERR((const void *)outbuf)
{
35 long tmp;
35 tmp = __builtin_expect(((unsigned long)ptr) > 18446744073709547520UL, 0L) { /* Function call is skipped due to function is undefined */}
35 return tmp != 0L;;
}
2034 assume(((int)tmp___1) == 0);
2039 -pci_map_single(dd->pdev, (void *)outbuf, (size_t )taskout, 1)
{
35 unsigned long long tmp;
34 struct device *__CPAchecker_TMP_0;
34 assume(((unsigned long)hwdev) != ((unsigned long)((struct pci_dev *)0)));
34 __CPAchecker_TMP_0 = &(hwdev->dev);
34 -ldv_dma_map_single_attrs_1(__CPAchecker_TMP_0, ptr, size, (enum dma_data_direction )direction, (struct dma_attrs *)0)
{
14 unsigned long long tmp;
13 -ldv_dma_map_single_attrs(dev, ptr, size, dir, attrs)
{
58 unsigned long long nonedetermined;
59 void *tmp;
58 tmp = ldv_undef_ptr() { /* Function call is skipped due to function is undefined */}
58 nonedetermined = (dma_addr_t )tmp;
61 assume(!(LDV_DMA_MAP_CALLS != 0));
63 LDV_DMA_MAP_CALLS = LDV_DMA_MAP_CALLS + 1;
65 return nonedetermined;;
}
13 return tmp;;
}
34 return tmp;;
}
2043 assume(!(outbuf_dma == 0ULL));
2047 dma_buffer = outbuf_dma;
2050 assume(taskin != 0U);
2051 tmp___2 = memdup_user(((const void *)buf) + ((unsigned long)intotal), (size_t )taskin) { /* Function call is skipped due to function is undefined */}
2051 inbuf = (u8 *)tmp___2;
2052 -IS_ERR((const void *)inbuf)
{
35 long tmp;
35 tmp = __builtin_expect(((unsigned long)ptr) > 18446744073709547520UL, 0L) { /* Function call is skipped due to function is undefined */}
35 return tmp != 0L;;
}
2052 assume(((int)tmp___4) == 0);
2057 -pci_map_single(dd->pdev, (void *)inbuf, (size_t )taskin, 2)
{
35 unsigned long long tmp;
34 struct device *__CPAchecker_TMP_0;
34 assume(((unsigned long)hwdev) != ((unsigned long)((struct pci_dev *)0)));
34 __CPAchecker_TMP_0 = &(hwdev->dev);
34 -ldv_dma_map_single_attrs_1(__CPAchecker_TMP_0, ptr, size, (enum dma_data_direction )direction, (struct dma_attrs *)0)
{
14 unsigned long long tmp;
13 -ldv_dma_map_single_attrs(dev, ptr, size, dir, attrs)
{
58 unsigned long long nonedetermined;
59 void *tmp;
58 tmp = ldv_undef_ptr() { /* Function call is skipped due to function is undefined */}
58 nonedetermined = (dma_addr_t )tmp;
61 assume(LDV_DMA_MAP_CALLS != 0);
61 -ldv_error()
{
15 LDV_ERROR:;
}
}
}
}
}
}
}
Source code
1 #ifndef _ASM_X86_BITOPS_H 2 #define _ASM_X86_BITOPS_H 3 4 /* 5 * Copyright 1992, Linus Torvalds. 6 * 7 * Note: inlines with more than a single statement should be marked 8 * __always_inline to avoid problems with older gcc's inlining heuristics. 9 */ 10 11 #ifndef _LINUX_BITOPS_H 12 #error only <linux/bitops.h> can be included directly 13 #endif 14 15 #include <linux/compiler.h> 16 #include <asm/alternative.h> 17 #include <asm/rmwcc.h> 18 #include <asm/barrier.h> 19 20 #if BITS_PER_LONG == 32 21 # define _BITOPS_LONG_SHIFT 5 22 #elif BITS_PER_LONG == 64 23 # define _BITOPS_LONG_SHIFT 6 24 #else 25 # error "Unexpected BITS_PER_LONG" 26 #endif 27 28 #define BIT_64(n) (U64_C(1) << (n)) 29 30 /* 31 * These have to be done with inline assembly: that way the bit-setting 32 * is guaranteed to be atomic. All bit operations return 0 if the bit 33 * was cleared before the operation and != 0 if it was not. 34 * 35 * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1). 36 */ 37 38 #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1) 39 /* Technically wrong, but this avoids compilation errors on some gcc 40 versions. */ 41 #define BITOP_ADDR(x) "=m" (*(volatile long *) (x)) 42 #else 43 #define BITOP_ADDR(x) "+m" (*(volatile long *) (x)) 44 #endif 45 46 #define ADDR BITOP_ADDR(addr) 47 48 /* 49 * We do the locked ops that don't return the old value as 50 * a mask operation on a byte. 51 */ 52 #define IS_IMMEDIATE(nr) (__builtin_constant_p(nr)) 53 #define CONST_MASK_ADDR(nr, addr) BITOP_ADDR((void *)(addr) + ((nr)>>3)) 54 #define CONST_MASK(nr) (1 << ((nr) & 7)) 55 56 /** 57 * set_bit - Atomically set a bit in memory 58 * @nr: the bit to set 59 * @addr: the address to start counting from 60 * 61 * This function is atomic and may not be reordered. See __set_bit() 62 * if you do not require the atomic guarantees. 63 * 64 * Note: there are no guarantees that this function will not be reordered 65 * on non x86 architectures, so if you are writing portable code, 66 * make sure not to rely on its reordering guarantees. 67 * 68 * Note that @nr may be almost arbitrarily large; this function is not 69 * restricted to acting on a single-word quantity. 70 */ 71 static __always_inline void 72 set_bit(long nr, volatile unsigned long *addr) 73 { 74 if (IS_IMMEDIATE(nr)) { 75 asm volatile(LOCK_PREFIX "orb %1,%0" 76 : CONST_MASK_ADDR(nr, addr) 77 : "iq" ((u8)CONST_MASK(nr)) 78 : "memory"); 79 } else { 80 asm volatile(LOCK_PREFIX "bts %1,%0" 81 : BITOP_ADDR(addr) : "Ir" (nr) : "memory"); 82 } 83 } 84 85 /** 86 * __set_bit - Set a bit in memory 87 * @nr: the bit to set 88 * @addr: the address to start counting from 89 * 90 * Unlike set_bit(), this function is non-atomic and may be reordered. 91 * If it's called on the same region of memory simultaneously, the effect 92 * may be that only one operation succeeds. 93 */ 94 static inline void __set_bit(long nr, volatile unsigned long *addr) 95 { 96 asm volatile("bts %1,%0" : ADDR : "Ir" (nr) : "memory"); 97 } 98 99 /** 100 * clear_bit - Clears a bit in memory 101 * @nr: Bit to clear 102 * @addr: Address to start counting from 103 * 104 * clear_bit() is atomic and may not be reordered. However, it does 105 * not contain a memory barrier, so if it is used for locking purposes, 106 * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic() 107 * in order to ensure changes are visible on other processors. 108 */ 109 static __always_inline void 110 clear_bit(long nr, volatile unsigned long *addr) 111 { 112 if (IS_IMMEDIATE(nr)) { 113 asm volatile(LOCK_PREFIX "andb %1,%0" 114 : CONST_MASK_ADDR(nr, addr) 115 : "iq" ((u8)~CONST_MASK(nr))); 116 } else { 117 asm volatile(LOCK_PREFIX "btr %1,%0" 118 : BITOP_ADDR(addr) 119 : "Ir" (nr)); 120 } 121 } 122 123 /* 124 * clear_bit_unlock - Clears a bit in memory 125 * @nr: Bit to clear 126 * @addr: Address to start counting from 127 * 128 * clear_bit() is atomic and implies release semantics before the memory 129 * operation. It can be used for an unlock. 130 */ 131 static inline void clear_bit_unlock(long nr, volatile unsigned long *addr) 132 { 133 barrier(); 134 clear_bit(nr, addr); 135 } 136 137 static inline void __clear_bit(long nr, volatile unsigned long *addr) 138 { 139 asm volatile("btr %1,%0" : ADDR : "Ir" (nr)); 140 } 141 142 /* 143 * __clear_bit_unlock - Clears a bit in memory 144 * @nr: Bit to clear 145 * @addr: Address to start counting from 146 * 147 * __clear_bit() is non-atomic and implies release semantics before the memory 148 * operation. It can be used for an unlock if no other CPUs can concurrently 149 * modify other bits in the word. 150 * 151 * No memory barrier is required here, because x86 cannot reorder stores past 152 * older loads. Same principle as spin_unlock. 153 */ 154 static inline void __clear_bit_unlock(long nr, volatile unsigned long *addr) 155 { 156 barrier(); 157 __clear_bit(nr, addr); 158 } 159 160 /** 161 * __change_bit - Toggle a bit in memory 162 * @nr: the bit to change 163 * @addr: the address to start counting from 164 * 165 * Unlike change_bit(), this function is non-atomic and may be reordered. 166 * If it's called on the same region of memory simultaneously, the effect 167 * may be that only one operation succeeds. 168 */ 169 static inline void __change_bit(long nr, volatile unsigned long *addr) 170 { 171 asm volatile("btc %1,%0" : ADDR : "Ir" (nr)); 172 } 173 174 /** 175 * change_bit - Toggle a bit in memory 176 * @nr: Bit to change 177 * @addr: Address to start counting from 178 * 179 * change_bit() is atomic and may not be reordered. 180 * Note that @nr may be almost arbitrarily large; this function is not 181 * restricted to acting on a single-word quantity. 182 */ 183 static inline void change_bit(long nr, volatile unsigned long *addr) 184 { 185 if (IS_IMMEDIATE(nr)) { 186 asm volatile(LOCK_PREFIX "xorb %1,%0" 187 : CONST_MASK_ADDR(nr, addr) 188 : "iq" ((u8)CONST_MASK(nr))); 189 } else { 190 asm volatile(LOCK_PREFIX "btc %1,%0" 191 : BITOP_ADDR(addr) 192 : "Ir" (nr)); 193 } 194 } 195 196 /** 197 * test_and_set_bit - Set a bit and return its old value 198 * @nr: Bit to set 199 * @addr: Address to count from 200 * 201 * This operation is atomic and cannot be reordered. 202 * It also implies a memory barrier. 203 */ 204 static inline int test_and_set_bit(long nr, volatile unsigned long *addr) 205 { 206 GEN_BINARY_RMWcc(LOCK_PREFIX "bts", *addr, "Ir", nr, "%0", "c"); 207 } 208 209 /** 210 * test_and_set_bit_lock - Set a bit and return its old value for lock 211 * @nr: Bit to set 212 * @addr: Address to count from 213 * 214 * This is the same as test_and_set_bit on x86. 215 */ 216 static __always_inline int 217 test_and_set_bit_lock(long nr, volatile unsigned long *addr) 218 { 219 return test_and_set_bit(nr, addr); 220 } 221 222 /** 223 * __test_and_set_bit - Set a bit and return its old value 224 * @nr: Bit to set 225 * @addr: Address to count from 226 * 227 * This operation is non-atomic and can be reordered. 228 * If two examples of this operation race, one can appear to succeed 229 * but actually fail. You must protect multiple accesses with a lock. 230 */ 231 static inline int __test_and_set_bit(long nr, volatile unsigned long *addr) 232 { 233 int oldbit; 234 235 asm("bts %2,%1\n\t" 236 "sbb %0,%0" 237 : "=r" (oldbit), ADDR 238 : "Ir" (nr)); 239 return oldbit; 240 } 241 242 /** 243 * test_and_clear_bit - Clear a bit and return its old value 244 * @nr: Bit to clear 245 * @addr: Address to count from 246 * 247 * This operation is atomic and cannot be reordered. 248 * It also implies a memory barrier. 249 */ 250 static inline int test_and_clear_bit(long nr, volatile unsigned long *addr) 251 { 252 GEN_BINARY_RMWcc(LOCK_PREFIX "btr", *addr, "Ir", nr, "%0", "c"); 253 } 254 255 /** 256 * __test_and_clear_bit - Clear a bit and return its old value 257 * @nr: Bit to clear 258 * @addr: Address to count from 259 * 260 * This operation is non-atomic and can be reordered. 261 * If two examples of this operation race, one can appear to succeed 262 * but actually fail. You must protect multiple accesses with a lock. 263 * 264 * Note: the operation is performed atomically with respect to 265 * the local CPU, but not other CPUs. Portable code should not 266 * rely on this behaviour. 267 * KVM relies on this behaviour on x86 for modifying memory that is also 268 * accessed from a hypervisor on the same CPU if running in a VM: don't change 269 * this without also updating arch/x86/kernel/kvm.c 270 */ 271 static inline int __test_and_clear_bit(long nr, volatile unsigned long *addr) 272 { 273 int oldbit; 274 275 asm volatile("btr %2,%1\n\t" 276 "sbb %0,%0" 277 : "=r" (oldbit), ADDR 278 : "Ir" (nr)); 279 return oldbit; 280 } 281 282 /* WARNING: non atomic and it can be reordered! */ 283 static inline int __test_and_change_bit(long nr, volatile unsigned long *addr) 284 { 285 int oldbit; 286 287 asm volatile("btc %2,%1\n\t" 288 "sbb %0,%0" 289 : "=r" (oldbit), ADDR 290 : "Ir" (nr) : "memory"); 291 292 return oldbit; 293 } 294 295 /** 296 * test_and_change_bit - Change a bit and return its old value 297 * @nr: Bit to change 298 * @addr: Address to count from 299 * 300 * This operation is atomic and cannot be reordered. 301 * It also implies a memory barrier. 302 */ 303 static inline int test_and_change_bit(long nr, volatile unsigned long *addr) 304 { 305 GEN_BINARY_RMWcc(LOCK_PREFIX "btc", *addr, "Ir", nr, "%0", "c"); 306 } 307 308 static __always_inline int constant_test_bit(long nr, const volatile unsigned long *addr) 309 { 310 return ((1UL << (nr & (BITS_PER_LONG-1))) & 311 (addr[nr >> _BITOPS_LONG_SHIFT])) != 0; 312 } 313 314 static inline int variable_test_bit(long nr, volatile const unsigned long *addr) 315 { 316 int oldbit; 317 318 asm volatile("bt %2,%1\n\t" 319 "sbb %0,%0" 320 : "=r" (oldbit) 321 : "m" (*(unsigned long *)addr), "Ir" (nr)); 322 323 return oldbit; 324 } 325 326 #if 0 /* Fool kernel-doc since it doesn't do macros yet */ 327 /** 328 * test_bit - Determine whether a bit is set 329 * @nr: bit number to test 330 * @addr: Address to start counting from 331 */ 332 static int test_bit(int nr, const volatile unsigned long *addr); 333 #endif 334 335 #define test_bit(nr, addr) \ 336 (__builtin_constant_p((nr)) \ 337 ? constant_test_bit((nr), (addr)) \ 338 : variable_test_bit((nr), (addr))) 339 340 /** 341 * __ffs - find first set bit in word 342 * @word: The word to search 343 * 344 * Undefined if no bit exists, so code should check against 0 first. 345 */ 346 static inline unsigned long __ffs(unsigned long word) 347 { 348 asm("rep; bsf %1,%0" 349 : "=r" (word) 350 : "rm" (word)); 351 return word; 352 } 353 354 /** 355 * ffz - find first zero bit in word 356 * @word: The word to search 357 * 358 * Undefined if no zero exists, so code should check against ~0UL first. 359 */ 360 static inline unsigned long ffz(unsigned long word) 361 { 362 asm("rep; bsf %1,%0" 363 : "=r" (word) 364 : "r" (~word)); 365 return word; 366 } 367 368 /* 369 * __fls: find last set bit in word 370 * @word: The word to search 371 * 372 * Undefined if no set bit exists, so code should check against 0 first. 373 */ 374 static inline unsigned long __fls(unsigned long word) 375 { 376 asm("bsr %1,%0" 377 : "=r" (word) 378 : "rm" (word)); 379 return word; 380 } 381 382 #undef ADDR 383 384 #ifdef __KERNEL__ 385 /** 386 * ffs - find first set bit in word 387 * @x: the word to search 388 * 389 * This is defined the same way as the libc and compiler builtin ffs 390 * routines, therefore differs in spirit from the other bitops. 391 * 392 * ffs(value) returns 0 if value is 0 or the position of the first 393 * set bit if value is nonzero. The first (least significant) bit 394 * is at position 1. 395 */ 396 static inline int ffs(int x) 397 { 398 int r; 399 400 #ifdef CONFIG_X86_64 401 /* 402 * AMD64 says BSFL won't clobber the dest reg if x==0; Intel64 says the 403 * dest reg is undefined if x==0, but their CPU architect says its 404 * value is written to set it to the same as before, except that the 405 * top 32 bits will be cleared. 406 * 407 * We cannot do this on 32 bits because at the very least some 408 * 486 CPUs did not behave this way. 409 */ 410 asm("bsfl %1,%0" 411 : "=r" (r) 412 : "rm" (x), "0" (-1)); 413 #elif defined(CONFIG_X86_CMOV) 414 asm("bsfl %1,%0\n\t" 415 "cmovzl %2,%0" 416 : "=&r" (r) : "rm" (x), "r" (-1)); 417 #else 418 asm("bsfl %1,%0\n\t" 419 "jnz 1f\n\t" 420 "movl $-1,%0\n" 421 "1:" : "=r" (r) : "rm" (x)); 422 #endif 423 return r + 1; 424 } 425 426 /** 427 * fls - find last set bit in word 428 * @x: the word to search 429 * 430 * This is defined in a similar way as the libc and compiler builtin 431 * ffs, but returns the position of the most significant set bit. 432 * 433 * fls(value) returns 0 if value is 0 or the position of the last 434 * set bit if value is nonzero. The last (most significant) bit is 435 * at position 32. 436 */ 437 static inline int fls(int x) 438 { 439 int r; 440 441 #ifdef CONFIG_X86_64 442 /* 443 * AMD64 says BSRL won't clobber the dest reg if x==0; Intel64 says the 444 * dest reg is undefined if x==0, but their CPU architect says its 445 * value is written to set it to the same as before, except that the 446 * top 32 bits will be cleared. 447 * 448 * We cannot do this on 32 bits because at the very least some 449 * 486 CPUs did not behave this way. 450 */ 451 asm("bsrl %1,%0" 452 : "=r" (r) 453 : "rm" (x), "0" (-1)); 454 #elif defined(CONFIG_X86_CMOV) 455 asm("bsrl %1,%0\n\t" 456 "cmovzl %2,%0" 457 : "=&r" (r) : "rm" (x), "rm" (-1)); 458 #else 459 asm("bsrl %1,%0\n\t" 460 "jnz 1f\n\t" 461 "movl $-1,%0\n" 462 "1:" : "=r" (r) : "rm" (x)); 463 #endif 464 return r + 1; 465 } 466 467 /** 468 * fls64 - find last set bit in a 64-bit word 469 * @x: the word to search 470 * 471 * This is defined in a similar way as the libc and compiler builtin 472 * ffsll, but returns the position of the most significant set bit. 473 * 474 * fls64(value) returns 0 if value is 0 or the position of the last 475 * set bit if value is nonzero. The last (most significant) bit is 476 * at position 64. 477 */ 478 #ifdef CONFIG_X86_64 479 static __always_inline int fls64(__u64 x) 480 { 481 int bitpos = -1; 482 /* 483 * AMD64 says BSRQ won't clobber the dest reg if x==0; Intel64 says the 484 * dest reg is undefined if x==0, but their CPU architect says its 485 * value is written to set it to the same as before. 486 */ 487 asm("bsrq %1,%q0" 488 : "+r" (bitpos) 489 : "rm" (x)); 490 return bitpos + 1; 491 } 492 #else 493 #include <asm-generic/bitops/fls64.h> 494 #endif 495 496 #include <asm-generic/bitops/find.h> 497 498 #include <asm-generic/bitops/sched.h> 499 500 #include <asm/arch_hweight.h> 501 502 #include <asm-generic/bitops/const_hweight.h> 503 504 #include <asm-generic/bitops/le.h> 505 506 #include <asm-generic/bitops/ext2-atomic-setbit.h> 507 508 #endif /* __KERNEL__ */ 509 #endif /* _ASM_X86_BITOPS_H */
1 #ifndef _ASM_X86_UACCESS_H 2 #define _ASM_X86_UACCESS_H 3 /* 4 * User space memory access functions 5 */ 6 #include <linux/errno.h> 7 #include <linux/compiler.h> 8 #include <linux/thread_info.h> 9 #include <linux/string.h> 10 #include <asm/asm.h> 11 #include <asm/page.h> 12 #include <asm/smap.h> 13 14 #define VERIFY_READ 0 15 #define VERIFY_WRITE 1 16 17 /* 18 * The fs value determines whether argument validity checking should be 19 * performed or not. If get_fs() == USER_DS, checking is performed, with 20 * get_fs() == KERNEL_DS, checking is bypassed. 21 * 22 * For historical reasons, these macros are grossly misnamed. 23 */ 24 25 #define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) 26 27 #define KERNEL_DS MAKE_MM_SEG(-1UL) 28 #define USER_DS MAKE_MM_SEG(TASK_SIZE_MAX) 29 30 #define get_ds() (KERNEL_DS) 31 #define get_fs() (current_thread_info()->addr_limit) 32 #define set_fs(x) (current_thread_info()->addr_limit = (x)) 33 34 #define segment_eq(a, b) ((a).seg == (b).seg) 35 36 #define user_addr_max() (current_thread_info()->addr_limit.seg) 37 #define __addr_ok(addr) \ 38 ((unsigned long __force)(addr) < user_addr_max()) 39 40 /* 41 * Test whether a block of memory is a valid user space address. 42 * Returns 0 if the range is valid, nonzero otherwise. 43 */ 44 static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, unsigned long limit) 45 { 46 /* 47 * If we have used "sizeof()" for the size, 48 * we know it won't overflow the limit (but 49 * it might overflow the 'addr', so it's 50 * important to subtract the size from the 51 * limit, not add it to the address). 52 */ 53 if (__builtin_constant_p(size)) 54 return unlikely(addr > limit - size); 55 56 /* Arbitrary sizes? Be careful about overflow */ 57 addr += size; 58 if (unlikely(addr < size)) 59 return true; 60 return unlikely(addr > limit); 61 } 62 63 #define __range_not_ok(addr, size, limit) \ 64 ({ \ 65 __chk_user_ptr(addr); \ 66 __chk_range_not_ok((unsigned long __force)(addr), size, limit); \ 67 }) 68 69 /** 70 * access_ok: - Checks if a user space pointer is valid 71 * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that 72 * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe 73 * to write to a block, it is always safe to read from it. 74 * @addr: User space pointer to start of block to check 75 * @size: Size of block to check 76 * 77 * Context: User context only. This function may sleep if pagefaults are 78 * enabled. 79 * 80 * Checks if a pointer to a block of memory in user space is valid. 81 * 82 * Returns true (nonzero) if the memory block may be valid, false (zero) 83 * if it is definitely invalid. 84 * 85 * Note that, depending on architecture, this function probably just 86 * checks that the pointer is in the user space range - after calling 87 * this function, memory access functions may still return -EFAULT. 88 */ 89 #define access_ok(type, addr, size) \ 90 likely(!__range_not_ok(addr, size, user_addr_max())) 91 92 /* 93 * The exception table consists of pairs of addresses relative to the 94 * exception table enty itself: the first is the address of an 95 * instruction that is allowed to fault, and the second is the address 96 * at which the program should continue. No registers are modified, 97 * so it is entirely up to the continuation code to figure out what to 98 * do. 99 * 100 * All the routines below use bits of fixup code that are out of line 101 * with the main instruction path. This means when everything is well, 102 * we don't even have to jump over them. Further, they do not intrude 103 * on our cache or tlb entries. 104 */ 105 106 struct exception_table_entry { 107 int insn, fixup; 108 }; 109 /* This is not the generic standard exception_table_entry format */ 110 #define ARCH_HAS_SORT_EXTABLE 111 #define ARCH_HAS_SEARCH_EXTABLE 112 113 extern int fixup_exception(struct pt_regs *regs); 114 extern int early_fixup_exception(unsigned long *ip); 115 116 /* 117 * These are the main single-value transfer routines. They automatically 118 * use the right size if we just have the right pointer type. 119 * 120 * This gets kind of ugly. We want to return _two_ values in "get_user()" 121 * and yet we don't want to do any pointers, because that is too much 122 * of a performance impact. Thus we have a few rather ugly macros here, 123 * and hide all the ugliness from the user. 124 * 125 * The "__xxx" versions of the user access functions are versions that 126 * do not verify the address space, that must have been done previously 127 * with a separate "access_ok()" call (this is used when we do multiple 128 * accesses to the same area of user memory). 129 */ 130 131 extern int __get_user_1(void); 132 extern int __get_user_2(void); 133 extern int __get_user_4(void); 134 extern int __get_user_8(void); 135 extern int __get_user_bad(void); 136 137 #define __uaccess_begin() stac() 138 #define __uaccess_end() clac() 139 140 /* 141 * This is a type: either unsigned long, if the argument fits into 142 * that type, or otherwise unsigned long long. 143 */ 144 #define __inttype(x) \ 145 __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL)) 146 147 /** 148 * get_user: - Get a simple variable from user space. 149 * @x: Variable to store result. 150 * @ptr: Source address, in user space. 151 * 152 * Context: User context only. This function may sleep if pagefaults are 153 * enabled. 154 * 155 * This macro copies a single simple variable from user space to kernel 156 * space. It supports simple types like char and int, but not larger 157 * data types like structures or arrays. 158 * 159 * @ptr must have pointer-to-simple-variable type, and the result of 160 * dereferencing @ptr must be assignable to @x without a cast. 161 * 162 * Returns zero on success, or -EFAULT on error. 163 * On error, the variable @x is set to zero. 164 */ 165 /* 166 * Careful: we have to cast the result to the type of the pointer 167 * for sign reasons. 168 * 169 * The use of _ASM_DX as the register specifier is a bit of a 170 * simplification, as gcc only cares about it as the starting point 171 * and not size: for a 64-bit value it will use %ecx:%edx on 32 bits 172 * (%ecx being the next register in gcc's x86 register sequence), and 173 * %rdx on 64 bits. 174 * 175 * Clang/LLVM cares about the size of the register, but still wants 176 * the base register for something that ends up being a pair. 177 */ 178 #define get_user(x, ptr) \ 179 ({ \ 180 int __ret_gu; \ 181 register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX); \ 182 __chk_user_ptr(ptr); \ 183 might_fault(); \ 184 asm volatile("call __get_user_%P3" \ 185 : "=a" (__ret_gu), "=r" (__val_gu) \ 186 : "0" (ptr), "i" (sizeof(*(ptr)))); \ 187 (x) = (__force __typeof__(*(ptr))) __val_gu; \ 188 __builtin_expect(__ret_gu, 0); \ 189 }) 190 191 #define __put_user_x(size, x, ptr, __ret_pu) \ 192 asm volatile("call __put_user_" #size : "=a" (__ret_pu) \ 193 : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") 194 195 196 197 #ifdef CONFIG_X86_32 198 #define __put_user_asm_u64(x, addr, err, errret) \ 199 asm volatile("\n" \ 200 "1: movl %%eax,0(%2)\n" \ 201 "2: movl %%edx,4(%2)\n" \ 202 "3:" \ 203 ".section .fixup,\"ax\"\n" \ 204 "4: movl %3,%0\n" \ 205 " jmp 3b\n" \ 206 ".previous\n" \ 207 _ASM_EXTABLE(1b, 4b) \ 208 _ASM_EXTABLE(2b, 4b) \ 209 : "=r" (err) \ 210 : "A" (x), "r" (addr), "i" (errret), "0" (err)) 211 212 #define __put_user_asm_ex_u64(x, addr) \ 213 asm volatile("\n" \ 214 "1: movl %%eax,0(%1)\n" \ 215 "2: movl %%edx,4(%1)\n" \ 216 "3:" \ 217 _ASM_EXTABLE_EX(1b, 2b) \ 218 _ASM_EXTABLE_EX(2b, 3b) \ 219 : : "A" (x), "r" (addr)) 220 221 #define __put_user_x8(x, ptr, __ret_pu) \ 222 asm volatile("call __put_user_8" : "=a" (__ret_pu) \ 223 : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") 224 #else 225 #define __put_user_asm_u64(x, ptr, retval, errret) \ 226 __put_user_asm(x, ptr, retval, "q", "", "er", errret) 227 #define __put_user_asm_ex_u64(x, addr) \ 228 __put_user_asm_ex(x, addr, "q", "", "er") 229 #define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu) 230 #endif 231 232 extern void __put_user_bad(void); 233 234 /* 235 * Strange magic calling convention: pointer in %ecx, 236 * value in %eax(:%edx), return value in %eax. clobbers %rbx 237 */ 238 extern void __put_user_1(void); 239 extern void __put_user_2(void); 240 extern void __put_user_4(void); 241 extern void __put_user_8(void); 242 243 /** 244 * put_user: - Write a simple value into user space. 245 * @x: Value to copy to user space. 246 * @ptr: Destination address, in user space. 247 * 248 * Context: User context only. This function may sleep if pagefaults are 249 * enabled. 250 * 251 * This macro copies a single simple value from kernel space to user 252 * space. It supports simple types like char and int, but not larger 253 * data types like structures or arrays. 254 * 255 * @ptr must have pointer-to-simple-variable type, and @x must be assignable 256 * to the result of dereferencing @ptr. 257 * 258 * Returns zero on success, or -EFAULT on error. 259 */ 260 #define put_user(x, ptr) \ 261 ({ \ 262 int __ret_pu; \ 263 __typeof__(*(ptr)) __pu_val; \ 264 __chk_user_ptr(ptr); \ 265 might_fault(); \ 266 __pu_val = x; \ 267 switch (sizeof(*(ptr))) { \ 268 case 1: \ 269 __put_user_x(1, __pu_val, ptr, __ret_pu); \ 270 break; \ 271 case 2: \ 272 __put_user_x(2, __pu_val, ptr, __ret_pu); \ 273 break; \ 274 case 4: \ 275 __put_user_x(4, __pu_val, ptr, __ret_pu); \ 276 break; \ 277 case 8: \ 278 __put_user_x8(__pu_val, ptr, __ret_pu); \ 279 break; \ 280 default: \ 281 __put_user_x(X, __pu_val, ptr, __ret_pu); \ 282 break; \ 283 } \ 284 __builtin_expect(__ret_pu, 0); \ 285 }) 286 287 #define __put_user_size(x, ptr, size, retval, errret) \ 288 do { \ 289 retval = 0; \ 290 __chk_user_ptr(ptr); \ 291 switch (size) { \ 292 case 1: \ 293 __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \ 294 break; \ 295 case 2: \ 296 __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \ 297 break; \ 298 case 4: \ 299 __put_user_asm(x, ptr, retval, "l", "k", "ir", errret); \ 300 break; \ 301 case 8: \ 302 __put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval, \ 303 errret); \ 304 break; \ 305 default: \ 306 __put_user_bad(); \ 307 } \ 308 } while (0) 309 310 /* 311 * This doesn't do __uaccess_begin/end - the exception handling 312 * around it must do that. 313 */ 314 #define __put_user_size_ex(x, ptr, size) \ 315 do { \ 316 __chk_user_ptr(ptr); \ 317 switch (size) { \ 318 case 1: \ 319 __put_user_asm_ex(x, ptr, "b", "b", "iq"); \ 320 break; \ 321 case 2: \ 322 __put_user_asm_ex(x, ptr, "w", "w", "ir"); \ 323 break; \ 324 case 4: \ 325 __put_user_asm_ex(x, ptr, "l", "k", "ir"); \ 326 break; \ 327 case 8: \ 328 __put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr); \ 329 break; \ 330 default: \ 331 __put_user_bad(); \ 332 } \ 333 } while (0) 334 335 #ifdef CONFIG_X86_32 336 #define __get_user_asm_u64(x, ptr, retval, errret) (x) = __get_user_bad() 337 #define __get_user_asm_ex_u64(x, ptr) (x) = __get_user_bad() 338 #else 339 #define __get_user_asm_u64(x, ptr, retval, errret) \ 340 __get_user_asm(x, ptr, retval, "q", "", "=r", errret) 341 #define __get_user_asm_ex_u64(x, ptr) \ 342 __get_user_asm_ex(x, ptr, "q", "", "=r") 343 #endif 344 345 #define __get_user_size(x, ptr, size, retval, errret) \ 346 do { \ 347 retval = 0; \ 348 __chk_user_ptr(ptr); \ 349 switch (size) { \ 350 case 1: \ 351 __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \ 352 break; \ 353 case 2: \ 354 __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \ 355 break; \ 356 case 4: \ 357 __get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \ 358 break; \ 359 case 8: \ 360 __get_user_asm_u64(x, ptr, retval, errret); \ 361 break; \ 362 default: \ 363 (x) = __get_user_bad(); \ 364 } \ 365 } while (0) 366 367 #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \ 368 asm volatile("\n" \ 369 "1: mov"itype" %2,%"rtype"1\n" \ 370 "2:\n" \ 371 ".section .fixup,\"ax\"\n" \ 372 "3: mov %3,%0\n" \ 373 " xor"itype" %"rtype"1,%"rtype"1\n" \ 374 " jmp 2b\n" \ 375 ".previous\n" \ 376 _ASM_EXTABLE(1b, 3b) \ 377 : "=r" (err), ltype(x) \ 378 : "m" (__m(addr)), "i" (errret), "0" (err)) 379 380 /* 381 * This doesn't do __uaccess_begin/end - the exception handling 382 * around it must do that. 383 */ 384 #define __get_user_size_ex(x, ptr, size) \ 385 do { \ 386 __chk_user_ptr(ptr); \ 387 switch (size) { \ 388 case 1: \ 389 __get_user_asm_ex(x, ptr, "b", "b", "=q"); \ 390 break; \ 391 case 2: \ 392 __get_user_asm_ex(x, ptr, "w", "w", "=r"); \ 393 break; \ 394 case 4: \ 395 __get_user_asm_ex(x, ptr, "l", "k", "=r"); \ 396 break; \ 397 case 8: \ 398 __get_user_asm_ex_u64(x, ptr); \ 399 break; \ 400 default: \ 401 (x) = __get_user_bad(); \ 402 } \ 403 } while (0) 404 405 #define __get_user_asm_ex(x, addr, itype, rtype, ltype) \ 406 asm volatile("1: mov"itype" %1,%"rtype"0\n" \ 407 "2:\n" \ 408 _ASM_EXTABLE_EX(1b, 2b) \ 409 : ltype(x) : "m" (__m(addr))) 410 411 #define __put_user_nocheck(x, ptr, size) \ 412 ({ \ 413 int __pu_err; \ 414 __uaccess_begin(); \ 415 __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \ 416 __uaccess_end(); \ 417 __builtin_expect(__pu_err, 0); \ 418 }) 419 420 #define __get_user_nocheck(x, ptr, size) \ 421 ({ \ 422 int __gu_err; \ 423 unsigned long __gu_val; \ 424 __uaccess_begin(); \ 425 __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \ 426 __uaccess_end(); \ 427 (x) = (__force __typeof__(*(ptr)))__gu_val; \ 428 __builtin_expect(__gu_err, 0); \ 429 }) 430 431 /* FIXME: this hack is definitely wrong -AK */ 432 struct __large_struct { unsigned long buf[100]; }; 433 #define __m(x) (*(struct __large_struct __user *)(x)) 434 435 /* 436 * Tell gcc we read from memory instead of writing: this is because 437 * we do not write to any memory gcc knows about, so there are no 438 * aliasing issues. 439 */ 440 #define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \ 441 asm volatile("\n" \ 442 "1: mov"itype" %"rtype"1,%2\n" \ 443 "2:\n" \ 444 ".section .fixup,\"ax\"\n" \ 445 "3: mov %3,%0\n" \ 446 " jmp 2b\n" \ 447 ".previous\n" \ 448 _ASM_EXTABLE(1b, 3b) \ 449 : "=r"(err) \ 450 : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err)) 451 452 #define __put_user_asm_ex(x, addr, itype, rtype, ltype) \ 453 asm volatile("1: mov"itype" %"rtype"0,%1\n" \ 454 "2:\n" \ 455 _ASM_EXTABLE_EX(1b, 2b) \ 456 : : ltype(x), "m" (__m(addr))) 457 458 /* 459 * uaccess_try and catch 460 */ 461 #define uaccess_try do { \ 462 current_thread_info()->uaccess_err = 0; \ 463 __uaccess_begin(); \ 464 barrier(); 465 466 #define uaccess_catch(err) \ 467 __uaccess_end(); \ 468 (err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0); \ 469 } while (0) 470 471 /** 472 * __get_user: - Get a simple variable from user space, with less checking. 473 * @x: Variable to store result. 474 * @ptr: Source address, in user space. 475 * 476 * Context: User context only. This function may sleep if pagefaults are 477 * enabled. 478 * 479 * This macro copies a single simple variable from user space to kernel 480 * space. It supports simple types like char and int, but not larger 481 * data types like structures or arrays. 482 * 483 * @ptr must have pointer-to-simple-variable type, and the result of 484 * dereferencing @ptr must be assignable to @x without a cast. 485 * 486 * Caller must check the pointer with access_ok() before calling this 487 * function. 488 * 489 * Returns zero on success, or -EFAULT on error. 490 * On error, the variable @x is set to zero. 491 */ 492 493 #define __get_user(x, ptr) \ 494 __get_user_nocheck((x), (ptr), sizeof(*(ptr))) 495 496 /** 497 * __put_user: - Write a simple value into user space, with less checking. 498 * @x: Value to copy to user space. 499 * @ptr: Destination address, in user space. 500 * 501 * Context: User context only. This function may sleep if pagefaults are 502 * enabled. 503 * 504 * This macro copies a single simple value from kernel space to user 505 * space. It supports simple types like char and int, but not larger 506 * data types like structures or arrays. 507 * 508 * @ptr must have pointer-to-simple-variable type, and @x must be assignable 509 * to the result of dereferencing @ptr. 510 * 511 * Caller must check the pointer with access_ok() before calling this 512 * function. 513 * 514 * Returns zero on success, or -EFAULT on error. 515 */ 516 517 #define __put_user(x, ptr) \ 518 __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) 519 520 #define __get_user_unaligned __get_user 521 #define __put_user_unaligned __put_user 522 523 /* 524 * {get|put}_user_try and catch 525 * 526 * get_user_try { 527 * get_user_ex(...); 528 * } get_user_catch(err) 529 */ 530 #define get_user_try uaccess_try 531 #define get_user_catch(err) uaccess_catch(err) 532 533 #define get_user_ex(x, ptr) do { \ 534 unsigned long __gue_val; \ 535 __get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr)))); \ 536 (x) = (__force __typeof__(*(ptr)))__gue_val; \ 537 } while (0) 538 539 #define put_user_try uaccess_try 540 #define put_user_catch(err) uaccess_catch(err) 541 542 #define put_user_ex(x, ptr) \ 543 __put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) 544 545 extern unsigned long 546 copy_from_user_nmi(void *to, const void __user *from, unsigned long n); 547 extern __must_check long 548 strncpy_from_user(char *dst, const char __user *src, long count); 549 550 extern __must_check long strlen_user(const char __user *str); 551 extern __must_check long strnlen_user(const char __user *str, long n); 552 553 unsigned long __must_check clear_user(void __user *mem, unsigned long len); 554 unsigned long __must_check __clear_user(void __user *mem, unsigned long len); 555 556 extern void __cmpxchg_wrong_size(void) 557 __compiletime_error("Bad argument size for cmpxchg"); 558 559 #define __user_atomic_cmpxchg_inatomic(uval, ptr, old, new, size) \ 560 ({ \ 561 int __ret = 0; \ 562 __typeof__(ptr) __uval = (uval); \ 563 __typeof__(*(ptr)) __old = (old); \ 564 __typeof__(*(ptr)) __new = (new); \ 565 __uaccess_begin(); \ 566 switch (size) { \ 567 case 1: \ 568 { \ 569 asm volatile("\n" \ 570 "1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n" \ 571 "2:\n" \ 572 "\t.section .fixup, \"ax\"\n" \ 573 "3:\tmov %3, %0\n" \ 574 "\tjmp 2b\n" \ 575 "\t.previous\n" \ 576 _ASM_EXTABLE(1b, 3b) \ 577 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ 578 : "i" (-EFAULT), "q" (__new), "1" (__old) \ 579 : "memory" \ 580 ); \ 581 break; \ 582 } \ 583 case 2: \ 584 { \ 585 asm volatile("\n" \ 586 "1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n" \ 587 "2:\n" \ 588 "\t.section .fixup, \"ax\"\n" \ 589 "3:\tmov %3, %0\n" \ 590 "\tjmp 2b\n" \ 591 "\t.previous\n" \ 592 _ASM_EXTABLE(1b, 3b) \ 593 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ 594 : "i" (-EFAULT), "r" (__new), "1" (__old) \ 595 : "memory" \ 596 ); \ 597 break; \ 598 } \ 599 case 4: \ 600 { \ 601 asm volatile("\n" \ 602 "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" \ 603 "2:\n" \ 604 "\t.section .fixup, \"ax\"\n" \ 605 "3:\tmov %3, %0\n" \ 606 "\tjmp 2b\n" \ 607 "\t.previous\n" \ 608 _ASM_EXTABLE(1b, 3b) \ 609 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ 610 : "i" (-EFAULT), "r" (__new), "1" (__old) \ 611 : "memory" \ 612 ); \ 613 break; \ 614 } \ 615 case 8: \ 616 { \ 617 if (!IS_ENABLED(CONFIG_X86_64)) \ 618 __cmpxchg_wrong_size(); \ 619 \ 620 asm volatile("\n" \ 621 "1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n" \ 622 "2:\n" \ 623 "\t.section .fixup, \"ax\"\n" \ 624 "3:\tmov %3, %0\n" \ 625 "\tjmp 2b\n" \ 626 "\t.previous\n" \ 627 _ASM_EXTABLE(1b, 3b) \ 628 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ 629 : "i" (-EFAULT), "r" (__new), "1" (__old) \ 630 : "memory" \ 631 ); \ 632 break; \ 633 } \ 634 default: \ 635 __cmpxchg_wrong_size(); \ 636 } \ 637 __uaccess_end(); \ 638 *__uval = __old; \ 639 __ret; \ 640 }) 641 642 #define user_atomic_cmpxchg_inatomic(uval, ptr, old, new) \ 643 ({ \ 644 access_ok(VERIFY_WRITE, (ptr), sizeof(*(ptr))) ? \ 645 __user_atomic_cmpxchg_inatomic((uval), (ptr), \ 646 (old), (new), sizeof(*(ptr))) : \ 647 -EFAULT; \ 648 }) 649 650 /* 651 * movsl can be slow when source and dest are not both 8-byte aligned 652 */ 653 #ifdef CONFIG_X86_INTEL_USERCOPY 654 extern struct movsl_mask { 655 int mask; 656 } ____cacheline_aligned_in_smp movsl_mask; 657 #endif 658 659 #define ARCH_HAS_NOCACHE_UACCESS 1 660 661 #ifdef CONFIG_X86_32 662 # include <asm/uaccess_32.h> 663 #else 664 # include <asm/uaccess_64.h> 665 #endif 666 667 unsigned long __must_check _copy_from_user(void *to, const void __user *from, 668 unsigned n); 669 unsigned long __must_check _copy_to_user(void __user *to, const void *from, 670 unsigned n); 671 672 #ifdef CONFIG_DEBUG_STRICT_USER_COPY_CHECKS 673 # define copy_user_diag __compiletime_error 674 #else 675 # define copy_user_diag __compiletime_warning 676 #endif 677 678 extern void copy_user_diag("copy_from_user() buffer size is too small") 679 copy_from_user_overflow(void); 680 extern void copy_user_diag("copy_to_user() buffer size is too small") 681 copy_to_user_overflow(void) __asm__("copy_from_user_overflow"); 682 683 #undef copy_user_diag 684 685 #ifdef CONFIG_DEBUG_STRICT_USER_COPY_CHECKS 686 687 extern void 688 __compiletime_warning("copy_from_user() buffer size is not provably correct") 689 __copy_from_user_overflow(void) __asm__("copy_from_user_overflow"); 690 #define __copy_from_user_overflow(size, count) __copy_from_user_overflow() 691 692 extern void 693 __compiletime_warning("copy_to_user() buffer size is not provably correct") 694 __copy_to_user_overflow(void) __asm__("copy_from_user_overflow"); 695 #define __copy_to_user_overflow(size, count) __copy_to_user_overflow() 696 697 #else 698 699 static inline void 700 __copy_from_user_overflow(int size, unsigned long count) 701 { 702 WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count); 703 } 704 705 #define __copy_to_user_overflow __copy_from_user_overflow 706 707 #endif 708 709 static inline unsigned long __must_check 710 copy_from_user(void *to, const void __user *from, unsigned long n) 711 { 712 int sz = __compiletime_object_size(to); 713 714 might_fault(); 715 716 /* 717 * While we would like to have the compiler do the checking for us 718 * even in the non-constant size case, any false positives there are 719 * a problem (especially when DEBUG_STRICT_USER_COPY_CHECKS, but even 720 * without - the [hopefully] dangerous looking nature of the warning 721 * would make people go look at the respecitive call sites over and 722 * over again just to find that there's no problem). 723 * 724 * And there are cases where it's just not realistic for the compiler 725 * to prove the count to be in range. For example when multiple call 726 * sites of a helper function - perhaps in different source files - 727 * all doing proper range checking, yet the helper function not doing 728 * so again. 729 * 730 * Therefore limit the compile time checking to the constant size 731 * case, and do only runtime checking for non-constant sizes. 732 */ 733 734 if (likely(sz < 0 || sz >= n)) 735 n = _copy_from_user(to, from, n); 736 else if(__builtin_constant_p(n)) 737 copy_from_user_overflow(); 738 else 739 __copy_from_user_overflow(sz, n); 740 741 return n; 742 } 743 744 static inline unsigned long __must_check 745 copy_to_user(void __user *to, const void *from, unsigned long n) 746 { 747 int sz = __compiletime_object_size(from); 748 749 might_fault(); 750 751 /* See the comment in copy_from_user() above. */ 752 if (likely(sz < 0 || sz >= n)) 753 n = _copy_to_user(to, from, n); 754 else if(__builtin_constant_p(n)) 755 copy_to_user_overflow(); 756 else 757 __copy_to_user_overflow(sz, n); 758 759 return n; 760 } 761 762 #undef __copy_from_user_overflow 763 #undef __copy_to_user_overflow 764 765 /* 766 * We rely on the nested NMI work to allow atomic faults from the NMI path; the 767 * nested NMI paths are careful to preserve CR2. 768 * 769 * Caller must use pagefault_enable/disable, or run in interrupt context, 770 * and also do a uaccess_ok() check 771 */ 772 #define __copy_from_user_nmi __copy_from_user_inatomic 773 774 /* 775 * The "unsafe" user accesses aren't really "unsafe", but the naming 776 * is a big fat warning: you have to not only do the access_ok() 777 * checking before using them, but you have to surround them with the 778 * user_access_begin/end() pair. 779 */ 780 #define user_access_begin() __uaccess_begin() 781 #define user_access_end() __uaccess_end() 782 783 #define unsafe_put_user(x, ptr) \ 784 ({ \ 785 int __pu_err; \ 786 __put_user_size((x), (ptr), sizeof(*(ptr)), __pu_err, -EFAULT); \ 787 __builtin_expect(__pu_err, 0); \ 788 }) 789 790 #define unsafe_get_user(x, ptr) \ 791 ({ \ 792 int __gu_err; \ 793 unsigned long __gu_val; \ 794 __get_user_size(__gu_val, (ptr), sizeof(*(ptr)), __gu_err, -EFAULT); \ 795 (x) = (__force __typeof__(*(ptr)))__gu_val; \ 796 __builtin_expect(__gu_err, 0); \ 797 }) 798 799 #endif /* _ASM_X86_UACCESS_H */
1 2 /* 3 * Driver for the Micron P320 SSD 4 * Copyright (C) 2011 Micron Technology, Inc. 5 * 6 * Portions of this code were derived from works subjected to the 7 * following copyright: 8 * Copyright (C) 2009 Integrated Device Technology, Inc. 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License as published by 12 * the Free Software Foundation; either version 2 of the License, or 13 * (at your option) any later version. 14 * 15 * This program is distributed in the hope that it will be useful, 16 * but WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 * GNU General Public License for more details. 19 * 20 */ 21 22 #include <linux/pci.h> 23 #include <linux/interrupt.h> 24 #include <linux/ata.h> 25 #include <linux/delay.h> 26 #include <linux/hdreg.h> 27 #include <linux/uaccess.h> 28 #include <linux/random.h> 29 #include <linux/smp.h> 30 #include <linux/compat.h> 31 #include <linux/fs.h> 32 #include <linux/module.h> 33 #include <linux/genhd.h> 34 #include <linux/blkdev.h> 35 #include <linux/blk-mq.h> 36 #include <linux/bio.h> 37 #include <linux/dma-mapping.h> 38 #include <linux/idr.h> 39 #include <linux/kthread.h> 40 #include <../drivers/ata/ahci.h> 41 #include <linux/export.h> 42 #include <linux/debugfs.h> 43 #include <linux/prefetch.h> 44 #include "mtip32xx.h" 45 46 #define HW_CMD_SLOT_SZ (MTIP_MAX_COMMAND_SLOTS * 32) 47 48 /* DMA region containing RX Fis, Identify, RLE10, and SMART buffers */ 49 #define AHCI_RX_FIS_SZ 0x100 50 #define AHCI_RX_FIS_OFFSET 0x0 51 #define AHCI_IDFY_SZ ATA_SECT_SIZE 52 #define AHCI_IDFY_OFFSET 0x400 53 #define AHCI_SECTBUF_SZ ATA_SECT_SIZE 54 #define AHCI_SECTBUF_OFFSET 0x800 55 #define AHCI_SMARTBUF_SZ ATA_SECT_SIZE 56 #define AHCI_SMARTBUF_OFFSET 0xC00 57 /* 0x100 + 0x200 + 0x200 + 0x200 is smaller than 4k but we pad it out */ 58 #define BLOCK_DMA_ALLOC_SZ 4096 59 60 /* DMA region containing command table (should be 8192 bytes) */ 61 #define AHCI_CMD_SLOT_SZ sizeof(struct mtip_cmd_hdr) 62 #define AHCI_CMD_TBL_SZ (MTIP_MAX_COMMAND_SLOTS * AHCI_CMD_SLOT_SZ) 63 #define AHCI_CMD_TBL_OFFSET 0x0 64 65 /* DMA region per command (contains header and SGL) */ 66 #define AHCI_CMD_TBL_HDR_SZ 0x80 67 #define AHCI_CMD_TBL_HDR_OFFSET 0x0 68 #define AHCI_CMD_TBL_SGL_SZ (MTIP_MAX_SG * sizeof(struct mtip_cmd_sg)) 69 #define AHCI_CMD_TBL_SGL_OFFSET AHCI_CMD_TBL_HDR_SZ 70 #define CMD_DMA_ALLOC_SZ (AHCI_CMD_TBL_SGL_SZ + AHCI_CMD_TBL_HDR_SZ) 71 72 73 #define HOST_CAP_NZDMA (1 << 19) 74 #define HOST_HSORG 0xFC 75 #define HSORG_DISABLE_SLOTGRP_INTR (1<<24) 76 #define HSORG_DISABLE_SLOTGRP_PXIS (1<<16) 77 #define HSORG_HWREV 0xFF00 78 #define HSORG_STYLE 0x8 79 #define HSORG_SLOTGROUPS 0x7 80 81 #define PORT_COMMAND_ISSUE 0x38 82 #define PORT_SDBV 0x7C 83 84 #define PORT_OFFSET 0x100 85 #define PORT_MEM_SIZE 0x80 86 87 #define PORT_IRQ_ERR \ 88 (PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR | PORT_IRQ_CONNECT | \ 89 PORT_IRQ_PHYRDY | PORT_IRQ_UNK_FIS | PORT_IRQ_BAD_PMP | \ 90 PORT_IRQ_TF_ERR | PORT_IRQ_HBUS_DATA_ERR | PORT_IRQ_IF_NONFATAL | \ 91 PORT_IRQ_OVERFLOW) 92 #define PORT_IRQ_LEGACY \ 93 (PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS) 94 #define PORT_IRQ_HANDLED \ 95 (PORT_IRQ_SDB_FIS | PORT_IRQ_LEGACY | \ 96 PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR | \ 97 PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY) 98 #define DEF_PORT_IRQ \ 99 (PORT_IRQ_ERR | PORT_IRQ_LEGACY | PORT_IRQ_SDB_FIS) 100 101 /* product numbers */ 102 #define MTIP_PRODUCT_UNKNOWN 0x00 103 #define MTIP_PRODUCT_ASICFPGA 0x11 104 105 /* Device instance number, incremented each time a device is probed. */ 106 static int instance; 107 108 static struct list_head online_list; 109 static struct list_head removing_list; 110 static spinlock_t dev_lock; 111 112 /* 113 * Global variable used to hold the major block device number 114 * allocated in mtip_init(). 115 */ 116 static int mtip_major; 117 static struct dentry *dfs_parent; 118 static struct dentry *dfs_device_status; 119 120 static u32 cpu_use[NR_CPUS]; 121 122 static DEFINE_SPINLOCK(rssd_index_lock); 123 static DEFINE_IDA(rssd_index_ida); 124 125 static int mtip_block_initialize(struct driver_data *dd); 126 127 #ifdef CONFIG_COMPAT 128 struct mtip_compat_ide_task_request_s { 129 __u8 io_ports[8]; 130 __u8 hob_ports[8]; 131 ide_reg_valid_t out_flags; 132 ide_reg_valid_t in_flags; 133 int data_phase; 134 int req_cmd; 135 compat_ulong_t out_size; 136 compat_ulong_t in_size; 137 }; 138 #endif 139 140 /* 141 * This function check_for_surprise_removal is called 142 * while card is removed from the system and it will 143 * read the vendor id from the configration space 144 * 145 * @pdev Pointer to the pci_dev structure. 146 * 147 * return value 148 * true if device removed, else false 149 */ 150 static bool mtip_check_surprise_removal(struct pci_dev *pdev) 151 { 152 u16 vendor_id = 0; 153 struct driver_data *dd = pci_get_drvdata(pdev); 154 155 if (dd->sr) 156 return true; 157 158 /* Read the vendorID from the configuration space */ 159 pci_read_config_word(pdev, 0x00, &vendor_id); 160 if (vendor_id == 0xFFFF) { 161 dd->sr = true; 162 if (dd->queue) 163 set_bit(QUEUE_FLAG_DEAD, &dd->queue->queue_flags); 164 else 165 dev_warn(&dd->pdev->dev, 166 "%s: dd->queue is NULL\n", __func__); 167 return true; /* device removed */ 168 } 169 170 return false; /* device present */ 171 } 172 173 static struct mtip_cmd *mtip_get_int_command(struct driver_data *dd) 174 { 175 struct request *rq; 176 177 rq = blk_mq_alloc_request(dd->queue, 0, BLK_MQ_REQ_RESERVED); 178 return blk_mq_rq_to_pdu(rq); 179 } 180 181 static void mtip_put_int_command(struct driver_data *dd, struct mtip_cmd *cmd) 182 { 183 blk_put_request(blk_mq_rq_from_pdu(cmd)); 184 } 185 186 /* 187 * Once we add support for one hctx per mtip group, this will change a bit 188 */ 189 static struct request *mtip_rq_from_tag(struct driver_data *dd, 190 unsigned int tag) 191 { 192 struct blk_mq_hw_ctx *hctx = dd->queue->queue_hw_ctx[0]; 193 194 return blk_mq_tag_to_rq(hctx->tags, tag); 195 } 196 197 static struct mtip_cmd *mtip_cmd_from_tag(struct driver_data *dd, 198 unsigned int tag) 199 { 200 struct request *rq = mtip_rq_from_tag(dd, tag); 201 202 return blk_mq_rq_to_pdu(rq); 203 } 204 205 /* 206 * IO completion function. 207 * 208 * This completion function is called by the driver ISR when a 209 * command that was issued by the kernel completes. It first calls the 210 * asynchronous completion function which normally calls back into the block 211 * layer passing the asynchronous callback data, then unmaps the 212 * scatter list associated with the completed command, and finally 213 * clears the allocated bit associated with the completed command. 214 * 215 * @port Pointer to the port data structure. 216 * @tag Tag of the command. 217 * @data Pointer to driver_data. 218 * @status Completion status. 219 * 220 * return value 221 * None 222 */ 223 static void mtip_async_complete(struct mtip_port *port, 224 int tag, struct mtip_cmd *cmd, int status) 225 { 226 struct driver_data *dd = port->dd; 227 struct request *rq; 228 229 if (unlikely(!dd) || unlikely(!port)) 230 return; 231 232 if (unlikely(status == PORT_IRQ_TF_ERR)) { 233 dev_warn(&port->dd->pdev->dev, 234 "Command tag %d failed due to TFE\n", tag); 235 } 236 237 /* Unmap the DMA scatter list entries */ 238 dma_unmap_sg(&dd->pdev->dev, cmd->sg, cmd->scatter_ents, cmd->direction); 239 240 rq = mtip_rq_from_tag(dd, tag); 241 242 if (unlikely(cmd->unaligned)) 243 up(&port->cmd_slot_unal); 244 245 blk_mq_end_request(rq, status ? -EIO : 0); 246 } 247 248 /* 249 * Reset the HBA (without sleeping) 250 * 251 * @dd Pointer to the driver data structure. 252 * 253 * return value 254 * 0 The reset was successful. 255 * -1 The HBA Reset bit did not clear. 256 */ 257 static int mtip_hba_reset(struct driver_data *dd) 258 { 259 unsigned long timeout; 260 261 /* Set the reset bit */ 262 writel(HOST_RESET, dd->mmio + HOST_CTL); 263 264 /* Flush */ 265 readl(dd->mmio + HOST_CTL); 266 267 /* 268 * Spin for up to 10 seconds waiting for reset acknowledgement. Spec 269 * is 1 sec but in LUN failure conditions, up to 10 secs are required 270 */ 271 timeout = jiffies + msecs_to_jiffies(10000); 272 do { 273 mdelay(10); 274 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) 275 return -1; 276 277 } while ((readl(dd->mmio + HOST_CTL) & HOST_RESET) 278 && time_before(jiffies, timeout)); 279 280 if (readl(dd->mmio + HOST_CTL) & HOST_RESET) 281 return -1; 282 283 return 0; 284 } 285 286 /* 287 * Issue a command to the hardware. 288 * 289 * Set the appropriate bit in the s_active and Command Issue hardware 290 * registers, causing hardware command processing to begin. 291 * 292 * @port Pointer to the port structure. 293 * @tag The tag of the command to be issued. 294 * 295 * return value 296 * None 297 */ 298 static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag) 299 { 300 int group = tag >> 5; 301 302 /* guard SACT and CI registers */ 303 spin_lock(&port->cmd_issue_lock[group]); 304 writel((1 << MTIP_TAG_BIT(tag)), 305 port->s_active[MTIP_TAG_INDEX(tag)]); 306 writel((1 << MTIP_TAG_BIT(tag)), 307 port->cmd_issue[MTIP_TAG_INDEX(tag)]); 308 spin_unlock(&port->cmd_issue_lock[group]); 309 } 310 311 /* 312 * Enable/disable the reception of FIS 313 * 314 * @port Pointer to the port data structure 315 * @enable 1 to enable, 0 to disable 316 * 317 * return value 318 * Previous state: 1 enabled, 0 disabled 319 */ 320 static int mtip_enable_fis(struct mtip_port *port, int enable) 321 { 322 u32 tmp; 323 324 /* enable FIS reception */ 325 tmp = readl(port->mmio + PORT_CMD); 326 if (enable) 327 writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD); 328 else 329 writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD); 330 331 /* Flush */ 332 readl(port->mmio + PORT_CMD); 333 334 return (((tmp & PORT_CMD_FIS_RX) == PORT_CMD_FIS_RX)); 335 } 336 337 /* 338 * Enable/disable the DMA engine 339 * 340 * @port Pointer to the port data structure 341 * @enable 1 to enable, 0 to disable 342 * 343 * return value 344 * Previous state: 1 enabled, 0 disabled. 345 */ 346 static int mtip_enable_engine(struct mtip_port *port, int enable) 347 { 348 u32 tmp; 349 350 /* enable FIS reception */ 351 tmp = readl(port->mmio + PORT_CMD); 352 if (enable) 353 writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD); 354 else 355 writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD); 356 357 readl(port->mmio + PORT_CMD); 358 return (((tmp & PORT_CMD_START) == PORT_CMD_START)); 359 } 360 361 /* 362 * Enables the port DMA engine and FIS reception. 363 * 364 * return value 365 * None 366 */ 367 static inline void mtip_start_port(struct mtip_port *port) 368 { 369 /* Enable FIS reception */ 370 mtip_enable_fis(port, 1); 371 372 /* Enable the DMA engine */ 373 mtip_enable_engine(port, 1); 374 } 375 376 /* 377 * Deinitialize a port by disabling port interrupts, the DMA engine, 378 * and FIS reception. 379 * 380 * @port Pointer to the port structure 381 * 382 * return value 383 * None 384 */ 385 static inline void mtip_deinit_port(struct mtip_port *port) 386 { 387 /* Disable interrupts on this port */ 388 writel(0, port->mmio + PORT_IRQ_MASK); 389 390 /* Disable the DMA engine */ 391 mtip_enable_engine(port, 0); 392 393 /* Disable FIS reception */ 394 mtip_enable_fis(port, 0); 395 } 396 397 /* 398 * Initialize a port. 399 * 400 * This function deinitializes the port by calling mtip_deinit_port() and 401 * then initializes it by setting the command header and RX FIS addresses, 402 * clearing the SError register and any pending port interrupts before 403 * re-enabling the default set of port interrupts. 404 * 405 * @port Pointer to the port structure. 406 * 407 * return value 408 * None 409 */ 410 static void mtip_init_port(struct mtip_port *port) 411 { 412 int i; 413 mtip_deinit_port(port); 414 415 /* Program the command list base and FIS base addresses */ 416 if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) { 417 writel((port->command_list_dma >> 16) >> 16, 418 port->mmio + PORT_LST_ADDR_HI); 419 writel((port->rxfis_dma >> 16) >> 16, 420 port->mmio + PORT_FIS_ADDR_HI); 421 } 422 423 writel(port->command_list_dma & 0xFFFFFFFF, 424 port->mmio + PORT_LST_ADDR); 425 writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR); 426 427 /* Clear SError */ 428 writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR); 429 430 /* reset the completed registers.*/ 431 for (i = 0; i < port->dd->slot_groups; i++) 432 writel(0xFFFFFFFF, port->completed[i]); 433 434 /* Clear any pending interrupts for this port */ 435 writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT); 436 437 /* Clear any pending interrupts on the HBA. */ 438 writel(readl(port->dd->mmio + HOST_IRQ_STAT), 439 port->dd->mmio + HOST_IRQ_STAT); 440 441 /* Enable port interrupts */ 442 writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK); 443 } 444 445 /* 446 * Restart a port 447 * 448 * @port Pointer to the port data structure. 449 * 450 * return value 451 * None 452 */ 453 static void mtip_restart_port(struct mtip_port *port) 454 { 455 unsigned long timeout; 456 457 /* Disable the DMA engine */ 458 mtip_enable_engine(port, 0); 459 460 /* Chip quirk: wait up to 500ms for PxCMD.CR == 0 */ 461 timeout = jiffies + msecs_to_jiffies(500); 462 while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) 463 && time_before(jiffies, timeout)) 464 ; 465 466 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 467 return; 468 469 /* 470 * Chip quirk: escalate to hba reset if 471 * PxCMD.CR not clear after 500 ms 472 */ 473 if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) { 474 dev_warn(&port->dd->pdev->dev, 475 "PxCMD.CR not clear, escalating reset\n"); 476 477 if (mtip_hba_reset(port->dd)) 478 dev_err(&port->dd->pdev->dev, 479 "HBA reset escalation failed.\n"); 480 481 /* 30 ms delay before com reset to quiesce chip */ 482 mdelay(30); 483 } 484 485 dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n"); 486 487 /* Set PxSCTL.DET */ 488 writel(readl(port->mmio + PORT_SCR_CTL) | 489 1, port->mmio + PORT_SCR_CTL); 490 readl(port->mmio + PORT_SCR_CTL); 491 492 /* Wait 1 ms to quiesce chip function */ 493 timeout = jiffies + msecs_to_jiffies(1); 494 while (time_before(jiffies, timeout)) 495 ; 496 497 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 498 return; 499 500 /* Clear PxSCTL.DET */ 501 writel(readl(port->mmio + PORT_SCR_CTL) & ~1, 502 port->mmio + PORT_SCR_CTL); 503 readl(port->mmio + PORT_SCR_CTL); 504 505 /* Wait 500 ms for bit 0 of PORT_SCR_STS to be set */ 506 timeout = jiffies + msecs_to_jiffies(500); 507 while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) 508 && time_before(jiffies, timeout)) 509 ; 510 511 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 512 return; 513 514 if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) 515 dev_warn(&port->dd->pdev->dev, 516 "COM reset failed\n"); 517 518 mtip_init_port(port); 519 mtip_start_port(port); 520 521 } 522 523 static int mtip_device_reset(struct driver_data *dd) 524 { 525 int rv = 0; 526 527 if (mtip_check_surprise_removal(dd->pdev)) 528 return 0; 529 530 if (mtip_hba_reset(dd) < 0) 531 rv = -EFAULT; 532 533 mdelay(1); 534 mtip_init_port(dd->port); 535 mtip_start_port(dd->port); 536 537 /* Enable interrupts on the HBA. */ 538 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, 539 dd->mmio + HOST_CTL); 540 return rv; 541 } 542 543 /* 544 * Helper function for tag logging 545 */ 546 static void print_tags(struct driver_data *dd, 547 char *msg, 548 unsigned long *tagbits, 549 int cnt) 550 { 551 unsigned char tagmap[128]; 552 int group, tagmap_len = 0; 553 554 memset(tagmap, 0, sizeof(tagmap)); 555 for (group = SLOTBITS_IN_LONGS; group > 0; group--) 556 tagmap_len += sprintf(tagmap + tagmap_len, "%016lX ", 557 tagbits[group-1]); 558 dev_warn(&dd->pdev->dev, 559 "%d command(s) %s: tagmap [%s]", cnt, msg, tagmap); 560 } 561 562 /* 563 * Internal command completion callback function. 564 * 565 * This function is normally called by the driver ISR when an internal 566 * command completed. This function signals the command completion by 567 * calling complete(). 568 * 569 * @port Pointer to the port data structure. 570 * @tag Tag of the command that has completed. 571 * @data Pointer to a completion structure. 572 * @status Completion status. 573 * 574 * return value 575 * None 576 */ 577 static void mtip_completion(struct mtip_port *port, 578 int tag, struct mtip_cmd *command, int status) 579 { 580 struct completion *waiting = command->comp_data; 581 if (unlikely(status == PORT_IRQ_TF_ERR)) 582 dev_warn(&port->dd->pdev->dev, 583 "Internal command %d completed with TFE\n", tag); 584 585 complete(waiting); 586 } 587 588 static void mtip_null_completion(struct mtip_port *port, 589 int tag, struct mtip_cmd *command, int status) 590 { 591 } 592 593 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer, 594 dma_addr_t buffer_dma, unsigned int sectors); 595 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id, 596 struct smart_attr *attrib); 597 /* 598 * Handle an error. 599 * 600 * @dd Pointer to the DRIVER_DATA structure. 601 * 602 * return value 603 * None 604 */ 605 static void mtip_handle_tfe(struct driver_data *dd) 606 { 607 int group, tag, bit, reissue, rv; 608 struct mtip_port *port; 609 struct mtip_cmd *cmd; 610 u32 completed; 611 struct host_to_dev_fis *fis; 612 unsigned long tagaccum[SLOTBITS_IN_LONGS]; 613 unsigned int cmd_cnt = 0; 614 unsigned char *buf; 615 char *fail_reason = NULL; 616 int fail_all_ncq_write = 0, fail_all_ncq_cmds = 0; 617 618 dev_warn(&dd->pdev->dev, "Taskfile error\n"); 619 620 port = dd->port; 621 622 set_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); 623 624 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags)) { 625 cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL); 626 dbg_printk(MTIP_DRV_NAME " TFE for the internal command\n"); 627 628 if (cmd->comp_data && cmd->comp_func) { 629 cmd->comp_func(port, MTIP_TAG_INTERNAL, 630 cmd, PORT_IRQ_TF_ERR); 631 } 632 goto handle_tfe_exit; 633 } 634 635 /* clear the tag accumulator */ 636 memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long)); 637 638 /* Loop through all the groups */ 639 for (group = 0; group < dd->slot_groups; group++) { 640 completed = readl(port->completed[group]); 641 642 dev_warn(&dd->pdev->dev, "g=%u, comp=%x\n", group, completed); 643 644 /* clear completed status register in the hardware.*/ 645 writel(completed, port->completed[group]); 646 647 /* Process successfully completed commands */ 648 for (bit = 0; bit < 32 && completed; bit++) { 649 if (!(completed & (1<<bit))) 650 continue; 651 tag = (group << 5) + bit; 652 653 /* Skip the internal command slot */ 654 if (tag == MTIP_TAG_INTERNAL) 655 continue; 656 657 cmd = mtip_cmd_from_tag(dd, tag); 658 if (likely(cmd->comp_func)) { 659 set_bit(tag, tagaccum); 660 cmd_cnt++; 661 cmd->comp_func(port, tag, cmd, 0); 662 } else { 663 dev_err(&port->dd->pdev->dev, 664 "Missing completion func for tag %d", 665 tag); 666 if (mtip_check_surprise_removal(dd->pdev)) { 667 /* don't proceed further */ 668 return; 669 } 670 } 671 } 672 } 673 674 print_tags(dd, "completed (TFE)", tagaccum, cmd_cnt); 675 676 /* Restart the port */ 677 mdelay(20); 678 mtip_restart_port(port); 679 680 /* Trying to determine the cause of the error */ 681 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, 682 dd->port->log_buf, 683 dd->port->log_buf_dma, 1); 684 if (rv) { 685 dev_warn(&dd->pdev->dev, 686 "Error in READ LOG EXT (10h) command\n"); 687 /* non-critical error, don't fail the load */ 688 } else { 689 buf = (unsigned char *)dd->port->log_buf; 690 if (buf[259] & 0x1) { 691 dev_info(&dd->pdev->dev, 692 "Write protect bit is set.\n"); 693 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); 694 fail_all_ncq_write = 1; 695 fail_reason = "write protect"; 696 } 697 if (buf[288] == 0xF7) { 698 dev_info(&dd->pdev->dev, 699 "Exceeded Tmax, drive in thermal shutdown.\n"); 700 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); 701 fail_all_ncq_cmds = 1; 702 fail_reason = "thermal shutdown"; 703 } 704 if (buf[288] == 0xBF) { 705 set_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag); 706 dev_info(&dd->pdev->dev, 707 "Drive indicates rebuild has failed. Secure erase required.\n"); 708 fail_all_ncq_cmds = 1; 709 fail_reason = "rebuild failed"; 710 } 711 } 712 713 /* clear the tag accumulator */ 714 memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long)); 715 716 /* Loop through all the groups */ 717 for (group = 0; group < dd->slot_groups; group++) { 718 for (bit = 0; bit < 32; bit++) { 719 reissue = 1; 720 tag = (group << 5) + bit; 721 cmd = mtip_cmd_from_tag(dd, tag); 722 723 fis = (struct host_to_dev_fis *)cmd->command; 724 725 /* Should re-issue? */ 726 if (tag == MTIP_TAG_INTERNAL || 727 fis->command == ATA_CMD_SET_FEATURES) 728 reissue = 0; 729 else { 730 if (fail_all_ncq_cmds || 731 (fail_all_ncq_write && 732 fis->command == ATA_CMD_FPDMA_WRITE)) { 733 dev_warn(&dd->pdev->dev, 734 " Fail: %s w/tag %d [%s].\n", 735 fis->command == ATA_CMD_FPDMA_WRITE ? 736 "write" : "read", 737 tag, 738 fail_reason != NULL ? 739 fail_reason : "unknown"); 740 if (cmd->comp_func) { 741 cmd->comp_func(port, tag, 742 cmd, -ENODATA); 743 } 744 continue; 745 } 746 } 747 748 /* 749 * First check if this command has 750 * exceeded its retries. 751 */ 752 if (reissue && (cmd->retries-- > 0)) { 753 754 set_bit(tag, tagaccum); 755 756 /* Re-issue the command. */ 757 mtip_issue_ncq_command(port, tag); 758 759 continue; 760 } 761 762 /* Retire a command that will not be reissued */ 763 dev_warn(&port->dd->pdev->dev, 764 "retiring tag %d\n", tag); 765 766 if (cmd->comp_func) 767 cmd->comp_func(port, tag, cmd, PORT_IRQ_TF_ERR); 768 else 769 dev_warn(&port->dd->pdev->dev, 770 "Bad completion for tag %d\n", 771 tag); 772 } 773 } 774 print_tags(dd, "reissued (TFE)", tagaccum, cmd_cnt); 775 776 handle_tfe_exit: 777 /* clear eh_active */ 778 clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); 779 wake_up_interruptible(&port->svc_wait); 780 } 781 782 /* 783 * Handle a set device bits interrupt 784 */ 785 static inline void mtip_workq_sdbfx(struct mtip_port *port, int group, 786 u32 completed) 787 { 788 struct driver_data *dd = port->dd; 789 int tag, bit; 790 struct mtip_cmd *command; 791 792 if (!completed) { 793 WARN_ON_ONCE(!completed); 794 return; 795 } 796 /* clear completed status register in the hardware.*/ 797 writel(completed, port->completed[group]); 798 799 /* Process completed commands. */ 800 for (bit = 0; (bit < 32) && completed; bit++) { 801 if (completed & 0x01) { 802 tag = (group << 5) | bit; 803 804 /* skip internal command slot. */ 805 if (unlikely(tag == MTIP_TAG_INTERNAL)) 806 continue; 807 808 command = mtip_cmd_from_tag(dd, tag); 809 if (likely(command->comp_func)) 810 command->comp_func(port, tag, command, 0); 811 else { 812 dev_dbg(&dd->pdev->dev, 813 "Null completion for tag %d", 814 tag); 815 816 if (mtip_check_surprise_removal( 817 dd->pdev)) { 818 return; 819 } 820 } 821 } 822 completed >>= 1; 823 } 824 825 /* If last, re-enable interrupts */ 826 if (atomic_dec_return(&dd->irq_workers_active) == 0) 827 writel(0xffffffff, dd->mmio + HOST_IRQ_STAT); 828 } 829 830 /* 831 * Process legacy pio and d2h interrupts 832 */ 833 static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat) 834 { 835 struct mtip_port *port = dd->port; 836 struct mtip_cmd *cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL); 837 838 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && 839 (cmd != NULL) && !(readl(port->cmd_issue[MTIP_TAG_INTERNAL]) 840 & (1 << MTIP_TAG_INTERNAL))) { 841 if (cmd->comp_func) { 842 cmd->comp_func(port, MTIP_TAG_INTERNAL, cmd, 0); 843 return; 844 } 845 } 846 847 return; 848 } 849 850 /* 851 * Demux and handle errors 852 */ 853 static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat) 854 { 855 856 if (unlikely(port_stat & PORT_IRQ_CONNECT)) { 857 dev_warn(&dd->pdev->dev, 858 "Clearing PxSERR.DIAG.x\n"); 859 writel((1 << 26), dd->port->mmio + PORT_SCR_ERR); 860 } 861 862 if (unlikely(port_stat & PORT_IRQ_PHYRDY)) { 863 dev_warn(&dd->pdev->dev, 864 "Clearing PxSERR.DIAG.n\n"); 865 writel((1 << 16), dd->port->mmio + PORT_SCR_ERR); 866 } 867 868 if (unlikely(port_stat & ~PORT_IRQ_HANDLED)) { 869 dev_warn(&dd->pdev->dev, 870 "Port stat errors %x unhandled\n", 871 (port_stat & ~PORT_IRQ_HANDLED)); 872 if (mtip_check_surprise_removal(dd->pdev)) 873 return; 874 } 875 if (likely(port_stat & (PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR))) { 876 set_bit(MTIP_PF_EH_ACTIVE_BIT, &dd->port->flags); 877 wake_up_interruptible(&dd->port->svc_wait); 878 } 879 } 880 881 static inline irqreturn_t mtip_handle_irq(struct driver_data *data) 882 { 883 struct driver_data *dd = (struct driver_data *) data; 884 struct mtip_port *port = dd->port; 885 u32 hba_stat, port_stat; 886 int rv = IRQ_NONE; 887 int do_irq_enable = 1, i, workers; 888 struct mtip_work *twork; 889 890 hba_stat = readl(dd->mmio + HOST_IRQ_STAT); 891 if (hba_stat) { 892 rv = IRQ_HANDLED; 893 894 /* Acknowledge the interrupt status on the port.*/ 895 port_stat = readl(port->mmio + PORT_IRQ_STAT); 896 if (unlikely(port_stat == 0xFFFFFFFF)) { 897 mtip_check_surprise_removal(dd->pdev); 898 return IRQ_HANDLED; 899 } 900 writel(port_stat, port->mmio + PORT_IRQ_STAT); 901 902 /* Demux port status */ 903 if (likely(port_stat & PORT_IRQ_SDB_FIS)) { 904 do_irq_enable = 0; 905 WARN_ON_ONCE(atomic_read(&dd->irq_workers_active) != 0); 906 907 /* Start at 1: group zero is always local? */ 908 for (i = 0, workers = 0; i < MTIP_MAX_SLOT_GROUPS; 909 i++) { 910 twork = &dd->work[i]; 911 twork->completed = readl(port->completed[i]); 912 if (twork->completed) 913 workers++; 914 } 915 916 atomic_set(&dd->irq_workers_active, workers); 917 if (workers) { 918 for (i = 1; i < MTIP_MAX_SLOT_GROUPS; i++) { 919 twork = &dd->work[i]; 920 if (twork->completed) 921 queue_work_on( 922 twork->cpu_binding, 923 dd->isr_workq, 924 &twork->work); 925 } 926 927 if (likely(dd->work[0].completed)) 928 mtip_workq_sdbfx(port, 0, 929 dd->work[0].completed); 930 931 } else { 932 /* 933 * Chip quirk: SDB interrupt but nothing 934 * to complete 935 */ 936 do_irq_enable = 1; 937 } 938 } 939 940 if (unlikely(port_stat & PORT_IRQ_ERR)) { 941 if (unlikely(mtip_check_surprise_removal(dd->pdev))) { 942 /* don't proceed further */ 943 return IRQ_HANDLED; 944 } 945 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 946 &dd->dd_flag)) 947 return rv; 948 949 mtip_process_errors(dd, port_stat & PORT_IRQ_ERR); 950 } 951 952 if (unlikely(port_stat & PORT_IRQ_LEGACY)) 953 mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY); 954 } 955 956 /* acknowledge interrupt */ 957 if (unlikely(do_irq_enable)) 958 writel(hba_stat, dd->mmio + HOST_IRQ_STAT); 959 960 return rv; 961 } 962 963 /* 964 * HBA interrupt subroutine. 965 * 966 * @irq IRQ number. 967 * @instance Pointer to the driver data structure. 968 * 969 * return value 970 * IRQ_HANDLED A HBA interrupt was pending and handled. 971 * IRQ_NONE This interrupt was not for the HBA. 972 */ 973 static irqreturn_t mtip_irq_handler(int irq, void *instance) 974 { 975 struct driver_data *dd = instance; 976 977 return mtip_handle_irq(dd); 978 } 979 980 static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag) 981 { 982 writel(1 << MTIP_TAG_BIT(tag), 983 port->cmd_issue[MTIP_TAG_INDEX(tag)]); 984 } 985 986 static bool mtip_pause_ncq(struct mtip_port *port, 987 struct host_to_dev_fis *fis) 988 { 989 struct host_to_dev_fis *reply; 990 unsigned long task_file_data; 991 992 reply = port->rxfis + RX_FIS_D2H_REG; 993 task_file_data = readl(port->mmio+PORT_TFDATA); 994 995 if ((task_file_data & 1)) 996 return false; 997 998 if (fis->command == ATA_CMD_SEC_ERASE_PREP) { 999 port->ic_pause_timer = jiffies; 1000 return true; 1001 } else if ((fis->command == ATA_CMD_DOWNLOAD_MICRO) && 1002 (fis->features == 0x03)) { 1003 set_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); 1004 port->ic_pause_timer = jiffies; 1005 return true; 1006 } else if ((fis->command == ATA_CMD_SEC_ERASE_UNIT) || 1007 ((fis->command == 0xFC) && 1008 (fis->features == 0x27 || fis->features == 0x72 || 1009 fis->features == 0x62 || fis->features == 0x26))) { 1010 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); 1011 /* Com reset after secure erase or lowlevel format */ 1012 mtip_restart_port(port); 1013 clear_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); 1014 return false; 1015 } 1016 1017 return false; 1018 } 1019 1020 /* 1021 * Wait for port to quiesce 1022 * 1023 * @port Pointer to port data structure 1024 * @timeout Max duration to wait (ms) 1025 * 1026 * return value 1027 * 0 Success 1028 * -EBUSY Commands still active 1029 */ 1030 static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout) 1031 { 1032 unsigned long to; 1033 unsigned int n; 1034 unsigned int active = 1; 1035 1036 blk_mq_stop_hw_queues(port->dd->queue); 1037 1038 to = jiffies + msecs_to_jiffies(timeout); 1039 do { 1040 if (test_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags) && 1041 test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { 1042 msleep(20); 1043 continue; /* svc thd is actively issuing commands */ 1044 } 1045 1046 msleep(100); 1047 if (mtip_check_surprise_removal(port->dd->pdev)) 1048 goto err_fault; 1049 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 1050 goto err_fault; 1051 1052 /* 1053 * Ignore s_active bit 0 of array element 0. 1054 * This bit will always be set 1055 */ 1056 active = readl(port->s_active[0]) & 0xFFFFFFFE; 1057 for (n = 1; n < port->dd->slot_groups; n++) 1058 active |= readl(port->s_active[n]); 1059 1060 if (!active) 1061 break; 1062 } while (time_before(jiffies, to)); 1063 1064 blk_mq_start_stopped_hw_queues(port->dd->queue, true); 1065 return active ? -EBUSY : 0; 1066 err_fault: 1067 blk_mq_start_stopped_hw_queues(port->dd->queue, true); 1068 return -EFAULT; 1069 } 1070 1071 /* 1072 * Execute an internal command and wait for the completion. 1073 * 1074 * @port Pointer to the port data structure. 1075 * @fis Pointer to the FIS that describes the command. 1076 * @fis_len Length in WORDS of the FIS. 1077 * @buffer DMA accessible for command data. 1078 * @buf_len Length, in bytes, of the data buffer. 1079 * @opts Command header options, excluding the FIS length 1080 * and the number of PRD entries. 1081 * @timeout Time in ms to wait for the command to complete. 1082 * 1083 * return value 1084 * 0 Command completed successfully. 1085 * -EFAULT The buffer address is not correctly aligned. 1086 * -EBUSY Internal command or other IO in progress. 1087 * -EAGAIN Time out waiting for command to complete. 1088 */ 1089 static int mtip_exec_internal_command(struct mtip_port *port, 1090 struct host_to_dev_fis *fis, 1091 int fis_len, 1092 dma_addr_t buffer, 1093 int buf_len, 1094 u32 opts, 1095 gfp_t atomic, 1096 unsigned long timeout) 1097 { 1098 struct mtip_cmd_sg *command_sg; 1099 DECLARE_COMPLETION_ONSTACK(wait); 1100 struct mtip_cmd *int_cmd; 1101 struct driver_data *dd = port->dd; 1102 int rv = 0; 1103 1104 /* Make sure the buffer is 8 byte aligned. This is asic specific. */ 1105 if (buffer & 0x00000007) { 1106 dev_err(&dd->pdev->dev, "SG buffer is not 8 byte aligned\n"); 1107 return -EFAULT; 1108 } 1109 1110 int_cmd = mtip_get_int_command(dd); 1111 1112 set_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); 1113 1114 if (fis->command == ATA_CMD_SEC_ERASE_PREP) 1115 set_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); 1116 1117 clear_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); 1118 1119 if (atomic == GFP_KERNEL) { 1120 if (fis->command != ATA_CMD_STANDBYNOW1) { 1121 /* wait for io to complete if non atomic */ 1122 if (mtip_quiesce_io(port, 1123 MTIP_QUIESCE_IO_TIMEOUT_MS) < 0) { 1124 dev_warn(&dd->pdev->dev, 1125 "Failed to quiesce IO\n"); 1126 mtip_put_int_command(dd, int_cmd); 1127 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); 1128 wake_up_interruptible(&port->svc_wait); 1129 return -EBUSY; 1130 } 1131 } 1132 1133 /* Set the completion function and data for the command. */ 1134 int_cmd->comp_data = &wait; 1135 int_cmd->comp_func = mtip_completion; 1136 1137 } else { 1138 /* Clear completion - we're going to poll */ 1139 int_cmd->comp_data = NULL; 1140 int_cmd->comp_func = mtip_null_completion; 1141 } 1142 1143 /* Copy the command to the command table */ 1144 memcpy(int_cmd->command, fis, fis_len*4); 1145 1146 /* Populate the SG list */ 1147 int_cmd->command_header->opts = 1148 __force_bit2int cpu_to_le32(opts | fis_len); 1149 if (buf_len) { 1150 command_sg = int_cmd->command + AHCI_CMD_TBL_HDR_SZ; 1151 1152 command_sg->info = 1153 __force_bit2int cpu_to_le32((buf_len-1) & 0x3FFFFF); 1154 command_sg->dba = 1155 __force_bit2int cpu_to_le32(buffer & 0xFFFFFFFF); 1156 command_sg->dba_upper = 1157 __force_bit2int cpu_to_le32((buffer >> 16) >> 16); 1158 1159 int_cmd->command_header->opts |= 1160 __force_bit2int cpu_to_le32((1 << 16)); 1161 } 1162 1163 /* Populate the command header */ 1164 int_cmd->command_header->byte_count = 0; 1165 1166 /* Issue the command to the hardware */ 1167 mtip_issue_non_ncq_command(port, MTIP_TAG_INTERNAL); 1168 1169 if (atomic == GFP_KERNEL) { 1170 /* Wait for the command to complete or timeout. */ 1171 if ((rv = wait_for_completion_interruptible_timeout( 1172 &wait, 1173 msecs_to_jiffies(timeout))) <= 0) { 1174 if (rv == -ERESTARTSYS) { /* interrupted */ 1175 dev_err(&dd->pdev->dev, 1176 "Internal command [%02X] was interrupted after %lu ms\n", 1177 fis->command, timeout); 1178 rv = -EINTR; 1179 goto exec_ic_exit; 1180 } else if (rv == 0) /* timeout */ 1181 dev_err(&dd->pdev->dev, 1182 "Internal command did not complete [%02X] within timeout of %lu ms\n", 1183 fis->command, timeout); 1184 else 1185 dev_err(&dd->pdev->dev, 1186 "Internal command [%02X] wait returned code [%d] after %lu ms - unhandled\n", 1187 fis->command, rv, timeout); 1188 1189 if (mtip_check_surprise_removal(dd->pdev) || 1190 test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 1191 &dd->dd_flag)) { 1192 dev_err(&dd->pdev->dev, 1193 "Internal command [%02X] wait returned due to SR\n", 1194 fis->command); 1195 rv = -ENXIO; 1196 goto exec_ic_exit; 1197 } 1198 mtip_device_reset(dd); /* recover from timeout issue */ 1199 rv = -EAGAIN; 1200 goto exec_ic_exit; 1201 } 1202 } else { 1203 u32 hba_stat, port_stat; 1204 1205 /* Spin for <timeout> checking if command still outstanding */ 1206 timeout = jiffies + msecs_to_jiffies(timeout); 1207 while ((readl(port->cmd_issue[MTIP_TAG_INTERNAL]) 1208 & (1 << MTIP_TAG_INTERNAL)) 1209 && time_before(jiffies, timeout)) { 1210 if (mtip_check_surprise_removal(dd->pdev)) { 1211 rv = -ENXIO; 1212 goto exec_ic_exit; 1213 } 1214 if ((fis->command != ATA_CMD_STANDBYNOW1) && 1215 test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 1216 &dd->dd_flag)) { 1217 rv = -ENXIO; 1218 goto exec_ic_exit; 1219 } 1220 port_stat = readl(port->mmio + PORT_IRQ_STAT); 1221 if (!port_stat) 1222 continue; 1223 1224 if (port_stat & PORT_IRQ_ERR) { 1225 dev_err(&dd->pdev->dev, 1226 "Internal command [%02X] failed\n", 1227 fis->command); 1228 mtip_device_reset(dd); 1229 rv = -EIO; 1230 goto exec_ic_exit; 1231 } else { 1232 writel(port_stat, port->mmio + PORT_IRQ_STAT); 1233 hba_stat = readl(dd->mmio + HOST_IRQ_STAT); 1234 if (hba_stat) 1235 writel(hba_stat, 1236 dd->mmio + HOST_IRQ_STAT); 1237 } 1238 break; 1239 } 1240 } 1241 1242 if (readl(port->cmd_issue[MTIP_TAG_INTERNAL]) 1243 & (1 << MTIP_TAG_INTERNAL)) { 1244 rv = -ENXIO; 1245 if (!test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { 1246 mtip_device_reset(dd); 1247 rv = -EAGAIN; 1248 } 1249 } 1250 exec_ic_exit: 1251 /* Clear the allocated and active bits for the internal command. */ 1252 mtip_put_int_command(dd, int_cmd); 1253 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); 1254 if (rv >= 0 && mtip_pause_ncq(port, fis)) { 1255 /* NCQ paused */ 1256 return rv; 1257 } 1258 wake_up_interruptible(&port->svc_wait); 1259 1260 return rv; 1261 } 1262 1263 /* 1264 * Byte-swap ATA ID strings. 1265 * 1266 * ATA identify data contains strings in byte-swapped 16-bit words. 1267 * They must be swapped (on all architectures) to be usable as C strings. 1268 * This function swaps bytes in-place. 1269 * 1270 * @buf The buffer location of the string 1271 * @len The number of bytes to swap 1272 * 1273 * return value 1274 * None 1275 */ 1276 static inline void ata_swap_string(u16 *buf, unsigned int len) 1277 { 1278 int i; 1279 for (i = 0; i < (len/2); i++) 1280 be16_to_cpus(&buf[i]); 1281 } 1282 1283 static void mtip_set_timeout(struct driver_data *dd, 1284 struct host_to_dev_fis *fis, 1285 unsigned int *timeout, u8 erasemode) 1286 { 1287 switch (fis->command) { 1288 case ATA_CMD_DOWNLOAD_MICRO: 1289 *timeout = 120000; /* 2 minutes */ 1290 break; 1291 case ATA_CMD_SEC_ERASE_UNIT: 1292 case 0xFC: 1293 if (erasemode) 1294 *timeout = ((*(dd->port->identify + 90) * 2) * 60000); 1295 else 1296 *timeout = ((*(dd->port->identify + 89) * 2) * 60000); 1297 break; 1298 case ATA_CMD_STANDBYNOW1: 1299 *timeout = 120000; /* 2 minutes */ 1300 break; 1301 case 0xF7: 1302 case 0xFA: 1303 *timeout = 60000; /* 60 seconds */ 1304 break; 1305 case ATA_CMD_SMART: 1306 *timeout = 15000; /* 15 seconds */ 1307 break; 1308 default: 1309 *timeout = MTIP_IOCTL_CMD_TIMEOUT_MS; 1310 break; 1311 } 1312 } 1313 1314 /* 1315 * Request the device identity information. 1316 * 1317 * If a user space buffer is not specified, i.e. is NULL, the 1318 * identify information is still read from the drive and placed 1319 * into the identify data buffer (@e port->identify) in the 1320 * port data structure. 1321 * When the identify buffer contains valid identify information @e 1322 * port->identify_valid is non-zero. 1323 * 1324 * @port Pointer to the port structure. 1325 * @user_buffer A user space buffer where the identify data should be 1326 * copied. 1327 * 1328 * return value 1329 * 0 Command completed successfully. 1330 * -EFAULT An error occurred while coping data to the user buffer. 1331 * -1 Command failed. 1332 */ 1333 static int mtip_get_identify(struct mtip_port *port, void __user *user_buffer) 1334 { 1335 int rv = 0; 1336 struct host_to_dev_fis fis; 1337 1338 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 1339 return -EFAULT; 1340 1341 /* Build the FIS. */ 1342 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1343 fis.type = 0x27; 1344 fis.opts = 1 << 7; 1345 fis.command = ATA_CMD_ID_ATA; 1346 1347 /* Set the identify information as invalid. */ 1348 port->identify_valid = 0; 1349 1350 /* Clear the identify information. */ 1351 memset(port->identify, 0, sizeof(u16) * ATA_ID_WORDS); 1352 1353 /* Execute the command. */ 1354 if (mtip_exec_internal_command(port, 1355 &fis, 1356 5, 1357 port->identify_dma, 1358 sizeof(u16) * ATA_ID_WORDS, 1359 0, 1360 GFP_KERNEL, 1361 MTIP_INT_CMD_TIMEOUT_MS) 1362 < 0) { 1363 rv = -1; 1364 goto out; 1365 } 1366 1367 /* 1368 * Perform any necessary byte-swapping. Yes, the kernel does in fact 1369 * perform field-sensitive swapping on the string fields. 1370 * See the kernel use of ata_id_string() for proof of this. 1371 */ 1372 #ifdef __LITTLE_ENDIAN 1373 ata_swap_string(port->identify + 27, 40); /* model string*/ 1374 ata_swap_string(port->identify + 23, 8); /* firmware string*/ 1375 ata_swap_string(port->identify + 10, 20); /* serial# string*/ 1376 #else 1377 { 1378 int i; 1379 for (i = 0; i < ATA_ID_WORDS; i++) 1380 port->identify[i] = le16_to_cpu(port->identify[i]); 1381 } 1382 #endif 1383 1384 /* Check security locked state */ 1385 if (port->identify[128] & 0x4) 1386 set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); 1387 else 1388 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); 1389 1390 #ifdef MTIP_TRIM /* Disabling TRIM support temporarily */ 1391 /* Demux ID.DRAT & ID.RZAT to determine trim support */ 1392 if (port->identify[69] & (1 << 14) && port->identify[69] & (1 << 5)) 1393 port->dd->trim_supp = true; 1394 else 1395 #endif 1396 port->dd->trim_supp = false; 1397 1398 /* Set the identify buffer as valid. */ 1399 port->identify_valid = 1; 1400 1401 if (user_buffer) { 1402 if (copy_to_user( 1403 user_buffer, 1404 port->identify, 1405 ATA_ID_WORDS * sizeof(u16))) { 1406 rv = -EFAULT; 1407 goto out; 1408 } 1409 } 1410 1411 out: 1412 return rv; 1413 } 1414 1415 /* 1416 * Issue a standby immediate command to the device. 1417 * 1418 * @port Pointer to the port structure. 1419 * 1420 * return value 1421 * 0 Command was executed successfully. 1422 * -1 An error occurred while executing the command. 1423 */ 1424 static int mtip_standby_immediate(struct mtip_port *port) 1425 { 1426 int rv; 1427 struct host_to_dev_fis fis; 1428 unsigned long start; 1429 unsigned int timeout; 1430 1431 /* Build the FIS. */ 1432 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1433 fis.type = 0x27; 1434 fis.opts = 1 << 7; 1435 fis.command = ATA_CMD_STANDBYNOW1; 1436 1437 mtip_set_timeout(port->dd, &fis, &timeout, 0); 1438 1439 start = jiffies; 1440 rv = mtip_exec_internal_command(port, 1441 &fis, 1442 5, 1443 0, 1444 0, 1445 0, 1446 GFP_ATOMIC, 1447 timeout); 1448 dbg_printk(MTIP_DRV_NAME "Time taken to complete standby cmd: %d ms\n", 1449 jiffies_to_msecs(jiffies - start)); 1450 if (rv) 1451 dev_warn(&port->dd->pdev->dev, 1452 "STANDBY IMMEDIATE command failed.\n"); 1453 1454 return rv; 1455 } 1456 1457 /* 1458 * Issue a READ LOG EXT command to the device. 1459 * 1460 * @port pointer to the port structure. 1461 * @page page number to fetch 1462 * @buffer pointer to buffer 1463 * @buffer_dma dma address corresponding to @buffer 1464 * @sectors page length to fetch, in sectors 1465 * 1466 * return value 1467 * @rv return value from mtip_exec_internal_command() 1468 */ 1469 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer, 1470 dma_addr_t buffer_dma, unsigned int sectors) 1471 { 1472 struct host_to_dev_fis fis; 1473 1474 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1475 fis.type = 0x27; 1476 fis.opts = 1 << 7; 1477 fis.command = ATA_CMD_READ_LOG_EXT; 1478 fis.sect_count = sectors & 0xFF; 1479 fis.sect_cnt_ex = (sectors >> 8) & 0xFF; 1480 fis.lba_low = page; 1481 fis.lba_mid = 0; 1482 fis.device = ATA_DEVICE_OBS; 1483 1484 memset(buffer, 0, sectors * ATA_SECT_SIZE); 1485 1486 return mtip_exec_internal_command(port, 1487 &fis, 1488 5, 1489 buffer_dma, 1490 sectors * ATA_SECT_SIZE, 1491 0, 1492 GFP_ATOMIC, 1493 MTIP_INT_CMD_TIMEOUT_MS); 1494 } 1495 1496 /* 1497 * Issue a SMART READ DATA command to the device. 1498 * 1499 * @port pointer to the port structure. 1500 * @buffer pointer to buffer 1501 * @buffer_dma dma address corresponding to @buffer 1502 * 1503 * return value 1504 * @rv return value from mtip_exec_internal_command() 1505 */ 1506 static int mtip_get_smart_data(struct mtip_port *port, u8 *buffer, 1507 dma_addr_t buffer_dma) 1508 { 1509 struct host_to_dev_fis fis; 1510 1511 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1512 fis.type = 0x27; 1513 fis.opts = 1 << 7; 1514 fis.command = ATA_CMD_SMART; 1515 fis.features = 0xD0; 1516 fis.sect_count = 1; 1517 fis.lba_mid = 0x4F; 1518 fis.lba_hi = 0xC2; 1519 fis.device = ATA_DEVICE_OBS; 1520 1521 return mtip_exec_internal_command(port, 1522 &fis, 1523 5, 1524 buffer_dma, 1525 ATA_SECT_SIZE, 1526 0, 1527 GFP_ATOMIC, 1528 15000); 1529 } 1530 1531 /* 1532 * Get the value of a smart attribute 1533 * 1534 * @port pointer to the port structure 1535 * @id attribute number 1536 * @attrib pointer to return attrib information corresponding to @id 1537 * 1538 * return value 1539 * -EINVAL NULL buffer passed or unsupported attribute @id. 1540 * -EPERM Identify data not valid, SMART not supported or not enabled 1541 */ 1542 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id, 1543 struct smart_attr *attrib) 1544 { 1545 int rv, i; 1546 struct smart_attr *pattr; 1547 1548 if (!attrib) 1549 return -EINVAL; 1550 1551 if (!port->identify_valid) { 1552 dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n"); 1553 return -EPERM; 1554 } 1555 if (!(port->identify[82] & 0x1)) { 1556 dev_warn(&port->dd->pdev->dev, "SMART not supported\n"); 1557 return -EPERM; 1558 } 1559 if (!(port->identify[85] & 0x1)) { 1560 dev_warn(&port->dd->pdev->dev, "SMART not enabled\n"); 1561 return -EPERM; 1562 } 1563 1564 memset(port->smart_buf, 0, ATA_SECT_SIZE); 1565 rv = mtip_get_smart_data(port, port->smart_buf, port->smart_buf_dma); 1566 if (rv) { 1567 dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n"); 1568 return rv; 1569 } 1570 1571 pattr = (struct smart_attr *)(port->smart_buf + 2); 1572 for (i = 0; i < 29; i++, pattr++) 1573 if (pattr->attr_id == id) { 1574 memcpy(attrib, pattr, sizeof(struct smart_attr)); 1575 break; 1576 } 1577 1578 if (i == 29) { 1579 dev_warn(&port->dd->pdev->dev, 1580 "Query for invalid SMART attribute ID\n"); 1581 rv = -EINVAL; 1582 } 1583 1584 return rv; 1585 } 1586 1587 /* 1588 * Trim unused sectors 1589 * 1590 * @dd pointer to driver_data structure 1591 * @lba starting lba 1592 * @len # of 512b sectors to trim 1593 * 1594 * return value 1595 * -ENOMEM Out of dma memory 1596 * -EINVAL Invalid parameters passed in, trim not supported 1597 * -EIO Error submitting trim request to hw 1598 */ 1599 static int mtip_send_trim(struct driver_data *dd, unsigned int lba, 1600 unsigned int len) 1601 { 1602 int i, rv = 0; 1603 u64 tlba, tlen, sect_left; 1604 struct mtip_trim_entry *buf; 1605 dma_addr_t dma_addr; 1606 struct host_to_dev_fis fis; 1607 1608 if (!len || dd->trim_supp == false) 1609 return -EINVAL; 1610 1611 /* Trim request too big */ 1612 WARN_ON(len > (MTIP_MAX_TRIM_ENTRY_LEN * MTIP_MAX_TRIM_ENTRIES)); 1613 1614 /* Trim request not aligned on 4k boundary */ 1615 WARN_ON(len % 8 != 0); 1616 1617 /* Warn if vu_trim structure is too big */ 1618 WARN_ON(sizeof(struct mtip_trim) > ATA_SECT_SIZE); 1619 1620 /* Allocate a DMA buffer for the trim structure */ 1621 buf = dmam_alloc_coherent(&dd->pdev->dev, ATA_SECT_SIZE, &dma_addr, 1622 GFP_KERNEL); 1623 if (!buf) 1624 return -ENOMEM; 1625 memset(buf, 0, ATA_SECT_SIZE); 1626 1627 for (i = 0, sect_left = len, tlba = lba; 1628 i < MTIP_MAX_TRIM_ENTRIES && sect_left; 1629 i++) { 1630 tlen = (sect_left >= MTIP_MAX_TRIM_ENTRY_LEN ? 1631 MTIP_MAX_TRIM_ENTRY_LEN : 1632 sect_left); 1633 buf[i].lba = __force_bit2int cpu_to_le32(tlba); 1634 buf[i].range = __force_bit2int cpu_to_le16(tlen); 1635 tlba += tlen; 1636 sect_left -= tlen; 1637 } 1638 WARN_ON(sect_left != 0); 1639 1640 /* Build the fis */ 1641 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1642 fis.type = 0x27; 1643 fis.opts = 1 << 7; 1644 fis.command = 0xfb; 1645 fis.features = 0x60; 1646 fis.sect_count = 1; 1647 fis.device = ATA_DEVICE_OBS; 1648 1649 if (mtip_exec_internal_command(dd->port, 1650 &fis, 1651 5, 1652 dma_addr, 1653 ATA_SECT_SIZE, 1654 0, 1655 GFP_KERNEL, 1656 MTIP_TRIM_TIMEOUT_MS) < 0) 1657 rv = -EIO; 1658 1659 dmam_free_coherent(&dd->pdev->dev, ATA_SECT_SIZE, buf, dma_addr); 1660 return rv; 1661 } 1662 1663 /* 1664 * Get the drive capacity. 1665 * 1666 * @dd Pointer to the device data structure. 1667 * @sectors Pointer to the variable that will receive the sector count. 1668 * 1669 * return value 1670 * 1 Capacity was returned successfully. 1671 * 0 The identify information is invalid. 1672 */ 1673 static bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors) 1674 { 1675 struct mtip_port *port = dd->port; 1676 u64 total, raw0, raw1, raw2, raw3; 1677 raw0 = port->identify[100]; 1678 raw1 = port->identify[101]; 1679 raw2 = port->identify[102]; 1680 raw3 = port->identify[103]; 1681 total = raw0 | raw1<<16 | raw2<<32 | raw3<<48; 1682 *sectors = total; 1683 return (bool) !!port->identify_valid; 1684 } 1685 1686 /* 1687 * Display the identify command data. 1688 * 1689 * @port Pointer to the port data structure. 1690 * 1691 * return value 1692 * None 1693 */ 1694 static void mtip_dump_identify(struct mtip_port *port) 1695 { 1696 sector_t sectors; 1697 unsigned short revid; 1698 char cbuf[42]; 1699 1700 if (!port->identify_valid) 1701 return; 1702 1703 strlcpy(cbuf, (char *)(port->identify+10), 21); 1704 dev_info(&port->dd->pdev->dev, 1705 "Serial No.: %s\n", cbuf); 1706 1707 strlcpy(cbuf, (char *)(port->identify+23), 9); 1708 dev_info(&port->dd->pdev->dev, 1709 "Firmware Ver.: %s\n", cbuf); 1710 1711 strlcpy(cbuf, (char *)(port->identify+27), 41); 1712 dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf); 1713 1714 dev_info(&port->dd->pdev->dev, "Security: %04x %s\n", 1715 port->identify[128], 1716 port->identify[128] & 0x4 ? "(LOCKED)" : ""); 1717 1718 if (mtip_hw_get_capacity(port->dd, &sectors)) 1719 dev_info(&port->dd->pdev->dev, 1720 "Capacity: %llu sectors (%llu MB)\n", 1721 (u64)sectors, 1722 ((u64)sectors) * ATA_SECT_SIZE >> 20); 1723 1724 pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid); 1725 switch (revid & 0xFF) { 1726 case 0x1: 1727 strlcpy(cbuf, "A0", 3); 1728 break; 1729 case 0x3: 1730 strlcpy(cbuf, "A2", 3); 1731 break; 1732 default: 1733 strlcpy(cbuf, "?", 2); 1734 break; 1735 } 1736 dev_info(&port->dd->pdev->dev, 1737 "Card Type: %s\n", cbuf); 1738 } 1739 1740 /* 1741 * Map the commands scatter list into the command table. 1742 * 1743 * @command Pointer to the command. 1744 * @nents Number of scatter list entries. 1745 * 1746 * return value 1747 * None 1748 */ 1749 static inline void fill_command_sg(struct driver_data *dd, 1750 struct mtip_cmd *command, 1751 int nents) 1752 { 1753 int n; 1754 unsigned int dma_len; 1755 struct mtip_cmd_sg *command_sg; 1756 struct scatterlist *sg = command->sg; 1757 1758 command_sg = command->command + AHCI_CMD_TBL_HDR_SZ; 1759 1760 for (n = 0; n < nents; n++) { 1761 dma_len = sg_dma_len(sg); 1762 if (dma_len > 0x400000) 1763 dev_err(&dd->pdev->dev, 1764 "DMA segment length truncated\n"); 1765 command_sg->info = __force_bit2int 1766 cpu_to_le32((dma_len-1) & 0x3FFFFF); 1767 command_sg->dba = __force_bit2int 1768 cpu_to_le32(sg_dma_address(sg)); 1769 command_sg->dba_upper = __force_bit2int 1770 cpu_to_le32((sg_dma_address(sg) >> 16) >> 16); 1771 command_sg++; 1772 sg++; 1773 } 1774 } 1775 1776 /* 1777 * @brief Execute a drive command. 1778 * 1779 * return value 0 The command completed successfully. 1780 * return value -1 An error occurred while executing the command. 1781 */ 1782 static int exec_drive_task(struct mtip_port *port, u8 *command) 1783 { 1784 struct host_to_dev_fis fis; 1785 struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG); 1786 unsigned int to; 1787 1788 /* Build the FIS. */ 1789 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1790 fis.type = 0x27; 1791 fis.opts = 1 << 7; 1792 fis.command = command[0]; 1793 fis.features = command[1]; 1794 fis.sect_count = command[2]; 1795 fis.sector = command[3]; 1796 fis.cyl_low = command[4]; 1797 fis.cyl_hi = command[5]; 1798 fis.device = command[6] & ~0x10; /* Clear the dev bit*/ 1799 1800 mtip_set_timeout(port->dd, &fis, &to, 0); 1801 1802 dbg_printk(MTIP_DRV_NAME " %s: User Command: cmd %x, feat %x, nsect %x, sect %x, lcyl %x, hcyl %x, sel %x\n", 1803 __func__, 1804 command[0], 1805 command[1], 1806 command[2], 1807 command[3], 1808 command[4], 1809 command[5], 1810 command[6]); 1811 1812 /* Execute the command. */ 1813 if (mtip_exec_internal_command(port, 1814 &fis, 1815 5, 1816 0, 1817 0, 1818 0, 1819 GFP_KERNEL, 1820 to) < 0) { 1821 return -1; 1822 } 1823 1824 command[0] = reply->command; /* Status*/ 1825 command[1] = reply->features; /* Error*/ 1826 command[4] = reply->cyl_low; 1827 command[5] = reply->cyl_hi; 1828 1829 dbg_printk(MTIP_DRV_NAME " %s: Completion Status: stat %x, err %x , cyl_lo %x cyl_hi %x\n", 1830 __func__, 1831 command[0], 1832 command[1], 1833 command[4], 1834 command[5]); 1835 1836 return 0; 1837 } 1838 1839 /* 1840 * @brief Execute a drive command. 1841 * 1842 * @param port Pointer to the port data structure. 1843 * @param command Pointer to the user specified command parameters. 1844 * @param user_buffer Pointer to the user space buffer where read sector 1845 * data should be copied. 1846 * 1847 * return value 0 The command completed successfully. 1848 * return value -EFAULT An error occurred while copying the completion 1849 * data to the user space buffer. 1850 * return value -1 An error occurred while executing the command. 1851 */ 1852 static int exec_drive_command(struct mtip_port *port, u8 *command, 1853 void __user *user_buffer) 1854 { 1855 struct host_to_dev_fis fis; 1856 struct host_to_dev_fis *reply; 1857 u8 *buf = NULL; 1858 dma_addr_t dma_addr = 0; 1859 int rv = 0, xfer_sz = command[3]; 1860 unsigned int to; 1861 1862 if (xfer_sz) { 1863 if (!user_buffer) 1864 return -EFAULT; 1865 1866 buf = dmam_alloc_coherent(&port->dd->pdev->dev, 1867 ATA_SECT_SIZE * xfer_sz, 1868 &dma_addr, 1869 GFP_KERNEL); 1870 if (!buf) { 1871 dev_err(&port->dd->pdev->dev, 1872 "Memory allocation failed (%d bytes)\n", 1873 ATA_SECT_SIZE * xfer_sz); 1874 return -ENOMEM; 1875 } 1876 memset(buf, 0, ATA_SECT_SIZE * xfer_sz); 1877 } 1878 1879 /* Build the FIS. */ 1880 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1881 fis.type = 0x27; 1882 fis.opts = 1 << 7; 1883 fis.command = command[0]; 1884 fis.features = command[2]; 1885 fis.sect_count = command[3]; 1886 if (fis.command == ATA_CMD_SMART) { 1887 fis.sector = command[1]; 1888 fis.cyl_low = 0x4F; 1889 fis.cyl_hi = 0xC2; 1890 } 1891 1892 mtip_set_timeout(port->dd, &fis, &to, 0); 1893 1894 if (xfer_sz) 1895 reply = (port->rxfis + RX_FIS_PIO_SETUP); 1896 else 1897 reply = (port->rxfis + RX_FIS_D2H_REG); 1898 1899 dbg_printk(MTIP_DRV_NAME 1900 " %s: User Command: cmd %x, sect %x, " 1901 "feat %x, sectcnt %x\n", 1902 __func__, 1903 command[0], 1904 command[1], 1905 command[2], 1906 command[3]); 1907 1908 /* Execute the command. */ 1909 if (mtip_exec_internal_command(port, 1910 &fis, 1911 5, 1912 (xfer_sz ? dma_addr : 0), 1913 (xfer_sz ? ATA_SECT_SIZE * xfer_sz : 0), 1914 0, 1915 GFP_KERNEL, 1916 to) 1917 < 0) { 1918 rv = -EFAULT; 1919 goto exit_drive_command; 1920 } 1921 1922 /* Collect the completion status. */ 1923 command[0] = reply->command; /* Status*/ 1924 command[1] = reply->features; /* Error*/ 1925 command[2] = reply->sect_count; 1926 1927 dbg_printk(MTIP_DRV_NAME 1928 " %s: Completion Status: stat %x, " 1929 "err %x, nsect %x\n", 1930 __func__, 1931 command[0], 1932 command[1], 1933 command[2]); 1934 1935 if (xfer_sz) { 1936 if (copy_to_user(user_buffer, 1937 buf, 1938 ATA_SECT_SIZE * command[3])) { 1939 rv = -EFAULT; 1940 goto exit_drive_command; 1941 } 1942 } 1943 exit_drive_command: 1944 if (buf) 1945 dmam_free_coherent(&port->dd->pdev->dev, 1946 ATA_SECT_SIZE * xfer_sz, buf, dma_addr); 1947 return rv; 1948 } 1949 1950 /* 1951 * Indicates whether a command has a single sector payload. 1952 * 1953 * @command passed to the device to perform the certain event. 1954 * @features passed to the device to perform the certain event. 1955 * 1956 * return value 1957 * 1 command is one that always has a single sector payload, 1958 * regardless of the value in the Sector Count field. 1959 * 0 otherwise 1960 * 1961 */ 1962 static unsigned int implicit_sector(unsigned char command, 1963 unsigned char features) 1964 { 1965 unsigned int rv = 0; 1966 1967 /* list of commands that have an implicit sector count of 1 */ 1968 switch (command) { 1969 case ATA_CMD_SEC_SET_PASS: 1970 case ATA_CMD_SEC_UNLOCK: 1971 case ATA_CMD_SEC_ERASE_PREP: 1972 case ATA_CMD_SEC_ERASE_UNIT: 1973 case ATA_CMD_SEC_FREEZE_LOCK: 1974 case ATA_CMD_SEC_DISABLE_PASS: 1975 case ATA_CMD_PMP_READ: 1976 case ATA_CMD_PMP_WRITE: 1977 rv = 1; 1978 break; 1979 case ATA_CMD_SET_MAX: 1980 if (features == ATA_SET_MAX_UNLOCK) 1981 rv = 1; 1982 break; 1983 case ATA_CMD_SMART: 1984 if ((features == ATA_SMART_READ_VALUES) || 1985 (features == ATA_SMART_READ_THRESHOLDS)) 1986 rv = 1; 1987 break; 1988 case ATA_CMD_CONF_OVERLAY: 1989 if ((features == ATA_DCO_IDENTIFY) || 1990 (features == ATA_DCO_SET)) 1991 rv = 1; 1992 break; 1993 } 1994 return rv; 1995 } 1996 1997 /* 1998 * Executes a taskfile 1999 * See ide_taskfile_ioctl() for derivation 2000 */ 2001 static int exec_drive_taskfile(struct driver_data *dd, 2002 void __user *buf, 2003 ide_task_request_t *req_task, 2004 int outtotal) 2005 { 2006 struct host_to_dev_fis fis; 2007 struct host_to_dev_fis *reply; 2008 u8 *outbuf = NULL; 2009 u8 *inbuf = NULL; 2010 dma_addr_t outbuf_dma = 0; 2011 dma_addr_t inbuf_dma = 0; 2012 dma_addr_t dma_buffer = 0; 2013 int err = 0; 2014 unsigned int taskin = 0; 2015 unsigned int taskout = 0; 2016 u8 nsect = 0; 2017 unsigned int timeout; 2018 unsigned int force_single_sector; 2019 unsigned int transfer_size; 2020 unsigned long task_file_data; 2021 int intotal = outtotal + req_task->out_size; 2022 int erasemode = 0; 2023 2024 taskout = req_task->out_size; 2025 taskin = req_task->in_size; 2026 /* 130560 = 512 * 0xFF*/ 2027 if (taskin > 130560 || taskout > 130560) { 2028 err = -EINVAL; 2029 goto abort; 2030 } 2031 2032 if (taskout) { 2033 outbuf = memdup_user(buf + outtotal, taskout); 2034 if (IS_ERR(outbuf)) { 2035 err = PTR_ERR(outbuf); 2036 outbuf = NULL; 2037 goto abort; 2038 } 2039 outbuf_dma = pci_map_single(dd->pdev, 2040 outbuf, 2041 taskout, 2042 DMA_TO_DEVICE); 2043 if (outbuf_dma == 0) { 2044 err = -ENOMEM; 2045 goto abort; 2046 } 2047 dma_buffer = outbuf_dma; 2048 } 2049 2050 if (taskin) { 2051 inbuf = memdup_user(buf + intotal, taskin); 2052 if (IS_ERR(inbuf)) { 2053 err = PTR_ERR(inbuf); 2054 inbuf = NULL; 2055 goto abort; 2056 } 2057 inbuf_dma = pci_map_single(dd->pdev, 2058 inbuf, 2059 taskin, DMA_FROM_DEVICE); 2060 if (inbuf_dma == 0) { 2061 err = -ENOMEM; 2062 goto abort; 2063 } 2064 dma_buffer = inbuf_dma; 2065 } 2066 2067 /* only supports PIO and non-data commands from this ioctl. */ 2068 switch (req_task->data_phase) { 2069 case TASKFILE_OUT: 2070 nsect = taskout / ATA_SECT_SIZE; 2071 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); 2072 break; 2073 case TASKFILE_IN: 2074 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); 2075 break; 2076 case TASKFILE_NO_DATA: 2077 reply = (dd->port->rxfis + RX_FIS_D2H_REG); 2078 break; 2079 default: 2080 err = -EINVAL; 2081 goto abort; 2082 } 2083 2084 /* Build the FIS. */ 2085 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 2086 2087 fis.type = 0x27; 2088 fis.opts = 1 << 7; 2089 fis.command = req_task->io_ports[7]; 2090 fis.features = req_task->io_ports[1]; 2091 fis.sect_count = req_task->io_ports[2]; 2092 fis.lba_low = req_task->io_ports[3]; 2093 fis.lba_mid = req_task->io_ports[4]; 2094 fis.lba_hi = req_task->io_ports[5]; 2095 /* Clear the dev bit*/ 2096 fis.device = req_task->io_ports[6] & ~0x10; 2097 2098 if ((req_task->in_flags.all == 0) && (req_task->out_flags.all & 1)) { 2099 req_task->in_flags.all = 2100 IDE_TASKFILE_STD_IN_FLAGS | 2101 (IDE_HOB_STD_IN_FLAGS << 8); 2102 fis.lba_low_ex = req_task->hob_ports[3]; 2103 fis.lba_mid_ex = req_task->hob_ports[4]; 2104 fis.lba_hi_ex = req_task->hob_ports[5]; 2105 fis.features_ex = req_task->hob_ports[1]; 2106 fis.sect_cnt_ex = req_task->hob_ports[2]; 2107 2108 } else { 2109 req_task->in_flags.all = IDE_TASKFILE_STD_IN_FLAGS; 2110 } 2111 2112 force_single_sector = implicit_sector(fis.command, fis.features); 2113 2114 if ((taskin || taskout) && (!fis.sect_count)) { 2115 if (nsect) 2116 fis.sect_count = nsect; 2117 else { 2118 if (!force_single_sector) { 2119 dev_warn(&dd->pdev->dev, 2120 "data movement but " 2121 "sect_count is 0\n"); 2122 err = -EINVAL; 2123 goto abort; 2124 } 2125 } 2126 } 2127 2128 dbg_printk(MTIP_DRV_NAME 2129 " %s: cmd %x, feat %x, nsect %x," 2130 " sect/lbal %x, lcyl/lbam %x, hcyl/lbah %x," 2131 " head/dev %x\n", 2132 __func__, 2133 fis.command, 2134 fis.features, 2135 fis.sect_count, 2136 fis.lba_low, 2137 fis.lba_mid, 2138 fis.lba_hi, 2139 fis.device); 2140 2141 /* check for erase mode support during secure erase.*/ 2142 if ((fis.command == ATA_CMD_SEC_ERASE_UNIT) && outbuf && 2143 (outbuf[0] & MTIP_SEC_ERASE_MODE)) { 2144 erasemode = 1; 2145 } 2146 2147 mtip_set_timeout(dd, &fis, &timeout, erasemode); 2148 2149 /* Determine the correct transfer size.*/ 2150 if (force_single_sector) 2151 transfer_size = ATA_SECT_SIZE; 2152 else 2153 transfer_size = ATA_SECT_SIZE * fis.sect_count; 2154 2155 /* Execute the command.*/ 2156 if (mtip_exec_internal_command(dd->port, 2157 &fis, 2158 5, 2159 dma_buffer, 2160 transfer_size, 2161 0, 2162 GFP_KERNEL, 2163 timeout) < 0) { 2164 err = -EIO; 2165 goto abort; 2166 } 2167 2168 task_file_data = readl(dd->port->mmio+PORT_TFDATA); 2169 2170 if ((req_task->data_phase == TASKFILE_IN) && !(task_file_data & 1)) { 2171 reply = dd->port->rxfis + RX_FIS_PIO_SETUP; 2172 req_task->io_ports[7] = reply->control; 2173 } else { 2174 reply = dd->port->rxfis + RX_FIS_D2H_REG; 2175 req_task->io_ports[7] = reply->command; 2176 } 2177 2178 /* reclaim the DMA buffers.*/ 2179 if (inbuf_dma) 2180 pci_unmap_single(dd->pdev, inbuf_dma, 2181 taskin, DMA_FROM_DEVICE); 2182 if (outbuf_dma) 2183 pci_unmap_single(dd->pdev, outbuf_dma, 2184 taskout, DMA_TO_DEVICE); 2185 inbuf_dma = 0; 2186 outbuf_dma = 0; 2187 2188 /* return the ATA registers to the caller.*/ 2189 req_task->io_ports[1] = reply->features; 2190 req_task->io_ports[2] = reply->sect_count; 2191 req_task->io_ports[3] = reply->lba_low; 2192 req_task->io_ports[4] = reply->lba_mid; 2193 req_task->io_ports[5] = reply->lba_hi; 2194 req_task->io_ports[6] = reply->device; 2195 2196 if (req_task->out_flags.all & 1) { 2197 2198 req_task->hob_ports[3] = reply->lba_low_ex; 2199 req_task->hob_ports[4] = reply->lba_mid_ex; 2200 req_task->hob_ports[5] = reply->lba_hi_ex; 2201 req_task->hob_ports[1] = reply->features_ex; 2202 req_task->hob_ports[2] = reply->sect_cnt_ex; 2203 } 2204 dbg_printk(MTIP_DRV_NAME 2205 " %s: Completion: stat %x," 2206 "err %x, sect_cnt %x, lbalo %x," 2207 "lbamid %x, lbahi %x, dev %x\n", 2208 __func__, 2209 req_task->io_ports[7], 2210 req_task->io_ports[1], 2211 req_task->io_ports[2], 2212 req_task->io_ports[3], 2213 req_task->io_ports[4], 2214 req_task->io_ports[5], 2215 req_task->io_ports[6]); 2216 2217 if (taskout) { 2218 if (copy_to_user(buf + outtotal, outbuf, taskout)) { 2219 err = -EFAULT; 2220 goto abort; 2221 } 2222 } 2223 if (taskin) { 2224 if (copy_to_user(buf + intotal, inbuf, taskin)) { 2225 err = -EFAULT; 2226 goto abort; 2227 } 2228 } 2229 abort: 2230 if (inbuf_dma) 2231 pci_unmap_single(dd->pdev, inbuf_dma, 2232 taskin, DMA_FROM_DEVICE); 2233 if (outbuf_dma) 2234 pci_unmap_single(dd->pdev, outbuf_dma, 2235 taskout, DMA_TO_DEVICE); 2236 kfree(outbuf); 2237 kfree(inbuf); 2238 2239 return err; 2240 } 2241 2242 /* 2243 * Handle IOCTL calls from the Block Layer. 2244 * 2245 * This function is called by the Block Layer when it receives an IOCTL 2246 * command that it does not understand. If the IOCTL command is not supported 2247 * this function returns -ENOTTY. 2248 * 2249 * @dd Pointer to the driver data structure. 2250 * @cmd IOCTL command passed from the Block Layer. 2251 * @arg IOCTL argument passed from the Block Layer. 2252 * 2253 * return value 2254 * 0 The IOCTL completed successfully. 2255 * -ENOTTY The specified command is not supported. 2256 * -EFAULT An error occurred copying data to a user space buffer. 2257 * -EIO An error occurred while executing the command. 2258 */ 2259 static int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd, 2260 unsigned long arg) 2261 { 2262 switch (cmd) { 2263 case HDIO_GET_IDENTITY: 2264 { 2265 if (copy_to_user((void __user *)arg, dd->port->identify, 2266 sizeof(u16) * ATA_ID_WORDS)) 2267 return -EFAULT; 2268 break; 2269 } 2270 case HDIO_DRIVE_CMD: 2271 { 2272 u8 drive_command[4]; 2273 2274 /* Copy the user command info to our buffer. */ 2275 if (copy_from_user(drive_command, 2276 (void __user *) arg, 2277 sizeof(drive_command))) 2278 return -EFAULT; 2279 2280 /* Execute the drive command. */ 2281 if (exec_drive_command(dd->port, 2282 drive_command, 2283 (void __user *) (arg+4))) 2284 return -EIO; 2285 2286 /* Copy the status back to the users buffer. */ 2287 if (copy_to_user((void __user *) arg, 2288 drive_command, 2289 sizeof(drive_command))) 2290 return -EFAULT; 2291 2292 break; 2293 } 2294 case HDIO_DRIVE_TASK: 2295 { 2296 u8 drive_command[7]; 2297 2298 /* Copy the user command info to our buffer. */ 2299 if (copy_from_user(drive_command, 2300 (void __user *) arg, 2301 sizeof(drive_command))) 2302 return -EFAULT; 2303 2304 /* Execute the drive command. */ 2305 if (exec_drive_task(dd->port, drive_command)) 2306 return -EIO; 2307 2308 /* Copy the status back to the users buffer. */ 2309 if (copy_to_user((void __user *) arg, 2310 drive_command, 2311 sizeof(drive_command))) 2312 return -EFAULT; 2313 2314 break; 2315 } 2316 case HDIO_DRIVE_TASKFILE: { 2317 ide_task_request_t req_task; 2318 int ret, outtotal; 2319 2320 if (copy_from_user(&req_task, (void __user *) arg, 2321 sizeof(req_task))) 2322 return -EFAULT; 2323 2324 outtotal = sizeof(req_task); 2325 2326 ret = exec_drive_taskfile(dd, (void __user *) arg, 2327 &req_task, outtotal); 2328 2329 if (copy_to_user((void __user *) arg, &req_task, 2330 sizeof(req_task))) 2331 return -EFAULT; 2332 2333 return ret; 2334 } 2335 2336 default: 2337 return -EINVAL; 2338 } 2339 return 0; 2340 } 2341 2342 /* 2343 * Submit an IO to the hw 2344 * 2345 * This function is called by the block layer to issue an io 2346 * to the device. Upon completion, the callback function will 2347 * be called with the data parameter passed as the callback data. 2348 * 2349 * @dd Pointer to the driver data structure. 2350 * @start First sector to read. 2351 * @nsect Number of sectors to read. 2352 * @nents Number of entries in scatter list for the read command. 2353 * @tag The tag of this read command. 2354 * @callback Pointer to the function that should be called 2355 * when the read completes. 2356 * @data Callback data passed to the callback function 2357 * when the read completes. 2358 * @dir Direction (read or write) 2359 * 2360 * return value 2361 * None 2362 */ 2363 static void mtip_hw_submit_io(struct driver_data *dd, struct request *rq, 2364 struct mtip_cmd *command, int nents, 2365 struct blk_mq_hw_ctx *hctx) 2366 { 2367 struct host_to_dev_fis *fis; 2368 struct mtip_port *port = dd->port; 2369 int dma_dir = rq_data_dir(rq) == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE; 2370 u64 start = blk_rq_pos(rq); 2371 unsigned int nsect = blk_rq_sectors(rq); 2372 2373 /* Map the scatter list for DMA access */ 2374 nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir); 2375 2376 prefetch(&port->flags); 2377 2378 command->scatter_ents = nents; 2379 2380 /* 2381 * The number of retries for this command before it is 2382 * reported as a failure to the upper layers. 2383 */ 2384 command->retries = MTIP_MAX_RETRIES; 2385 2386 /* Fill out fis */ 2387 fis = command->command; 2388 fis->type = 0x27; 2389 fis->opts = 1 << 7; 2390 if (dma_dir == DMA_FROM_DEVICE) 2391 fis->command = ATA_CMD_FPDMA_READ; 2392 else 2393 fis->command = ATA_CMD_FPDMA_WRITE; 2394 fis->lba_low = start & 0xFF; 2395 fis->lba_mid = (start >> 8) & 0xFF; 2396 fis->lba_hi = (start >> 16) & 0xFF; 2397 fis->lba_low_ex = (start >> 24) & 0xFF; 2398 fis->lba_mid_ex = (start >> 32) & 0xFF; 2399 fis->lba_hi_ex = (start >> 40) & 0xFF; 2400 fis->device = 1 << 6; 2401 fis->features = nsect & 0xFF; 2402 fis->features_ex = (nsect >> 8) & 0xFF; 2403 fis->sect_count = ((rq->tag << 3) | (rq->tag >> 5)); 2404 fis->sect_cnt_ex = 0; 2405 fis->control = 0; 2406 fis->res2 = 0; 2407 fis->res3 = 0; 2408 fill_command_sg(dd, command, nents); 2409 2410 if (unlikely(command->unaligned)) 2411 fis->device |= 1 << 7; 2412 2413 /* Populate the command header */ 2414 command->command_header->opts = 2415 __force_bit2int cpu_to_le32( 2416 (nents << 16) | 5 | AHCI_CMD_PREFETCH); 2417 command->command_header->byte_count = 0; 2418 2419 /* 2420 * Set the completion function and data for the command 2421 * within this layer. 2422 */ 2423 command->comp_data = dd; 2424 command->comp_func = mtip_async_complete; 2425 command->direction = dma_dir; 2426 2427 /* 2428 * To prevent this command from being issued 2429 * if an internal command is in progress or error handling is active. 2430 */ 2431 if (unlikely(port->flags & MTIP_PF_PAUSE_IO)) { 2432 set_bit(rq->tag, port->cmds_to_issue); 2433 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); 2434 return; 2435 } 2436 2437 /* Issue the command to the hardware */ 2438 mtip_issue_ncq_command(port, rq->tag); 2439 } 2440 2441 /* 2442 * Sysfs status dump. 2443 * 2444 * @dev Pointer to the device structure, passed by the kernrel. 2445 * @attr Pointer to the device_attribute structure passed by the kernel. 2446 * @buf Pointer to the char buffer that will receive the stats info. 2447 * 2448 * return value 2449 * The size, in bytes, of the data copied into buf. 2450 */ 2451 static ssize_t mtip_hw_show_status(struct device *dev, 2452 struct device_attribute *attr, 2453 char *buf) 2454 { 2455 struct driver_data *dd = dev_to_disk(dev)->private_data; 2456 int size = 0; 2457 2458 if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag)) 2459 size += sprintf(buf, "%s", "thermal_shutdown\n"); 2460 else if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag)) 2461 size += sprintf(buf, "%s", "write_protect\n"); 2462 else 2463 size += sprintf(buf, "%s", "online\n"); 2464 2465 return size; 2466 } 2467 2468 static DEVICE_ATTR(status, S_IRUGO, mtip_hw_show_status, NULL); 2469 2470 /* debugsfs entries */ 2471 2472 static ssize_t show_device_status(struct device_driver *drv, char *buf) 2473 { 2474 int size = 0; 2475 struct driver_data *dd, *tmp; 2476 unsigned long flags; 2477 char id_buf[42]; 2478 u16 status = 0; 2479 2480 spin_lock_irqsave(&dev_lock, flags); 2481 size += sprintf(&buf[size], "Devices Present:\n"); 2482 list_for_each_entry_safe(dd, tmp, &online_list, online_list) { 2483 if (dd->pdev) { 2484 if (dd->port && 2485 dd->port->identify && 2486 dd->port->identify_valid) { 2487 strlcpy(id_buf, 2488 (char *) (dd->port->identify + 10), 21); 2489 status = *(dd->port->identify + 141); 2490 } else { 2491 memset(id_buf, 0, 42); 2492 status = 0; 2493 } 2494 2495 if (dd->port && 2496 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { 2497 size += sprintf(&buf[size], 2498 " device %s %s (ftl rebuild %d %%)\n", 2499 dev_name(&dd->pdev->dev), 2500 id_buf, 2501 status); 2502 } else { 2503 size += sprintf(&buf[size], 2504 " device %s %s\n", 2505 dev_name(&dd->pdev->dev), 2506 id_buf); 2507 } 2508 } 2509 } 2510 2511 size += sprintf(&buf[size], "Devices Being Removed:\n"); 2512 list_for_each_entry_safe(dd, tmp, &removing_list, remove_list) { 2513 if (dd->pdev) { 2514 if (dd->port && 2515 dd->port->identify && 2516 dd->port->identify_valid) { 2517 strlcpy(id_buf, 2518 (char *) (dd->port->identify+10), 21); 2519 status = *(dd->port->identify + 141); 2520 } else { 2521 memset(id_buf, 0, 42); 2522 status = 0; 2523 } 2524 2525 if (dd->port && 2526 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { 2527 size += sprintf(&buf[size], 2528 " device %s %s (ftl rebuild %d %%)\n", 2529 dev_name(&dd->pdev->dev), 2530 id_buf, 2531 status); 2532 } else { 2533 size += sprintf(&buf[size], 2534 " device %s %s\n", 2535 dev_name(&dd->pdev->dev), 2536 id_buf); 2537 } 2538 } 2539 } 2540 spin_unlock_irqrestore(&dev_lock, flags); 2541 2542 return size; 2543 } 2544 2545 static ssize_t mtip_hw_read_device_status(struct file *f, char __user *ubuf, 2546 size_t len, loff_t *offset) 2547 { 2548 struct driver_data *dd = (struct driver_data *)f->private_data; 2549 int size = *offset; 2550 char *buf; 2551 int rv = 0; 2552 2553 if (!len || *offset) 2554 return 0; 2555 2556 buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL); 2557 if (!buf) { 2558 dev_err(&dd->pdev->dev, 2559 "Memory allocation: status buffer\n"); 2560 return -ENOMEM; 2561 } 2562 2563 size += show_device_status(NULL, buf); 2564 2565 *offset = size <= len ? size : len; 2566 size = copy_to_user(ubuf, buf, *offset); 2567 if (size) 2568 rv = -EFAULT; 2569 2570 kfree(buf); 2571 return rv ? rv : *offset; 2572 } 2573 2574 static ssize_t mtip_hw_read_registers(struct file *f, char __user *ubuf, 2575 size_t len, loff_t *offset) 2576 { 2577 struct driver_data *dd = (struct driver_data *)f->private_data; 2578 char *buf; 2579 u32 group_allocated; 2580 int size = *offset; 2581 int n, rv = 0; 2582 2583 if (!len || size) 2584 return 0; 2585 2586 buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL); 2587 if (!buf) { 2588 dev_err(&dd->pdev->dev, 2589 "Memory allocation: register buffer\n"); 2590 return -ENOMEM; 2591 } 2592 2593 size += sprintf(&buf[size], "H/ S ACTive : [ 0x"); 2594 2595 for (n = dd->slot_groups-1; n >= 0; n--) 2596 size += sprintf(&buf[size], "%08X ", 2597 readl(dd->port->s_active[n])); 2598 2599 size += sprintf(&buf[size], "]\n"); 2600 size += sprintf(&buf[size], "H/ Command Issue : [ 0x"); 2601 2602 for (n = dd->slot_groups-1; n >= 0; n--) 2603 size += sprintf(&buf[size], "%08X ", 2604 readl(dd->port->cmd_issue[n])); 2605 2606 size += sprintf(&buf[size], "]\n"); 2607 size += sprintf(&buf[size], "H/ Completed : [ 0x"); 2608 2609 for (n = dd->slot_groups-1; n >= 0; n--) 2610 size += sprintf(&buf[size], "%08X ", 2611 readl(dd->port->completed[n])); 2612 2613 size += sprintf(&buf[size], "]\n"); 2614 size += sprintf(&buf[size], "H/ PORT IRQ STAT : [ 0x%08X ]\n", 2615 readl(dd->port->mmio + PORT_IRQ_STAT)); 2616 size += sprintf(&buf[size], "H/ HOST IRQ STAT : [ 0x%08X ]\n", 2617 readl(dd->mmio + HOST_IRQ_STAT)); 2618 size += sprintf(&buf[size], "\n"); 2619 2620 size += sprintf(&buf[size], "L/ Commands in Q : [ 0x"); 2621 2622 for (n = dd->slot_groups-1; n >= 0; n--) { 2623 if (sizeof(long) > sizeof(u32)) 2624 group_allocated = 2625 dd->port->cmds_to_issue[n/2] >> (32*(n&1)); 2626 else 2627 group_allocated = dd->port->cmds_to_issue[n]; 2628 size += sprintf(&buf[size], "%08X ", group_allocated); 2629 } 2630 size += sprintf(&buf[size], "]\n"); 2631 2632 *offset = size <= len ? size : len; 2633 size = copy_to_user(ubuf, buf, *offset); 2634 if (size) 2635 rv = -EFAULT; 2636 2637 kfree(buf); 2638 return rv ? rv : *offset; 2639 } 2640 2641 static ssize_t mtip_hw_read_flags(struct file *f, char __user *ubuf, 2642 size_t len, loff_t *offset) 2643 { 2644 struct driver_data *dd = (struct driver_data *)f->private_data; 2645 char *buf; 2646 int size = *offset; 2647 int rv = 0; 2648 2649 if (!len || size) 2650 return 0; 2651 2652 buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL); 2653 if (!buf) { 2654 dev_err(&dd->pdev->dev, 2655 "Memory allocation: flag buffer\n"); 2656 return -ENOMEM; 2657 } 2658 2659 size += sprintf(&buf[size], "Flag-port : [ %08lX ]\n", 2660 dd->port->flags); 2661 size += sprintf(&buf[size], "Flag-dd : [ %08lX ]\n", 2662 dd->dd_flag); 2663 2664 *offset = size <= len ? size : len; 2665 size = copy_to_user(ubuf, buf, *offset); 2666 if (size) 2667 rv = -EFAULT; 2668 2669 kfree(buf); 2670 return rv ? rv : *offset; 2671 } 2672 2673 static const struct file_operations mtip_device_status_fops = { 2674 .owner = THIS_MODULE, 2675 .open = simple_open, 2676 .read = mtip_hw_read_device_status, 2677 .llseek = no_llseek, 2678 }; 2679 2680 static const struct file_operations mtip_regs_fops = { 2681 .owner = THIS_MODULE, 2682 .open = simple_open, 2683 .read = mtip_hw_read_registers, 2684 .llseek = no_llseek, 2685 }; 2686 2687 static const struct file_operations mtip_flags_fops = { 2688 .owner = THIS_MODULE, 2689 .open = simple_open, 2690 .read = mtip_hw_read_flags, 2691 .llseek = no_llseek, 2692 }; 2693 2694 /* 2695 * Create the sysfs related attributes. 2696 * 2697 * @dd Pointer to the driver data structure. 2698 * @kobj Pointer to the kobj for the block device. 2699 * 2700 * return value 2701 * 0 Operation completed successfully. 2702 * -EINVAL Invalid parameter. 2703 */ 2704 static int mtip_hw_sysfs_init(struct driver_data *dd, struct kobject *kobj) 2705 { 2706 if (!kobj || !dd) 2707 return -EINVAL; 2708 2709 if (sysfs_create_file(kobj, &dev_attr_status.attr)) 2710 dev_warn(&dd->pdev->dev, 2711 "Error creating 'status' sysfs entry\n"); 2712 return 0; 2713 } 2714 2715 /* 2716 * Remove the sysfs related attributes. 2717 * 2718 * @dd Pointer to the driver data structure. 2719 * @kobj Pointer to the kobj for the block device. 2720 * 2721 * return value 2722 * 0 Operation completed successfully. 2723 * -EINVAL Invalid parameter. 2724 */ 2725 static int mtip_hw_sysfs_exit(struct driver_data *dd, struct kobject *kobj) 2726 { 2727 if (!kobj || !dd) 2728 return -EINVAL; 2729 2730 sysfs_remove_file(kobj, &dev_attr_status.attr); 2731 2732 return 0; 2733 } 2734 2735 static int mtip_hw_debugfs_init(struct driver_data *dd) 2736 { 2737 if (!dfs_parent) 2738 return -1; 2739 2740 dd->dfs_node = debugfs_create_dir(dd->disk->disk_name, dfs_parent); 2741 if (IS_ERR_OR_NULL(dd->dfs_node)) { 2742 dev_warn(&dd->pdev->dev, 2743 "Error creating node %s under debugfs\n", 2744 dd->disk->disk_name); 2745 dd->dfs_node = NULL; 2746 return -1; 2747 } 2748 2749 debugfs_create_file("flags", S_IRUGO, dd->dfs_node, dd, 2750 &mtip_flags_fops); 2751 debugfs_create_file("registers", S_IRUGO, dd->dfs_node, dd, 2752 &mtip_regs_fops); 2753 2754 return 0; 2755 } 2756 2757 static void mtip_hw_debugfs_exit(struct driver_data *dd) 2758 { 2759 if (dd->dfs_node) 2760 debugfs_remove_recursive(dd->dfs_node); 2761 } 2762 2763 /* 2764 * Perform any init/resume time hardware setup 2765 * 2766 * @dd Pointer to the driver data structure. 2767 * 2768 * return value 2769 * None 2770 */ 2771 static inline void hba_setup(struct driver_data *dd) 2772 { 2773 u32 hwdata; 2774 hwdata = readl(dd->mmio + HOST_HSORG); 2775 2776 /* interrupt bug workaround: use only 1 IS bit.*/ 2777 writel(hwdata | 2778 HSORG_DISABLE_SLOTGRP_INTR | 2779 HSORG_DISABLE_SLOTGRP_PXIS, 2780 dd->mmio + HOST_HSORG); 2781 } 2782 2783 static int mtip_device_unaligned_constrained(struct driver_data *dd) 2784 { 2785 return (dd->pdev->device == P420M_DEVICE_ID ? 1 : 0); 2786 } 2787 2788 /* 2789 * Detect the details of the product, and store anything needed 2790 * into the driver data structure. This includes product type and 2791 * version and number of slot groups. 2792 * 2793 * @dd Pointer to the driver data structure. 2794 * 2795 * return value 2796 * None 2797 */ 2798 static void mtip_detect_product(struct driver_data *dd) 2799 { 2800 u32 hwdata; 2801 unsigned int rev, slotgroups; 2802 2803 /* 2804 * HBA base + 0xFC [15:0] - vendor-specific hardware interface 2805 * info register: 2806 * [15:8] hardware/software interface rev# 2807 * [ 3] asic-style interface 2808 * [ 2:0] number of slot groups, minus 1 (only valid for asic-style). 2809 */ 2810 hwdata = readl(dd->mmio + HOST_HSORG); 2811 2812 dd->product_type = MTIP_PRODUCT_UNKNOWN; 2813 dd->slot_groups = 1; 2814 2815 if (hwdata & 0x8) { 2816 dd->product_type = MTIP_PRODUCT_ASICFPGA; 2817 rev = (hwdata & HSORG_HWREV) >> 8; 2818 slotgroups = (hwdata & HSORG_SLOTGROUPS) + 1; 2819 dev_info(&dd->pdev->dev, 2820 "ASIC-FPGA design, HS rev 0x%x, " 2821 "%i slot groups [%i slots]\n", 2822 rev, 2823 slotgroups, 2824 slotgroups * 32); 2825 2826 if (slotgroups > MTIP_MAX_SLOT_GROUPS) { 2827 dev_warn(&dd->pdev->dev, 2828 "Warning: driver only supports " 2829 "%i slot groups.\n", MTIP_MAX_SLOT_GROUPS); 2830 slotgroups = MTIP_MAX_SLOT_GROUPS; 2831 } 2832 dd->slot_groups = slotgroups; 2833 return; 2834 } 2835 2836 dev_warn(&dd->pdev->dev, "Unrecognized product id\n"); 2837 } 2838 2839 /* 2840 * Blocking wait for FTL rebuild to complete 2841 * 2842 * @dd Pointer to the DRIVER_DATA structure. 2843 * 2844 * return value 2845 * 0 FTL rebuild completed successfully 2846 * -EFAULT FTL rebuild error/timeout/interruption 2847 */ 2848 static int mtip_ftl_rebuild_poll(struct driver_data *dd) 2849 { 2850 unsigned long timeout, cnt = 0, start; 2851 2852 dev_warn(&dd->pdev->dev, 2853 "FTL rebuild in progress. Polling for completion.\n"); 2854 2855 start = jiffies; 2856 timeout = jiffies + msecs_to_jiffies(MTIP_FTL_REBUILD_TIMEOUT_MS); 2857 2858 do { 2859 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 2860 &dd->dd_flag))) 2861 return -EFAULT; 2862 if (mtip_check_surprise_removal(dd->pdev)) 2863 return -EFAULT; 2864 2865 if (mtip_get_identify(dd->port, NULL) < 0) 2866 return -EFAULT; 2867 2868 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == 2869 MTIP_FTL_REBUILD_MAGIC) { 2870 ssleep(1); 2871 /* Print message every 3 minutes */ 2872 if (cnt++ >= 180) { 2873 dev_warn(&dd->pdev->dev, 2874 "FTL rebuild in progress (%d secs).\n", 2875 jiffies_to_msecs(jiffies - start) / 1000); 2876 cnt = 0; 2877 } 2878 } else { 2879 dev_warn(&dd->pdev->dev, 2880 "FTL rebuild complete (%d secs).\n", 2881 jiffies_to_msecs(jiffies - start) / 1000); 2882 mtip_block_initialize(dd); 2883 return 0; 2884 } 2885 } while (time_before(jiffies, timeout)); 2886 2887 /* Check for timeout */ 2888 dev_err(&dd->pdev->dev, 2889 "Timed out waiting for FTL rebuild to complete (%d secs).\n", 2890 jiffies_to_msecs(jiffies - start) / 1000); 2891 return -EFAULT; 2892 } 2893 2894 /* 2895 * service thread to issue queued commands 2896 * 2897 * @data Pointer to the driver data structure. 2898 * 2899 * return value 2900 * 0 2901 */ 2902 2903 static int mtip_service_thread(void *data) 2904 { 2905 struct driver_data *dd = (struct driver_data *)data; 2906 unsigned long slot, slot_start, slot_wrap; 2907 unsigned int num_cmd_slots = dd->slot_groups * 32; 2908 struct mtip_port *port = dd->port; 2909 2910 while (1) { 2911 if (kthread_should_stop() || 2912 test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) 2913 goto st_out; 2914 clear_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); 2915 2916 /* 2917 * the condition is to check neither an internal command is 2918 * is in progress nor error handling is active 2919 */ 2920 wait_event_interruptible(port->svc_wait, (port->flags) && 2921 !(port->flags & MTIP_PF_PAUSE_IO)); 2922 2923 set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); 2924 2925 if (kthread_should_stop() || 2926 test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) 2927 goto st_out; 2928 2929 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 2930 &dd->dd_flag))) 2931 goto st_out; 2932 2933 restart_eh: 2934 /* Demux bits: start with error handling */ 2935 if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) { 2936 mtip_handle_tfe(dd); 2937 clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); 2938 } 2939 2940 if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) 2941 goto restart_eh; 2942 2943 if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { 2944 slot = 1; 2945 /* used to restrict the loop to one iteration */ 2946 slot_start = num_cmd_slots; 2947 slot_wrap = 0; 2948 while (1) { 2949 slot = find_next_bit(port->cmds_to_issue, 2950 num_cmd_slots, slot); 2951 if (slot_wrap == 1) { 2952 if ((slot_start >= slot) || 2953 (slot >= num_cmd_slots)) 2954 break; 2955 } 2956 if (unlikely(slot_start == num_cmd_slots)) 2957 slot_start = slot; 2958 2959 if (unlikely(slot == num_cmd_slots)) { 2960 slot = 1; 2961 slot_wrap = 1; 2962 continue; 2963 } 2964 2965 /* Issue the command to the hardware */ 2966 mtip_issue_ncq_command(port, slot); 2967 2968 clear_bit(slot, port->cmds_to_issue); 2969 } 2970 2971 clear_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); 2972 } 2973 2974 if (test_bit(MTIP_PF_REBUILD_BIT, &port->flags)) { 2975 if (mtip_ftl_rebuild_poll(dd) < 0) 2976 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, 2977 &dd->dd_flag); 2978 clear_bit(MTIP_PF_REBUILD_BIT, &port->flags); 2979 } 2980 } 2981 2982 st_out: 2983 return 0; 2984 } 2985 2986 /* 2987 * DMA region teardown 2988 * 2989 * @dd Pointer to driver_data structure 2990 * 2991 * return value 2992 * None 2993 */ 2994 static void mtip_dma_free(struct driver_data *dd) 2995 { 2996 struct mtip_port *port = dd->port; 2997 2998 if (port->block1) 2999 dmam_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, 3000 port->block1, port->block1_dma); 3001 3002 if (port->command_list) { 3003 dmam_free_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, 3004 port->command_list, port->command_list_dma); 3005 } 3006 } 3007 3008 /* 3009 * DMA region setup 3010 * 3011 * @dd Pointer to driver_data structure 3012 * 3013 * return value 3014 * -ENOMEM Not enough free DMA region space to initialize driver 3015 */ 3016 static int mtip_dma_alloc(struct driver_data *dd) 3017 { 3018 struct mtip_port *port = dd->port; 3019 3020 /* Allocate dma memory for RX Fis, Identify, and Sector Bufffer */ 3021 port->block1 = 3022 dmam_alloc_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, 3023 &port->block1_dma, GFP_KERNEL); 3024 if (!port->block1) 3025 return -ENOMEM; 3026 memset(port->block1, 0, BLOCK_DMA_ALLOC_SZ); 3027 3028 /* Allocate dma memory for command list */ 3029 port->command_list = 3030 dmam_alloc_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, 3031 &port->command_list_dma, GFP_KERNEL); 3032 if (!port->command_list) { 3033 dmam_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, 3034 port->block1, port->block1_dma); 3035 port->block1 = NULL; 3036 port->block1_dma = 0; 3037 return -ENOMEM; 3038 } 3039 memset(port->command_list, 0, AHCI_CMD_TBL_SZ); 3040 3041 /* Setup all pointers into first DMA region */ 3042 port->rxfis = port->block1 + AHCI_RX_FIS_OFFSET; 3043 port->rxfis_dma = port->block1_dma + AHCI_RX_FIS_OFFSET; 3044 port->identify = port->block1 + AHCI_IDFY_OFFSET; 3045 port->identify_dma = port->block1_dma + AHCI_IDFY_OFFSET; 3046 port->log_buf = port->block1 + AHCI_SECTBUF_OFFSET; 3047 port->log_buf_dma = port->block1_dma + AHCI_SECTBUF_OFFSET; 3048 port->smart_buf = port->block1 + AHCI_SMARTBUF_OFFSET; 3049 port->smart_buf_dma = port->block1_dma + AHCI_SMARTBUF_OFFSET; 3050 3051 return 0; 3052 } 3053 3054 static int mtip_hw_get_identify(struct driver_data *dd) 3055 { 3056 struct smart_attr attr242; 3057 unsigned char *buf; 3058 int rv; 3059 3060 if (mtip_get_identify(dd->port, NULL) < 0) 3061 return -EFAULT; 3062 3063 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == 3064 MTIP_FTL_REBUILD_MAGIC) { 3065 set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags); 3066 return MTIP_FTL_REBUILD_MAGIC; 3067 } 3068 mtip_dump_identify(dd->port); 3069 3070 /* check write protect, over temp and rebuild statuses */ 3071 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, 3072 dd->port->log_buf, 3073 dd->port->log_buf_dma, 1); 3074 if (rv) { 3075 dev_warn(&dd->pdev->dev, 3076 "Error in READ LOG EXT (10h) command\n"); 3077 /* non-critical error, don't fail the load */ 3078 } else { 3079 buf = (unsigned char *)dd->port->log_buf; 3080 if (buf[259] & 0x1) { 3081 dev_info(&dd->pdev->dev, 3082 "Write protect bit is set.\n"); 3083 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); 3084 } 3085 if (buf[288] == 0xF7) { 3086 dev_info(&dd->pdev->dev, 3087 "Exceeded Tmax, drive in thermal shutdown.\n"); 3088 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); 3089 } 3090 if (buf[288] == 0xBF) { 3091 dev_info(&dd->pdev->dev, 3092 "Drive indicates rebuild has failed.\n"); 3093 /* TODO */ 3094 } 3095 } 3096 3097 /* get write protect progess */ 3098 memset(&attr242, 0, sizeof(struct smart_attr)); 3099 if (mtip_get_smart_attr(dd->port, 242, &attr242)) 3100 dev_warn(&dd->pdev->dev, 3101 "Unable to check write protect progress\n"); 3102 else 3103 dev_info(&dd->pdev->dev, 3104 "Write protect progress: %u%% (%u blocks)\n", 3105 attr242.cur, le32_to_cpu(attr242.data)); 3106 3107 return rv; 3108 } 3109 3110 /* 3111 * Called once for each card. 3112 * 3113 * @dd Pointer to the driver data structure. 3114 * 3115 * return value 3116 * 0 on success, else an error code. 3117 */ 3118 static int mtip_hw_init(struct driver_data *dd) 3119 { 3120 int i; 3121 int rv; 3122 unsigned int num_command_slots; 3123 unsigned long timeout, timetaken; 3124 3125 dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR]; 3126 3127 mtip_detect_product(dd); 3128 if (dd->product_type == MTIP_PRODUCT_UNKNOWN) { 3129 rv = -EIO; 3130 goto out1; 3131 } 3132 num_command_slots = dd->slot_groups * 32; 3133 3134 hba_setup(dd); 3135 3136 dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL, 3137 dd->numa_node); 3138 if (!dd->port) { 3139 dev_err(&dd->pdev->dev, 3140 "Memory allocation: port structure\n"); 3141 return -ENOMEM; 3142 } 3143 3144 /* Continue workqueue setup */ 3145 for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++) 3146 dd->work[i].port = dd->port; 3147 3148 /* Enable unaligned IO constraints for some devices */ 3149 if (mtip_device_unaligned_constrained(dd)) 3150 dd->unal_qdepth = MTIP_MAX_UNALIGNED_SLOTS; 3151 else 3152 dd->unal_qdepth = 0; 3153 3154 sema_init(&dd->port->cmd_slot_unal, dd->unal_qdepth); 3155 3156 /* Spinlock to prevent concurrent issue */ 3157 for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++) 3158 spin_lock_init(&dd->port->cmd_issue_lock[i]); 3159 3160 /* Set the port mmio base address. */ 3161 dd->port->mmio = dd->mmio + PORT_OFFSET; 3162 dd->port->dd = dd; 3163 3164 /* DMA allocations */ 3165 rv = mtip_dma_alloc(dd); 3166 if (rv < 0) 3167 goto out1; 3168 3169 /* Setup the pointers to the extended s_active and CI registers. */ 3170 for (i = 0; i < dd->slot_groups; i++) { 3171 dd->port->s_active[i] = 3172 dd->port->mmio + i*0x80 + PORT_SCR_ACT; 3173 dd->port->cmd_issue[i] = 3174 dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE; 3175 dd->port->completed[i] = 3176 dd->port->mmio + i*0x80 + PORT_SDBV; 3177 } 3178 3179 timetaken = jiffies; 3180 timeout = jiffies + msecs_to_jiffies(30000); 3181 while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) && 3182 time_before(jiffies, timeout)) { 3183 mdelay(100); 3184 } 3185 if (unlikely(mtip_check_surprise_removal(dd->pdev))) { 3186 timetaken = jiffies - timetaken; 3187 dev_warn(&dd->pdev->dev, 3188 "Surprise removal detected at %u ms\n", 3189 jiffies_to_msecs(timetaken)); 3190 rv = -ENODEV; 3191 goto out2 ; 3192 } 3193 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) { 3194 timetaken = jiffies - timetaken; 3195 dev_warn(&dd->pdev->dev, 3196 "Removal detected at %u ms\n", 3197 jiffies_to_msecs(timetaken)); 3198 rv = -EFAULT; 3199 goto out2; 3200 } 3201 3202 /* Conditionally reset the HBA. */ 3203 if (!(readl(dd->mmio + HOST_CAP) & HOST_CAP_NZDMA)) { 3204 if (mtip_hba_reset(dd) < 0) { 3205 dev_err(&dd->pdev->dev, 3206 "Card did not reset within timeout\n"); 3207 rv = -EIO; 3208 goto out2; 3209 } 3210 } else { 3211 /* Clear any pending interrupts on the HBA */ 3212 writel(readl(dd->mmio + HOST_IRQ_STAT), 3213 dd->mmio + HOST_IRQ_STAT); 3214 } 3215 3216 mtip_init_port(dd->port); 3217 mtip_start_port(dd->port); 3218 3219 /* Setup the ISR and enable interrupts. */ 3220 rv = devm_request_irq(&dd->pdev->dev, 3221 dd->pdev->irq, 3222 mtip_irq_handler, 3223 IRQF_SHARED, 3224 dev_driver_string(&dd->pdev->dev), 3225 dd); 3226 3227 if (rv) { 3228 dev_err(&dd->pdev->dev, 3229 "Unable to allocate IRQ %d\n", dd->pdev->irq); 3230 goto out2; 3231 } 3232 irq_set_affinity_hint(dd->pdev->irq, get_cpu_mask(dd->isr_binding)); 3233 3234 /* Enable interrupts on the HBA. */ 3235 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, 3236 dd->mmio + HOST_CTL); 3237 3238 init_waitqueue_head(&dd->port->svc_wait); 3239 3240 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { 3241 rv = -EFAULT; 3242 goto out3; 3243 } 3244 3245 return rv; 3246 3247 out3: 3248 /* Disable interrupts on the HBA. */ 3249 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, 3250 dd->mmio + HOST_CTL); 3251 3252 /* Release the IRQ. */ 3253 irq_set_affinity_hint(dd->pdev->irq, NULL); 3254 devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd); 3255 3256 out2: 3257 mtip_deinit_port(dd->port); 3258 mtip_dma_free(dd); 3259 3260 out1: 3261 /* Free the memory allocated for the for structure. */ 3262 kfree(dd->port); 3263 3264 return rv; 3265 } 3266 3267 static void mtip_standby_drive(struct driver_data *dd) 3268 { 3269 if (dd->sr) 3270 return; 3271 3272 /* 3273 * Send standby immediate (E0h) to the drive so that it 3274 * saves its state. 3275 */ 3276 if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) && 3277 !test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) 3278 if (mtip_standby_immediate(dd->port)) 3279 dev_warn(&dd->pdev->dev, 3280 "STANDBY IMMEDIATE failed\n"); 3281 } 3282 3283 /* 3284 * Called to deinitialize an interface. 3285 * 3286 * @dd Pointer to the driver data structure. 3287 * 3288 * return value 3289 * 0 3290 */ 3291 static int mtip_hw_exit(struct driver_data *dd) 3292 { 3293 /* 3294 * Send standby immediate (E0h) to the drive so that it 3295 * saves its state. 3296 */ 3297 if (!dd->sr) { 3298 /* de-initialize the port. */ 3299 mtip_deinit_port(dd->port); 3300 3301 /* Disable interrupts on the HBA. */ 3302 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, 3303 dd->mmio + HOST_CTL); 3304 } 3305 3306 /* Release the IRQ. */ 3307 irq_set_affinity_hint(dd->pdev->irq, NULL); 3308 devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd); 3309 msleep(1000); 3310 3311 /* Free dma regions */ 3312 mtip_dma_free(dd); 3313 3314 /* Free the memory allocated for the for structure. */ 3315 kfree(dd->port); 3316 dd->port = NULL; 3317 3318 return 0; 3319 } 3320 3321 /* 3322 * Issue a Standby Immediate command to the device. 3323 * 3324 * This function is called by the Block Layer just before the 3325 * system powers off during a shutdown. 3326 * 3327 * @dd Pointer to the driver data structure. 3328 * 3329 * return value 3330 * 0 3331 */ 3332 static int mtip_hw_shutdown(struct driver_data *dd) 3333 { 3334 /* 3335 * Send standby immediate (E0h) to the drive so that it 3336 * saves its state. 3337 */ 3338 if (!dd->sr && dd->port) 3339 mtip_standby_immediate(dd->port); 3340 3341 return 0; 3342 } 3343 3344 /* 3345 * Suspend function 3346 * 3347 * This function is called by the Block Layer just before the 3348 * system hibernates. 3349 * 3350 * @dd Pointer to the driver data structure. 3351 * 3352 * return value 3353 * 0 Suspend was successful 3354 * -EFAULT Suspend was not successful 3355 */ 3356 static int mtip_hw_suspend(struct driver_data *dd) 3357 { 3358 /* 3359 * Send standby immediate (E0h) to the drive 3360 * so that it saves its state. 3361 */ 3362 if (mtip_standby_immediate(dd->port) != 0) { 3363 dev_err(&dd->pdev->dev, 3364 "Failed standby-immediate command\n"); 3365 return -EFAULT; 3366 } 3367 3368 /* Disable interrupts on the HBA.*/ 3369 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, 3370 dd->mmio + HOST_CTL); 3371 mtip_deinit_port(dd->port); 3372 3373 return 0; 3374 } 3375 3376 /* 3377 * Resume function 3378 * 3379 * This function is called by the Block Layer as the 3380 * system resumes. 3381 * 3382 * @dd Pointer to the driver data structure. 3383 * 3384 * return value 3385 * 0 Resume was successful 3386 * -EFAULT Resume was not successful 3387 */ 3388 static int mtip_hw_resume(struct driver_data *dd) 3389 { 3390 /* Perform any needed hardware setup steps */ 3391 hba_setup(dd); 3392 3393 /* Reset the HBA */ 3394 if (mtip_hba_reset(dd) != 0) { 3395 dev_err(&dd->pdev->dev, 3396 "Unable to reset the HBA\n"); 3397 return -EFAULT; 3398 } 3399 3400 /* 3401 * Enable the port, DMA engine, and FIS reception specific 3402 * h/w in controller. 3403 */ 3404 mtip_init_port(dd->port); 3405 mtip_start_port(dd->port); 3406 3407 /* Enable interrupts on the HBA.*/ 3408 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, 3409 dd->mmio + HOST_CTL); 3410 3411 return 0; 3412 } 3413 3414 /* 3415 * Helper function for reusing disk name 3416 * upon hot insertion. 3417 */ 3418 static int rssd_disk_name_format(char *prefix, 3419 int index, 3420 char *buf, 3421 int buflen) 3422 { 3423 const int base = 'z' - 'a' + 1; 3424 char *begin = buf + strlen(prefix); 3425 char *end = buf + buflen; 3426 char *p; 3427 int unit; 3428 3429 p = end - 1; 3430 *p = '\0'; 3431 unit = base; 3432 do { 3433 if (p == begin) 3434 return -EINVAL; 3435 *--p = 'a' + (index % unit); 3436 index = (index / unit) - 1; 3437 } while (index >= 0); 3438 3439 memmove(begin, p, end - p); 3440 memcpy(buf, prefix, strlen(prefix)); 3441 3442 return 0; 3443 } 3444 3445 /* 3446 * Block layer IOCTL handler. 3447 * 3448 * @dev Pointer to the block_device structure. 3449 * @mode ignored 3450 * @cmd IOCTL command passed from the user application. 3451 * @arg Argument passed from the user application. 3452 * 3453 * return value 3454 * 0 IOCTL completed successfully. 3455 * -ENOTTY IOCTL not supported or invalid driver data 3456 * structure pointer. 3457 */ 3458 static int mtip_block_ioctl(struct block_device *dev, 3459 fmode_t mode, 3460 unsigned cmd, 3461 unsigned long arg) 3462 { 3463 struct driver_data *dd = dev->bd_disk->private_data; 3464 3465 if (!capable(CAP_SYS_ADMIN)) 3466 return -EACCES; 3467 3468 if (!dd) 3469 return -ENOTTY; 3470 3471 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) 3472 return -ENOTTY; 3473 3474 switch (cmd) { 3475 case BLKFLSBUF: 3476 return -ENOTTY; 3477 default: 3478 return mtip_hw_ioctl(dd, cmd, arg); 3479 } 3480 } 3481 3482 #ifdef CONFIG_COMPAT 3483 /* 3484 * Block layer compat IOCTL handler. 3485 * 3486 * @dev Pointer to the block_device structure. 3487 * @mode ignored 3488 * @cmd IOCTL command passed from the user application. 3489 * @arg Argument passed from the user application. 3490 * 3491 * return value 3492 * 0 IOCTL completed successfully. 3493 * -ENOTTY IOCTL not supported or invalid driver data 3494 * structure pointer. 3495 */ 3496 static int mtip_block_compat_ioctl(struct block_device *dev, 3497 fmode_t mode, 3498 unsigned cmd, 3499 unsigned long arg) 3500 { 3501 struct driver_data *dd = dev->bd_disk->private_data; 3502 3503 if (!capable(CAP_SYS_ADMIN)) 3504 return -EACCES; 3505 3506 if (!dd) 3507 return -ENOTTY; 3508 3509 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) 3510 return -ENOTTY; 3511 3512 switch (cmd) { 3513 case BLKFLSBUF: 3514 return -ENOTTY; 3515 case HDIO_DRIVE_TASKFILE: { 3516 struct mtip_compat_ide_task_request_s __user *compat_req_task; 3517 ide_task_request_t req_task; 3518 int compat_tasksize, outtotal, ret; 3519 3520 compat_tasksize = 3521 sizeof(struct mtip_compat_ide_task_request_s); 3522 3523 compat_req_task = 3524 (struct mtip_compat_ide_task_request_s __user *) arg; 3525 3526 if (copy_from_user(&req_task, (void __user *) arg, 3527 compat_tasksize - (2 * sizeof(compat_long_t)))) 3528 return -EFAULT; 3529 3530 if (get_user(req_task.out_size, &compat_req_task->out_size)) 3531 return -EFAULT; 3532 3533 if (get_user(req_task.in_size, &compat_req_task->in_size)) 3534 return -EFAULT; 3535 3536 outtotal = sizeof(struct mtip_compat_ide_task_request_s); 3537 3538 ret = exec_drive_taskfile(dd, (void __user *) arg, 3539 &req_task, outtotal); 3540 3541 if (copy_to_user((void __user *) arg, &req_task, 3542 compat_tasksize - 3543 (2 * sizeof(compat_long_t)))) 3544 return -EFAULT; 3545 3546 if (put_user(req_task.out_size, &compat_req_task->out_size)) 3547 return -EFAULT; 3548 3549 if (put_user(req_task.in_size, &compat_req_task->in_size)) 3550 return -EFAULT; 3551 3552 return ret; 3553 } 3554 default: 3555 return mtip_hw_ioctl(dd, cmd, arg); 3556 } 3557 } 3558 #endif 3559 3560 /* 3561 * Obtain the geometry of the device. 3562 * 3563 * You may think that this function is obsolete, but some applications, 3564 * fdisk for example still used CHS values. This function describes the 3565 * device as having 224 heads and 56 sectors per cylinder. These values are 3566 * chosen so that each cylinder is aligned on a 4KB boundary. Since a 3567 * partition is described in terms of a start and end cylinder this means 3568 * that each partition is also 4KB aligned. Non-aligned partitions adversely 3569 * affects performance. 3570 * 3571 * @dev Pointer to the block_device strucutre. 3572 * @geo Pointer to a hd_geometry structure. 3573 * 3574 * return value 3575 * 0 Operation completed successfully. 3576 * -ENOTTY An error occurred while reading the drive capacity. 3577 */ 3578 static int mtip_block_getgeo(struct block_device *dev, 3579 struct hd_geometry *geo) 3580 { 3581 struct driver_data *dd = dev->bd_disk->private_data; 3582 sector_t capacity; 3583 3584 if (!dd) 3585 return -ENOTTY; 3586 3587 if (!(mtip_hw_get_capacity(dd, &capacity))) { 3588 dev_warn(&dd->pdev->dev, 3589 "Could not get drive capacity.\n"); 3590 return -ENOTTY; 3591 } 3592 3593 geo->heads = 224; 3594 geo->sectors = 56; 3595 sector_div(capacity, (geo->heads * geo->sectors)); 3596 geo->cylinders = capacity; 3597 return 0; 3598 } 3599 3600 /* 3601 * Block device operation function. 3602 * 3603 * This structure contains pointers to the functions required by the block 3604 * layer. 3605 */ 3606 static const struct block_device_operations mtip_block_ops = { 3607 .ioctl = mtip_block_ioctl, 3608 #ifdef CONFIG_COMPAT 3609 .compat_ioctl = mtip_block_compat_ioctl, 3610 #endif 3611 .getgeo = mtip_block_getgeo, 3612 .owner = THIS_MODULE 3613 }; 3614 3615 static inline bool is_se_active(struct driver_data *dd) 3616 { 3617 if (unlikely(test_bit(MTIP_PF_SE_ACTIVE_BIT, &dd->port->flags))) { 3618 if (dd->port->ic_pause_timer) { 3619 unsigned long to = dd->port->ic_pause_timer + 3620 msecs_to_jiffies(1000); 3621 if (time_after(jiffies, to)) { 3622 clear_bit(MTIP_PF_SE_ACTIVE_BIT, 3623 &dd->port->flags); 3624 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag); 3625 dd->port->ic_pause_timer = 0; 3626 wake_up_interruptible(&dd->port->svc_wait); 3627 return false; 3628 } 3629 } 3630 return true; 3631 } 3632 return false; 3633 } 3634 3635 /* 3636 * Block layer make request function. 3637 * 3638 * This function is called by the kernel to process a BIO for 3639 * the P320 device. 3640 * 3641 * @queue Pointer to the request queue. Unused other than to obtain 3642 * the driver data structure. 3643 * @rq Pointer to the request. 3644 * 3645 */ 3646 static int mtip_submit_request(struct blk_mq_hw_ctx *hctx, struct request *rq) 3647 { 3648 struct driver_data *dd = hctx->queue->queuedata; 3649 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3650 unsigned int nents; 3651 3652 if (is_se_active(dd)) 3653 return -ENODATA; 3654 3655 if (unlikely(dd->dd_flag & MTIP_DDF_STOP_IO)) { 3656 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 3657 &dd->dd_flag))) { 3658 return -ENXIO; 3659 } 3660 if (unlikely(test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))) { 3661 return -ENODATA; 3662 } 3663 if (unlikely(test_bit(MTIP_DDF_WRITE_PROTECT_BIT, 3664 &dd->dd_flag) && 3665 rq_data_dir(rq))) { 3666 return -ENODATA; 3667 } 3668 if (unlikely(test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag))) 3669 return -ENODATA; 3670 if (test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag)) 3671 return -ENXIO; 3672 } 3673 3674 if (rq->cmd_flags & REQ_DISCARD) { 3675 int err; 3676 3677 err = mtip_send_trim(dd, blk_rq_pos(rq), blk_rq_sectors(rq)); 3678 blk_mq_end_request(rq, err); 3679 return 0; 3680 } 3681 3682 /* Create the scatter list for this request. */ 3683 nents = blk_rq_map_sg(hctx->queue, rq, cmd->sg); 3684 3685 /* Issue the read/write. */ 3686 mtip_hw_submit_io(dd, rq, cmd, nents, hctx); 3687 return 0; 3688 } 3689 3690 static bool mtip_check_unal_depth(struct blk_mq_hw_ctx *hctx, 3691 struct request *rq) 3692 { 3693 struct driver_data *dd = hctx->queue->queuedata; 3694 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3695 3696 if (rq_data_dir(rq) == READ || !dd->unal_qdepth) 3697 return false; 3698 3699 /* 3700 * If unaligned depth must be limited on this controller, mark it 3701 * as unaligned if the IO isn't on a 4k boundary (start of length). 3702 */ 3703 if (blk_rq_sectors(rq) <= 64) { 3704 if ((blk_rq_pos(rq) & 7) || (blk_rq_sectors(rq) & 7)) 3705 cmd->unaligned = 1; 3706 } 3707 3708 if (cmd->unaligned && down_trylock(&dd->port->cmd_slot_unal)) 3709 return true; 3710 3711 return false; 3712 } 3713 3714 static int mtip_queue_rq(struct blk_mq_hw_ctx *hctx, 3715 const struct blk_mq_queue_data *bd) 3716 { 3717 struct request *rq = bd->rq; 3718 int ret; 3719 3720 if (unlikely(mtip_check_unal_depth(hctx, rq))) 3721 return BLK_MQ_RQ_QUEUE_BUSY; 3722 3723 blk_mq_start_request(rq); 3724 3725 ret = mtip_submit_request(hctx, rq); 3726 if (likely(!ret)) 3727 return BLK_MQ_RQ_QUEUE_OK; 3728 3729 rq->errors = ret; 3730 return BLK_MQ_RQ_QUEUE_ERROR; 3731 } 3732 3733 static void mtip_free_cmd(void *data, struct request *rq, 3734 unsigned int hctx_idx, unsigned int request_idx) 3735 { 3736 struct driver_data *dd = data; 3737 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3738 3739 if (!cmd->command) 3740 return; 3741 3742 dmam_free_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, 3743 cmd->command, cmd->command_dma); 3744 } 3745 3746 static int mtip_init_cmd(void *data, struct request *rq, unsigned int hctx_idx, 3747 unsigned int request_idx, unsigned int numa_node) 3748 { 3749 struct driver_data *dd = data; 3750 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3751 u32 host_cap_64 = readl(dd->mmio + HOST_CAP) & HOST_CAP_64; 3752 3753 /* 3754 * For flush requests, request_idx starts at the end of the 3755 * tag space. Since we don't support FLUSH/FUA, simply return 3756 * 0 as there's nothing to be done. 3757 */ 3758 if (request_idx >= MTIP_MAX_COMMAND_SLOTS) 3759 return 0; 3760 3761 cmd->command = dmam_alloc_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, 3762 &cmd->command_dma, GFP_KERNEL); 3763 if (!cmd->command) 3764 return -ENOMEM; 3765 3766 memset(cmd->command, 0, CMD_DMA_ALLOC_SZ); 3767 3768 /* Point the command headers at the command tables. */ 3769 cmd->command_header = dd->port->command_list + 3770 (sizeof(struct mtip_cmd_hdr) * request_idx); 3771 cmd->command_header_dma = dd->port->command_list_dma + 3772 (sizeof(struct mtip_cmd_hdr) * request_idx); 3773 3774 if (host_cap_64) 3775 cmd->command_header->ctbau = __force_bit2int cpu_to_le32((cmd->command_dma >> 16) >> 16); 3776 3777 cmd->command_header->ctba = __force_bit2int cpu_to_le32(cmd->command_dma & 0xFFFFFFFF); 3778 3779 sg_init_table(cmd->sg, MTIP_MAX_SG); 3780 return 0; 3781 } 3782 3783 static struct blk_mq_ops mtip_mq_ops = { 3784 .queue_rq = mtip_queue_rq, 3785 .map_queue = blk_mq_map_queue, 3786 .init_request = mtip_init_cmd, 3787 .exit_request = mtip_free_cmd, 3788 }; 3789 3790 /* 3791 * Block layer initialization function. 3792 * 3793 * This function is called once by the PCI layer for each P320 3794 * device that is connected to the system. 3795 * 3796 * @dd Pointer to the driver data structure. 3797 * 3798 * return value 3799 * 0 on success else an error code. 3800 */ 3801 static int mtip_block_initialize(struct driver_data *dd) 3802 { 3803 int rv = 0, wait_for_rebuild = 0; 3804 sector_t capacity; 3805 unsigned int index = 0; 3806 struct kobject *kobj; 3807 3808 if (dd->disk) 3809 goto skip_create_disk; /* hw init done, before rebuild */ 3810 3811 if (mtip_hw_init(dd)) { 3812 rv = -EINVAL; 3813 goto protocol_init_error; 3814 } 3815 3816 dd->disk = alloc_disk_node(MTIP_MAX_MINORS, dd->numa_node); 3817 if (dd->disk == NULL) { 3818 dev_err(&dd->pdev->dev, 3819 "Unable to allocate gendisk structure\n"); 3820 rv = -EINVAL; 3821 goto alloc_disk_error; 3822 } 3823 3824 /* Generate the disk name, implemented same as in sd.c */ 3825 do { 3826 if (!ida_pre_get(&rssd_index_ida, GFP_KERNEL)) 3827 goto ida_get_error; 3828 3829 spin_lock(&rssd_index_lock); 3830 rv = ida_get_new(&rssd_index_ida, &index); 3831 spin_unlock(&rssd_index_lock); 3832 } while (rv == -EAGAIN); 3833 3834 if (rv) 3835 goto ida_get_error; 3836 3837 rv = rssd_disk_name_format("rssd", 3838 index, 3839 dd->disk->disk_name, 3840 DISK_NAME_LEN); 3841 if (rv) 3842 goto disk_index_error; 3843 3844 dd->disk->driverfs_dev = &dd->pdev->dev; 3845 dd->disk->major = dd->major; 3846 dd->disk->first_minor = index * MTIP_MAX_MINORS; 3847 dd->disk->minors = MTIP_MAX_MINORS; 3848 dd->disk->fops = &mtip_block_ops; 3849 dd->disk->private_data = dd; 3850 dd->index = index; 3851 3852 mtip_hw_debugfs_init(dd); 3853 3854 skip_create_disk: 3855 memset(&dd->tags, 0, sizeof(dd->tags)); 3856 dd->tags.ops = &mtip_mq_ops; 3857 dd->tags.nr_hw_queues = 1; 3858 dd->tags.queue_depth = MTIP_MAX_COMMAND_SLOTS; 3859 dd->tags.reserved_tags = 1; 3860 dd->tags.cmd_size = sizeof(struct mtip_cmd); 3861 dd->tags.numa_node = dd->numa_node; 3862 dd->tags.flags = BLK_MQ_F_SHOULD_MERGE; 3863 dd->tags.driver_data = dd; 3864 3865 rv = blk_mq_alloc_tag_set(&dd->tags); 3866 if (rv) { 3867 dev_err(&dd->pdev->dev, 3868 "Unable to allocate request queue\n"); 3869 goto block_queue_alloc_init_error; 3870 } 3871 3872 /* Allocate the request queue. */ 3873 dd->queue = blk_mq_init_queue(&dd->tags); 3874 if (IS_ERR(dd->queue)) { 3875 dev_err(&dd->pdev->dev, 3876 "Unable to allocate request queue\n"); 3877 rv = -ENOMEM; 3878 goto block_queue_alloc_init_error; 3879 } 3880 3881 dd->disk->queue = dd->queue; 3882 dd->queue->queuedata = dd; 3883 3884 /* Initialize the protocol layer. */ 3885 wait_for_rebuild = mtip_hw_get_identify(dd); 3886 if (wait_for_rebuild < 0) { 3887 dev_err(&dd->pdev->dev, 3888 "Protocol layer initialization failed\n"); 3889 rv = -EINVAL; 3890 goto init_hw_cmds_error; 3891 } 3892 3893 /* 3894 * if rebuild pending, start the service thread, and delay the block 3895 * queue creation and add_disk() 3896 */ 3897 if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC) 3898 goto start_service_thread; 3899 3900 /* Set device limits. */ 3901 set_bit(QUEUE_FLAG_NONROT, &dd->queue->queue_flags); 3902 clear_bit(QUEUE_FLAG_ADD_RANDOM, &dd->queue->queue_flags); 3903 blk_queue_max_segments(dd->queue, MTIP_MAX_SG); 3904 blk_queue_physical_block_size(dd->queue, 4096); 3905 blk_queue_max_hw_sectors(dd->queue, 0xffff); 3906 blk_queue_max_segment_size(dd->queue, 0x400000); 3907 blk_queue_io_min(dd->queue, 4096); 3908 blk_queue_bounce_limit(dd->queue, dd->pdev->dma_mask); 3909 3910 /* 3911 * write back cache is not supported in the device. FUA depends on 3912 * write back cache support, hence setting flush support to zero. 3913 */ 3914 blk_queue_flush(dd->queue, 0); 3915 3916 /* Signal trim support */ 3917 if (dd->trim_supp == true) { 3918 set_bit(QUEUE_FLAG_DISCARD, &dd->queue->queue_flags); 3919 dd->queue->limits.discard_granularity = 4096; 3920 blk_queue_max_discard_sectors(dd->queue, 3921 MTIP_MAX_TRIM_ENTRY_LEN * MTIP_MAX_TRIM_ENTRIES); 3922 dd->queue->limits.discard_zeroes_data = 0; 3923 } 3924 3925 /* Set the capacity of the device in 512 byte sectors. */ 3926 if (!(mtip_hw_get_capacity(dd, &capacity))) { 3927 dev_warn(&dd->pdev->dev, 3928 "Could not read drive capacity\n"); 3929 rv = -EIO; 3930 goto read_capacity_error; 3931 } 3932 set_capacity(dd->disk, capacity); 3933 3934 /* Enable the block device and add it to /dev */ 3935 add_disk(dd->disk); 3936 3937 dd->bdev = bdget_disk(dd->disk, 0); 3938 /* 3939 * Now that the disk is active, initialize any sysfs attributes 3940 * managed by the protocol layer. 3941 */ 3942 kobj = kobject_get(&disk_to_dev(dd->disk)->kobj); 3943 if (kobj) { 3944 mtip_hw_sysfs_init(dd, kobj); 3945 kobject_put(kobj); 3946 } 3947 3948 if (dd->mtip_svc_handler) { 3949 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); 3950 return rv; /* service thread created for handling rebuild */ 3951 } 3952 3953 start_service_thread: 3954 dd->mtip_svc_handler = kthread_create_on_node(mtip_service_thread, 3955 dd, dd->numa_node, 3956 "mtip_svc_thd_%02d", index); 3957 3958 if (IS_ERR(dd->mtip_svc_handler)) { 3959 dev_err(&dd->pdev->dev, "service thread failed to start\n"); 3960 dd->mtip_svc_handler = NULL; 3961 rv = -EFAULT; 3962 goto kthread_run_error; 3963 } 3964 wake_up_process(dd->mtip_svc_handler); 3965 if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC) 3966 rv = wait_for_rebuild; 3967 3968 return rv; 3969 3970 kthread_run_error: 3971 bdput(dd->bdev); 3972 dd->bdev = NULL; 3973 3974 /* Delete our gendisk. This also removes the device from /dev */ 3975 del_gendisk(dd->disk); 3976 3977 read_capacity_error: 3978 init_hw_cmds_error: 3979 blk_cleanup_queue(dd->queue); 3980 blk_mq_free_tag_set(&dd->tags); 3981 block_queue_alloc_init_error: 3982 mtip_hw_debugfs_exit(dd); 3983 disk_index_error: 3984 spin_lock(&rssd_index_lock); 3985 ida_remove(&rssd_index_ida, index); 3986 spin_unlock(&rssd_index_lock); 3987 3988 ida_get_error: 3989 put_disk(dd->disk); 3990 3991 alloc_disk_error: 3992 mtip_hw_exit(dd); /* De-initialize the protocol layer. */ 3993 3994 protocol_init_error: 3995 return rv; 3996 } 3997 3998 /* 3999 * Block layer deinitialization function. 4000 * 4001 * Called by the PCI layer as each P320 device is removed. 4002 * 4003 * @dd Pointer to the driver data structure. 4004 * 4005 * return value 4006 * 0 4007 */ 4008 static int mtip_block_remove(struct driver_data *dd) 4009 { 4010 struct kobject *kobj; 4011 4012 mtip_hw_debugfs_exit(dd); 4013 4014 if (dd->mtip_svc_handler) { 4015 set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags); 4016 wake_up_interruptible(&dd->port->svc_wait); 4017 kthread_stop(dd->mtip_svc_handler); 4018 } 4019 4020 /* Clean up the sysfs attributes, if created */ 4021 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) { 4022 kobj = kobject_get(&disk_to_dev(dd->disk)->kobj); 4023 if (kobj) { 4024 mtip_hw_sysfs_exit(dd, kobj); 4025 kobject_put(kobj); 4026 } 4027 } 4028 4029 if (!dd->sr) 4030 mtip_standby_drive(dd); 4031 else 4032 dev_info(&dd->pdev->dev, "device %s surprise removal\n", 4033 dd->disk->disk_name); 4034 4035 /* 4036 * Delete our gendisk structure. This also removes the device 4037 * from /dev 4038 */ 4039 if (dd->bdev) { 4040 bdput(dd->bdev); 4041 dd->bdev = NULL; 4042 } 4043 if (dd->disk) { 4044 del_gendisk(dd->disk); 4045 if (dd->disk->queue) { 4046 blk_cleanup_queue(dd->queue); 4047 blk_mq_free_tag_set(&dd->tags); 4048 dd->queue = NULL; 4049 } 4050 put_disk(dd->disk); 4051 } 4052 dd->disk = NULL; 4053 4054 spin_lock(&rssd_index_lock); 4055 ida_remove(&rssd_index_ida, dd->index); 4056 spin_unlock(&rssd_index_lock); 4057 4058 /* De-initialize the protocol layer. */ 4059 mtip_hw_exit(dd); 4060 4061 return 0; 4062 } 4063 4064 /* 4065 * Function called by the PCI layer when just before the 4066 * machine shuts down. 4067 * 4068 * If a protocol layer shutdown function is present it will be called 4069 * by this function. 4070 * 4071 * @dd Pointer to the driver data structure. 4072 * 4073 * return value 4074 * 0 4075 */ 4076 static int mtip_block_shutdown(struct driver_data *dd) 4077 { 4078 mtip_hw_shutdown(dd); 4079 4080 /* Delete our gendisk structure, and cleanup the blk queue. */ 4081 if (dd->disk) { 4082 dev_info(&dd->pdev->dev, 4083 "Shutting down %s ...\n", dd->disk->disk_name); 4084 4085 del_gendisk(dd->disk); 4086 if (dd->disk->queue) { 4087 blk_cleanup_queue(dd->queue); 4088 blk_mq_free_tag_set(&dd->tags); 4089 } 4090 put_disk(dd->disk); 4091 dd->disk = NULL; 4092 dd->queue = NULL; 4093 } 4094 4095 spin_lock(&rssd_index_lock); 4096 ida_remove(&rssd_index_ida, dd->index); 4097 spin_unlock(&rssd_index_lock); 4098 return 0; 4099 } 4100 4101 static int mtip_block_suspend(struct driver_data *dd) 4102 { 4103 dev_info(&dd->pdev->dev, 4104 "Suspending %s ...\n", dd->disk->disk_name); 4105 mtip_hw_suspend(dd); 4106 return 0; 4107 } 4108 4109 static int mtip_block_resume(struct driver_data *dd) 4110 { 4111 dev_info(&dd->pdev->dev, "Resuming %s ...\n", 4112 dd->disk->disk_name); 4113 mtip_hw_resume(dd); 4114 return 0; 4115 } 4116 4117 static void drop_cpu(int cpu) 4118 { 4119 cpu_use[cpu]--; 4120 } 4121 4122 static int get_least_used_cpu_on_node(int node) 4123 { 4124 int cpu, least_used_cpu, least_cnt; 4125 const struct cpumask *node_mask; 4126 4127 node_mask = cpumask_of_node(node); 4128 least_used_cpu = cpumask_first(node_mask); 4129 least_cnt = cpu_use[least_used_cpu]; 4130 cpu = least_used_cpu; 4131 4132 for_each_cpu(cpu, node_mask) { 4133 if (cpu_use[cpu] < least_cnt) { 4134 least_used_cpu = cpu; 4135 least_cnt = cpu_use[cpu]; 4136 } 4137 } 4138 cpu_use[least_used_cpu]++; 4139 return least_used_cpu; 4140 } 4141 4142 /* Helper for selecting a node in round robin mode */ 4143 static inline int mtip_get_next_rr_node(void) 4144 { 4145 static int next_node = -1; 4146 4147 if (next_node == -1) { 4148 next_node = first_online_node; 4149 return next_node; 4150 } 4151 4152 next_node = next_online_node(next_node); 4153 if (next_node == MAX_NUMNODES) 4154 next_node = first_online_node; 4155 return next_node; 4156 } 4157 4158 static DEFINE_HANDLER(0); 4159 static DEFINE_HANDLER(1); 4160 static DEFINE_HANDLER(2); 4161 static DEFINE_HANDLER(3); 4162 static DEFINE_HANDLER(4); 4163 static DEFINE_HANDLER(5); 4164 static DEFINE_HANDLER(6); 4165 static DEFINE_HANDLER(7); 4166 4167 static void mtip_disable_link_opts(struct driver_data *dd, struct pci_dev *pdev) 4168 { 4169 int pos; 4170 unsigned short pcie_dev_ctrl; 4171 4172 pos = pci_find_capability(pdev, PCI_CAP_ID_EXP); 4173 if (pos) { 4174 pci_read_config_word(pdev, 4175 pos + PCI_EXP_DEVCTL, 4176 &pcie_dev_ctrl); 4177 if (pcie_dev_ctrl & (1 << 11) || 4178 pcie_dev_ctrl & (1 << 4)) { 4179 dev_info(&dd->pdev->dev, 4180 "Disabling ERO/No-Snoop on bridge device %04x:%04x\n", 4181 pdev->vendor, pdev->device); 4182 pcie_dev_ctrl &= ~(PCI_EXP_DEVCTL_NOSNOOP_EN | 4183 PCI_EXP_DEVCTL_RELAX_EN); 4184 pci_write_config_word(pdev, 4185 pos + PCI_EXP_DEVCTL, 4186 pcie_dev_ctrl); 4187 } 4188 } 4189 } 4190 4191 static void mtip_fix_ero_nosnoop(struct driver_data *dd, struct pci_dev *pdev) 4192 { 4193 /* 4194 * This workaround is specific to AMD/ATI chipset with a PCI upstream 4195 * device with device id 0x5aXX 4196 */ 4197 if (pdev->bus && pdev->bus->self) { 4198 if (pdev->bus->self->vendor == PCI_VENDOR_ID_ATI && 4199 ((pdev->bus->self->device & 0xff00) == 0x5a00)) { 4200 mtip_disable_link_opts(dd, pdev->bus->self); 4201 } else { 4202 /* Check further up the topology */ 4203 struct pci_dev *parent_dev = pdev->bus->self; 4204 if (parent_dev->bus && 4205 parent_dev->bus->parent && 4206 parent_dev->bus->parent->self && 4207 parent_dev->bus->parent->self->vendor == 4208 PCI_VENDOR_ID_ATI && 4209 (parent_dev->bus->parent->self->device & 4210 0xff00) == 0x5a00) { 4211 mtip_disable_link_opts(dd, 4212 parent_dev->bus->parent->self); 4213 } 4214 } 4215 } 4216 } 4217 4218 /* 4219 * Called for each supported PCI device detected. 4220 * 4221 * This function allocates the private data structure, enables the 4222 * PCI device and then calls the block layer initialization function. 4223 * 4224 * return value 4225 * 0 on success else an error code. 4226 */ 4227 static int mtip_pci_probe(struct pci_dev *pdev, 4228 const struct pci_device_id *ent) 4229 { 4230 int rv = 0; 4231 struct driver_data *dd = NULL; 4232 char cpu_list[256]; 4233 const struct cpumask *node_mask; 4234 int cpu, i = 0, j = 0; 4235 int my_node = NUMA_NO_NODE; 4236 unsigned long flags; 4237 4238 /* Allocate memory for this devices private data. */ 4239 my_node = pcibus_to_node(pdev->bus); 4240 if (my_node != NUMA_NO_NODE) { 4241 if (!node_online(my_node)) 4242 my_node = mtip_get_next_rr_node(); 4243 } else { 4244 dev_info(&pdev->dev, "Kernel not reporting proximity, choosing a node\n"); 4245 my_node = mtip_get_next_rr_node(); 4246 } 4247 dev_info(&pdev->dev, "NUMA node %d (closest: %d,%d, probe on %d:%d)\n", 4248 my_node, pcibus_to_node(pdev->bus), dev_to_node(&pdev->dev), 4249 cpu_to_node(raw_smp_processor_id()), raw_smp_processor_id()); 4250 4251 dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node); 4252 if (dd == NULL) { 4253 dev_err(&pdev->dev, 4254 "Unable to allocate memory for driver data\n"); 4255 return -ENOMEM; 4256 } 4257 4258 /* Attach the private data to this PCI device. */ 4259 pci_set_drvdata(pdev, dd); 4260 4261 rv = pcim_enable_device(pdev); 4262 if (rv < 0) { 4263 dev_err(&pdev->dev, "Unable to enable device\n"); 4264 goto iomap_err; 4265 } 4266 4267 /* Map BAR5 to memory. */ 4268 rv = pcim_iomap_regions(pdev, 1 << MTIP_ABAR, MTIP_DRV_NAME); 4269 if (rv < 0) { 4270 dev_err(&pdev->dev, "Unable to map regions\n"); 4271 goto iomap_err; 4272 } 4273 4274 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { 4275 rv = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 4276 4277 if (rv) { 4278 rv = pci_set_consistent_dma_mask(pdev, 4279 DMA_BIT_MASK(32)); 4280 if (rv) { 4281 dev_warn(&pdev->dev, 4282 "64-bit DMA enable failed\n"); 4283 goto setmask_err; 4284 } 4285 } 4286 } 4287 4288 /* Copy the info we may need later into the private data structure. */ 4289 dd->major = mtip_major; 4290 dd->instance = instance; 4291 dd->pdev = pdev; 4292 dd->numa_node = my_node; 4293 4294 INIT_LIST_HEAD(&dd->online_list); 4295 INIT_LIST_HEAD(&dd->remove_list); 4296 4297 memset(dd->workq_name, 0, 32); 4298 snprintf(dd->workq_name, 31, "mtipq%d", dd->instance); 4299 4300 dd->isr_workq = create_workqueue(dd->workq_name); 4301 if (!dd->isr_workq) { 4302 dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance); 4303 rv = -ENOMEM; 4304 goto block_initialize_err; 4305 } 4306 4307 memset(cpu_list, 0, sizeof(cpu_list)); 4308 4309 node_mask = cpumask_of_node(dd->numa_node); 4310 if (!cpumask_empty(node_mask)) { 4311 for_each_cpu(cpu, node_mask) 4312 { 4313 snprintf(&cpu_list[j], 256 - j, "%d ", cpu); 4314 j = strlen(cpu_list); 4315 } 4316 4317 dev_info(&pdev->dev, "Node %d on package %d has %d cpu(s): %s\n", 4318 dd->numa_node, 4319 topology_physical_package_id(cpumask_first(node_mask)), 4320 nr_cpus_node(dd->numa_node), 4321 cpu_list); 4322 } else 4323 dev_dbg(&pdev->dev, "mtip32xx: node_mask empty\n"); 4324 4325 dd->isr_binding = get_least_used_cpu_on_node(dd->numa_node); 4326 dev_info(&pdev->dev, "Initial IRQ binding node:cpu %d:%d\n", 4327 cpu_to_node(dd->isr_binding), dd->isr_binding); 4328 4329 /* first worker context always runs in ISR */ 4330 dd->work[0].cpu_binding = dd->isr_binding; 4331 dd->work[1].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); 4332 dd->work[2].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); 4333 dd->work[3].cpu_binding = dd->work[0].cpu_binding; 4334 dd->work[4].cpu_binding = dd->work[1].cpu_binding; 4335 dd->work[5].cpu_binding = dd->work[2].cpu_binding; 4336 dd->work[6].cpu_binding = dd->work[2].cpu_binding; 4337 dd->work[7].cpu_binding = dd->work[1].cpu_binding; 4338 4339 /* Log the bindings */ 4340 for_each_present_cpu(cpu) { 4341 memset(cpu_list, 0, sizeof(cpu_list)); 4342 for (i = 0, j = 0; i < MTIP_MAX_SLOT_GROUPS; i++) { 4343 if (dd->work[i].cpu_binding == cpu) { 4344 snprintf(&cpu_list[j], 256 - j, "%d ", i); 4345 j = strlen(cpu_list); 4346 } 4347 } 4348 if (j) 4349 dev_info(&pdev->dev, "CPU %d: WQs %s\n", cpu, cpu_list); 4350 } 4351 4352 INIT_WORK(&dd->work[0].work, mtip_workq_sdbf0); 4353 INIT_WORK(&dd->work[1].work, mtip_workq_sdbf1); 4354 INIT_WORK(&dd->work[2].work, mtip_workq_sdbf2); 4355 INIT_WORK(&dd->work[3].work, mtip_workq_sdbf3); 4356 INIT_WORK(&dd->work[4].work, mtip_workq_sdbf4); 4357 INIT_WORK(&dd->work[5].work, mtip_workq_sdbf5); 4358 INIT_WORK(&dd->work[6].work, mtip_workq_sdbf6); 4359 INIT_WORK(&dd->work[7].work, mtip_workq_sdbf7); 4360 4361 pci_set_master(pdev); 4362 rv = pci_enable_msi(pdev); 4363 if (rv) { 4364 dev_warn(&pdev->dev, 4365 "Unable to enable MSI interrupt.\n"); 4366 goto msi_initialize_err; 4367 } 4368 4369 mtip_fix_ero_nosnoop(dd, pdev); 4370 4371 /* Initialize the block layer. */ 4372 rv = mtip_block_initialize(dd); 4373 if (rv < 0) { 4374 dev_err(&pdev->dev, 4375 "Unable to initialize block layer\n"); 4376 goto block_initialize_err; 4377 } 4378 4379 /* 4380 * Increment the instance count so that each device has a unique 4381 * instance number. 4382 */ 4383 instance++; 4384 if (rv != MTIP_FTL_REBUILD_MAGIC) 4385 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); 4386 else 4387 rv = 0; /* device in rebuild state, return 0 from probe */ 4388 4389 /* Add to online list even if in ftl rebuild */ 4390 spin_lock_irqsave(&dev_lock, flags); 4391 list_add(&dd->online_list, &online_list); 4392 spin_unlock_irqrestore(&dev_lock, flags); 4393 4394 goto done; 4395 4396 block_initialize_err: 4397 pci_disable_msi(pdev); 4398 4399 msi_initialize_err: 4400 if (dd->isr_workq) { 4401 flush_workqueue(dd->isr_workq); 4402 destroy_workqueue(dd->isr_workq); 4403 drop_cpu(dd->work[0].cpu_binding); 4404 drop_cpu(dd->work[1].cpu_binding); 4405 drop_cpu(dd->work[2].cpu_binding); 4406 } 4407 setmask_err: 4408 pcim_iounmap_regions(pdev, 1 << MTIP_ABAR); 4409 4410 iomap_err: 4411 kfree(dd); 4412 pci_set_drvdata(pdev, NULL); 4413 return rv; 4414 done: 4415 return rv; 4416 } 4417 4418 /* 4419 * Called for each probed device when the device is removed or the 4420 * driver is unloaded. 4421 * 4422 * return value 4423 * None 4424 */ 4425 static void mtip_pci_remove(struct pci_dev *pdev) 4426 { 4427 struct driver_data *dd = pci_get_drvdata(pdev); 4428 unsigned long flags, to; 4429 4430 set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); 4431 4432 spin_lock_irqsave(&dev_lock, flags); 4433 list_del_init(&dd->online_list); 4434 list_add(&dd->remove_list, &removing_list); 4435 spin_unlock_irqrestore(&dev_lock, flags); 4436 4437 mtip_check_surprise_removal(pdev); 4438 synchronize_irq(dd->pdev->irq); 4439 4440 /* Spin until workers are done */ 4441 to = jiffies + msecs_to_jiffies(4000); 4442 do { 4443 msleep(20); 4444 } while (atomic_read(&dd->irq_workers_active) != 0 && 4445 time_before(jiffies, to)); 4446 4447 if (atomic_read(&dd->irq_workers_active) != 0) { 4448 dev_warn(&dd->pdev->dev, 4449 "Completion workers still active!\n"); 4450 } 4451 4452 blk_mq_stop_hw_queues(dd->queue); 4453 /* Clean up the block layer. */ 4454 mtip_block_remove(dd); 4455 4456 if (dd->isr_workq) { 4457 flush_workqueue(dd->isr_workq); 4458 destroy_workqueue(dd->isr_workq); 4459 drop_cpu(dd->work[0].cpu_binding); 4460 drop_cpu(dd->work[1].cpu_binding); 4461 drop_cpu(dd->work[2].cpu_binding); 4462 } 4463 4464 pci_disable_msi(pdev); 4465 4466 spin_lock_irqsave(&dev_lock, flags); 4467 list_del_init(&dd->remove_list); 4468 spin_unlock_irqrestore(&dev_lock, flags); 4469 4470 kfree(dd); 4471 4472 pcim_iounmap_regions(pdev, 1 << MTIP_ABAR); 4473 pci_set_drvdata(pdev, NULL); 4474 } 4475 4476 /* 4477 * Called for each probed device when the device is suspended. 4478 * 4479 * return value 4480 * 0 Success 4481 * <0 Error 4482 */ 4483 static int mtip_pci_suspend(struct pci_dev *pdev, pm_message_t mesg) 4484 { 4485 int rv = 0; 4486 struct driver_data *dd = pci_get_drvdata(pdev); 4487 4488 if (!dd) { 4489 dev_err(&pdev->dev, 4490 "Driver private datastructure is NULL\n"); 4491 return -EFAULT; 4492 } 4493 4494 set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); 4495 4496 /* Disable ports & interrupts then send standby immediate */ 4497 rv = mtip_block_suspend(dd); 4498 if (rv < 0) { 4499 dev_err(&pdev->dev, 4500 "Failed to suspend controller\n"); 4501 return rv; 4502 } 4503 4504 /* 4505 * Save the pci config space to pdev structure & 4506 * disable the device 4507 */ 4508 pci_save_state(pdev); 4509 pci_disable_device(pdev); 4510 4511 /* Move to Low power state*/ 4512 pci_set_power_state(pdev, PCI_D3hot); 4513 4514 return rv; 4515 } 4516 4517 /* 4518 * Called for each probed device when the device is resumed. 4519 * 4520 * return value 4521 * 0 Success 4522 * <0 Error 4523 */ 4524 static int mtip_pci_resume(struct pci_dev *pdev) 4525 { 4526 int rv = 0; 4527 struct driver_data *dd; 4528 4529 dd = pci_get_drvdata(pdev); 4530 if (!dd) { 4531 dev_err(&pdev->dev, 4532 "Driver private datastructure is NULL\n"); 4533 return -EFAULT; 4534 } 4535 4536 /* Move the device to active State */ 4537 pci_set_power_state(pdev, PCI_D0); 4538 4539 /* Restore PCI configuration space */ 4540 pci_restore_state(pdev); 4541 4542 /* Enable the PCI device*/ 4543 rv = pcim_enable_device(pdev); 4544 if (rv < 0) { 4545 dev_err(&pdev->dev, 4546 "Failed to enable card during resume\n"); 4547 goto err; 4548 } 4549 pci_set_master(pdev); 4550 4551 /* 4552 * Calls hbaReset, initPort, & startPort function 4553 * then enables interrupts 4554 */ 4555 rv = mtip_block_resume(dd); 4556 if (rv < 0) 4557 dev_err(&pdev->dev, "Unable to resume\n"); 4558 4559 err: 4560 clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); 4561 4562 return rv; 4563 } 4564 4565 /* 4566 * Shutdown routine 4567 * 4568 * return value 4569 * None 4570 */ 4571 static void mtip_pci_shutdown(struct pci_dev *pdev) 4572 { 4573 struct driver_data *dd = pci_get_drvdata(pdev); 4574 if (dd) 4575 mtip_block_shutdown(dd); 4576 } 4577 4578 /* Table of device ids supported by this driver. */ 4579 static const struct pci_device_id mtip_pci_tbl[] = { 4580 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320H_DEVICE_ID) }, 4581 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320M_DEVICE_ID) }, 4582 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320S_DEVICE_ID) }, 4583 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P325M_DEVICE_ID) }, 4584 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420H_DEVICE_ID) }, 4585 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420M_DEVICE_ID) }, 4586 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P425M_DEVICE_ID) }, 4587 { 0 } 4588 }; 4589 4590 /* Structure that describes the PCI driver functions. */ 4591 static struct pci_driver mtip_pci_driver = { 4592 .name = MTIP_DRV_NAME, 4593 .id_table = mtip_pci_tbl, 4594 .probe = mtip_pci_probe, 4595 .remove = mtip_pci_remove, 4596 .suspend = mtip_pci_suspend, 4597 .resume = mtip_pci_resume, 4598 .shutdown = mtip_pci_shutdown, 4599 }; 4600 4601 MODULE_DEVICE_TABLE(pci, mtip_pci_tbl); 4602 4603 /* 4604 * Module initialization function. 4605 * 4606 * Called once when the module is loaded. This function allocates a major 4607 * block device number to the Cyclone devices and registers the PCI layer 4608 * of the driver. 4609 * 4610 * Return value 4611 * 0 on success else error code. 4612 */ 4613 static int __init mtip_init(void) 4614 { 4615 int error; 4616 4617 pr_info(MTIP_DRV_NAME " Version " MTIP_DRV_VERSION "\n"); 4618 4619 spin_lock_init(&dev_lock); 4620 4621 INIT_LIST_HEAD(&online_list); 4622 INIT_LIST_HEAD(&removing_list); 4623 4624 /* Allocate a major block device number to use with this driver. */ 4625 error = register_blkdev(0, MTIP_DRV_NAME); 4626 if (error <= 0) { 4627 pr_err("Unable to register block device (%d)\n", 4628 error); 4629 return -EBUSY; 4630 } 4631 mtip_major = error; 4632 4633 dfs_parent = debugfs_create_dir("rssd", NULL); 4634 if (IS_ERR_OR_NULL(dfs_parent)) { 4635 pr_warn("Error creating debugfs parent\n"); 4636 dfs_parent = NULL; 4637 } 4638 if (dfs_parent) { 4639 dfs_device_status = debugfs_create_file("device_status", 4640 S_IRUGO, dfs_parent, NULL, 4641 &mtip_device_status_fops); 4642 if (IS_ERR_OR_NULL(dfs_device_status)) { 4643 pr_err("Error creating device_status node\n"); 4644 dfs_device_status = NULL; 4645 } 4646 } 4647 4648 /* Register our PCI operations. */ 4649 error = pci_register_driver(&mtip_pci_driver); 4650 if (error) { 4651 debugfs_remove(dfs_parent); 4652 unregister_blkdev(mtip_major, MTIP_DRV_NAME); 4653 } 4654 4655 return error; 4656 } 4657 4658 /* 4659 * Module de-initialization function. 4660 * 4661 * Called once when the module is unloaded. This function deallocates 4662 * the major block device number allocated by mtip_init() and 4663 * unregisters the PCI layer of the driver. 4664 * 4665 * Return value 4666 * none 4667 */ 4668 static void __exit mtip_exit(void) 4669 { 4670 /* Release the allocated major block device number. */ 4671 unregister_blkdev(mtip_major, MTIP_DRV_NAME); 4672 4673 /* Unregister the PCI driver. */ 4674 pci_unregister_driver(&mtip_pci_driver); 4675 4676 debugfs_remove_recursive(dfs_parent); 4677 } 4678 4679 MODULE_AUTHOR("Micron Technology, Inc"); 4680 MODULE_DESCRIPTION("Micron RealSSD PCIe Block Driver"); 4681 MODULE_LICENSE("GPL"); 4682 MODULE_VERSION(MTIP_DRV_VERSION); 4683 4684 module_init(mtip_init); 4685 module_exit(mtip_exit); 4686 4687 4688 4689 4690 4691 /* LDV_COMMENT_BEGIN_MAIN */ 4692 #ifdef LDV_MAIN0_sequence_infinite_withcheck_stateful 4693 4694 /*###########################################################################*/ 4695 4696 /*############## Driver Environment Generator 0.2 output ####################*/ 4697 4698 /*###########################################################################*/ 4699 4700 4701 4702 /* LDV_COMMENT_FUNCTION_DECLARE_LDV Special function for LDV verifier. Test if all kernel resources are correctly released by driver before driver will be unloaded. */ 4703 void ldv_check_final_state(void); 4704 4705 /* LDV_COMMENT_FUNCTION_DECLARE_LDV Special function for LDV verifier. Test correct return result. */ 4706 void ldv_check_return_value(int res); 4707 4708 /* LDV_COMMENT_FUNCTION_DECLARE_LDV Special function for LDV verifier. Test correct return result of probe() function. */ 4709 void ldv_check_return_value_probe(int res); 4710 4711 /* LDV_COMMENT_FUNCTION_DECLARE_LDV Special function for LDV verifier. Initializes the model. */ 4712 void ldv_initialize(void); 4713 4714 /* LDV_COMMENT_FUNCTION_DECLARE_LDV Special function for LDV verifier. Reinitializes the model between distinct model function calls. */ 4715 void ldv_handler_precall(void); 4716 4717 /* LDV_COMMENT_FUNCTION_DECLARE_LDV Special function for LDV verifier. Returns arbitrary interger value. */ 4718 int nondet_int(void); 4719 4720 /* LDV_COMMENT_VAR_DECLARE_LDV Special variable for LDV verifier. */ 4721 int LDV_IN_INTERRUPT; 4722 4723 /* LDV_COMMENT_FUNCTION_MAIN Main function for LDV verifier. */ 4724 void ldv_main0_sequence_infinite_withcheck_stateful(void) { 4725 4726 4727 4728 /* LDV_COMMENT_BEGIN_VARIABLE_DECLARATION_PART */ 4729 /*============================= VARIABLE DECLARATION PART =============================*/ 4730 /** STRUCT: struct type: file_operations, struct name: mtip_device_status_fops **/ 4731 /* content: static ssize_t mtip_hw_read_device_status(struct file *f, char __user *ubuf, size_t len, loff_t *offset)*/ 4732 /* LDV_COMMENT_BEGIN_PREP */ 4733 #define HW_CMD_SLOT_SZ (MTIP_MAX_COMMAND_SLOTS * 32) 4734 #define AHCI_RX_FIS_SZ 0x100 4735 #define AHCI_RX_FIS_OFFSET 0x0 4736 #define AHCI_IDFY_SZ ATA_SECT_SIZE 4737 #define AHCI_IDFY_OFFSET 0x400 4738 #define AHCI_SECTBUF_SZ ATA_SECT_SIZE 4739 #define AHCI_SECTBUF_OFFSET 0x800 4740 #define AHCI_SMARTBUF_SZ ATA_SECT_SIZE 4741 #define AHCI_SMARTBUF_OFFSET 0xC00 4742 #define BLOCK_DMA_ALLOC_SZ 4096 4743 #define AHCI_CMD_SLOT_SZ sizeof(struct mtip_cmd_hdr) 4744 #define AHCI_CMD_TBL_SZ (MTIP_MAX_COMMAND_SLOTS * AHCI_CMD_SLOT_SZ) 4745 #define AHCI_CMD_TBL_OFFSET 0x0 4746 #define AHCI_CMD_TBL_HDR_SZ 0x80 4747 #define AHCI_CMD_TBL_HDR_OFFSET 0x0 4748 #define AHCI_CMD_TBL_SGL_SZ (MTIP_MAX_SG * sizeof(struct mtip_cmd_sg)) 4749 #define AHCI_CMD_TBL_SGL_OFFSET AHCI_CMD_TBL_HDR_SZ 4750 #define CMD_DMA_ALLOC_SZ (AHCI_CMD_TBL_SGL_SZ + AHCI_CMD_TBL_HDR_SZ) 4751 #define HOST_CAP_NZDMA (1 << 19) 4752 #define HOST_HSORG 0xFC 4753 #define HSORG_DISABLE_SLOTGRP_INTR (1<<24) 4754 #define HSORG_DISABLE_SLOTGRP_PXIS (1<<16) 4755 #define HSORG_HWREV 0xFF00 4756 #define HSORG_STYLE 0x8 4757 #define HSORG_SLOTGROUPS 0x7 4758 #define PORT_COMMAND_ISSUE 0x38 4759 #define PORT_SDBV 0x7C 4760 #define PORT_OFFSET 0x100 4761 #define PORT_MEM_SIZE 0x80 4762 #define PORT_IRQ_ERR \ 4763 (PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR | PORT_IRQ_CONNECT | \ 4764 PORT_IRQ_PHYRDY | PORT_IRQ_UNK_FIS | PORT_IRQ_BAD_PMP | \ 4765 PORT_IRQ_TF_ERR | PORT_IRQ_HBUS_DATA_ERR | PORT_IRQ_IF_NONFATAL | \ 4766 PORT_IRQ_OVERFLOW) 4767 #define PORT_IRQ_LEGACY \ 4768 (PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS) 4769 #define PORT_IRQ_HANDLED \ 4770 (PORT_IRQ_SDB_FIS | PORT_IRQ_LEGACY | \ 4771 PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR | \ 4772 PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY) 4773 #define DEF_PORT_IRQ \ 4774 (PORT_IRQ_ERR | PORT_IRQ_LEGACY | PORT_IRQ_SDB_FIS) 4775 #define MTIP_PRODUCT_UNKNOWN 0x00 4776 #define MTIP_PRODUCT_ASICFPGA 0x11 4777 #ifdef CONFIG_COMPAT 4778 #endif 4779 #ifdef __LITTLE_ENDIAN 4780 #else 4781 #endif 4782 #ifdef MTIP_TRIM 4783 #endif 4784 /* LDV_COMMENT_END_PREP */ 4785 /* LDV_COMMENT_VAR_DECLARE Variable declaration for function "mtip_hw_read_device_status" */ 4786 struct file * var_group1; 4787 /* LDV_COMMENT_VAR_DECLARE Variable declaration for function "mtip_hw_read_device_status" */ 4788 char __user * var_mtip_hw_read_device_status_47_p1; 4789 /* LDV_COMMENT_VAR_DECLARE Variable declaration for function "mtip_hw_read_device_status" */ 4790 size_t var_mtip_hw_read_device_status_47_p2; 4791 /* LDV_COMMENT_VAR_DECLARE Variable declaration for function "mtip_hw_read_device_status" */ 4792 loff_t * var_mtip_hw_read_device_status_47_p3; 4793 /* LDV_COMMENT_VAR_DECLARE Variable declaration for test return result from function call "mtip_hw_read_device_status" */ 4794 static ssize_t res_mtip_hw_read_device_status_47; 4795 /* LDV_COMMENT_BEGIN_PREP */ 4796 #ifdef CONFIG_COMPAT 4797 #endif 4798 #ifdef CONFIG_COMPAT 4799 #endif 4800 /* LDV_COMMENT_END_PREP */ 4801 4802 /** STRUCT: struct type: file_operations, struct name: mtip_regs_fops **/ 4803 /* content: static ssize_t mtip_hw_read_registers(struct file *f, char __user *ubuf, size_t len, loff_t *offset)*/ 4804 /* LDV_COMMENT_BEGIN_PREP */ 4805 #define HW_CMD_SLOT_SZ (MTIP_MAX_COMMAND_SLOTS * 32) 4806 #define AHCI_RX_FIS_SZ 0x100 4807 #define AHCI_RX_FIS_OFFSET 0x0 4808 #define AHCI_IDFY_SZ ATA_SECT_SIZE 4809 #define AHCI_IDFY_OFFSET 0x400 4810 #define AHCI_SECTBUF_SZ ATA_SECT_SIZE 4811 #define AHCI_SECTBUF_OFFSET 0x800 4812 #define AHCI_SMARTBUF_SZ ATA_SECT_SIZE 4813 #define AHCI_SMARTBUF_OFFSET 0xC00 4814 #define BLOCK_DMA_ALLOC_SZ 4096 4815 #define AHCI_CMD_SLOT_SZ sizeof(struct mtip_cmd_hdr) 4816 #define AHCI_CMD_TBL_SZ (MTIP_MAX_COMMAND_SLOTS * AHCI_CMD_SLOT_SZ) 4817 #define AHCI_CMD_TBL_OFFSET 0x0 4818 #define AHCI_CMD_TBL_HDR_SZ 0x80 4819 #define AHCI_CMD_TBL_HDR_OFFSET 0x0 4820 #define AHCI_CMD_TBL_SGL_SZ (MTIP_MAX_SG * sizeof(struct mtip_cmd_sg)) 4821 #define AHCI_CMD_TBL_SGL_OFFSET AHCI_CMD_TBL_HDR_SZ 4822 #define CMD_DMA_ALLOC_SZ (AHCI_CMD_TBL_SGL_SZ + AHCI_CMD_TBL_HDR_SZ) 4823 #define HOST_CAP_NZDMA (1 << 19) 4824 #define HOST_HSORG 0xFC 4825 #define HSORG_DISABLE_SLOTGRP_INTR (1<<24) 4826 #define HSORG_DISABLE_SLOTGRP_PXIS (1<<16) 4827 #define HSORG_HWREV 0xFF00 4828 #define HSORG_STYLE 0x8 4829 #define HSORG_SLOTGROUPS 0x7 4830 #define PORT_COMMAND_ISSUE 0x38 4831 #define PORT_SDBV 0x7C 4832 #define PORT_OFFSET 0x100 4833 #define PORT_MEM_SIZE 0x80 4834 #define PORT_IRQ_ERR \ 4835 (PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR | PORT_IRQ_CONNECT | \ 4836 PORT_IRQ_PHYRDY | PORT_IRQ_UNK_FIS | PORT_IRQ_BAD_PMP | \ 4837 PORT_IRQ_TF_ERR | PORT_IRQ_HBUS_DATA_ERR | PORT_IRQ_IF_NONFATAL | \ 4838 PORT_IRQ_OVERFLOW) 4839 #define PORT_IRQ_LEGACY \ 4840 (PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS) 4841 #define PORT_IRQ_HANDLED \ 4842 (PORT_IRQ_SDB_FIS | PORT_IRQ_LEGACY | \ 4843 PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR | \ 4844 PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY) 4845 #define DEF_PORT_IRQ \ 4846 (PORT_IRQ_ERR | PORT_IRQ_LEGACY | PORT_IRQ_SDB_FIS) 4847 #define MTIP_PRODUCT_UNKNOWN 0x00 4848 #define MTIP_PRODUCT_ASICFPGA 0x11 4849 #ifdef CONFIG_COMPAT 4850 #endif 4851 #ifdef __LITTLE_ENDIAN 4852 #else 4853 #endif 4854 #ifdef MTIP_TRIM 4855 #endif 4856 /* LDV_COMMENT_END_PREP */ 4857 /* LDV_COMMENT_VAR_DECLARE Variable declaration for function "mtip_hw_read_registers" */ 4858 char __user * var_mtip_hw_read_registers_48_p1; 4859 /* LDV_COMMENT_VAR_DECLARE Variable declaration for function "mtip_hw_read_registers" */ 4860 size_t var_mtip_hw_read_registers_48_p2; 4861 /* LDV_COMMENT_VAR_DECLARE Variable declaration for function "mtip_hw_read_registers" */ 4862 loff_t * var_mtip_hw_read_registers_48_p3; 4863 /* LDV_COMMENT_VAR_DECLARE Variable declaration for test return result from function call "mtip_hw_read_registers" */ 4864 static ssize_t res_mtip_hw_read_registers_48; 4865 /* LDV_COMMENT_BEGIN_PREP */ 4866 #ifdef CONFIG_COMPAT 4867 #endif 4868 #ifdef CONFIG_COMPAT 4869 #endif 4870 /* LDV_COMMENT_END_PREP */ 4871 4872 /** STRUCT: struct type: file_operations, struct name: mtip_flags_fops **/ 4873 /* content: static ssize_t mtip_hw_read_flags(struct file *f, char __user *ubuf, size_t len, loff_t *offset)*/ 4874 /* LDV_COMMENT_BEGIN_PREP */ 4875 #define HW_CMD_SLOT_SZ (MTIP_MAX_COMMAND_SLOTS * 32) 4876 #define AHCI_RX_FIS_SZ 0x100 4877 #define AHCI_RX_FIS_OFFSET 0x0 4878 #define AHCI_IDFY_SZ ATA_SECT_SIZE 4879 #define AHCI_IDFY_OFFSET 0x400 4880 #define AHCI_SECTBUF_SZ ATA_SECT_SIZE 4881 #define AHCI_SECTBUF_OFFSET 0x800 4882 #define AHCI_SMARTBUF_SZ ATA_SECT_SIZE 4883 #define AHCI_SMARTBUF_OFFSET 0xC00 4884 #define BLOCK_DMA_ALLOC_SZ 4096 4885 #define AHCI_CMD_SLOT_SZ sizeof(struct mtip_cmd_hdr) 4886 #define AHCI_CMD_TBL_SZ (MTIP_MAX_COMMAND_SLOTS * AHCI_CMD_SLOT_SZ) 4887 #define AHCI_CMD_TBL_OFFSET 0x0 4888 #define AHCI_CMD_TBL_HDR_SZ 0x80 4889 #define AHCI_CMD_TBL_HDR_OFFSET 0x0 4890 #define AHCI_CMD_TBL_SGL_SZ (MTIP_MAX_SG * sizeof(struct mtip_cmd_sg)) 4891 #define AHCI_CMD_TBL_SGL_OFFSET AHCI_CMD_TBL_HDR_SZ 4892 #define CMD_DMA_ALLOC_SZ (AHCI_CMD_TBL_SGL_SZ + AHCI_CMD_TBL_HDR_SZ) 4893 #define HOST_CAP_NZDMA (1 << 19) 4894 #define HOST_HSORG 0xFC 4895 #define HSORG_DISABLE_SLOTGRP_INTR (1<<24) 4896 #define HSORG_DISABLE_SLOTGRP_PXIS (1<<16) 4897 #define HSORG_HWREV 0xFF00 4898 #define HSORG_STYLE 0x8 4899 #define HSORG_SLOTGROUPS 0x7 4900 #define PORT_COMMAND_ISSUE 0x38 4901 #define PORT_SDBV 0x7C 4902 #define PORT_OFFSET 0x100 4903 #define PORT_MEM_SIZE 0x80